diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/io/ByteBufferNetworkInputStream.java b/fe/fe-common/src/main/java/org/apache/doris/common/io/ByteBufferNetworkInputStream.java index 41bac05329..15d61cdd1a 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/io/ByteBufferNetworkInputStream.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/io/ByteBufferNetworkInputStream.java @@ -42,10 +42,16 @@ public class ByteBufferNetworkInputStream extends InputStream { if (closed) { throw new IOException("Stream is already closed."); } - ByteArrayInputStream inputStream = new ByteArrayInputStream(buffer.array(), buffer.position(), buffer.limit()); + ByteArrayInputStream inputStream = new ByteArrayInputStream(bytesCopy(buffer)); queue.offer(inputStream, 300, TimeUnit.SECONDS); } + public byte[] bytesCopy(ByteBuffer buffer) { + byte[] result = new byte[buffer.limit() - buffer.position()]; + System.arraycopy(buffer.array(), buffer.position(), result, 0, result.length); + return result; + } + public void markFinished() { this.finished = true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MysqlLoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MysqlLoadManager.java index c0bef652f2..30a25cbeb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MysqlLoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MysqlLoadManager.java @@ -81,10 +81,10 @@ public class MysqlLoadManager { InputStreamEntity entity = getInputStreamEntity(context, dataDesc.isClientLocal(), file); HttpPut request = generateRequestForMySqlLoad(entity, dataDesc, database, table, token); try (final CloseableHttpResponse response = httpclient.execute(request)) { - JsonObject result = JsonParser.parseString(EntityUtils.toString(response.getEntity())) - .getAsJsonObject(); + String body = EntityUtils.toString(response.getEntity()); + JsonObject result = JsonParser.parseString(body).getAsJsonObject(); if (!result.get("Status").getAsString().equalsIgnoreCase("Success")) { - LOG.warn("Execute stream load for mysql data load failed with message: " + request); + LOG.warn("Execute mysql data load failed with request: {} and response: {}", request, body); throw new LoadException(result.get("Message").getAsString()); } loadResult.incRecords(result.get("NumberLoadedRows").getAsLong()); diff --git a/regression-test/data/load_p0/mysql_load/test_mysql_load_big_file.csv b/regression-test/data/load_p0/mysql_load/test_mysql_load_big_file.csv new file mode 100644 index 0000000000..3bdc7c1aef --- /dev/null +++ b/regression-test/data/load_p0/mysql_load/test_mysql_load_big_file.csv @@ -0,0 +1,416 @@ +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 +1,2,3,abc,2022-12-01,2022-12-01:09:30:31 +2,3,3,abc,2022-12-01,2022-12-01:09:30:31 +3,4,3,abc,2022-12-01,2022-12-01:09:30:31 +4,5,3,abc,2022-12-01,2022-12-01:09:30:31 diff --git a/regression-test/data/load_p0/mysql_load/test_mysql_load_big_file.out b/regression-test/data/load_p0/mysql_load/test_mysql_load_big_file.out new file mode 100644 index 0000000000..7805ad6982 --- /dev/null +++ b/regression-test/data/load_p0/mysql_load/test_mysql_load_big_file.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 2 104 +2 3 104 +3 4 104 +4 5 104 + diff --git a/regression-test/suites/load_p0/mysql_load/test_mysql_load_big_file.groovy b/regression-test/suites/load_p0/mysql_load/test_mysql_load_big_file.groovy new file mode 100644 index 0000000000..2017326bda --- /dev/null +++ b/regression-test/suites/load_p0/mysql_load/test_mysql_load_big_file.groovy @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_mysql_load_big_file", "p0") { + sql "show tables" + + def tableName = "test_mysql_load_big_file" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(20) NULL, + `k2` bigint(20) NULL, + `v1` tinyint(4) NULL, + `v2` string NULL, + `v3` date NULL, + `v4` datetime NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 3 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + // If file size is bigger than 16384(16 * 1024), mysql client will spilt it into two packets. In this test, the file size is 17056. + def mysql_load_skip_lines = getLoalFilePath "test_mysql_load_big_file.csv" + + // no any skip + sql """ + LOAD DATA + LOCAL + INFILE '${mysql_load_skip_lines}' + INTO TABLE ${tableName} + COLUMNS TERMINATED BY ','; + """ + + sql "sync" + qt_sql "select k1,k2,count(*) from ${tableName} group by k1, k2 order by k1, k2" +} +