From 4e92f63d7bdc29864d1cc03c69a600a009bd60bd Mon Sep 17 00:00:00 2001 From: huangzhaowei Date: Wed, 1 Feb 2023 16:32:11 +0800 Subject: [PATCH] [Fix](Load) Disable for the developer to import fast json in fe (#16235) --- fe/check/checkstyle/import-control.xml | 1 + .../doris/load/loadv2/MysqlLoadManager.java | 15 +++++----- .../clone/TabletRepairAndBalanceTest.java | 6 +++- .../doris/statistics/HistogramTest.java | 28 +++++++++---------- 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/fe/check/checkstyle/import-control.xml b/fe/check/checkstyle/import-control.xml index 53c524c288..f38810a364 100644 --- a/fe/check/checkstyle/import-control.xml +++ b/fe/check/checkstyle/import-control.xml @@ -30,6 +30,7 @@ under the License. + diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MysqlLoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MysqlLoadManager.java index 44d9aaf139..a394ee34d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MysqlLoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MysqlLoadManager.java @@ -31,9 +31,9 @@ import org.apache.doris.system.Backend; import org.apache.doris.system.BeSelectionPolicy; import org.apache.doris.system.SystemInfoService; -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONObject; import com.google.common.base.Joiner; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPut; import org.apache.http.entity.ContentType; @@ -78,13 +78,14 @@ public class MysqlLoadManager { InputStreamEntity entity = getInputStreamEntity(context, dataDesc.isClientLocal(), file); HttpPut request = generateRequestForMySqlLoad(entity, dataDesc, database, table); try (final CloseableHttpResponse response = httpclient.execute(request)) { - JSONObject result = JSON.parseObject(EntityUtils.toString(response.getEntity())); - if (!result.getString("Status").equalsIgnoreCase("Success")) { + JsonObject result = JsonParser.parseString(EntityUtils.toString(response.getEntity())) + .getAsJsonObject(); + if (!result.get("Status").getAsString().equalsIgnoreCase("Success")) { LOG.warn("Execute stream load for mysql data load failed with message: " + request); - throw new LoadException(result.getString("Message")); + throw new LoadException(result.get("Message").getAsString()); } - loadResult.incRecords(result.getLong("NumberLoadedRows")); - loadResult.incSkipped(result.getIntValue("NumberFilteredRows")); + loadResult.incRecords(result.get("NumberLoadedRows").getAsLong()); + loadResult.incSkipped(result.get("NumberFilteredRows").getAsInt()); } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java b/fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java index 658b9d8ba1..3bb3c641ca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java @@ -73,6 +73,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -584,7 +585,10 @@ public class TabletRepairAndBalanceTest { try { tbl.checkReplicaAllocation(); break; - } catch (UserException e) { + } catch (UserException | NoSuchElementException e) { + // Why do we add no such element exception because hash map is not a thread safe struct. + // In this ut using a big loop to iterate the hash map, + // it will increase the probability of map to throw NoSuchElementException exception. System.out.println(e.getMessage()); } Thread.sleep(1000); diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/HistogramTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/HistogramTest.java index d115b6c5ec..219256c4ec 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/HistogramTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/HistogramTest.java @@ -23,9 +23,9 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.statistics.util.StatisticsUtil; -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONArray; -import com.alibaba.fastjson2.JSONObject; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -95,18 +95,18 @@ class HistogramTest { @Test void testSerializeToJson() throws AnalysisException { String json = Histogram.serializeToJson(histogramUnderTest); - JSONObject histogramJson = JSON.parseObject(json); + JsonObject histogramJson = JsonParser.parseString(json).getAsJsonObject(); - int maxBucketSize = histogramJson.getIntValue("max_bucket_num"); + int maxBucketSize = histogramJson.get("max_bucket_num").getAsInt(); Assertions.assertEquals(128, maxBucketSize); - int bucketSize = histogramJson.getIntValue("bucket_num"); + int bucketSize = histogramJson.get("bucket_num").getAsInt(); Assertions.assertEquals(5, bucketSize); - float sampleRate = histogramJson.getFloat("sample_rate"); + float sampleRate = histogramJson.get("sample_rate").getAsFloat(); Assertions.assertEquals(1.0, sampleRate); - JSONArray jsonArray = histogramJson.getJSONArray("buckets"); + JsonArray jsonArray = histogramJson.get("buckets").getAsJsonArray(); Assertions.assertEquals(5, jsonArray.size()); // test first bucket @@ -118,13 +118,13 @@ class HistogramTest { boolean flag = false; for (int i = 0; i < jsonArray.size(); i++) { - JSONObject bucketJson = jsonArray.getJSONObject(i); + JsonObject bucketJson = jsonArray.get(i).getAsJsonObject(); assert datatype != null; - LiteralExpr lower = StatisticsUtil.readableValue(datatype, bucketJson.get("lower").toString()); - LiteralExpr upper = StatisticsUtil.readableValue(datatype, bucketJson.get("upper").toString()); - int count = bucketJson.getIntValue("count"); - int preSum = bucketJson.getIntValue("pre_sum"); - int ndv = bucketJson.getIntValue("ndv"); + LiteralExpr lower = StatisticsUtil.readableValue(datatype, bucketJson.get("lower").getAsString()); + LiteralExpr upper = StatisticsUtil.readableValue(datatype, bucketJson.get("upper").getAsString()); + int count = bucketJson.get("count").getAsInt(); + int preSum = bucketJson.get("pre_sum").getAsInt(); + int ndv = bucketJson.get("ndv").getAsInt(); if (expectedLower.equals(lower) && expectedUpper.equals(upper) && count == 9 && preSum == 0 && ndv == 1) { flag = true; break;