database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Fix](Load) Disable for the developer to import fast json in fe (#16235)

Browse Source
This commit is contained in:
huangzhaowei
2023-02-01 16:32:11 +08:00
committed by GitHub
parent bf16228851
commit 4e92f63d7b
4 changed files with 28 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fe/check/checkstyle/import-control.xml
View File

@ -30,6 +30,7 @@ under the License.
<disallow pkg="io.fabric8.zjsonpatch.internal.guava" />
<disallow pkg="org.checkerframework.com.google" />
<disallow pkg="org.apache.iceberg.relocated" />
<disallow pkg="com.alibaba.fastjson2" />
<subpackage name="nereids">
<allow pkg="org.junit.jupiter"/>
<disallow pkg="org.junit"/>

15
fe/fe-core/src/main/java/org/apache/doris/load/loadv2/MysqlLoadManager.java
View File

@ -31,9 +31,9 @@ import org.apache.doris.system.Backend;
import org.apache.doris.system.BeSelectionPolicy;
import org.apache.doris.system.SystemInfoService;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONObject;
import com.google.common.base.Joiner;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.ContentType;
@ -78,13 +78,14 @@ public class MysqlLoadManager {
InputStreamEntity entity = getInputStreamEntity(context, dataDesc.isClientLocal(), file);
HttpPut request = generateRequestForMySqlLoad(entity, dataDesc, database, table);
try (final CloseableHttpResponse response = httpclient.execute(request)) {
JSONObject result = JSON.parseObject(EntityUtils.toString(response.getEntity()));
if (!result.getString("Status").equalsIgnoreCase("Success")) {
JsonObject result = JsonParser.parseString(EntityUtils.toString(response.getEntity()))
.getAsJsonObject();
if (!result.get("Status").getAsString().equalsIgnoreCase("Success")) {
LOG.warn("Execute stream load for mysql data load failed with message: " + request);
throw new LoadException(result.getString("Message"));
throw new LoadException(result.get("Message").getAsString());
}
loadResult.incRecords(result.getLong("NumberLoadedRows"));
loadResult.incSkipped(result.getIntValue("NumberFilteredRows"));
loadResult.incRecords(result.get("NumberLoadedRows").getAsLong());
loadResult.incSkipped(result.get("NumberFilteredRows").getAsInt());
}
}
}

6
fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java
View File

@ -73,6 +73,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
@ -584,7 +585,10 @@ public class TabletRepairAndBalanceTest {
try {
tbl.checkReplicaAllocation();
break;
} catch (UserException e) {
} catch (UserException | NoSuchElementException e) {
// Why do we add no such element exception because hash map is not a thread safe struct.
// In this ut using a big loop to iterate the hash map,
// it will increase the probability of map to throw NoSuchElementException exception.
System.out.println(e.getMessage());
}
Thread.sleep(1000);

28
fe/fe-core/src/test/java/org/apache/doris/statistics/HistogramTest.java
View File

@ -23,9 +23,9 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.statistics.util.StatisticsUtil;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@ -95,18 +95,18 @@ class HistogramTest {
@Test
void testSerializeToJson() throws AnalysisException {
String json = Histogram.serializeToJson(histogramUnderTest);
JSONObject histogramJson = JSON.parseObject(json);
JsonObject histogramJson = JsonParser.parseString(json).getAsJsonObject();
int maxBucketSize = histogramJson.getIntValue("max_bucket_num");
int maxBucketSize = histogramJson.get("max_bucket_num").getAsInt();
Assertions.assertEquals(128, maxBucketSize);
int bucketSize = histogramJson.getIntValue("bucket_num");
int bucketSize = histogramJson.get("bucket_num").getAsInt();
Assertions.assertEquals(5, bucketSize);
float sampleRate = histogramJson.getFloat("sample_rate");
float sampleRate = histogramJson.get("sample_rate").getAsFloat();
Assertions.assertEquals(1.0, sampleRate);
JSONArray jsonArray = histogramJson.getJSONArray("buckets");
JsonArray jsonArray = histogramJson.get("buckets").getAsJsonArray();
Assertions.assertEquals(5, jsonArray.size());
// test first bucket
@ -118,13 +118,13 @@ class HistogramTest {
boolean flag = false;
for (int i = 0; i < jsonArray.size(); i++) {
JSONObject bucketJson = jsonArray.getJSONObject(i);
JsonObject bucketJson = jsonArray.get(i).getAsJsonObject();
assert datatype != null;
LiteralExpr lower = StatisticsUtil.readableValue(datatype, bucketJson.get("lower").toString());
LiteralExpr upper = StatisticsUtil.readableValue(datatype, bucketJson.get("upper").toString());
int count = bucketJson.getIntValue("count");
int preSum = bucketJson.getIntValue("pre_sum");
int ndv = bucketJson.getIntValue("ndv");
LiteralExpr lower = StatisticsUtil.readableValue(datatype, bucketJson.get("lower").getAsString());
LiteralExpr upper = StatisticsUtil.readableValue(datatype, bucketJson.get("upper").getAsString());
int count = bucketJson.get("count").getAsInt();
int preSum = bucketJson.get("pre_sum").getAsInt();
int ndv = bucketJson.get("ndv").getAsInt();
if (expectedLower.equals(lower) && expectedUpper.equals(upper) && count == 9 && preSum == 0 && ndv == 1) {
flag = true;
break;