database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Fix](Nereids) Fix minidump connect context loading and concurrency bug (#19578)

Browse Source 
There are two problems of mini dump:
1、minidump do not load connect context to ThreadInfo, so it can not be get easily
2、minidump write maps with not concurrent protection, so the map size with change when we iterating map iterator

Solution:
1、loading connect context to minidump threading
2、use immutable map copy a new map before we actually doing iteration
This commit is contained in:
LiBinfeng
2023-05-17 15:09:00 +08:00
committed by GitHub
parent 3e661a30c2
commit cc9d340400
4 changed files with 8 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java
View File

@ -31,6 +31,7 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.HashBasedTable;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
@ -654,7 +655,7 @@ public class ColocateTableIndex implements Writable {
public void write(DataOutput out) throws IOException {
int size = groupName2Id.size();
out.writeInt(size);
for (Map.Entry<String, GroupId> entry : groupName2Id.entrySet()) {
for (Map.Entry<String, GroupId> entry : ImmutableMap.copyOf(groupName2Id).entrySet()) {
Text.writeString(out, entry.getKey()); // group name
entry.getValue().write(out); // group id
Collection<Long> tableIds = group2Tables.get(entry.getValue());

3
fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java
View File

@ -141,12 +141,13 @@ public class Minidump {
connectContext.getSessionVariable().setDumpNereids(false);
connectContext.setDatabase(minidump.getDbName());
connectContext.getSessionVariable().setPlanNereidsDump(true);
connectContext.getSessionVariable().enableNereidsTimeout = true;
connectContext.getSessionVariable().enableNereidsTimeout = false;
connectContext.getSessionVariable().setEnableNereidsPlanner(true);
connectContext.getSessionVariable().setEnableNereidsTrace(false);
connectContext.getSessionVariable().setNereidsTraceEventMode("all");
connectContext.getTotalColumnStatisticMap().putAll(minidump.getTotalColumnStatisticMap());
connectContext.getTotalHistogramMap().putAll(minidump.getTotalHistogramMap());
connectContext.setThreadLocalInfo();
Env.getCurrentEnv().setColocateTableIndex(minidump.getColocateTableIndex());
NereidsParser nereidsParser = new NereidsParser();
LogicalPlan parsed = nereidsParser.parseSingle(minidump.getSql());

3
fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.qe.SessionVariable;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.Histogram;
import com.google.common.collect.ImmutableMap;
import org.json.JSONArray;
import org.json.JSONObject;
@ -159,7 +160,7 @@ public class MinidumpUtils {
*/
public static JSONArray serializeColumnStatistic(Map<String, ColumnStatistic> totalColumnStatisticMap) {
JSONArray columnStatistics = new JSONArray();
for (Map.Entry<String, ColumnStatistic> entry : totalColumnStatisticMap.entrySet()) {
for (Map.Entry<String, ColumnStatistic> entry : ImmutableMap.copyOf(totalColumnStatisticMap).entrySet()) {
ColumnStatistic columnStatistic = entry.getValue();
String colName = entry.getKey();
JSONObject oneColumnStats = new JSONObject();

4
fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
View File

@ -506,10 +506,10 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
new ColumnStatisticBuilder(cache).setHistogram(histogram);
columnStatisticMap.put(slotReference, columnStatisticBuilder.build());
cache = columnStatisticBuilder.build();
totalHistogramMap.put(table.getName() + colName, histogram);
totalHistogramMap.put(table.getName() + ":" + colName, histogram);
}
columnStatisticMap.put(slotReference, cache);
totalColumnStatisticMap.put(table.getName() + colName, cache);
totalColumnStatisticMap.put(table.getName() + ":" + colName, cache);
totalHistogramMap.put(table.getName() + colName, histogram);
}
return new Statistics(rowCount, columnStatisticMap);