From cc9d3404006684a57fedf0738cd2d3e20b2b66f9 Mon Sep 17 00:00:00 2001 From: LiBinfeng <46676950+LiBinfeng-01@users.noreply.github.com> Date: Wed, 17 May 2023 15:09:00 +0800 Subject: [PATCH] [Fix](Nereids) Fix minidump connect context loading and concurrency bug (#19578) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are two problems of mini dump: 1、minidump do not load connect context to ThreadInfo, so it can not be get easily 2、minidump write maps with not concurrent protection, so the map size with change when we iterating map iterator Solution: 1、loading connect context to minidump threading 2、use immutable map copy a new map before we actually doing iteration --- .../java/org/apache/doris/catalog/ColocateTableIndex.java | 3 ++- .../main/java/org/apache/doris/nereids/minidump/Minidump.java | 3 ++- .../java/org/apache/doris/nereids/minidump/MinidumpUtils.java | 3 ++- .../java/org/apache/doris/nereids/stats/StatsCalculator.java | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java index e0e327c2c3..82c6f04183 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java @@ -31,6 +31,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.HashBasedTable; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; @@ -654,7 +655,7 @@ public class ColocateTableIndex implements Writable { public void write(DataOutput out) throws IOException { int size = groupName2Id.size(); out.writeInt(size); - for (Map.Entry entry : groupName2Id.entrySet()) { + for (Map.Entry entry : ImmutableMap.copyOf(groupName2Id).entrySet()) { Text.writeString(out, entry.getKey()); // group name entry.getValue().write(out); // group id Collection tableIds = group2Tables.get(entry.getValue()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java index 44bd848fb0..0185da1a38 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java @@ -141,12 +141,13 @@ public class Minidump { connectContext.getSessionVariable().setDumpNereids(false); connectContext.setDatabase(minidump.getDbName()); connectContext.getSessionVariable().setPlanNereidsDump(true); - connectContext.getSessionVariable().enableNereidsTimeout = true; + connectContext.getSessionVariable().enableNereidsTimeout = false; connectContext.getSessionVariable().setEnableNereidsPlanner(true); connectContext.getSessionVariable().setEnableNereidsTrace(false); connectContext.getSessionVariable().setNereidsTraceEventMode("all"); connectContext.getTotalColumnStatisticMap().putAll(minidump.getTotalColumnStatisticMap()); connectContext.getTotalHistogramMap().putAll(minidump.getTotalHistogramMap()); + connectContext.setThreadLocalInfo(); Env.getCurrentEnv().setColocateTableIndex(minidump.getColocateTableIndex()); NereidsParser nereidsParser = new NereidsParser(); LogicalPlan parsed = nereidsParser.parseSingle(minidump.getSql()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java index ebb7a957ed..1fcfae0952 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java @@ -24,6 +24,7 @@ import org.apache.doris.qe.SessionVariable; import org.apache.doris.statistics.ColumnStatistic; import org.apache.doris.statistics.Histogram; +import com.google.common.collect.ImmutableMap; import org.json.JSONArray; import org.json.JSONObject; @@ -159,7 +160,7 @@ public class MinidumpUtils { */ public static JSONArray serializeColumnStatistic(Map totalColumnStatisticMap) { JSONArray columnStatistics = new JSONArray(); - for (Map.Entry entry : totalColumnStatisticMap.entrySet()) { + for (Map.Entry entry : ImmutableMap.copyOf(totalColumnStatisticMap).entrySet()) { ColumnStatistic columnStatistic = entry.getValue(); String colName = entry.getKey(); JSONObject oneColumnStats = new JSONObject(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index 2ee3d6cb89..95cedf3be9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -506,10 +506,10 @@ public class StatsCalculator extends DefaultPlanVisitor { new ColumnStatisticBuilder(cache).setHistogram(histogram); columnStatisticMap.put(slotReference, columnStatisticBuilder.build()); cache = columnStatisticBuilder.build(); - totalHistogramMap.put(table.getName() + colName, histogram); + totalHistogramMap.put(table.getName() + ":" + colName, histogram); } columnStatisticMap.put(slotReference, cache); - totalColumnStatisticMap.put(table.getName() + colName, cache); + totalColumnStatisticMap.put(table.getName() + ":" + colName, cache); totalHistogramMap.put(table.getName() + colName, histogram); } return new Statistics(rowCount, columnStatisticMap);