From a1c6ea876fbe9519674c522c7977cb115cc17a7d Mon Sep 17 00:00:00 2001 From: luozenglin <37725793+luozenglin@users.noreply.github.com> Date: Mon, 26 Dec 2022 23:14:32 +0800 Subject: [PATCH] [fix](inbitmap) fix core dump caused by bitmap filter with union (#15333) The join node need project operation to remove unnecessary columns from the output tuples. For SetOperationNode output tuple and input tuple is consistent and do not need project, but the children of SetOperationNode may be join nodes, so the children of the SetOperationNode need to do the project operation. --- .../doris/analysis/BitmapFilterPredicate.java | 5 ++++ .../apache/doris/planner/OriginalPlanner.java | 2 +- .../doris/planner/SetOperationNode.java | 23 +++++++++++++++++++ .../data/query_p0/join/test_bitmap_filter.out | 16 +++++++++++++ .../query_p0/join/test_bitmap_filter.groovy | 5 ++++ 5 files changed, 50 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BitmapFilterPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BitmapFilterPredicate.java index 1802f0917b..8e6b703977 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BitmapFilterPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BitmapFilterPredicate.java @@ -77,6 +77,11 @@ public class BitmapFilterPredicate extends Predicate { + "Please `set runtime_filter_type = 'xxx, bitmap_filter'` first."); } + if (ConnectContext.get() == null || !ConnectContext.get().getSessionVariable().isEnableProjection()) { + throw new AnalysisException( + "Please enable the session variable 'enable_projection' through `set enable_projection = true;`"); + } + if (!VectorizedUtil.isVectorized()) { throw new AnalysisException("In bitmap syntax is currently only supported in the vectorization engine."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java index 7d7595dc7c..6f6050b190 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java @@ -153,7 +153,7 @@ public class OriginalPlanner extends Planner { if (analyzer.getContext() != null && analyzer.getContext().getSessionVariable().isEnableProjection() - && statement instanceof SelectStmt) { + && statement instanceof QueryStmt) { ProjectPlanner projectPlanner = new ProjectPlanner(analyzer); projectPlanner.projectSingleNodePlan(queryStmt.getResultExprs(), singleNodePlan); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java index cfa32316b7..7c365f7e13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java @@ -20,6 +20,7 @@ package org.apache.doris.planner; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.SlotDescriptor; +import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; @@ -38,12 +39,14 @@ import org.apache.doris.thrift.TUnionNode; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; /** @@ -453,6 +456,26 @@ public abstract class SetOperationNode extends PlanNode { return numInstances; } + public void initOutputSlotIds(Set requiredSlotIdSet, Analyzer analyzer) { + } + + public void projectOutputTuple() { + } + + public Set computeInputSlotIds(Analyzer analyzer) { + Set results = Sets.newHashSet(); + for (int i = 0; i < resultExprLists.size(); ++i) { + List substituteList = + Expr.substituteList(resultExprLists.get(i), children.get(i).getOutputSmap(), analyzer, true); + for (Expr expr : substituteList) { + List slotIdList = Lists.newArrayList(); + expr.getIds(null, slotIdList); + results.addAll(slotIdList); + } + } + return results; + } + /** * just for Nereids. */ diff --git a/regression-test/data/query_p0/join/test_bitmap_filter.out b/regression-test/data/query_p0/join/test_bitmap_filter.out index bf8be1dca2..9484c6770d 100644 --- a/regression-test/data/query_p0/join/test_bitmap_filter.out +++ b/regression-test/data/query_p0/join/test_bitmap_filter.out @@ -75,3 +75,19 @@ 2015-04-02 2015-04-02 +-- !sql12 -- +1 +3 +5 +7 +9 +10 +11 +12 +13 +14 +255 +1985 +1991 +32767 + diff --git a/regression-test/suites/query_p0/join/test_bitmap_filter.groovy b/regression-test/suites/query_p0/join/test_bitmap_filter.groovy index bb09505889..788f26ec3a 100644 --- a/regression-test/suites/query_p0/join/test_bitmap_filter.groovy +++ b/regression-test/suites/query_p0/join/test_bitmap_filter.groovy @@ -60,6 +60,11 @@ suite("test_bitmap_filter", "query_p0") { qt_sql11 "select k10 from ${tbl1} where cast(k10 as bigint) in (select bitmap_or(k2, to_bitmap(20120314)) from ${tbl2} b) order by 1;" + qt_sql12 """ + with w1 as (select k1 from ${tbl1} where k1 in (select k2 from ${tbl2})), w2 as (select k2 from ${tbl1} where k2 in (select k3 from ${tbl2})) + select * from (select * from w1 union select * from w2) tmp order by 1; + """ + test { sql "select k1, k2 from ${tbl1} b1 where k1 in (select k2 from ${tbl2} b2 where b1.k2 = b2.k1) order by k1;" exception "In bitmap does not support correlated subquery"