From a89477e8b5e8adb70c03a5e553a2ef62c2a7d804 Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Fri, 3 Nov 2023 14:24:01 +0800 Subject: [PATCH] [fix](Nereids) could not run query with repeat node in cte (#26330) ExpressionDeepCopier not process VirtualReference, so we generate inline plan with mistake. --- .../trees/copier/ExpressionDeepCopier.java | 30 +++++++++++++++++++ .../nereids/trees/plans/algebra/Repeat.java | 14 ++------- .../data/nereids_syntax_p0/cte.out | 11 +++++++ .../suites/nereids_syntax_p0/cte.groovy | 7 ++++- 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/ExpressionDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/ExpressionDeepCopier.java index 0038b9e4cc..84cdb9cbaa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/ExpressionDeepCopier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/ExpressionDeepCopier.java @@ -26,9 +26,15 @@ import org.apache.doris.nereids.trees.expressions.ListQuery; import org.apache.doris.nereids.trees.expressions.ScalarSubquery; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.expressions.VirtualSlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.trees.plans.algebra.Repeat.GroupingSetShapes; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import com.google.common.base.Function; + import java.util.List; import java.util.Map; import java.util.Optional; @@ -75,6 +81,30 @@ public class ExpressionDeepCopier extends DefaultExpressionRewriter exprIdReplaceMap = context.exprIdReplaceMap; + ExprId newExprId; + if (exprIdReplaceMap.containsKey(virtualSlotReference.getExprId())) { + newExprId = exprIdReplaceMap.get(virtualSlotReference.getExprId()); + } else { + newExprId = StatementScopeIdGenerator.newExprId(); + } + // according to VirtualReference generating logic in Repeat.java + // generateVirtualGroupingIdSlot and generateVirtualSlotByFunction + Optional newOriginExpression = virtualSlotReference.getOriginExpression() + .map(func -> (GroupingScalarFunction) func.accept(this, context)); + Function> newFunction = newOriginExpression + .>>map(f -> f::computeVirtualSlotValue) + .orElseGet(() -> GroupingSetShapes::computeVirtualGroupingIdValue); + VirtualSlotReference newOne = new VirtualSlotReference(newExprId, + virtualSlotReference.getName(), virtualSlotReference.getDataType(), + virtualSlotReference.nullable(), virtualSlotReference.getQualifier(), + newOriginExpression, newFunction); + exprIdReplaceMap.put(virtualSlotReference.getExprId(), newOne.getExprId()); + return newOne; + } + @Override public Expression visitExistsSubquery(Exists exists, DeepCopierContext context) { LogicalPlan logicalPlan = LogicalPlanDeepCopier.INSTANCE.deepCopy(exists.getQueryPlan(), context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java index 267e60c629..388cb42643 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java @@ -60,13 +60,13 @@ public interface Repeat extends Aggregate { static VirtualSlotReference generateVirtualGroupingIdSlot() { return new VirtualSlotReference(COL_GROUPING_ID, BigIntType.INSTANCE, Optional.empty(), - shapes -> shapes.computeVirtualGroupingIdValue()); + GroupingSetShapes::computeVirtualGroupingIdValue); } static VirtualSlotReference generateVirtualSlotByFunction(GroupingScalarFunction function) { return new VirtualSlotReference( generateVirtualSlotName(function), function.getDataType(), Optional.of(function), - shapes -> function.computeVirtualSlotValue(shapes)); + function::computeVirtualSlotValue); } /** @@ -175,7 +175,7 @@ public interface Repeat extends Aggregate { if (index == null) { throw new AnalysisException("Can not find grouping set expression in output: " + expression); } - if (groupingSetsIndex.contains(index)) { + if (groupingSetIndex.contains(index)) { throw new AnalysisException("expression duplicate in grouping set: " + expression); } groupingSetIndex.add(index); @@ -228,14 +228,6 @@ public interface Repeat extends Aggregate { this.shapes = ImmutableList.copyOf(shapes); } - public GroupingSetShape getGroupingSetShape(int index) { - return shapes.get(index); - } - - public Expression getExpression(int index) { - return flattenGroupingSetExpression.get(index); - } - // compute a long value that backend need to fill to the GROUPING_ID slot public List computeVirtualGroupingIdValue() { return shapes.stream() diff --git a/regression-test/data/nereids_syntax_p0/cte.out b/regression-test/data/nereids_syntax_p0/cte.out index b9efc81fdc..aae6a8964c 100644 --- a/regression-test/data/nereids_syntax_p0/cte.out +++ b/regression-test/data/nereids_syntax_p0/cte.out @@ -82,6 +82,17 @@ ASIA 1 15 29 +-- !cte_with_repeat -- +\N \N 1 +\N 1 1 +\N 2 1 +\N 6 1 +1309892 \N 0 +1309892 1 0 +1309892 2 0 +1310179 \N 0 +1310179 6 0 + -- !test -- 1 2023-08-25 00:00:00 10 10 1 2023-08-25 01:00:00 20 30 diff --git a/regression-test/suites/nereids_syntax_p0/cte.groovy b/regression-test/suites/nereids_syntax_p0/cte.groovy index ba94556991..56056117bc 100644 --- a/regression-test/suites/nereids_syntax_p0/cte.groovy +++ b/regression-test/suites/nereids_syntax_p0/cte.groovy @@ -253,7 +253,7 @@ suite("cte") { ORDER BY dd.s_suppkey; """ - sql "set experimental_enable_pipeline_engine=true" + sql "set enable_pipeline_engine=true" qt_cte14 """ SELECT abs(dd.s_suppkey) @@ -308,6 +308,11 @@ suite("cte") { sql "WITH cte_0 AS ( SELECT 1 AS a ) SELECT * from cte_0 t1 LIMIT 10 UNION SELECT * from cte_0 t1 LIMIT 10" + qt_cte_with_repeat """ + with cte_0 as (select lo_orderkey, lo_linenumber, grouping_id(lo_orderkey) as id from lineorder group by cube(lo_orderkey, lo_linenumber)) + select * from cte_0 order by lo_orderkey, lo_linenumber, id + """ + qt_test """ SELECT * FROM ( WITH temptable as (