From d50c8b6d3ac39780e8bd0a235da2ec79cea93308 Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Tue, 9 Jan 2024 11:54:48 +0800 Subject: [PATCH] [Improvement](nereids) Query rewrite by mv support bitmap_union and bitmap_union_count roll up (#29418) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Query rewrite by mv support bitmap_union and bitmap_union_count roll up, aggregate functions which supports roll up is listed as following: | 查询中函数 | 物化视图中函数 | 函数上卷后 | |------------------|--------------|--------------------| | max | max | max | | min | min | min | | sum | sum | sum | | count | count | sum | | count(distinct ) | bitmap_union | bitmap_union_count | | bitmap_union | bitmap_union | bitmap_union| | bitmap_union_count | bitmap_union | bitmap_union_count | this depends on https://github.com/apache/doris/pull/29256 --- ...AbstractMaterializedViewAggregateRule.java | 210 ++++++++------- .../mv/AbstractMaterializedViewRule.java | 15 +- .../mv/MaterializationContext.java | 11 +- .../functions/agg/BitmapUnion.java | 8 +- .../functions/agg/BitmapUnionCount.java | 8 +- .../aggregate_with_roll_up.out | 76 ++++++ .../aggregate_with_roll_up.groovy | 253 ++++++++++++++++-- 7 files changed, 459 insertions(+), 122 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index c977a55ce7..2804a0814f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnion; +import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount; import org.apache.doris.nereids.trees.expressions.functions.agg.CouldRollUp; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmap; @@ -65,18 +66,33 @@ import java.util.stream.Collectors; */ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMaterializedViewRule { - // we only support roll up function which has only one argument currently protected static final Multimap AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP = ArrayListMultimap.create(); protected final String currentClassName = this.getClass().getSimpleName(); - private final Logger logger = LogManager.getLogger(this.getClass()); static { - AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(new Count(true, Any.INSTANCE), - new BitmapUnion(new ToBitmap(new Cast(Any.INSTANCE, BigIntType.INSTANCE)))); + // support count distinct roll up + // with bitmap_union and to_bitmap AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(new Count(true, Any.INSTANCE), new BitmapUnion(new ToBitmap(Any.INSTANCE))); + // with bitmap_union, to_bitmap and cast + AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(new Count(true, Any.INSTANCE), + new BitmapUnion(new ToBitmap(new Cast(Any.INSTANCE, BigIntType.INSTANCE)))); + + // support bitmap_union_count roll up + // field is already bitmap with only bitmap_union + AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put( + new BitmapUnionCount(Any.INSTANCE), + new BitmapUnion(Any.INSTANCE)); + // with bitmap_union and to_bitmap + AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put( + new BitmapUnionCount(new ToBitmap(Any.INSTANCE)), + new BitmapUnion(new ToBitmap(Any.INSTANCE))); + // with bitmap_union, to_bitmap and cast + AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put( + new BitmapUnionCount(new ToBitmap(new Cast(Any.INSTANCE, BigIntType.INSTANCE))), + new BitmapUnion(new ToBitmap(new Cast(Any.INSTANCE, BigIntType.INSTANCE)))); } @Override @@ -101,89 +117,54 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate String.format("query plan = %s\n", queryStructInfo.getOriginalPlan().treeString()))); return null; } - // Firstly, handle query group by expression rewrite - LogicalAggregate queryAggregate = queryTopPlanAndAggPair.value(); + // Firstly,if group by expression between query and view is equals, try to rewrite expression directly Plan queryTopPlan = queryTopPlanAndAggPair.key(); - // query and view have the same dimension, try to rewrite rewrittenQueryGroupExpr - LogicalAggregate viewAggregate = viewTopPlanAndAggPair.value(); - Plan viewTopPlan = viewTopPlanAndAggPair.key(); - boolean needRollUp = - queryAggregate.getGroupByExpressions().size() != viewAggregate.getGroupByExpressions().size(); - if (queryAggregate.getGroupByExpressions().size() == viewAggregate.getGroupByExpressions().size()) { - List queryGroupShuttledExpression = ExpressionUtils.shuttleExpressionWithLineage( - queryAggregate.getGroupByExpressions(), queryTopPlan); - List viewGroupShuttledExpression = ExpressionUtils.shuttleExpressionWithLineage( - viewAggregate.getGroupByExpressions(), viewTopPlan) - .stream() - .map(expr -> ExpressionUtils.replace(expr, queryToViewSlotMapping.inverse().toSlotReferenceMap())) - .collect(Collectors.toList()); - needRollUp = !queryGroupShuttledExpression.equals(viewGroupShuttledExpression); - } - if (!needRollUp) { - List rewrittenQueryGroupExpr = rewriteExpression(queryTopPlan.getExpressions(), + SlotMapping viewToQurySlotMapping = queryToViewSlotMapping.inverse(); + if (isGroupByEquals(queryTopPlanAndAggPair, viewTopPlanAndAggPair, viewToQurySlotMapping)) { + List rewrittenQueryExpressions = rewriteExpression(queryTopPlan.getExpressions(), queryTopPlan, materializationContext.getMvExprToMvScanExprMapping(), queryToViewSlotMapping, true); - if (rewrittenQueryGroupExpr.isEmpty()) { - // can not rewrite, bail out. - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Can not rewrite expression when no roll up", - String.format("expressionToWrite = %s,\n mvExprToMvScanExprMapping = %s,\n" - + "queryToViewSlotMapping = %s", - queryTopPlan.getExpressions(), - materializationContext.getMvExprToMvScanExprMapping(), - queryToViewSlotMapping))); - return null; + if (!rewrittenQueryExpressions.isEmpty()) { + return new LogicalProject<>( + rewrittenQueryExpressions.stream().map(NamedExpression.class::cast) + .collect(Collectors.toList()), + tempRewritedPlan); + } - return new LogicalProject<>( - rewrittenQueryGroupExpr.stream().map(NamedExpression.class::cast).collect(Collectors.toList()), - tempRewritedPlan); - } - // the dimension in query and view are different, try to roll up - // Split query aggregate to group expression and agg function - // Firstly, find the query top output rewrite function expr list which only use query aggregate function, - // This will be used to roll up - if (viewAggregate.getOutputExpressions().stream().anyMatch( - viewExpr -> viewExpr.anyMatch(expr -> expr instanceof AggregateFunction - && ((AggregateFunction) expr).isDistinct()))) { - // if mv aggregate function contains distinct, can not roll up, bail out. + // if fails, record the reason and then try to roll up aggregate function materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("View contains distinct function so can not roll up", - String.format("view plan = %s", viewAggregate.getOutputExpressions()))); - return null; + Pair.of("Can not rewrite expression when no roll up", + String.format("expressionToWrite = %s,\n mvExprToMvScanExprMapping = %s,\n" + + "queryToViewSlotMapping = %s", + queryTopPlan.getExpressions(), + materializationContext.getMvExprToMvScanExprMapping(), + queryToViewSlotMapping))); } + // try to roll up. // split the query top plan expressions to group expressions and functions, if can not, bail out. Pair, Set> queryGroupAndFunctionPair = topPlanSplitToGroupAndFunction(queryTopPlanAndAggPair); - if (queryGroupAndFunctionPair == null) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Query top plan split to group by and function fail", - String.format("queryTopPlan = %s,\n agg = %s", - queryTopPlanAndAggPair.key().treeString(), - queryTopPlanAndAggPair.value().treeString()))); - return null; - } - // Secondly, try to roll up the agg functions // this map will be used to rewrite expression Multimap needRollupExprMap = HashMultimap.create(); Multimap groupRewrittenExprMap = HashMultimap.create(); + // permute the mv expr mapping to query based Map mvExprToMvScanExprQueryBased = - materializationContext.getMvExprToMvScanExprMapping().keyPermute( - queryToViewSlotMapping.inverse()).flattenMap().get(0); - + materializationContext.getMvExprToMvScanExprMapping().keyPermute(viewToQurySlotMapping) + .flattenMap().get(0); Set queryTopPlanFunctionSet = queryGroupAndFunctionPair.value(); // try to rewrite, contains both roll up aggregate functions and aggregate group expression List finalAggregateExpressions = new ArrayList<>(); List finalGroupExpressions = new ArrayList<>(); for (Expression topExpression : queryTopPlan.getExpressions()) { - // is agg function, try to roll up and rewrite + // if agg function, try to roll up and rewrite if (queryTopPlanFunctionSet.contains(topExpression)) { Expression queryFunctionShuttled = ExpressionUtils.shuttleExpressionWithLineage( topExpression, queryTopPlan); // try to roll up - AggregateFunction queryFunction = (AggregateFunction) topExpression.firstMatch( + AggregateFunction queryFunction = (AggregateFunction) queryFunctionShuttled.firstMatch( expr -> expr instanceof AggregateFunction); Function rollupAggregateFunction = rollup(queryFunction, queryFunctionShuttled, mvExprToMvScanExprQueryBased); @@ -213,7 +194,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate } finalAggregateExpressions.add((NamedExpression) rewrittenFunctionExpression); } else { - // try to rewrite group expression + // if group by expression, try to rewrite group by expression Expression queryGroupShuttledExpr = ExpressionUtils.shuttleExpressionWithLineage(topExpression, queryTopPlan); if (!mvExprToMvScanExprQueryBased.containsKey(queryGroupShuttledExpr)) { @@ -277,15 +258,25 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate return expr; }) .collect(Collectors.toList()); - LogicalAggregate rewrittenAggregate = new LogicalAggregate(finalGroupExpressions, - finalAggregateExpressions, mvProject); - // record the group id in materializationContext, and when rewrite again in - // the same group, bail out quickly. - if (queryStructInfo.getOriginalPlan().getGroupExpression().isPresent()) { - materializationContext.addMatchedGroup( - queryStructInfo.getOriginalPlan().getGroupExpression().get().getOwnerGroup().getGroupId()); - } - return rewrittenAggregate; + return new LogicalAggregate(finalGroupExpressions, finalAggregateExpressions, mvProject); + } + + private boolean isGroupByEquals(Pair> queryTopPlanAndAggPair, + Pair> viewTopPlanAndAggPair, + SlotMapping viewToQurySlotMapping) { + Plan queryTopPlan = queryTopPlanAndAggPair.key(); + Plan viewTopPlan = viewTopPlanAndAggPair.key(); + LogicalAggregate queryAggregate = queryTopPlanAndAggPair.value(); + LogicalAggregate viewAggregate = viewTopPlanAndAggPair.value(); + List queryGroupShuttledExpression = ExpressionUtils.shuttleExpressionWithLineage( + queryAggregate.getGroupByExpressions(), queryTopPlan); + List viewGroupShuttledExpression = ExpressionUtils.shuttleExpressionWithLineage( + viewAggregate.getGroupByExpressions(), viewTopPlan) + .stream() + .map(expr -> ExpressionUtils.replace(expr, viewToQurySlotMapping.toSlotReferenceMap())) + .collect(Collectors.toList()); + return queryAggregate.getGroupByExpressions().size() == viewAggregate.getGroupByExpressions().size() + && queryGroupShuttledExpression.equals(viewGroupShuttledExpression); } /** @@ -309,9 +300,11 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate return null; } Expression rollupParam = null; + Expression viewRollupFunction = null; if (mvExprToMvScanExprQueryBased.containsKey(queryAggregateFunctionShuttled)) { // function can rewrite by view rollupParam = mvExprToMvScanExprQueryBased.get(queryAggregateFunctionShuttled); + viewRollupFunction = queryAggregateFunctionShuttled; } else { // function can not rewrite by view, try to use complex roll up param // eg: query is count(distinct param), mv sql is bitmap_union(to_bitmap(param)) @@ -322,43 +315,59 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate if (isAggregateFunctionEquivalent(queryAggregateFunction, queryAggregateFunctionShuttled, (Function) mvExprShuttled)) { rollupParam = mvExprToMvScanExprQueryBased.get(mvExprShuttled); + viewRollupFunction = mvExprShuttled; } } } - if (rollupParam == null) { + if (rollupParam == null || !canRollup(viewRollupFunction)) { return null; } // do roll up return ((CouldRollUp) queryAggregateFunction).constructRollUp(rollupParam); } + // Check the aggregate function can roll up or not, return true if could roll up + // if view aggregate function is distinct or is in the un supported rollup functions, it doesn't support + // roll up. + private boolean canRollup(Expression rollupExpression) { + if (rollupExpression == null) { + return false; + } + if (rollupExpression instanceof Function && !(rollupExpression instanceof AggregateFunction)) { + return false; + } + if (rollupExpression instanceof AggregateFunction) { + AggregateFunction aggregateFunction = (AggregateFunction) rollupExpression; + return !aggregateFunction.isDistinct() && aggregateFunction instanceof CouldRollUp; + } + return true; + } + private Pair, Set> topPlanSplitToGroupAndFunction( Pair> topPlanAndAggPair) { - LogicalAggregate queryAggregate = topPlanAndAggPair.value(); Set queryAggGroupSet = new HashSet<>(queryAggregate.getGroupByExpressions()); - Set queryAggFunctionSet = queryAggregate.getOutputExpressions().stream() + // when query is bitmap_count(bitmap_union), the plan is as following: + // project(bitmap_count()#1) + // aggregate(bitmap_union()#2) + // we should use exprId which query top plan used to decide the query top plan is use the + // bottom agg function or not + Set queryAggFunctionSet = queryAggregate.getOutputExpressions().stream() .filter(expr -> !queryAggGroupSet.contains(expr)) + .map(NamedExpression::getExprId) .collect(Collectors.toSet()); Plan queryTopPlan = topPlanAndAggPair.key(); Set topGroupByExpressions = new HashSet<>(); Set topFunctionExpressions = new HashSet<>(); - queryTopPlan.getExpressions().forEach( - expression -> { - if (expression.anyMatch(expr -> expr instanceof NamedExpression - && queryAggFunctionSet.contains((NamedExpression) expr))) { - topFunctionExpressions.add(expression); - } else { - topGroupByExpressions.add(expression); - } - }); - // only support to reference the aggregate function directly in top, will support expression later. - if (topFunctionExpressions.stream().anyMatch( - topAggFunc -> !(topAggFunc instanceof NamedExpression) && (!queryAggFunctionSet.contains(topAggFunc) - || !queryAggFunctionSet.contains(topAggFunc.child(0))))) { - return null; - } + queryTopPlan.getExpressions().forEach(expression -> { + if (expression.anyMatch(expr -> expr instanceof NamedExpression + && queryAggFunctionSet.contains(((NamedExpression) expr).getExprId()))) { + topFunctionExpressions.add(expression); + } else { + topGroupByExpressions.add(expression); + } + }); return Pair.of(topGroupByExpressions, topFunctionExpressions); } @@ -427,11 +436,14 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate continue; } // check param in query function is same as the view function - List viewFunctionArguments = extractViewArguments(equivalentFunction, viewFunction); - if (queryFunctionShuttled.getArguments().size() != 1 || viewFunctionArguments.size() != 1) { + List viewFunctionArguments = extractArguments(equivalentFunction, viewFunction); + List queryFunctionArguments = + extractArguments(equivalentFunctionEntry.getKey(), queryFunction); + // check argument size,we only support roll up function which has only one argument currently + if (queryFunctionArguments.size() != 1 || viewFunctionArguments.size() != 1) { continue; } - if (Objects.equals(queryFunctionShuttled.getArguments().get(0), viewFunctionArguments.get(0))) { + if (Objects.equals(queryFunctionArguments.get(0), viewFunctionArguments.get(0))) { return true; } } @@ -441,14 +453,14 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate } /** - * Extract the view function arguments by equivalentFunction pattern - * Such as equivalentFunction def is bitmap_union(to_bitmap(Any.INSTANCE)), - * viewFunction is bitmap_union(to_bitmap(case when a = 5 then 1 else 2 end)) + * Extract the function arguments by functionWithAny pattern + * Such as functionWithAny def is bitmap_union(to_bitmap(Any.INSTANCE)), + * actualFunction is bitmap_union(to_bitmap(case when a = 5 then 1 else 2 end)) * after extracting, the return argument is: case when a = 5 then 1 else 2 end */ - private List extractViewArguments(Expression equivalentFunction, Function viewFunction) { - Set exprSetToRemove = equivalentFunction.collectToSet(expr -> !(expr instanceof Any)); - return viewFunction.collectFirst(expr -> + private List extractArguments(Expression functionWithAny, Function actualFunction) { + Set exprSetToRemove = functionWithAny.collectToSet(expr -> !(expr instanceof Any)); + return actualFunction.collectFirst(expr -> exprSetToRemove.stream().noneMatch(exprToRemove -> exprToRemove.equals(expr))); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 6b6d7553c5..39807a8ce2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -99,8 +99,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac } for (MaterializationContext materializationContext : materializationContexts) { // already rewrite, bail out - if (queryPlan.getGroupExpression().isPresent() && materializationContext.alreadyRewrite( - queryPlan.getGroupExpression().get().getOwnerGroup().getGroupId())) { + if (checkIfRewritten(queryPlan, materializationContext)) { continue; } List viewStructInfos = extractStructInfo(materializationContext.getMvPlan(), cascadesContext); @@ -212,6 +211,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac Rewriter.getWholeTreeRewriter(rewrittenPlanContext).execute(); rewrittenPlan = rewrittenPlanContext.getRewritePlan(); materializationContext.setSuccess(true); + recordIfRewritten(queryPlan, materializationContext); rewriteResults.add(rewrittenPlan); } } @@ -532,6 +532,17 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac return true; } + protected void recordIfRewritten(Plan plan, MaterializationContext context) { + if (plan.getGroupExpression().isPresent()) { + context.addMatchedGroup(plan.getGroupExpression().get().getOwnerGroup().getGroupId()); + } + } + + protected boolean checkIfRewritten(Plan plan, MaterializationContext context) { + return plan.getGroupExpression().isPresent() + && context.alreadyRewrite(plan.getGroupExpression().get().getOwnerGroup().getGroupId()); + } + /** * Query and mv match node */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index fe893e60ab..0414f7007f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -196,29 +196,32 @@ public class MaterializationContext { */ public static String toSummaryString(List materializationContexts, List chosenMaterializationNames) { + if (materializationContexts.isEmpty()) { + return ""; + } Set materializationChosenNameSet = chosenMaterializationNames.stream() .map(MTMV::getName) .collect(Collectors.toSet()); StringBuilder builder = new StringBuilder(); - builder.append("\n\nMaterializedView\n"); + builder.append("\nMaterializedView"); builder.append("\nMaterializedViewRewriteFail:"); for (MaterializationContext ctx : materializationContexts) { if (!ctx.isSuccess()) { Set failReasonSet = ctx.getFailReason().values().stream().map(Pair::key).collect(Collectors.toSet()); - builder.append("\n\n") + builder.append("\n") .append(" Name: ").append(ctx.getMTMV().getName()) .append("\n") .append(" FailSummary: ").append(String.join(", ", failReasonSet)); } } - builder.append("\n\nMaterializedViewRewriteSuccessButNotChose:\n"); + builder.append("\nMaterializedViewRewriteSuccessButNotChose:\n"); builder.append(" Names: ").append(materializationContexts.stream() .filter(materializationContext -> materializationContext.isSuccess() && !materializationChosenNameSet.contains(materializationContext.getMTMV().getName())) .map(materializationContext -> materializationContext.getMTMV().getName()) .collect(Collectors.joining(", "))); - builder.append("\n\nMaterializedViewRewriteSuccessAndChose:\n"); + builder.append("\nMaterializedViewRewriteSuccessAndChose:\n"); builder.append(" Names: ").append(String.join(", ", materializationChosenNameSet)); return builder.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnion.java index 6809cd530d..bd2a071697 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnion.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BitmapType; @@ -35,7 +36,7 @@ import java.util.List; * AggregateFunction 'bitmap_union'. This class is generated by GenerateFunction. */ public class BitmapUnion extends AggregateFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, BitmapFunction { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, BitmapFunction, CouldRollUp { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BitmapType.INSTANCE).args(BitmapType.INSTANCE) @@ -78,4 +79,9 @@ public class BitmapUnion extends AggregateFunction public List getSignatures() { return SIGNATURES; } + + @Override + public Function constructRollUp(Expression param, Expression... varParams) { + return new BitmapUnion(this.isDistinct(), param); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionCount.java index de8456ee01..667fe2021f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionCount.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionCount.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BigIntType; @@ -36,7 +37,7 @@ import java.util.List; * AggregateFunction 'bitmap_union_count'. This class is generated by GenerateFunction. */ public class BitmapUnionCount extends AggregateFunction - implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, BitmapFunction { + implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, BitmapFunction, CouldRollUp { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(BigIntType.INSTANCE).args(BitmapType.INSTANCE) @@ -79,4 +80,9 @@ public class BitmapUnionCount extends AggregateFunction public List getSignatures() { return SIGNATURES; } + + @Override + public Function constructRollUp(Expression param, Expression... varParams) { + return new BitmapUnionCount(param); + } } diff --git a/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out b/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out index 334980ed00..f20fc74747 100644 --- a/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out +++ b/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out @@ -31,6 +31,12 @@ -- !query15_0_after -- 3 3 2023-12-11 43.20 43.20 43.20 1 0 +-- !query15_1_before -- +2023-12-11 2023-12-11 3 3 43.20 43.20 43.20 1 0 \N 0 + +-- !query15_1_after -- +2023-12-11 2023-12-11 3 3 43.20 43.20 43.20 1 0 \N 0 + -- !query16_0_before -- 3 3 2023-12-11 43.20 43.20 43.20 1 0 @@ -73,6 +79,20 @@ 2023-12-11 3 2023-12-11 43.20 43.20 43.20 1 0 2023-12-12 3 2023-12-12 57.40 56.20 1.20 2 0 +-- !query20_1_before -- +2023-12-08 2023-12-08 2 3 20.00 10.50 9.50 2 0 \N 0 +2023-12-09 2023-12-09 4 3 11.50 11.50 11.50 1 0 \N 0 +2023-12-10 2023-12-10 2 4 46.00 33.50 12.50 2 0 \N 0 +2023-12-11 2023-12-11 3 3 43.20 43.20 43.20 1 0 \N 0 +2023-12-12 2023-12-12 2 3 57.40 56.20 1.20 2 0 \N 0 + +-- !query20_1_after -- +2023-12-08 2023-12-08 2 3 20.00 10.50 9.50 2 0 \N 0 +2023-12-09 2023-12-09 4 3 11.50 11.50 11.50 1 0 \N 0 +2023-12-10 2023-12-10 2 4 46.00 33.50 12.50 2 0 \N 0 +2023-12-11 2023-12-11 3 3 43.20 43.20 43.20 1 0 \N 0 +2023-12-12 2023-12-12 2 3 57.40 56.20 1.20 2 0 \N 0 + -- !query21_0_before -- 2 3 2023-12-08 20.00 10.50 9.50 2 0 2 3 2023-12-12 57.40 56.20 1.20 2 0 @@ -129,6 +149,34 @@ 3 3 2023-12-11 43.20 43.20 43.20 1 4 3 2023-12-09 11.50 11.50 11.50 1 +-- !query25_1_before -- +2023-12-08 3 20.00 10.50 9.50 2 \N \N +2023-12-09 3 11.50 11.50 11.50 1 \N \N +2023-12-10 4 46.00 33.50 12.50 2 \N \N +2023-12-11 3 43.20 43.20 43.20 1 \N \N +2023-12-12 3 57.40 56.20 1.20 2 \N \N + +-- !query25_1_after -- +2023-12-08 3 20.00 10.50 9.50 2 \N \N +2023-12-09 3 11.50 11.50 11.50 1 \N \N +2023-12-10 4 46.00 33.50 12.50 2 \N \N +2023-12-11 3 43.20 43.20 43.20 1 \N \N +2023-12-12 3 57.40 56.20 1.20 2 \N \N + +-- !query25_2_before -- +2023-12-08 3 20.00 10.50 9.50 2 \N \N 1 0 0 +2023-12-09 3 11.50 11.50 11.50 1 \N \N 1 0 0 +2023-12-10 4 46.00 33.50 12.50 2 \N \N 1 0 0 +2023-12-11 3 43.20 43.20 43.20 1 \N \N 0 1 1 +2023-12-12 3 57.40 56.20 1.20 2 \N \N 0 1 1 + +-- !query25_2_after -- +2023-12-08 3 20.00 10.50 9.50 2 \N \N 1 0 0 +2023-12-09 3 11.50 11.50 11.50 1 \N \N 1 0 0 +2023-12-10 4 46.00 33.50 12.50 2 \N \N 1 0 0 +2023-12-11 3 43.20 43.20 43.20 1 \N \N 0 1 1 +2023-12-12 3 57.40 56.20 1.20 2 \N \N 0 1 1 + -- !query1_1_before -- 1 yy 0 0 11.50 11.50 11.50 1 @@ -143,3 +191,31 @@ 2 mi 0 0 57.40 56.20 1.20 2 2 mm 0 0 43.20 43.20 43.20 1 +-- !query26_0_before -- +2023-12-08 1 20.00 10.50 9.50 2 0 0 +2023-12-09 1 11.50 11.50 11.50 1 0 0 +2023-12-10 1 46.00 33.50 12.50 2 0 0 +2023-12-11 2 43.20 43.20 43.20 1 0 0 +2023-12-12 2 57.40 56.20 1.20 2 0 0 + +-- !query26_0_after -- +2023-12-08 1 20.00 10.50 9.50 2 0 0 +2023-12-09 1 11.50 11.50 11.50 1 0 0 +2023-12-10 1 46.00 33.50 12.50 2 0 0 +2023-12-11 2 43.20 43.20 43.20 1 0 0 +2023-12-12 2 57.40 56.20 1.20 2 0 0 + +-- !query27_0_before -- +2023-12-08 1 20.00 10.50 9.50 2 0 0 +2023-12-09 1 11.50 11.50 11.50 1 0 0 +2023-12-10 1 46.00 33.50 12.50 2 0 0 +2023-12-11 2 43.20 43.20 43.20 1 0 0 +2023-12-12 2 57.40 56.20 1.20 2 0 0 + +-- !query27_0_after -- +2023-12-08 1 20.00 10.50 9.50 2 0 0 +2023-12-09 1 11.50 11.50 11.50 1 0 0 +2023-12-10 1 46.00 33.50 12.50 2 0 0 +2023-12-11 2 43.20 43.20 43.20 1 0 0 +2023-12-12 2 57.40 56.20 1.20 2 0 0 + diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy index b9d3864751..7a111bc169 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy @@ -50,7 +50,7 @@ suite("aggregate_with_roll_up") { DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 PROPERTIES ( "replication_num" = "1" - ) + ); """ sql """ @@ -248,18 +248,18 @@ suite("aggregate_with_roll_up") { def mv13_1 = """ - select l_shipdate, o_orderdate, l_partkey, l_suppkey, - sum(o_totalprice) as sum_total, - max(o_totalprice) as max_total, - min(o_totalprice) as min_total, - count(*) as count_all, - bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic - from lineitem - left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate - group by - l_shipdate, - o_orderdate, - l_partkey, + select l_shipdate, o_orderdate, l_partkey, l_suppkey, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, l_suppkey; """ def query13_1 = """ @@ -272,7 +272,7 @@ suite("aggregate_with_roll_up") { from (select * from lineitem where l_shipdate = '2023-12-11') t1 left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate group by - o_orderdate, + o_orderdate, l_partkey, l_suppkey; """ @@ -349,6 +349,45 @@ suite("aggregate_with_roll_up") { order_qt_query15_0_after "${query15_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv15_0""" + def mv15_1 = """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from lineitem + left join (select * from orders where o_orderstatus = 'o') t2 + on lineitem.l_orderkey = o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query15_1 = """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + bitmap_union_count(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)), + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from (select * from lineitem where l_shipdate = '2023-12-11') t1 + left join (select * from orders where o_orderstatus = 'o') t2 + on t1.l_orderkey = o_orderkey and t1.l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + + order_qt_query15_1_before "${query15_1}" + check_rewrite_with_mv_partition(mv15_1, query15_1, "mv15_1", "l_shipdate") + order_qt_query15_1_after "${query15_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv15_1""" // filter outside + left + use roll up dimension def mv16_0 = "select l_shipdate, o_orderdate, l_partkey, l_suppkey, " + @@ -534,6 +573,47 @@ suite("aggregate_with_roll_up") { order_qt_query20_0_after "${query20_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv20_0""" + def mv20_1 = """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from lineitem + left join (select * from orders where o_orderstatus = 'o') t2 + on lineitem.l_orderkey = o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + + def query20_1 = """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + bitmap_union_count(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)), + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from lineitem + left join (select * from orders where o_orderstatus = 'o') t2 + on lineitem.l_orderkey = o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + order_qt_query20_1_before "${query20_1}" + check_rewrite(mv20_1, query20_1, "mv20_1") + order_qt_query20_1_after "${query20_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv20_1""" + + // filter inside + right + left + use not roll up dimension def mv21_0 = "select l_shipdate, o_orderdate, l_partkey, l_suppkey, " + "sum(o_totalprice) as sum_total, " + @@ -731,6 +811,83 @@ suite("aggregate_with_roll_up") { order_qt_query25_0_after "${query25_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv25_0""" + + // bitmap_union roll up to bitmap_union + def mv25_1 = """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 2, 3) then o_custkey else null end)) cnt_1, + bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (3, 4, 5) then o_custkey else null end)) as cnt_2 + from lineitem + left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query25_1 = """ + select o_orderdate, l_suppkey, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 2, 3) then o_custkey else null end)) cnt_1, + bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (3, 4, 5) then o_custkey else null end)) as cnt_2 + from lineitem + left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate + group by + o_orderdate, + l_suppkey; + """ + order_qt_query25_1_before "${query25_1}" + check_rewrite(mv25_1, query25_1, "mv25_1") + order_qt_query25_1_after "${query25_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv25_1""" + + // bitmap_union roll up to bitmap_union_count + def mv25_2 = """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 0 and o_orderkey IN (1, 2, 3) then o_custkey else null end)) cnt_1, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (3, 4, 5) then o_custkey else null end)) as cnt_2 + from lineitem + left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query25_2 = """ + select o_orderdate, l_suppkey, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 0 and o_orderkey IN (1, 2, 3) then o_custkey else null end)), + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (3, 4, 5) then o_custkey else null end)), + bitmap_union_count(to_bitmap(case when o_shippriority > 0 and o_orderkey IN (1, 2, 3) then o_custkey else null end)), + bitmap_union_count(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (3, 4, 5) then o_custkey else null end)), + count(distinct case when o_shippriority > 1 and o_orderkey IN (3, 4, 5) then o_custkey else null end) + from lineitem + left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate + group by + o_orderdate, + l_suppkey; + """ + order_qt_query25_2_before "${query25_2}" + check_rewrite(mv25_2, query25_2, "mv25_2") + order_qt_query25_2_after "${query25_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv25_2""" + + // single table // filter + use roll up dimension def mv1_1 = "select o_orderdate, o_shippriority, o_comment, " + @@ -798,5 +955,71 @@ suite("aggregate_with_roll_up") { order_qt_query2_0_after "${query2_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_0""" - // can not rewrite, todo + // can not rewrite + // bitmap_union_count is aggregate function but not support roll up + def mv26_0 = """ + select o_orderdate, o_shippriority, o_comment, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union_count(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, + bitmap_union_count(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 + from orders + group by + o_orderdate, + o_shippriority, + o_comment; + """ + def query26_0 = """ + select o_orderdate, o_shippriority, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union_count(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, + bitmap_union_count(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 + from orders + group by + o_orderdate, + o_shippriority; + """ + order_qt_query26_0_before "${query26_0}" + check_not_match(mv26_0, query26_0, "mv26_0") + order_qt_query26_0_after "${query26_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv26_0""" + + + // bitmap_count is not aggregate function, so doesn't not support roll up + def mv27_0 = """ + select o_orderdate, o_shippriority, o_comment, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_count(bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 2, 3) then o_custkey else null end))) cnt_1, + bitmap_count(bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (3, 4, 5) then o_custkey else null end))) as cnt_2 + from orders + group by + o_orderdate, + o_shippriority, + o_comment; + """ + def query27_0 = """ + select o_orderdate, o_shippriority, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_count(bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 2, 3) then o_custkey else null end))) cnt_1, + bitmap_count(bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (3, 4, 5) then o_custkey else null end))) as cnt_2 + from orders + group by + o_orderdate, + o_shippriority; + """ + order_qt_query27_0_before "${query27_0}" + check_not_match(mv27_0, query27_0, "mv27_0") + order_qt_query27_0_after "${query27_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv27_0""" }