database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Improvement](nereids) Query rewrite by mv support bitmap_union and bitmap_union_count roll up (#29418)

Browse Source 
Query rewrite by mv support bitmap_union and bitmap_union_count roll up, aggregate functions which supports roll up is listed as following:

| 查询中函数            | 物化视图中函数      | 函数上卷后              |
|------------------|--------------|--------------------|
| max              | max          | max                |
| min              | min          | min                |
| sum              | sum          | sum                |
| count            | count        | sum                |
| count(distinct ) | bitmap_union | bitmap_union_count |
| bitmap_union | bitmap_union | bitmap_union|
| bitmap_union_count | bitmap_union | bitmap_union_count |

this depends on  https://github.com/apache/doris/pull/29256
This commit is contained in:
seawinde
2024-01-09 11:54:48 +08:00
committed by yiguolei
parent 0a55376a48
commit d50c8b6d3a
7 changed files with 459 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

210
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
View File

@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnion;
import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount;
import org.apache.doris.nereids.trees.expressions.functions.agg.CouldRollUp;
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmap;
@ -65,18 +66,33 @@ import java.util.stream.Collectors;
*/
public abstract class AbstractMaterializedViewAggregateRule extends AbstractMaterializedViewRule {
// we only support roll up function which has only one argument currently
protected static final Multimap<Expression, Expression>
AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP = ArrayListMultimap.create();
protected final String currentClassName = this.getClass().getSimpleName();
private final Logger logger = LogManager.getLogger(this.getClass());
static {
AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(new Count(true, Any.INSTANCE),
new BitmapUnion(new ToBitmap(new Cast(Any.INSTANCE, BigIntType.INSTANCE))));
// support count distinct roll up
// with bitmap_union and to_bitmap
AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(new Count(true, Any.INSTANCE),
new BitmapUnion(new ToBitmap(Any.INSTANCE)));
// with bitmap_union, to_bitmap and cast
AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(new Count(true, Any.INSTANCE),
new BitmapUnion(new ToBitmap(new Cast(Any.INSTANCE, BigIntType.INSTANCE))));
// support bitmap_union_count roll up
// field is already bitmap with only bitmap_union
AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(
new BitmapUnionCount(Any.INSTANCE),
new BitmapUnion(Any.INSTANCE));
// with bitmap_union and to_bitmap
AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(
new BitmapUnionCount(new ToBitmap(Any.INSTANCE)),
new BitmapUnion(new ToBitmap(Any.INSTANCE)));
// with bitmap_union, to_bitmap and cast
AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(
new BitmapUnionCount(new ToBitmap(new Cast(Any.INSTANCE, BigIntType.INSTANCE))),
new BitmapUnion(new ToBitmap(new Cast(Any.INSTANCE, BigIntType.INSTANCE))));
}
@Override
@ -101,89 +117,54 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
String.format("query plan = %s\n", queryStructInfo.getOriginalPlan().treeString())));
return null;
}
// Firstly, handle query group by expression rewrite
LogicalAggregate<Plan> queryAggregate = queryTopPlanAndAggPair.value();
// Firstly,if group by expression between query and view is equals, try to rewrite expression directly
Plan queryTopPlan = queryTopPlanAndAggPair.key();
// query and view have the same dimension, try to rewrite rewrittenQueryGroupExpr
LogicalAggregate<Plan> viewAggregate = viewTopPlanAndAggPair.value();
Plan viewTopPlan = viewTopPlanAndAggPair.key();
boolean needRollUp =
queryAggregate.getGroupByExpressions().size() != viewAggregate.getGroupByExpressions().size();
if (queryAggregate.getGroupByExpressions().size() == viewAggregate.getGroupByExpressions().size()) {
List<? extends Expression> queryGroupShuttledExpression = ExpressionUtils.shuttleExpressionWithLineage(
queryAggregate.getGroupByExpressions(), queryTopPlan);
List<? extends Expression> viewGroupShuttledExpression = ExpressionUtils.shuttleExpressionWithLineage(
viewAggregate.getGroupByExpressions(), viewTopPlan)
.stream()
.map(expr -> ExpressionUtils.replace(expr, queryToViewSlotMapping.inverse().toSlotReferenceMap()))
.collect(Collectors.toList());
needRollUp = !queryGroupShuttledExpression.equals(viewGroupShuttledExpression);
}
if (!needRollUp) {
List<Expression> rewrittenQueryGroupExpr = rewriteExpression(queryTopPlan.getExpressions(),
SlotMapping viewToQurySlotMapping = queryToViewSlotMapping.inverse();
if (isGroupByEquals(queryTopPlanAndAggPair, viewTopPlanAndAggPair, viewToQurySlotMapping)) {
List<Expression> rewrittenQueryExpressions = rewriteExpression(queryTopPlan.getExpressions(),
queryTopPlan,
materializationContext.getMvExprToMvScanExprMapping(),
queryToViewSlotMapping,
true);
if (rewrittenQueryGroupExpr.isEmpty()) {
// can not rewrite, bail out.
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
Pair.of("Can not rewrite expression when no roll up",
String.format("expressionToWrite = %s,\n mvExprToMvScanExprMapping = %s,\n"
+ "queryToViewSlotMapping = %s",
queryTopPlan.getExpressions(),
materializationContext.getMvExprToMvScanExprMapping(),
queryToViewSlotMapping)));
return null;
if (!rewrittenQueryExpressions.isEmpty()) {
return new LogicalProject<>(
rewrittenQueryExpressions.stream().map(NamedExpression.class::cast)
.collect(Collectors.toList()),
tempRewritedPlan);
}
return new LogicalProject<>(
rewrittenQueryGroupExpr.stream().map(NamedExpression.class::cast).collect(Collectors.toList()),
tempRewritedPlan);
}
// the dimension in query and view are different, try to roll up
// Split query aggregate to group expression and agg function
// Firstly, find the query top output rewrite function expr list which only use query aggregate function,
// This will be used to roll up
if (viewAggregate.getOutputExpressions().stream().anyMatch(
viewExpr -> viewExpr.anyMatch(expr -> expr instanceof AggregateFunction
&& ((AggregateFunction) expr).isDistinct()))) {
// if mv aggregate function contains distinct, can not roll up, bail out.
// if fails, record the reason and then try to roll up aggregate function
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
Pair.of("View contains distinct function so can not roll up",
String.format("view plan = %s", viewAggregate.getOutputExpressions())));
return null;
Pair.of("Can not rewrite expression when no roll up",
String.format("expressionToWrite = %s,\n mvExprToMvScanExprMapping = %s,\n"
+ "queryToViewSlotMapping = %s",
queryTopPlan.getExpressions(),
materializationContext.getMvExprToMvScanExprMapping(),
queryToViewSlotMapping)));
}
// try to roll up.
// split the query top plan expressions to group expressions and functions, if can not, bail out.
Pair<Set<? extends Expression>, Set<? extends Expression>> queryGroupAndFunctionPair
= topPlanSplitToGroupAndFunction(queryTopPlanAndAggPair);
if (queryGroupAndFunctionPair == null) {
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
Pair.of("Query top plan split to group by and function fail",
String.format("queryTopPlan = %s,\n agg = %s",
queryTopPlanAndAggPair.key().treeString(),
queryTopPlanAndAggPair.value().treeString())));
return null;
}
// Secondly, try to roll up the agg functions
// this map will be used to rewrite expression
Multimap<Expression, Expression> needRollupExprMap = HashMultimap.create();
Multimap<Expression, Expression> groupRewrittenExprMap = HashMultimap.create();
// permute the mv expr mapping to query based
Map<Expression, Expression> mvExprToMvScanExprQueryBased =
materializationContext.getMvExprToMvScanExprMapping().keyPermute(
queryToViewSlotMapping.inverse()).flattenMap().get(0);
materializationContext.getMvExprToMvScanExprMapping().keyPermute(viewToQurySlotMapping)
.flattenMap().get(0);
Set<? extends Expression> queryTopPlanFunctionSet = queryGroupAndFunctionPair.value();
// try to rewrite, contains both roll up aggregate functions and aggregate group expression
List<NamedExpression> finalAggregateExpressions = new ArrayList<>();
List<Expression> finalGroupExpressions = new ArrayList<>();
for (Expression topExpression : queryTopPlan.getExpressions()) {
// is agg function, try to roll up and rewrite
// if agg function, try to roll up and rewrite
if (queryTopPlanFunctionSet.contains(topExpression)) {
Expression queryFunctionShuttled = ExpressionUtils.shuttleExpressionWithLineage(
topExpression,
queryTopPlan);
// try to roll up
AggregateFunction queryFunction = (AggregateFunction) topExpression.firstMatch(
AggregateFunction queryFunction = (AggregateFunction) queryFunctionShuttled.firstMatch(
expr -> expr instanceof AggregateFunction);
Function rollupAggregateFunction = rollup(queryFunction, queryFunctionShuttled,
mvExprToMvScanExprQueryBased);
@ -213,7 +194,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
}
finalAggregateExpressions.add((NamedExpression) rewrittenFunctionExpression);
} else {
// try to rewrite group expression
// if group by expression, try to rewrite group by expression
Expression queryGroupShuttledExpr =
ExpressionUtils.shuttleExpressionWithLineage(topExpression, queryTopPlan);
if (!mvExprToMvScanExprQueryBased.containsKey(queryGroupShuttledExpr)) {
@ -277,15 +258,25 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
return expr;
})
.collect(Collectors.toList());
LogicalAggregate rewrittenAggregate = new LogicalAggregate(finalGroupExpressions,
finalAggregateExpressions, mvProject);
// record the group id in materializationContext, and when rewrite again in
// the same group, bail out quickly.
if (queryStructInfo.getOriginalPlan().getGroupExpression().isPresent()) {
materializationContext.addMatchedGroup(
queryStructInfo.getOriginalPlan().getGroupExpression().get().getOwnerGroup().getGroupId());
}
return rewrittenAggregate;
return new LogicalAggregate(finalGroupExpressions, finalAggregateExpressions, mvProject);
}
private boolean isGroupByEquals(Pair<Plan, LogicalAggregate<Plan>> queryTopPlanAndAggPair,
Pair<Plan, LogicalAggregate<Plan>> viewTopPlanAndAggPair,
SlotMapping viewToQurySlotMapping) {
Plan queryTopPlan = queryTopPlanAndAggPair.key();
Plan viewTopPlan = viewTopPlanAndAggPair.key();
LogicalAggregate<Plan> queryAggregate = queryTopPlanAndAggPair.value();
LogicalAggregate<Plan> viewAggregate = viewTopPlanAndAggPair.value();
List<? extends Expression> queryGroupShuttledExpression = ExpressionUtils.shuttleExpressionWithLineage(
queryAggregate.getGroupByExpressions(), queryTopPlan);
List<? extends Expression> viewGroupShuttledExpression = ExpressionUtils.shuttleExpressionWithLineage(
viewAggregate.getGroupByExpressions(), viewTopPlan)
.stream()
.map(expr -> ExpressionUtils.replace(expr, viewToQurySlotMapping.toSlotReferenceMap()))
.collect(Collectors.toList());
return queryAggregate.getGroupByExpressions().size() == viewAggregate.getGroupByExpressions().size()
&& queryGroupShuttledExpression.equals(viewGroupShuttledExpression);
}
/**
@ -309,9 +300,11 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
return null;
}
Expression rollupParam = null;
Expression viewRollupFunction = null;
if (mvExprToMvScanExprQueryBased.containsKey(queryAggregateFunctionShuttled)) {
// function can rewrite by view
rollupParam = mvExprToMvScanExprQueryBased.get(queryAggregateFunctionShuttled);
viewRollupFunction = queryAggregateFunctionShuttled;
} else {
// function can not rewrite by view, try to use complex roll up param
// eg: query is count(distinct param), mv sql is bitmap_union(to_bitmap(param))
@ -322,43 +315,59 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
if (isAggregateFunctionEquivalent(queryAggregateFunction, queryAggregateFunctionShuttled,
(Function) mvExprShuttled)) {
rollupParam = mvExprToMvScanExprQueryBased.get(mvExprShuttled);
viewRollupFunction = mvExprShuttled;
}
}
}
if (rollupParam == null) {
if (rollupParam == null || !canRollup(viewRollupFunction)) {
return null;
}
// do roll up
return ((CouldRollUp) queryAggregateFunction).constructRollUp(rollupParam);
}
// Check the aggregate function can roll up or not, return true if could roll up
// if view aggregate function is distinct or is in the un supported rollup functions, it doesn't support
// roll up.
private boolean canRollup(Expression rollupExpression) {
if (rollupExpression == null) {
return false;
}
if (rollupExpression instanceof Function && !(rollupExpression instanceof AggregateFunction)) {
return false;
}
if (rollupExpression instanceof AggregateFunction) {
AggregateFunction aggregateFunction = (AggregateFunction) rollupExpression;
return !aggregateFunction.isDistinct() && aggregateFunction instanceof CouldRollUp;
}
return true;
}
private Pair<Set<? extends Expression>, Set<? extends Expression>> topPlanSplitToGroupAndFunction(
Pair<Plan, LogicalAggregate<Plan>> topPlanAndAggPair) {
LogicalAggregate<Plan> queryAggregate = topPlanAndAggPair.value();
Set<Expression> queryAggGroupSet = new HashSet<>(queryAggregate.getGroupByExpressions());
Set<Expression> queryAggFunctionSet = queryAggregate.getOutputExpressions().stream()
// when query is bitmap_count(bitmap_union), the plan is as following:
// project(bitmap_count()#1)
// aggregate(bitmap_union()#2)
// we should use exprId which query top plan used to decide the query top plan is use the
// bottom agg function or not
Set<ExprId> queryAggFunctionSet = queryAggregate.getOutputExpressions().stream()
.filter(expr -> !queryAggGroupSet.contains(expr))
.map(NamedExpression::getExprId)
.collect(Collectors.toSet());
Plan queryTopPlan = topPlanAndAggPair.key();
Set<Expression> topGroupByExpressions = new HashSet<>();
Set<Expression> topFunctionExpressions = new HashSet<>();
queryTopPlan.getExpressions().forEach(
expression -> {
if (expression.anyMatch(expr -> expr instanceof NamedExpression
&& queryAggFunctionSet.contains((NamedExpression) expr))) {
topFunctionExpressions.add(expression);
} else {
topGroupByExpressions.add(expression);
}
});
// only support to reference the aggregate function directly in top, will support expression later.
if (topFunctionExpressions.stream().anyMatch(
topAggFunc -> !(topAggFunc instanceof NamedExpression) && (!queryAggFunctionSet.contains(topAggFunc)
|| !queryAggFunctionSet.contains(topAggFunc.child(0))))) {
return null;
}
queryTopPlan.getExpressions().forEach(expression -> {
if (expression.anyMatch(expr -> expr instanceof NamedExpression
&& queryAggFunctionSet.contains(((NamedExpression) expr).getExprId()))) {
topFunctionExpressions.add(expression);
} else {
topGroupByExpressions.add(expression);
}
});
return Pair.of(topGroupByExpressions, topFunctionExpressions);
}
@ -427,11 +436,14 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
continue;
}
// check param in query function is same as the view function
List<Expression> viewFunctionArguments = extractViewArguments(equivalentFunction, viewFunction);
if (queryFunctionShuttled.getArguments().size() != 1 || viewFunctionArguments.size() != 1) {
List<Expression> viewFunctionArguments = extractArguments(equivalentFunction, viewFunction);
List<Expression> queryFunctionArguments =
extractArguments(equivalentFunctionEntry.getKey(), queryFunction);
// check argument size,we only support roll up function which has only one argument currently
if (queryFunctionArguments.size() != 1 || viewFunctionArguments.size() != 1) {
continue;
}
if (Objects.equals(queryFunctionShuttled.getArguments().get(0), viewFunctionArguments.get(0))) {
if (Objects.equals(queryFunctionArguments.get(0), viewFunctionArguments.get(0))) {
return true;
}
}
@ -441,14 +453,14 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
}
/**
* Extract the view function arguments by equivalentFunction pattern
* Such as equivalentFunction def is bitmap_union(to_bitmap(Any.INSTANCE)),
* viewFunction is bitmap_union(to_bitmap(case when a = 5 then 1 else 2 end))
* Extract the function arguments by functionWithAny pattern
* Such as functionWithAny def is bitmap_union(to_bitmap(Any.INSTANCE)),
* actualFunction is bitmap_union(to_bitmap(case when a = 5 then 1 else 2 end))
* after extracting, the return argument is: case when a = 5 then 1 else 2 end
*/
private List<Expression> extractViewArguments(Expression equivalentFunction, Function viewFunction) {
Set<Object> exprSetToRemove = equivalentFunction.collectToSet(expr -> !(expr instanceof Any));
return viewFunction.collectFirst(expr ->
private List<Expression> extractArguments(Expression functionWithAny, Function actualFunction) {
Set<Object> exprSetToRemove = functionWithAny.collectToSet(expr -> !(expr instanceof Any));
return actualFunction.collectFirst(expr ->
exprSetToRemove.stream().noneMatch(exprToRemove -> exprToRemove.equals(expr)));
}
}

15
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -99,8 +99,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
}
for (MaterializationContext materializationContext : materializationContexts) {
// already rewrite, bail out
if (queryPlan.getGroupExpression().isPresent() && materializationContext.alreadyRewrite(
queryPlan.getGroupExpression().get().getOwnerGroup().getGroupId())) {
if (checkIfRewritten(queryPlan, materializationContext)) {
continue;
}
List<StructInfo> viewStructInfos = extractStructInfo(materializationContext.getMvPlan(), cascadesContext);
@ -212,6 +211,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
Rewriter.getWholeTreeRewriter(rewrittenPlanContext).execute();
rewrittenPlan = rewrittenPlanContext.getRewritePlan();
materializationContext.setSuccess(true);
recordIfRewritten(queryPlan, materializationContext);
rewriteResults.add(rewrittenPlan);
}
}
@ -532,6 +532,17 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
return true;
}
protected void recordIfRewritten(Plan plan, MaterializationContext context) {
if (plan.getGroupExpression().isPresent()) {
context.addMatchedGroup(plan.getGroupExpression().get().getOwnerGroup().getGroupId());
}
}
protected boolean checkIfRewritten(Plan plan, MaterializationContext context) {
return plan.getGroupExpression().isPresent()
&& context.alreadyRewrite(plan.getGroupExpression().get().getOwnerGroup().getGroupId());
}
/**
* Query and mv match node
*/

11
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
View File

@ -196,29 +196,32 @@ public class MaterializationContext {
*/
public static String toSummaryString(List<MaterializationContext> materializationContexts,
List<MTMV> chosenMaterializationNames) {
if (materializationContexts.isEmpty()) {
return "";
}
Set<String> materializationChosenNameSet = chosenMaterializationNames.stream()
.map(MTMV::getName)
.collect(Collectors.toSet());
StringBuilder builder = new StringBuilder();
builder.append("\n\nMaterializedView\n");
builder.append("\nMaterializedView");
builder.append("\nMaterializedViewRewriteFail:");
for (MaterializationContext ctx : materializationContexts) {
if (!ctx.isSuccess()) {
Set<String> failReasonSet =
ctx.getFailReason().values().stream().map(Pair::key).collect(Collectors.toSet());
builder.append("\n\n")
builder.append("\n")
.append(" Name: ").append(ctx.getMTMV().getName())
.append("\n")
.append(" FailSummary: ").append(String.join(", ", failReasonSet));
}
}
builder.append("\n\nMaterializedViewRewriteSuccessButNotChose:\n");
builder.append("\nMaterializedViewRewriteSuccessButNotChose:\n");
builder.append(" Names: ").append(materializationContexts.stream()
.filter(materializationContext -> materializationContext.isSuccess()
&& !materializationChosenNameSet.contains(materializationContext.getMTMV().getName()))
.map(materializationContext -> materializationContext.getMTMV().getName())
.collect(Collectors.joining(", ")));
builder.append("\n\nMaterializedViewRewriteSuccessAndChose:\n");
builder.append("\nMaterializedViewRewriteSuccessAndChose:\n");
builder.append(" Names: ").append(String.join(", ", materializationChosenNameSet));
return builder.toString();
}

8
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnion.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BitmapType;
@ -35,7 +36,7 @@ import java.util.List;
* AggregateFunction 'bitmap_union'. This class is generated by GenerateFunction.
*/
public class BitmapUnion extends AggregateFunction
implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, BitmapFunction {
implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, BitmapFunction, CouldRollUp {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(BitmapType.INSTANCE).args(BitmapType.INSTANCE)
@ -78,4 +79,9 @@ public class BitmapUnion extends AggregateFunction
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}
@Override
public Function constructRollUp(Expression param, Expression... varParams) {
return new BitmapUnion(this.isDistinct(), param);
}
}

8
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionCount.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BigIntType;
@ -36,7 +37,7 @@ import java.util.List;
* AggregateFunction 'bitmap_union_count'. This class is generated by GenerateFunction.
*/
public class BitmapUnionCount extends AggregateFunction
implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, BitmapFunction {
implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, BitmapFunction, CouldRollUp {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(BigIntType.INSTANCE).args(BitmapType.INSTANCE)
@ -79,4 +80,9 @@ public class BitmapUnionCount extends AggregateFunction
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}
@Override
public Function constructRollUp(Expression param, Expression... varParams) {
return new BitmapUnionCount(param);
}
}

76
regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out
View File

@ -31,6 +31,12 @@
-- !query15_0_after --
3 3 2023-12-11 43.20 43.20 43.20 1 0
-- !query15_1_before --
2023-12-11 2023-12-11 3 3 43.20 43.20 43.20 1 0 \N 0
-- !query15_1_after --
2023-12-11 2023-12-11 3 3 43.20 43.20 43.20 1 0 \N 0
-- !query16_0_before --
3 3 2023-12-11 43.20 43.20 43.20 1 0
@ -73,6 +79,20 @@
2023-12-11 3 2023-12-11 43.20 43.20 43.20 1 0
2023-12-12 3 2023-12-12 57.40 56.20 1.20 2 0
-- !query20_1_before --
2023-12-08 2023-12-08 2 3 20.00 10.50 9.50 2 0 \N 0
2023-12-09 2023-12-09 4 3 11.50 11.50 11.50 1 0 \N 0
2023-12-10 2023-12-10 2 4 46.00 33.50 12.50 2 0 \N 0
2023-12-11 2023-12-11 3 3 43.20 43.20 43.20 1 0 \N 0
2023-12-12 2023-12-12 2 3 57.40 56.20 1.20 2 0 \N 0
-- !query20_1_after --
2023-12-08 2023-12-08 2 3 20.00 10.50 9.50 2 0 \N 0
2023-12-09 2023-12-09 4 3 11.50 11.50 11.50 1 0 \N 0
2023-12-10 2023-12-10 2 4 46.00 33.50 12.50 2 0 \N 0
2023-12-11 2023-12-11 3 3 43.20 43.20 43.20 1 0 \N 0
2023-12-12 2023-12-12 2 3 57.40 56.20 1.20 2 0 \N 0
-- !query21_0_before --
2 3 2023-12-08 20.00 10.50 9.50 2 0
2 3 2023-12-12 57.40 56.20 1.20 2 0
@ -129,6 +149,34 @@
3 3 2023-12-11 43.20 43.20 43.20 1
4 3 2023-12-09 11.50 11.50 11.50 1
-- !query25_1_before --
2023-12-08 3 20.00 10.50 9.50 2 \N \N
2023-12-09 3 11.50 11.50 11.50 1 \N \N
2023-12-10 4 46.00 33.50 12.50 2 \N \N
2023-12-11 3 43.20 43.20 43.20 1 \N \N
2023-12-12 3 57.40 56.20 1.20 2 \N \N
-- !query25_1_after --
2023-12-08 3 20.00 10.50 9.50 2 \N \N
2023-12-09 3 11.50 11.50 11.50 1 \N \N
2023-12-10 4 46.00 33.50 12.50 2 \N \N
2023-12-11 3 43.20 43.20 43.20 1 \N \N
2023-12-12 3 57.40 56.20 1.20 2 \N \N
-- !query25_2_before --
2023-12-08 3 20.00 10.50 9.50 2 \N \N 1 0 0
2023-12-09 3 11.50 11.50 11.50 1 \N \N 1 0 0
2023-12-10 4 46.00 33.50 12.50 2 \N \N 1 0 0
2023-12-11 3 43.20 43.20 43.20 1 \N \N 0 1 1
2023-12-12 3 57.40 56.20 1.20 2 \N \N 0 1 1
-- !query25_2_after --
2023-12-08 3 20.00 10.50 9.50 2 \N \N 1 0 0
2023-12-09 3 11.50 11.50 11.50 1 \N \N 1 0 0
2023-12-10 4 46.00 33.50 12.50 2 \N \N 1 0 0
2023-12-11 3 43.20 43.20 43.20 1 \N \N 0 1 1
2023-12-12 3 57.40 56.20 1.20 2 \N \N 0 1 1
-- !query1_1_before --
1 yy 0 0 11.50 11.50 11.50 1
@ -143,3 +191,31 @@
2 mi 0 0 57.40 56.20 1.20 2
2 mm 0 0 43.20 43.20 43.20 1
-- !query26_0_before --
2023-12-08 1 20.00 10.50 9.50 2 0 0
2023-12-09 1 11.50 11.50 11.50 1 0 0
2023-12-10 1 46.00 33.50 12.50 2 0 0
2023-12-11 2 43.20 43.20 43.20 1 0 0
2023-12-12 2 57.40 56.20 1.20 2 0 0
-- !query26_0_after --
2023-12-08 1 20.00 10.50 9.50 2 0 0
2023-12-09 1 11.50 11.50 11.50 1 0 0
2023-12-10 1 46.00 33.50 12.50 2 0 0
2023-12-11 2 43.20 43.20 43.20 1 0 0
2023-12-12 2 57.40 56.20 1.20 2 0 0
-- !query27_0_before --
2023-12-08 1 20.00 10.50 9.50 2 0 0
2023-12-09 1 11.50 11.50 11.50 1 0 0
2023-12-10 1 46.00 33.50 12.50 2 0 0
2023-12-11 2 43.20 43.20 43.20 1 0 0
2023-12-12 2 57.40 56.20 1.20 2 0 0
-- !query27_0_after --
2023-12-08 1 20.00 10.50 9.50 2 0 0
2023-12-09 1 11.50 11.50 11.50 1 0 0
2023-12-10 1 46.00 33.50 12.50 2 0 0
2023-12-11 2 43.20 43.20 43.20 1 0 0
2023-12-12 2 57.40 56.20 1.20 2 0 0

253
regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
View File

@ -50,7 +50,7 @@ suite("aggregate_with_roll_up") {
DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3
PROPERTIES (
"replication_num" = "1"
)
);
"""
sql """
@ -248,18 +248,18 @@ suite("aggregate_with_roll_up") {
def mv13_1 = """
select l_shipdate, o_orderdate, l_partkey, l_suppkey,
sum(o_totalprice) as sum_total,
max(o_totalprice) as max_total,
min(o_totalprice) as min_total,
count(*) as count_all,
bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic
from lineitem
left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate
group by
l_shipdate,
o_orderdate,
l_partkey,
select l_shipdate, o_orderdate, l_partkey, l_suppkey,
sum(o_totalprice) as sum_total,
max(o_totalprice) as max_total,
min(o_totalprice) as min_total,
count(*) as count_all,
bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic
from lineitem
left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate
group by
l_shipdate,
o_orderdate,
l_partkey,
l_suppkey;
"""
def query13_1 = """
@ -272,7 +272,7 @@ suite("aggregate_with_roll_up") {
from (select * from lineitem where l_shipdate = '2023-12-11') t1
left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate
group by
o_orderdate,
o_orderdate,
l_partkey,
l_suppkey;
"""
@ -349,6 +349,45 @@ suite("aggregate_with_roll_up") {
order_qt_query15_0_after "${query15_0}"
sql """ DROP MATERIALIZED VIEW IF EXISTS mv15_0"""
def mv15_1 = """
select l_shipdate, o_orderdate, l_partkey, l_suppkey,
sum(o_totalprice) as sum_total,
max(o_totalprice) as max_total,
min(o_totalprice) as min_total,
count(*) as count_all,
bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic
from lineitem
left join (select * from orders where o_orderstatus = 'o') t2
on lineitem.l_orderkey = o_orderkey and l_shipdate = o_orderdate
group by
l_shipdate,
o_orderdate,
l_partkey,
l_suppkey;
"""
def query15_1 = """
select l_shipdate, o_orderdate, l_partkey, l_suppkey,
sum(o_totalprice),
max(o_totalprice),
min(o_totalprice),
count(*),
bitmap_union_count(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)),
bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)),
count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)
from (select * from lineitem where l_shipdate = '2023-12-11') t1
left join (select * from orders where o_orderstatus = 'o') t2
on t1.l_orderkey = o_orderkey and t1.l_shipdate = o_orderdate
group by
l_shipdate,
o_orderdate,
l_partkey,
l_suppkey;
"""
order_qt_query15_1_before "${query15_1}"
check_rewrite_with_mv_partition(mv15_1, query15_1, "mv15_1", "l_shipdate")
order_qt_query15_1_after "${query15_1}"
sql """ DROP MATERIALIZED VIEW IF EXISTS mv15_1"""
// filter outside + left + use roll up dimension
def mv16_0 = "select l_shipdate, o_orderdate, l_partkey, l_suppkey, " +
@ -534,6 +573,47 @@ suite("aggregate_with_roll_up") {
order_qt_query20_0_after "${query20_0}"
sql """ DROP MATERIALIZED VIEW IF EXISTS mv20_0"""
def mv20_1 = """
select l_shipdate, o_orderdate, l_partkey, l_suppkey,
sum(o_totalprice) as sum_total,
max(o_totalprice) as max_total,
min(o_totalprice) as min_total,
count(*) as count_all,
bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic
from lineitem
left join (select * from orders where o_orderstatus = 'o') t2
on lineitem.l_orderkey = o_orderkey and l_shipdate = o_orderdate
group by
l_shipdate,
o_orderdate,
l_partkey,
l_suppkey;
"""
def query20_1 = """
select l_shipdate, o_orderdate, l_partkey, l_suppkey,
sum(o_totalprice),
max(o_totalprice),
min(o_totalprice),
count(*),
bitmap_union_count(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)),
bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)),
count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)
from lineitem
left join (select * from orders where o_orderstatus = 'o') t2
on lineitem.l_orderkey = o_orderkey and l_shipdate = o_orderdate
group by
l_shipdate,
o_orderdate,
l_partkey,
l_suppkey;
"""
order_qt_query20_1_before "${query20_1}"
check_rewrite(mv20_1, query20_1, "mv20_1")
order_qt_query20_1_after "${query20_1}"
sql """ DROP MATERIALIZED VIEW IF EXISTS mv20_1"""
// filter inside + right + left + use not roll up dimension
def mv21_0 = "select l_shipdate, o_orderdate, l_partkey, l_suppkey, " +
"sum(o_totalprice) as sum_total, " +
@ -731,6 +811,83 @@ suite("aggregate_with_roll_up") {
order_qt_query25_0_after "${query25_0}"
sql """ DROP MATERIALIZED VIEW IF EXISTS mv25_0"""
// bitmap_union roll up to bitmap_union
def mv25_1 = """
select l_shipdate, o_orderdate, l_partkey, l_suppkey,
sum(o_totalprice) as sum_total,
max(o_totalprice) as max_total,
min(o_totalprice) as min_total,
count(*) as count_all,
bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 2, 3) then o_custkey else null end)) cnt_1,
bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (3, 4, 5) then o_custkey else null end)) as cnt_2
from lineitem
left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate
group by
l_shipdate,
o_orderdate,
l_partkey,
l_suppkey;
"""
def query25_1 = """
select o_orderdate, l_suppkey,
sum(o_totalprice) as sum_total,
max(o_totalprice) as max_total,
min(o_totalprice) as min_total,
count(*) as count_all,
bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 2, 3) then o_custkey else null end)) cnt_1,
bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (3, 4, 5) then o_custkey else null end)) as cnt_2
from lineitem
left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate
group by
o_orderdate,
l_suppkey;
"""
order_qt_query25_1_before "${query25_1}"
check_rewrite(mv25_1, query25_1, "mv25_1")
order_qt_query25_1_after "${query25_1}"
sql """ DROP MATERIALIZED VIEW IF EXISTS mv25_1"""
// bitmap_union roll up to bitmap_union_count
def mv25_2 = """
select l_shipdate, o_orderdate, l_partkey, l_suppkey,
sum(o_totalprice) as sum_total,
max(o_totalprice) as max_total,
min(o_totalprice) as min_total,
count(*) as count_all,
bitmap_union(to_bitmap(case when o_shippriority > 0 and o_orderkey IN (1, 2, 3) then o_custkey else null end)) cnt_1,
bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (3, 4, 5) then o_custkey else null end)) as cnt_2
from lineitem
left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate
group by
l_shipdate,
o_orderdate,
l_partkey,
l_suppkey;
"""
def query25_2 = """
select o_orderdate, l_suppkey,
sum(o_totalprice) as sum_total,
max(o_totalprice) as max_total,
min(o_totalprice) as min_total,
count(*) as count_all,
bitmap_union(to_bitmap(case when o_shippriority > 0 and o_orderkey IN (1, 2, 3) then o_custkey else null end)),
bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (3, 4, 5) then o_custkey else null end)),
bitmap_union_count(to_bitmap(case when o_shippriority > 0 and o_orderkey IN (1, 2, 3) then o_custkey else null end)),
bitmap_union_count(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (3, 4, 5) then o_custkey else null end)),
count(distinct case when o_shippriority > 1 and o_orderkey IN (3, 4, 5) then o_custkey else null end)
from lineitem
left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate
group by
o_orderdate,
l_suppkey;
"""
order_qt_query25_2_before "${query25_2}"
check_rewrite(mv25_2, query25_2, "mv25_2")
order_qt_query25_2_after "${query25_2}"
sql """ DROP MATERIALIZED VIEW IF EXISTS mv25_2"""
// single table
// filter + use roll up dimension
def mv1_1 = "select o_orderdate, o_shippriority, o_comment, " +
@ -798,5 +955,71 @@ suite("aggregate_with_roll_up") {
order_qt_query2_0_after "${query2_0}"
sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_0"""
// can not rewrite, todo
// can not rewrite
// bitmap_union_count is aggregate function but not support roll up
def mv26_0 = """
select o_orderdate, o_shippriority, o_comment,
sum(o_totalprice) as sum_total,
max(o_totalprice) as max_total,
min(o_totalprice) as min_total,
count(*) as count_all,
bitmap_union_count(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1,
bitmap_union_count(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2
from orders
group by
o_orderdate,
o_shippriority,
o_comment;
"""
def query26_0 = """
select o_orderdate, o_shippriority,
sum(o_totalprice) as sum_total,
max(o_totalprice) as max_total,
min(o_totalprice) as min_total,
count(*) as count_all,
bitmap_union_count(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1,
bitmap_union_count(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2
from orders
group by
o_orderdate,
o_shippriority;
"""
order_qt_query26_0_before "${query26_0}"
check_not_match(mv26_0, query26_0, "mv26_0")
order_qt_query26_0_after "${query26_0}"
sql """ DROP MATERIALIZED VIEW IF EXISTS mv26_0"""
// bitmap_count is not aggregate function, so doesn't not support roll up
def mv27_0 = """
select o_orderdate, o_shippriority, o_comment,
sum(o_totalprice) as sum_total,
max(o_totalprice) as max_total,
min(o_totalprice) as min_total,
count(*) as count_all,
bitmap_count(bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 2, 3) then o_custkey else null end))) cnt_1,
bitmap_count(bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (3, 4, 5) then o_custkey else null end))) as cnt_2
from orders
group by
o_orderdate,
o_shippriority,
o_comment;
"""
def query27_0 = """
select o_orderdate, o_shippriority,
sum(o_totalprice) as sum_total,
max(o_totalprice) as max_total,
min(o_totalprice) as min_total,
count(*) as count_all,
bitmap_count(bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 2, 3) then o_custkey else null end))) cnt_1,
bitmap_count(bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (3, 4, 5) then o_custkey else null end))) as cnt_2
from orders
group by
o_orderdate,
o_shippriority;
"""
order_qt_query27_0_before "${query27_0}"
check_not_match(mv27_0, query27_0, "mv27_0")
order_qt_query27_0_after "${query27_0}"
sql """ DROP MATERIALIZED VIEW IF EXISTS mv27_0"""
}