database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Feature](materialized-view) support count(1) on materialized view (#28135)

Browse Source 
support count(1) on materialized view
fix match failed like select k1, sum(k1) from t group by k1
This commit is contained in:
Pxl
2023-12-09 01:36:46 +08:00
committed by GitHub
parent b6e72d57c5
commit 027b06059a
9 changed files with 114 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
View File

@ -258,14 +258,6 @@ public class CreateMaterializedViewStmt extends DdlStmt {
throw new AnalysisException("The materialized view only support the single column or function expr. "
+ "Error column: " + selectListItemExpr.toSql());
}
List<SlotRef> slots = new ArrayList<>();
selectListItemExpr.collect(SlotRef.class, slots);
if (!isReplay && slots.size() == 0) {
throw new AnalysisException(
"The materialized view contain constant expr is disallowed, expr: "
+ selectListItemExpr.toSql());
}
if (selectListItemExpr instanceof FunctionCallExpr
&& ((FunctionCallExpr) selectListItemExpr).isAggregateFunction()) {
@ -278,6 +270,13 @@ public class CreateMaterializedViewStmt extends DdlStmt {
// build mv column item
mvColumnItemList.add(buildMVColumnItem(analyzer, functionCallExpr));
} else {
List<SlotRef> slots = new ArrayList<>();
selectListItemExpr.collect(SlotRef.class, slots);
if (!isReplay && slots.size() == 0) {
throw new AnalysisException(
"The materialized view contain constant expr is disallowed, expr: "
+ selectListItemExpr.toSql());
}
if (meetAggregate) {
throw new AnalysisException("The aggregate column should be after the single column");
}

98
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java
View File

@ -55,6 +55,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapHash;
import org.apache.doris.nereids.trees.expressions.functions.scalar.HllHash;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmap;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmapWithCheck;
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.trees.plans.Plan;
@ -1072,24 +1073,26 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
// has rewritten agg functions
Map<Slot, Slot> slotMap = exprRewriteMap.slotMap;
if (!slotMap.isEmpty()) {
// Note that the slots in the rewritten agg functions shouldn't appear in filters or grouping expressions.
// For example: we have a duplicated-type table t(c1, c2) and a materialized index that has
// a bitmap_union column `mv_bitmap_union_c2` for the column c2.
// The query `select c1, count(distinct c2) from t where c2 > 0 group by c1` can't use the materialized
// index because we have a filter `c2 > 0` for the aggregated column c2.
Set<Slot> slotsToReplace = slotMap.keySet();
Set<String> indexConjuncts = PlanNode
// Note that the slots in the rewritten agg functions shouldn't appear in filters or grouping expressions.
// For example: we have a duplicated-type table t(c1, c2) and a materialized index that has
// a bitmap_union column `mv_bitmap_union_c2` for the column c2.
// The query `select c1, count(distinct c2) from t where c2 > 0 group by c1` can't use the materialized
// index because we have a filter `c2 > 0` for the aggregated column c2.
Set<Slot> slotsToReplace = slotMap.keySet();
Set<String> indexConjuncts;
try {
indexConjuncts = PlanNode
.splitAndCompoundPredicateToConjuncts(context.checkContext.getMeta().getWhereClause()).stream()
.map(e -> new NereidsParser().parseExpression(e.toSql()).toSql()).collect(Collectors.toSet());
if (isInputSlotsContainsNone(
predicates.stream().filter(e -> !indexConjuncts.contains(e.toSql())).collect(Collectors.toList()),
slotsToReplace) && isInputSlotsContainsNone(groupingExprs, slotsToReplace)) {
ImmutableSet<Slot> newRequiredSlots = requiredScanOutput.stream()
.map(slot -> (Slot) ExpressionUtils.replace(slot, slotMap))
.collect(ImmutableSet.toImmutableSet());
return new AggRewriteResult(index, true, newRequiredSlots, exprRewriteMap);
}
} catch (Exception e) {
return new AggRewriteResult(index, false, null, null);
}
if (isInputSlotsContainsNone(
predicates.stream().filter(e -> !indexConjuncts.contains(e.toSql())).collect(Collectors.toList()),
slotsToReplace) && isInputSlotsContainsNone(groupingExprs, slotsToReplace)) {
ImmutableSet<Slot> newRequiredSlots = requiredScanOutput.stream()
.map(slot -> (Slot) ExpressionUtils.replace(slot, slotMap)).collect(ImmutableSet.toImmutableSet());
return new AggRewriteResult(index, true, newRequiredSlots, exprRewriteMap);
}
return new AggRewriteResult(index, false, null, null);
@ -1207,8 +1210,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
Expression expr = new ToBitmapWithCheck(castIfNeed(count.child(0), BigIntType.INSTANCE));
// count distinct a value column.
if (slotOpt.isPresent() && !context.checkContext.keyNameToColumn.containsKey(
normalizeName(expr.toSql()))) {
if (slotOpt.isPresent()) {
String bitmapUnionColumn = normalizeName(CreateMaterializedViewStmt.mvColumnBuilder(
AggregateType.BITMAP_UNION, CreateMaterializedViewStmt.mvColumnBuilder(expr.toSql())));
@ -1229,33 +1231,37 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
return bitmapUnionCount;
}
}
} else if (!count.isDistinct() && count.arity() == 1) {
}
Expression child = null;
if (!count.isDistinct() && count.arity() == 1) {
// count(col) -> sum(mva_SUM__CASE WHEN col IS NULL THEN 0 ELSE 1 END)
Optional<Slot> slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(count.child(0));
// count a value column.
if (slotOpt.isPresent() && !context.checkContext.keyNameToColumn.containsKey(
normalizeName(slotOpt.get().toSql()))) {
String countColumn = normalizeName(CreateMaterializedViewStmt
.mvColumnBuilder(AggregateType.SUM,
CreateMaterializedViewStmt.mvColumnBuilder(slotToCaseWhen(slotOpt.get()).toSql())));
if (slotOpt.isPresent()) {
child = slotOpt.get();
}
} else if (count.arity() == 0) {
// count(*) / count(1) -> sum(mva_SUM__CASE WHEN 1 IS NULL THEN 0 ELSE 1 END)
child = new TinyIntLiteral((byte) 1);
}
Column mvColumn = context.checkContext.getColumn(countColumn);
// has bitmap_union_count column
if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) {
Slot countSlot = context.checkContext.scan.getOutputByIndex(context.checkContext.index)
.stream()
.filter(s -> countColumn.equalsIgnoreCase(normalizeName(s.getName())))
.findFirst()
.orElseThrow(() -> new AnalysisException(
"cannot find count slot when select mv"));
if (child != null) {
String countColumn = normalizeName(CreateMaterializedViewStmt.mvColumnBuilder(AggregateType.SUM,
CreateMaterializedViewStmt.mvColumnBuilder(slotToCaseWhen(child).toSql())));
context.exprRewriteMap.slotMap.put(slotOpt.get(), countSlot);
context.exprRewriteMap.projectExprMap.put(slotOpt.get(), countSlot);
Sum sum = new Sum(countSlot);
context.exprRewriteMap.aggFuncMap.put(count, sum);
return sum;
Column mvColumn = context.checkContext.getColumn(countColumn);
if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) {
Slot countSlot = context.checkContext.scan.getOutputByIndex(context.checkContext.index).stream()
.filter(s -> countColumn.equalsIgnoreCase(normalizeName(s.getName()))).findFirst()
.orElseThrow(() -> new AnalysisException("cannot find count slot when select mv"));
if (child instanceof Slot) {
context.exprRewriteMap.slotMap.put((Slot) child, countSlot);
}
context.exprRewriteMap.projectExprMap.put(child, countSlot);
Sum sum = new Sum(countSlot);
context.exprRewriteMap.aggFuncMap.put(count, sum);
return sum;
}
}
return count;
@ -1416,8 +1422,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
}
Optional<Slot> slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(ndv.child(0));
// ndv on a value column.
if (slotOpt.isPresent() && !context.checkContext.keyNameToColumn.containsKey(
normalizeName(slotOpt.get().toSql()))) {
if (slotOpt.isPresent()) {
Expression expr = castIfNeed(ndv.child(), VarcharType.SYSTEM_DEFAULT);
String hllUnionColumn = normalizeName(
CreateMaterializedViewStmt.mvColumnBuilder(AggregateType.HLL_UNION,
@ -1450,8 +1455,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
return result;
}
Optional<Slot> slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(sum.child(0));
if (!sum.isDistinct() && slotOpt.isPresent()
&& !context.checkContext.keyNameToColumn.containsKey(normalizeName(slotOpt.get().toSql()))) {
if (!sum.isDistinct() && slotOpt.isPresent()) {
Expression expr = castIfNeed(sum.child(), BigIntType.INSTANCE);
String sumColumn = normalizeName(CreateMaterializedViewStmt.mvColumnBuilder(AggregateType.SUM,
CreateMaterializedViewStmt.mvColumnBuilder(expr.toSql())));
@ -1487,10 +1491,8 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
Set<Slot> slots = aggregateFunction.collect(SlotReference.class::isInstance);
for (Slot slot : slots) {
if (!context.checkContext.keyNameToColumn.containsKey(normalizeName(slot.toSql()))) {
context.exprRewriteMap.slotMap.put(slot, aggStateSlot);
context.exprRewriteMap.projectExprMap.put(slot, aggStateSlot);
}
context.exprRewriteMap.slotMap.put(slot, aggStateSlot);
context.exprRewriteMap.projectExprMap.put(slot, aggStateSlot);
}
MergeCombinator mergeCombinator = new MergeCombinator(Arrays.asList(aggStateSlot), aggregateFunction);

1
fe/fe-core/src/main/java/org/apache/doris/task/AlterReplicaTask.java
View File

@ -128,7 +128,6 @@ public class AlterReplicaTask extends AgentTask {
List<SlotRef> slots = Lists.newArrayList();
entry.getValue().collect(SlotRef.class, slots);
TAlterMaterializedViewParam mvParam = new TAlterMaterializedViewParam(entry.getKey());
mvParam.setOriginColumnName(slots.get(0).getColumnName());
mvParam.setMvExpr(entry.getValue().treeToThrift());
req.addToMaterializedViewParams(mvParam);
}