From f8b368a85e5396cdd59f2879fbf4ff149110a1fe Mon Sep 17 00:00:00 2001 From: Shuo Wang Date: Thu, 22 Dec 2022 14:40:25 +0800 Subject: [PATCH] [Feature](Nereids) Support bitmap for materialized index. (#14863) This PR adds the rewriting and matching logic for the bitmap_union column in materialized index. If a materialized index has bitmap_union column, we try to rewrite count distinct or bitmap_union_count to the bitmap_union column in materialized index. --- .../translator/PhysicalPlanTranslator.java | 5 +- .../jobs/batch/NereidsRewriteJobExecutor.java | 2 + .../nereids/properties/LogicalProperties.java | 17 +- .../apache/doris/nereids/rules/RuleType.java | 1 + .../rules/analysis/CheckAfterRewrite.java | 4 +- .../AbstractSelectMaterializedIndexRule.java | 56 ++- .../SelectMaterializedIndexWithAggregate.java | 472 ++++++++++++++++-- ...lectMaterializedIndexWithoutAggregate.java | 23 +- .../rewrite/logical/CountDistinctRewrite.java | 70 +++ .../functions/agg/BitmapUnionCount.java | 6 + .../visitor/AggregateFunctionVisitor.java | 5 + .../nereids/trees/plans/AbstractPlan.java | 5 + .../doris/nereids/trees/plans/FakePlan.java | 7 + .../doris/nereids/trees/plans/Plan.java | 6 + .../nereids/trees/plans/commands/Command.java | 5 + .../plans/logical/AbstractLogicalPlan.java | 2 +- .../trees/plans/logical/LogicalOlapScan.java | 111 +++- .../apache/doris/nereids/types/DataType.java | 4 + .../doris/nereids/util/ExpressionUtils.java | 20 +- .../nereids/rules/mv/SelectMvIndexTest.java | 310 +++++++----- .../nereids/trees/plans/PlanToStringTest.java | 3 +- 21 files changed, 911 insertions(+), 223 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/CountDistinctRewrite.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 6237bef417..bf1d327830 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -437,7 +437,10 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor slotList = olapScan.getOutput(); + List slotList = new ImmutableList.Builder() + .addAll(olapScan.getOutput()) + .addAll(olapScan.getNonUserVisibleOutput()) + .build(); OlapTable olapTable = olapScan.getTable(); TupleDescriptor tupleDescriptor = generateTupleDesc(slotList, olapTable, context); tupleDescriptor.setTable(olapTable); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java index 587ae5b6ce..54a3ff465f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithAggregate; import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithoutAggregate; import org.apache.doris.nereids.rules.rewrite.logical.BuildAggForUnion; import org.apache.doris.nereids.rules.rewrite.logical.ColumnPruning; +import org.apache.doris.nereids.rules.rewrite.logical.CountDistinctRewrite; import org.apache.doris.nereids.rules.rewrite.logical.EliminateAggregate; import org.apache.doris.nereids.rules.rewrite.logical.EliminateFilter; import org.apache.doris.nereids.rules.rewrite.logical.EliminateGroupByConstant; @@ -95,6 +96,7 @@ public class NereidsRewriteJobExecutor extends BatchRulesJob { .add(topDownBatch(ImmutableList.of(new EliminateLimit()))) .add(topDownBatch(ImmutableList.of(new EliminateFilter()))) .add(topDownBatch(ImmutableList.of(new PruneOlapScanPartition()))) + .add(topDownBatch(ImmutableList.of(new CountDistinctRewrite()))) .add(topDownBatch(ImmutableList.of(new SelectMaterializedIndexWithAggregate()))) .add(topDownBatch(ImmutableList.of(new SelectMaterializedIndexWithoutAggregate()))) .add(topDownBatch(ImmutableList.of(new PruneOlapScanTablet()))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java index bdfd22cbe1..77da03452c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import java.util.HashSet; @@ -37,21 +38,29 @@ import java.util.stream.Collectors; */ public class LogicalProperties { protected final Supplier> outputSupplier; + protected final Supplier> nonUserVisibleOutputSupplier; protected final Supplier> outputExprIdsSupplier; protected final Supplier> outputSetSupplier; protected final Supplier> outputExprIdSetSupplier; private Integer hashCode = null; + public LogicalProperties(Supplier> outputSupplier) { + this(outputSupplier, ImmutableList::of); + } + /** * constructor of LogicalProperties. * * @param outputSupplier provide the output. Supplier can lazy compute output without * throw exception for which children have UnboundRelation */ - public LogicalProperties(Supplier> outputSupplier) { + public LogicalProperties(Supplier> outputSupplier, Supplier> nonUserVisibleOutputSupplier) { this.outputSupplier = Suppliers.memoize( Objects.requireNonNull(outputSupplier, "outputSupplier can not be null") ); + this.nonUserVisibleOutputSupplier = Suppliers.memoize( + Objects.requireNonNull(nonUserVisibleOutputSupplier, "nonUserVisibleOutputSupplier can not be null") + ); this.outputExprIdsSupplier = Suppliers.memoize( () -> this.outputSupplier.get().stream().map(NamedExpression::getExprId).map(Id.class::cast) .collect(Collectors.toList()) @@ -69,6 +78,10 @@ public class LogicalProperties { return outputSupplier.get(); } + public List getNonUserVisibleOutput() { + return nonUserVisibleOutputSupplier.get(); + } + public Set getOutputSet() { return outputSetSupplier.get(); } @@ -82,7 +95,7 @@ public class LogicalProperties { } public LogicalProperties withOutput(List output) { - return new LogicalProperties(Suppliers.ofInstance(output)); + return new LogicalProperties(Suppliers.ofInstance(output), nonUserVisibleOutputSupplier); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index a4b4afed4d..ffd863be08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -158,6 +158,7 @@ public enum RuleType { HIDE_ONE_ROW_RELATION_UNDER_UNION(RuleTypeClass.REWRITE), MERGE_SET_OPERATION(RuleTypeClass.REWRITE), BUILD_AGG_FOR_UNION(RuleTypeClass.REWRITE), + COUNT_DISTINCT_REWRITE(RuleTypeClass.REWRITE), REWRITE_SENTINEL(RuleTypeClass.REWRITE), // limit push down diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java index 40ea9300a6..d913f381ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java @@ -31,6 +31,7 @@ import org.apache.commons.lang.StringUtils; import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * some check need to do after analyze whole plan. @@ -49,8 +50,7 @@ public class CheckAfterRewrite extends OneAnalysisRuleFactory { .flatMap(expr -> expr.getInputSlots().stream()) .collect(Collectors.toSet()); Set childrenOutput = plan.children().stream() - .map(Plan::getOutput) - .flatMap(List::stream) + .flatMap(child -> Stream.concat(child.getOutput().stream(), child.getNonUserVisibleOutput().stream())) .collect(Collectors.toSet()); notFromChildren.removeAll(childrenOutput); notFromChildren = removeValidVirtualSlots(notFromChildren, childrenOutput); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/AbstractSelectMaterializedIndexRule.java index e6471d2353..03fe7de431 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/AbstractSelectMaterializedIndexRule.java @@ -35,6 +35,7 @@ import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import org.apache.commons.collections.CollectionUtils; import java.util.Comparator; import java.util.List; @@ -61,21 +62,14 @@ public abstract class AbstractSelectMaterializedIndexRule { } } - /** - * 1. indexes have all the required columns. - * 2. find matching key prefix most. - * 3. sort by row count, column count and index id. - */ - protected List filterAndOrder( - Stream candidates, + protected boolean containAllRequiredColumns( + MaterializedIndex index, LogicalOlapScan scan, - Set requiredScanOutput, - List predicates) { - + Set requiredScanOutput) { OlapTable table = scan.getTable(); // Scan slot exprId -> slot name - Map exprIdToName = scan.getOutput() - .stream() + Map exprIdToName = Stream.concat( + scan.getOutput().stream(), scan.getNonUserVisibleOutput().stream()) .collect(Collectors.toMap(NamedExpression::getExprId, NamedExpression::getName)); // get required column names in metadata. @@ -84,22 +78,32 @@ public abstract class AbstractSelectMaterializedIndexRule { .map(slot -> exprIdToName.get(slot.getExprId())) .collect(Collectors.toSet()); - // 1. filter index contains all the required columns by column name. - List containAllRequiredColumns = candidates - .filter(index -> table.getSchemaByIndexId(index.getId(), true) - .stream() - .map(Column::getName) - .collect(Collectors.toSet()) - .containsAll(requiredColumnNames) - ).collect(Collectors.toList()); + return table.getSchemaByIndexId(index.getId(), true).stream() + .map(Column::getName) + .collect(Collectors.toSet()) + .containsAll(requiredColumnNames); + } - // 2. find matching key prefix most. - List matchingKeyPrefixMost = matchPrefixMost(scan, containAllRequiredColumns, predicates, - exprIdToName); + /** + * 1. find matching key prefix most. + * 2. sort by row count, column count and index id. + */ + protected long selectBestIndex( + List candidates, + LogicalOlapScan scan, + List predicates) { + OlapTable table = scan.getTable(); + // Scan slot exprId -> slot name + Map exprIdToName = scan.getOutput() + .stream() + .collect(Collectors.toMap(NamedExpression::getExprId, NamedExpression::getName)); + + // find matching key prefix most. + List matchingKeyPrefixMost = matchPrefixMost(scan, candidates, predicates, exprIdToName); List partitionIds = scan.getSelectedPartitionIds(); - // 3. sort by row count, column count and index id. - return matchingKeyPrefixMost.stream() + // sort by row count, column count and index id. + List sortedIndexIds = matchingKeyPrefixMost.stream() .map(MaterializedIndex::getId) .sorted(Comparator // compare by row count @@ -111,6 +115,8 @@ public abstract class AbstractSelectMaterializedIndexRule { // compare by index id .thenComparing(rid -> (Long) rid)) .collect(Collectors.toList()); + + return CollectionUtils.isEmpty(sortedIndexIds) ? scan.getTable().getBaseIndexId() : sortedIndexIds.get(0); } protected List matchPrefixMost( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java index f67886dafa..5c0a017196 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.rules.mv; +import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.MaterializedIndex; @@ -26,16 +27,21 @@ import org.apache.doris.nereids.annotation.Developing; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.agg.Max; import org.apache.doris.nereids.trees.expressions.functions.agg.Min; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmap; +import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PreAggStatus; import org.apache.doris.nereids.trees.plans.algebra.Project; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; @@ -48,6 +54,9 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.collect.Streams; import java.util.List; import java.util.Map; @@ -76,15 +85,25 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial // Aggregate(Scan) logicalAggregate(logicalOlapScan().when(this::shouldSelectIndex)).then(agg -> { LogicalOlapScan scan = agg.child(); - Pair> result = select( + SelectResult result = select( scan, agg.getInputSlots(), ImmutableList.of(), extractAggFunctionAndReplaceSlot(agg, Optional.empty()), agg.getGroupByExpressions()); - return agg.withChildren( - scan.withMaterializedIndexSelected(result.key(), result.value()) - ); + if (result.exprRewriteMap.isEmpty()) { + return agg.withChildren( + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId) + ); + } else { + return new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.empty(), Optional.empty(), result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId) + ); + } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_SCAN), // filter could push down scan. @@ -98,16 +117,31 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial .addAll(filter.getInputSlots()) .build(); - Pair> result = select( + SelectResult result = select( scan, requiredSlots, filter.getConjuncts(), extractAggFunctionAndReplaceSlot(agg, Optional.empty()), agg.getGroupByExpressions() ); - return agg.withChildren(filter.withChildren( - scan.withMaterializedIndexSelected(result.key(), result.value()) - )); + + if (result.exprRewriteMap.isEmpty()) { + return agg.withChildren(filter.withChildren( + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId) + )); + } else { + return new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.empty(), Optional.empty(), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + // Not that no need to replace slots in the filter, because the slots to replace + // are value columns, which shouldn't appear in filters. + filter.withChildren( + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)) + ); + } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_FILTER_SCAN), // column pruning or other projections such as alias, etc. @@ -116,7 +150,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial .then(agg -> { LogicalProject project = agg.child(); LogicalOlapScan scan = project.child(); - Pair> result = select( + SelectResult result = select( scan, project.getInputSlots(), ImmutableList.of(), @@ -124,11 +158,28 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial Optional.of(project)), agg.getGroupByExpressions() ); - return agg.withChildren( - project.withChildren( - scan.withMaterializedIndexSelected(result.key(), result.value()) - ) - ); + + if (result.exprRewriteMap.isEmpty()) { + return agg.withChildren( + project.withChildren( + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId) + ) + ); + } else { + List newProjectList = replaceProjectList(project, + result.exprRewriteMap.projectExprMap); + LogicalProject newProject = new LogicalProject<>( + newProjectList, + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)); + return new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + newProject + ); + } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_PROJECT_SCAN), // filter could push down and project. @@ -141,7 +192,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial Set requiredSlots = Stream.concat( project.getInputSlots().stream(), filter.getInputSlots().stream()) .collect(Collectors.toSet()); - Pair> result = select( + SelectResult result = select( scan, requiredSlots, filter.getConjuncts(), @@ -149,9 +200,27 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial ExpressionUtils.replace(agg.getGroupByExpressions(), project.getAliasToProducer()) ); - return agg.withChildren(project.withChildren(filter.withChildren( - scan.withMaterializedIndexSelected(result.key(), result.value()) - ))); + + if (result.exprRewriteMap.isEmpty()) { + return agg.withChildren(project.withChildren(filter.withChildren( + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId) + ))); + } else { + List newProjectList = replaceProjectList(project, + result.exprRewriteMap.projectExprMap); + LogicalProject newProject = new LogicalProject<>(newProjectList, + filter.withChildren(scan.withMaterializedIndexSelected(result.preAggStatus, + result.indexId))); + + return new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + newProject + ); + } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_PROJECT_FILTER_SCAN), // filter can't push down @@ -161,7 +230,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial LogicalFilter> filter = agg.child(); LogicalProject project = filter.child(); LogicalOlapScan scan = project.child(); - Pair> result = select( + SelectResult result = select( scan, project.getInputSlots(), ImmutableList.of(), @@ -169,9 +238,26 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial ExpressionUtils.replace(agg.getGroupByExpressions(), project.getAliasToProducer()) ); - return agg.withChildren(filter.withChildren(project.withChildren( - scan.withMaterializedIndexSelected(result.key(), result.value()) - ))); + + if (result.exprRewriteMap.isEmpty()) { + return agg.withChildren(filter.withChildren(project.withChildren( + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId) + ))); + } else { + List newProjectList = replaceProjectList(project, + result.exprRewriteMap.projectExprMap); + LogicalProject newProject = new LogicalProject<>(newProjectList, + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)); + + return new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + filter.withChildren(newProject) + ); + } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_FILTER_PROJECT_SCAN) ); } @@ -183,11 +269,12 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial /** * Select materialized index ids. *

- * 1. find candidate indexes by pre-agg status: checking input aggregate functions and group by expressions - * and pushdown predicates. - * 2. filter and order the candidate indexes. + * 1. find candidate indexes by pre-agg status: + * checking input aggregate functions and group by expressions and pushdown predicates. + * 2. filter indexes that have all the required columns. + * 3. select best index from all the candidate indexes that could use. */ - private Pair> select( + private SelectResult select( LogicalOlapScan scan, Set requiredScanOutput, List predicates, @@ -199,28 +286,32 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial OlapTable table = scan.getTable(); - // 0. check pre-aggregation status. - final PreAggStatus preAggStatus; - final Stream checkPreAggResult; switch (table.getKeysType()) { case AGG_KEYS: - case UNIQUE_KEYS: - // Check pre-aggregation status by base index for aggregate-keys and unique-keys OLAP table. - preAggStatus = checkPreAggStatus(scan, table.getBaseIndexId(), predicates, + case UNIQUE_KEYS: { + // Only checking pre-aggregation status by base index is enough for aggregate-keys and + // unique-keys OLAP table. + // Because the schemas in non-base materialized index are subsets of the schema of base index. + PreAggStatus preAggStatus = checkPreAggStatus(scan, table.getBaseIndexId(), predicates, aggregateFunctions, groupingExprs); if (preAggStatus.isOff()) { // return early if pre agg status if off. - return Pair.of(preAggStatus, ImmutableList.of(scan.getTable().getBaseIndexId())); + return new SelectResult(preAggStatus, scan.getTable().getBaseIndexId(), new ExprRewriteMap()); + } else { + List rollupsWithAllRequiredCols = table.getVisibleIndex().stream() + .filter(index -> containAllRequiredColumns(index, scan, requiredScanOutput)) + .collect(Collectors.toList()); + return new SelectResult(preAggStatus, selectBestIndex(rollupsWithAllRequiredCols, scan, predicates), + new ExprRewriteMap()); } - checkPreAggResult = table.getVisibleIndex().stream(); - break; - case DUP_KEYS: + } + case DUP_KEYS: { Map> indexesGroupByIsBaseOrNot = table.getVisibleIndex() .stream() .collect(Collectors.groupingBy(index -> index.getId() == table.getBaseIndexId())); // Duplicate-keys table could use base index and indexes that pre-aggregation status is on. - checkPreAggResult = Stream.concat( + Stream checkPreAggResult = Stream.concat( indexesGroupByIsBaseOrNot.get(true).stream(), indexesGroupByIsBaseOrNot.getOrDefault(false, ImmutableList.of()) .stream() @@ -228,15 +319,51 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial aggregateFunctions, groupingExprs).isOn()) ); + Set candidatesWithoutRewriting = checkPreAggResult.collect(Collectors.toSet()); + + // try to rewrite bitmap, hll by materialized index columns. + List candidatesWithRewriting = indexesGroupByIsBaseOrNot.getOrDefault(false, + ImmutableList.of()) + .stream() + .filter(index -> !candidatesWithoutRewriting.contains(index)) + .map(index -> rewriteAgg(index, scan, requiredScanOutput, predicates, aggregateFunctions, + groupingExprs)) + .filter(result -> result.success) + .collect(Collectors.toList()); + + List haveAllRequiredColumns = Streams.concat( + candidatesWithoutRewriting.stream() + .filter(index -> containAllRequiredColumns(index, scan, requiredScanOutput)), + candidatesWithRewriting + .stream() + .filter(aggRewriteResult -> containAllRequiredColumns(aggRewriteResult.index, scan, + aggRewriteResult.requiredScanOutput)) + .map(aggRewriteResult -> aggRewriteResult.index) + ).collect(Collectors.toList()); + + long selectIndexId = selectBestIndex(haveAllRequiredColumns, scan, predicates); + Optional rewriteResultOpt = candidatesWithRewriting.stream() + .filter(aggRewriteResult -> aggRewriteResult.index.getId() == selectIndexId) + .findAny(); // Pre-aggregation is set to `on` by default for duplicate-keys table. - preAggStatus = PreAggStatus.on(); - break; + return new SelectResult(PreAggStatus.on(), selectIndexId, + rewriteResultOpt.map(r -> r.exprRewriteMap).orElse(new ExprRewriteMap())); + } default: throw new RuntimeException("Not supported keys type: " + table.getKeysType()); } + } - List sortedIndexId = filterAndOrder(checkPreAggResult, scan, requiredScanOutput, predicates); - return Pair.of(preAggStatus, sortedIndexId); + private static class SelectResult { + public final PreAggStatus preAggStatus; + public final long indexId; + public ExprRewriteMap exprRewriteMap; + + public SelectResult(PreAggStatus preAggStatus, long indexId, ExprRewriteMap exprRewriteMap) { + this.preAggStatus = preAggStatus; + this.indexId = indexId; + this.exprRewriteMap = exprRewriteMap; + } } /** @@ -271,6 +398,13 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial .collect(Collectors.toList()); } + private static AggregateFunction replaceAggFuncInput(AggregateFunction aggFunc, + Optional> slotToProducerOpt) { + return slotToProducerOpt.map( + slotToExpressions -> (AggregateFunction) ExpressionUtils.replace(aggFunc, slotToExpressions)) + .orElse(aggFunc); + } + /////////////////////////////////////////////////////////////////////////// // Set pre-aggregation status. /////////////////////////////////////////////////////////////////////////// @@ -338,7 +472,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial @Override public PreAggStatus visitCount(Count count, CheckContext context) { // Now count(distinct key_column) is only supported for aggregate-keys and unique-keys OLAP table. - if (count.isDistinct()) { + if (count.isDistinct() && count.arity() == 1) { Optional exprIdOpt = extractSlotId(count.child(0)); if (exprIdOpt.isPresent() && context.exprIdToKeyColumn.containsKey(exprIdOpt.get())) { return PreAggStatus.on(); @@ -348,6 +482,16 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial "Count distinct is only valid for key columns, but meet %s.", count.toSql())); } + @Override + public PreAggStatus visitBitmapUnionCount(BitmapUnionCount bitmapUnionCount, CheckContext context) { + Optional slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(bitmapUnionCount.child()); + if (slotOpt.isPresent() && context.exprIdToValueColumn.containsKey(slotOpt.get().getExprId())) { + return PreAggStatus.on(); + } else { + return PreAggStatus.off("invalid bitmap_union_count: " + bitmapUnionCount.toSql()); + } + } + private PreAggStatus checkAggFunc( AggregateFunction aggFunc, AggregateType matchingAggType, @@ -389,7 +533,10 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial public final Map exprIdToKeyColumn; public final Map exprIdToValueColumn; + public final LogicalOlapScan scan; + public CheckContext(LogicalOlapScan scan, long indexId) { + this.scan = scan; // map> Map> nameToColumnGroupingByIsKey = scan.getTable().getSchemaByIndexId(indexId) @@ -400,8 +547,8 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial )); Map keyNameToColumn = nameToColumnGroupingByIsKey.get(true); Map valueNameToColumn = nameToColumnGroupingByIsKey.getOrDefault(false, ImmutableMap.of()); - Map nameToExprId = scan.getOutput() - .stream() + Map nameToExprId = Stream.concat( + scan.getOutput().stream(), scan.getNonUserVisibleOutput().stream()) .collect(Collectors.toMap( NamedExpression::getName, NamedExpression::getExprId) @@ -461,4 +608,241 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial exprToColumn.key().toSql(), exprToColumn.value().getName()))) .orElse(PreAggStatus.on()); } + + /** + * rewrite for bitmap and hll + */ + private AggRewriteResult rewriteAgg(MaterializedIndex index, + LogicalOlapScan scan, + Set requiredScanOutput, + List predicates, + List aggregateFunctions, + List groupingExprs) { + ExprRewriteMap exprRewriteMap = new ExprRewriteMap(); + RewriteContext context = new RewriteContext(new CheckContext(scan, index.getId()), exprRewriteMap); + aggregateFunctions.forEach(aggFun -> AggFuncRewriter.rewrite(aggFun, context)); + + // has rewritten agg functions + Map slotMap = exprRewriteMap.slotMap; + if (!slotMap.isEmpty()) { + // Note that the slots in the rewritten agg functions shouldn't appear in filters or grouping expressions. + // For example: we have a duplicated-type table t(c1, c2) and a materialized index that has + // a bitmap_union column `mv_bitmap_union_c2` for the column c2. + // The query `select c1, count(distinct c2) from t where c2 > 0 group by c1` can't use the materialized + // index because we have a filter `c2 > 0` for the aggregated column c2. + Set slotsToReplace = slotMap.keySet(); + if (!isInputSlotsContainsAny(predicates, slotsToReplace) + && !isInputSlotsContainsAny(groupingExprs, slotsToReplace)) { + ImmutableSet newRequiredSlots = requiredScanOutput.stream() + .map(slot -> (Slot) ExpressionUtils.replace(slot, slotMap)) + .collect(ImmutableSet.toImmutableSet()); + return new AggRewriteResult(index, true, newRequiredSlots, exprRewriteMap); + } + } + + return new AggRewriteResult(index, false, null, null); + } + + private static class ExprRewriteMap { + + /** + * Replace map for scan output slot. + */ + public final Map slotMap; + + /** + * Replace map for expressions in project. + */ + public final Map projectExprMap; + /** + * Replace map for aggregate functions. + */ + public final Map aggFuncMap; + + public ExprRewriteMap() { + this.slotMap = Maps.newHashMap(); + this.projectExprMap = Maps.newHashMap(); + this.aggFuncMap = Maps.newHashMap(); + } + + public boolean isEmpty() { + return slotMap.isEmpty(); + } + } + + private static class AggRewriteResult { + public final MaterializedIndex index; + public final boolean success; + public final Set requiredScanOutput; + public ExprRewriteMap exprRewriteMap; + + public AggRewriteResult(MaterializedIndex index, + boolean success, + Set requiredScanOutput, + ExprRewriteMap exprRewriteMap) { + this.index = index; + this.success = success; + this.requiredScanOutput = requiredScanOutput; + this.exprRewriteMap = exprRewriteMap; + } + } + + private boolean isInputSlotsContainsAny(List expressions, Set slotsToCheck) { + Set inputSlotSet = ExpressionUtils.getInputSlotSet(expressions); + return !Sets.intersection(inputSlotSet, slotsToCheck).isEmpty(); + } + + private static class RewriteContext { + public final CheckContext checkContext; + public final ExprRewriteMap exprRewriteMap; + + public RewriteContext(CheckContext context, ExprRewriteMap exprRewriteMap) { + this.checkContext = context; + this.exprRewriteMap = exprRewriteMap; + } + } + + private static class AggFuncRewriter extends DefaultExpressionRewriter { + public static final AggFuncRewriter INSTANCE = new AggFuncRewriter(); + + private static Expression rewrite(Expression expr, RewriteContext context) { + return expr.accept(INSTANCE, context); + } + + /** + * count(distinct col) -> bitmap_union_count(mv_bitmap_union_col) + */ + @Override + public Expression visitCount(Count count, RewriteContext context) { + if (count.isDistinct() && count.arity() == 1) { + Optional slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(count.child(0)); + + // count distinct a value column. + if (slotOpt.isPresent() && !context.checkContext.exprIdToKeyColumn.containsKey( + slotOpt.get().getExprId())) { + String bitmapUnionCountColumn = CreateMaterializedViewStmt + .mvColumnBuilder(AggregateType.BITMAP_UNION.name().toLowerCase(), slotOpt.get().getName()); + + Column mvColumn = context.checkContext.scan.getTable().getVisibleColumn(bitmapUnionCountColumn); + // has bitmap_union_count column + if (mvColumn != null && context.checkContext.exprIdToValueColumn.containsValue(mvColumn)) { + Slot bitmapUnionCountSlot = context.checkContext.scan.getNonUserVisibleOutput() + .stream() + .filter(s -> s.getName().equals(bitmapUnionCountColumn)) + .findFirst() + .get(); + + context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionCountSlot); + context.exprRewriteMap.projectExprMap.put(slotOpt.get(), bitmapUnionCountSlot); + BitmapUnionCount bitmapUnionCount = new BitmapUnionCount(bitmapUnionCountSlot); + context.exprRewriteMap.aggFuncMap.put(count, bitmapUnionCount); + return bitmapUnionCount; + } + } + } + return count; + } + + /** + * bitmap_union_count(to_bitmap(col)) -> bitmap_union_count(mv_bitmap_union_col) + */ + @Override + public Expression visitBitmapUnionCount(BitmapUnionCount bitmapUnionCount, RewriteContext context) { + if (bitmapUnionCount.child() instanceof ToBitmap) { + ToBitmap toBitmap = (ToBitmap) bitmapUnionCount.child(); + Optional slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(toBitmap.child()); + if (slotOpt.isPresent()) { + String bitmapUnionCountColumn = CreateMaterializedViewStmt + .mvColumnBuilder(AggregateType.BITMAP_UNION.name().toLowerCase(), slotOpt.get().getName()); + + Column mvColumn = context.checkContext.scan.getTable().getVisibleColumn(bitmapUnionCountColumn); + // has bitmap_union_count column + if (mvColumn != null && context.checkContext.exprIdToValueColumn.containsValue(mvColumn)) { + + Slot bitmapUnionCountSlot = context.checkContext.scan.getNonUserVisibleOutput() + .stream() + .filter(s -> s.getName().equals(bitmapUnionCountColumn)) + .findFirst() + .get(); + + context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionCountSlot); + context.exprRewriteMap.projectExprMap.put(toBitmap, bitmapUnionCountSlot); + BitmapUnionCount newBitmapUnionCount = new BitmapUnionCount(bitmapUnionCountSlot); + context.exprRewriteMap.aggFuncMap.put(bitmapUnionCount, newBitmapUnionCount); + return newBitmapUnionCount; + } + } + } + + return bitmapUnionCount; + } + } + + private List replaceAggOutput( + LogicalAggregate agg, + Optional oldProjectOpt, + Optional newProjectOpt, + ExprRewriteMap exprRewriteMap) { + ResultAggFuncRewriteCtx ctx = new ResultAggFuncRewriteCtx(oldProjectOpt, newProjectOpt, exprRewriteMap); + return agg.getOutputExpressions() + .stream() + .map(expr -> (NamedExpression) ResultAggFuncRewriter.rewrite(expr, ctx)) + .collect(ImmutableList.toImmutableList()); + } + + private static class ResultAggFuncRewriteCtx { + public final Optional> oldProjectSlotToProducerOpt; + public final Optional> newProjectExprMapOpt; + public final ExprRewriteMap exprRewriteMap; + + public ResultAggFuncRewriteCtx( + Optional oldProject, + Optional newProject, + ExprRewriteMap exprRewriteMap) { + this.oldProjectSlotToProducerOpt = oldProject.map(Project::getAliasToProducer); + this.newProjectExprMapOpt = newProject.map(project -> project.getProjects() + .stream() + .filter(Alias.class::isInstance) + .collect( + Collectors.toMap( + // Avoid cast to alias, retrieving the first child expression. + alias -> alias.child(0), + NamedExpression::toSlot + ) + )); + this.exprRewriteMap = exprRewriteMap; + } + } + + private static class ResultAggFuncRewriter extends DefaultExpressionRewriter { + public static final ResultAggFuncRewriter INSTANCE = new ResultAggFuncRewriter(); + + public static Expression rewrite(Expression expr, ResultAggFuncRewriteCtx ctx) { + return expr.accept(INSTANCE, ctx); + } + + @Override + public Expression visitAggregateFunction(AggregateFunction aggregateFunction, + ResultAggFuncRewriteCtx ctx) { + // normalize aggregate function to match the agg func replace map. + AggregateFunction aggFunc = replaceAggFuncInput(aggregateFunction, ctx.oldProjectSlotToProducerOpt); + Map aggFuncMap = ctx.exprRewriteMap.aggFuncMap; + if (aggFuncMap.containsKey(aggFunc)) { + AggregateFunction replacedAggFunc = aggFuncMap.get(aggFunc); + // replace the input slot by new project expr mapping. + return ctx.newProjectExprMapOpt.map(map -> ExpressionUtils.replace(replacedAggFunc, map)) + .orElse(replacedAggFunc); + } else { + return aggregateFunction; + } + } + } + + private List replaceProjectList( + LogicalProject project, + Map projectMap) { + return project.getProjects().stream() + .map(expr -> (NamedExpression) ExpressionUtils.replace(expr, projectMap)) + .collect(ImmutableList.toImmutableList()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java index 9cb63789fd..1976a4adfc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.rules.mv; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; import org.apache.doris.nereids.rules.Rule; @@ -124,24 +125,34 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater List candidates = table.getVisibleIndex().stream() .filter(index -> index.getId() == baseIndexId || table.getKeyColumnsByIndexId(index.getId()).size() == baseIndexKeySize) + .filter(index -> containAllRequiredColumns(index, scan, requiredScanOutputSupplier.get())) .collect(Collectors.toList()); + PreAggStatus preAgg = PreAggStatus.off("No aggregate on scan."); if (candidates.size() == 1) { // `candidates` only have base index. - return scan.withMaterializedIndexSelected(preAgg, ImmutableList.of(baseIndexId)); + return scan.withMaterializedIndexSelected(preAgg, baseIndexId); } else { return scan.withMaterializedIndexSelected(preAgg, - filterAndOrder(candidates.stream(), scan, requiredScanOutputSupplier.get(), - predicatesSupplier.get())); + selectBestIndex(candidates, scan, predicatesSupplier.get())); } case DUP_KEYS: // Set pre-aggregation to `on` to keep consistency with legacy logic. + List candidate = scan.getTable().getVisibleIndex().stream() + .filter(index -> !indexHasAggregate(index, scan)) + .filter(index -> containAllRequiredColumns(index, scan, + requiredScanOutputSupplier.get())) + .collect(Collectors.toList()); return scan.withMaterializedIndexSelected(PreAggStatus.on(), - filterAndOrder(scan.getTable().getVisibleIndex().stream(), scan, - requiredScanOutputSupplier.get(), - predicatesSupplier.get())); + selectBestIndex(candidate, scan, predicatesSupplier.get())); default: throw new RuntimeException("Not supported keys type: " + scan.getTable().getKeysType()); } } + + private boolean indexHasAggregate(MaterializedIndex index, LogicalOlapScan scan) { + return scan.getTable().getSchemaByIndexId(index.getId()) + .stream() + .anyMatch(Column::isAggregated); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/CountDistinctRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/CountDistinctRewrite.java new file mode 100644 index 0000000000..652783f5d4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/CountDistinctRewrite.java @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.logical; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * Rewrite count distinct for bitmap and hll type value. + *

+ * count(distinct bitmap_col) -> bitmap_union_count(bitmap col) + * todo: add support for HLL type. + */ +public class CountDistinctRewrite extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalAggregate().then(agg -> { + List output = agg.getOutputExpressions() + .stream() + .map(CountDistinctRewriter::rewrite) + .map(NamedExpression.class::cast) + .collect(ImmutableList.toImmutableList()); + return agg.withAggOutput(output); + }).toRule(RuleType.COUNT_DISTINCT_REWRITE); + } + + private static class CountDistinctRewriter extends DefaultExpressionRewriter { + private static final CountDistinctRewriter INSTANCE = new CountDistinctRewriter(); + + public static Expression rewrite(Expression expr) { + return expr.accept(INSTANCE, null); + } + + @Override + public Expression visitCount(Count count, Void context) { + if (count.isDistinct() && count.arity() == 1) { + Expression child = count.child(0); + if (child.getDataType().isBitmap()) { + return new BitmapUnionCount(child); + } + } + return count; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionCount.java index b1dd6345e8..269eec3260 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionCount.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionCount.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BitmapType; import org.apache.doris.nereids.types.DataType; @@ -62,4 +63,9 @@ public class BitmapUnionCount extends AggregateFunction public List getSignatures() { return SIGNATURES; } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitBitmapUnionCount(this, context); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java index b9abc86ed5..77254fd830 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.visitor; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.Avg; +import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.agg.GroupBitAnd; import org.apache.doris.nereids.trees.expressions.functions.agg.GroupBitOr; @@ -72,4 +73,8 @@ public interface AggregateFunctionVisitor { default R visitSum(Sum sum, C context) { return visitAggregateFunction(sum, context); } + + default R visitBitmapUnionCount(BitmapUnionCount bitmapUnionCount, C context) { + return visitAggregateFunction(bitmapUnionCount, context); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java index 6235f2b7a5..f3f829eade 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java @@ -139,6 +139,11 @@ public abstract class AbstractPlan extends AbstractTreeNode implements Pla return getLogicalProperties().getOutput(); } + @Override + public List getNonUserVisibleOutput() { + return getLogicalProperties().getNonUserVisibleOutput(); + } + @Override public Set getOutputSet() { return getLogicalProperties().getOutputSet(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java index 2058ad37d2..91a392732f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java @@ -23,6 +23,8 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import com.google.common.collect.ImmutableList; + import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -87,6 +89,11 @@ public class FakePlan implements Plan { return new ArrayList<>(); } + @Override + public List getNonUserVisibleOutput() { + return ImmutableList.of(); + } + @Override public String treeString() { return "DUMMY"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java index 1db9446f57..e0702937a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java @@ -87,6 +87,8 @@ public interface Plan extends TreeNode { */ List getOutput(); + List getNonUserVisibleOutput(); + /** * Get output slot set of the plan. */ @@ -114,6 +116,10 @@ public interface Plan extends TreeNode { throw new IllegalStateException("Not support compute output for " + getClass().getName()); } + default List computeNonUserVisibleOutput() { + return ImmutableList.of(); + } + String treeString(); default Plan withOutput(List output) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java index 1326169f45..9cec62708b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java @@ -83,6 +83,11 @@ public interface Command extends LogicalPlan { throw new RuntimeException("Command do not implement getOutput"); } + @Override + default List getNonUserVisibleOutput() { + throw new RuntimeException("Command do not implement getNonUserVisibleOutput"); + } + @Override default String treeString() { throw new RuntimeException("Command do not implement treeString"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java index bd75d9624c..feef64e3a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java @@ -51,7 +51,7 @@ public abstract class AbstractLogicalPlan extends AbstractPlan implements Logica if (hasUnboundChild || hasUnboundExpression()) { return UnboundLogicalProperties.INSTANCE; } else { - return new LogicalProperties(this::computeOutput); + return new LogicalProperties(this::computeOutput, this::computeNonUserVisibleOutput); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 4b19cd2d63..7d5a88876c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.logical; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; @@ -24,6 +25,8 @@ import org.apache.doris.catalog.Table; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.PreAggStatus; @@ -36,41 +39,75 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import org.apache.commons.collections.CollectionUtils; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; /** * Logical OlapScan. */ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, OlapScan { - private final long selectedIndexId; - private final List selectedTabletIds; - private final boolean partitionPruned; - private final boolean tabletPruned; + /////////////////////////////////////////////////////////////////////////// + // Members for materialized index. + /////////////////////////////////////////////////////////////////////////// - private final List candidateIndexIds; + /** + * The select materialized index id to read data from. + */ + private final long selectedIndexId; + + /** + * Status to indicate materialized index id is selected or not. + */ private final boolean indexSelected; + /** + * Status to indicate using pre-aggregation or not. + */ private final PreAggStatus preAggStatus; + /////////////////////////////////////////////////////////////////////////// + // Members for tablet ids. + /////////////////////////////////////////////////////////////////////////// + + /** + * Selected tablet ids to read data from. + */ + private final ImmutableList selectedTabletIds; + + /** + * Status to indicate tablets are pruned or not. + */ + private final boolean tabletPruned; + + /////////////////////////////////////////////////////////////////////////// + // Members for partition ids. + /////////////////////////////////////////////////////////////////////////// + /** + * Status to indicate partitions are pruned or not. + * todo: should be pulled up to base class? + */ + private final boolean partitionPruned; + public LogicalOlapScan(RelationId id, OlapTable table) { this(id, table, ImmutableList.of()); } public LogicalOlapScan(RelationId id, OlapTable table, List qualifier) { this(id, table, qualifier, Optional.empty(), Optional.empty(), - table.getPartitionIds(), false, ImmutableList.of(), false, - ImmutableList.of(), false, PreAggStatus.on()); + table.getPartitionIds(), false, + ImmutableList.of(), false, + -1, false, PreAggStatus.on()); } public LogicalOlapScan(RelationId id, Table table, List qualifier) { this(id, table, qualifier, Optional.empty(), Optional.empty(), ((OlapTable) table).getPartitionIds(), false, ImmutableList.of(), false, - ImmutableList.of(), false, PreAggStatus.on()); + -1, false, PreAggStatus.on()); } /** @@ -80,17 +117,13 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, Optional groupExpression, Optional logicalProperties, List selectedPartitionIds, boolean partitionPruned, List selectedTabletIds, boolean tabletPruned, - List candidateIndexIds, boolean indexSelected, PreAggStatus preAggStatus) { + long selectedIndexId, boolean indexSelected, PreAggStatus preAggStatus) { super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier, groupExpression, logicalProperties, selectedPartitionIds); - // TODO: use CBO manner to select best index id, according to index's statistics info, - // revisit this after rollup and materialized view selection are fully supported. - this.selectedIndexId = CollectionUtils.isEmpty(candidateIndexIds) - ? getTable().getBaseIndexId() : candidateIndexIds.get(0); this.selectedTabletIds = ImmutableList.copyOf(selectedTabletIds); this.partitionPruned = partitionPruned; this.tabletPruned = tabletPruned; - this.candidateIndexIds = ImmutableList.copyOf(candidateIndexIds); + this.selectedIndexId = selectedIndexId <= 0 ? getTable().getBaseIndexId() : selectedIndexId; this.indexSelected = indexSelected; this.preAggStatus = preAggStatus; } @@ -113,7 +146,7 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, return Utils.toSqlString("LogicalOlapScan", "qualified", qualifiedName(), "output", getOutput(), - "candidateIndexIds", candidateIndexIds, + "indexName", getSelectedMaterializedIndexName().orElse(""), "selectedIndexId", selectedIndexId, "preAgg", preAggStatus ); @@ -127,46 +160,53 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, if (o == null || getClass() != o.getClass() || !super.equals(o)) { return false; } - return Objects.equals(selectedPartitionIds, ((LogicalOlapScan) o).selectedPartitionIds) - && Objects.equals(candidateIndexIds, ((LogicalOlapScan) o).candidateIndexIds) - && Objects.equals(selectedTabletIds, ((LogicalOlapScan) o).selectedTabletIds); + return Objects.equals(id, ((LogicalOlapScan) o).id) + && Objects.equals(selectedPartitionIds, ((LogicalOlapScan) o).selectedPartitionIds) + && Objects.equals(partitionPruned, ((LogicalOlapScan) o).partitionPruned) + && Objects.equals(selectedIndexId, ((LogicalOlapScan) o).selectedIndexId) + && Objects.equals(indexSelected, ((LogicalOlapScan) o).indexSelected) + && Objects.equals(selectedTabletIds, ((LogicalOlapScan) o).selectedTabletIds) + && Objects.equals(tabletPruned, ((LogicalOlapScan) o).tabletPruned); } @Override public int hashCode() { - return Objects.hash(id, selectedPartitionIds, candidateIndexIds, selectedTabletIds); + return Objects.hash(id, + selectedPartitionIds, partitionPruned, + selectedIndexId, indexSelected, + selectedTabletIds, tabletPruned); } @Override public Plan withGroupExpression(Optional groupExpression) { return new LogicalOlapScan(id, table, qualifier, groupExpression, Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned, - candidateIndexIds, indexSelected, preAggStatus); + selectedIndexId, indexSelected, preAggStatus); } @Override public LogicalOlapScan withLogicalProperties(Optional logicalProperties) { return new LogicalOlapScan(id, table, qualifier, Optional.empty(), logicalProperties, selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned, - candidateIndexIds, indexSelected, preAggStatus); + selectedIndexId, indexSelected, preAggStatus); } public LogicalOlapScan withSelectedPartitionIds(List selectedPartitionIds) { return new LogicalOlapScan(id, table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, true, selectedTabletIds, tabletPruned, - candidateIndexIds, indexSelected, preAggStatus); + selectedIndexId, indexSelected, preAggStatus); } - public LogicalOlapScan withMaterializedIndexSelected(PreAggStatus preAgg, List candidateIndexIds) { + public LogicalOlapScan withMaterializedIndexSelected(PreAggStatus preAgg, long indexId) { return new LogicalOlapScan(id, table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned, - candidateIndexIds, true, preAgg); + indexId, true, preAgg); } public LogicalOlapScan withSelectedTabletIds(List selectedTabletIds) { return new LogicalOlapScan(id, table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, true, - candidateIndexIds, indexSelected, preAggStatus); + selectedIndexId, indexSelected, preAggStatus); } @Override @@ -204,4 +244,23 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, return indexSelected ? Optional.ofNullable(((OlapTable) table).getIndexNameById(selectedIndexId)) : Optional.empty(); } + + @Override + public List computeNonUserVisibleOutput() { + Set baseSchemaColNames = table.getBaseSchema().stream() + .map(Column::getName) + .collect(Collectors.toSet()); + + OlapTable olapTable = (OlapTable) table; + // extra columns in materialized index, such as `mv_bitmap_union_xxx` + return olapTable.getVisibleIndexIdToMeta().values() + .stream() + .filter(index -> index.getIndexId() != ((OlapTable) table).getBaseIndexId()) + .flatMap(index -> index.getSchema() + .stream() + .filter(col -> !baseSchemaColNames.contains(col.getName())) + ) + .map(col -> SlotReference.fromColumn(col, qualified())) + .collect(ImmutableList.toImmutableList()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java index 2e14638460..ab1fe13fc7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java @@ -445,6 +445,10 @@ public abstract class DataType implements AbstractDataType { return this instanceof DateTimeV2Type; } + public boolean isBitmap() { + return this instanceof BitmapType; + } + public DataType promotion() { if (PROMOTION_MAP.containsKey(this.getClass())) { return PROMOTION_MAP.get(this.getClass()).get(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 17f45d452b..f177ef4532 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -44,6 +44,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; @@ -235,12 +236,20 @@ public class ExpressionUtils { * Otherwise, return empty optional result. */ public static Optional isSlotOrCastOnSlot(Expression expr) { + return extractSlotOrCastOnSlot(expr).map(Slot::getExprId); + } + + /** + * Check whether the input expression is a {@link org.apache.doris.nereids.trees.expressions.Slot} + * or at least one {@link Cast} on a {@link org.apache.doris.nereids.trees.expressions.Slot} + */ + public static Optional extractSlotOrCastOnSlot(Expression expr) { while (expr instanceof Cast) { expr = expr.child(0); } if (expr instanceof SlotReference) { - return Optional.of(((SlotReference) expr).getExprId()); + return Optional.of((Slot) expr); } else { return Optional.empty(); } @@ -425,4 +434,13 @@ public class ExpressionUtils { // skip current expression cubeToGroupingSets(cubeExpressions, activeIndex + 1, currentGroupingSet, groupingSets); } + + /** + * Get input slot set from list of expressions. + */ + public static Set getInputSlotSet(Collection exprs) { + return exprs.stream() + .flatMap(expr -> expr.getInputSlots().stream()) + .collect(ImmutableSet.toImmutableSet()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java index 6ec7ab3764..0c5042b936 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java @@ -19,6 +19,11 @@ package org.apache.doris.nereids.rules.mv; import org.apache.doris.catalog.FunctionSet; import org.apache.doris.common.FeConstants; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.planner.OlapScanNode; @@ -34,6 +39,8 @@ import org.junit.jupiter.api.Test; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; /** * Tests ported from {@link org.apache.doris.planner.MaterializedViewFunctionTest} @@ -47,6 +54,7 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement private static final String DEPTS_MV_NAME = "depts_mv"; private static final String USER_TAG_TABLE_NAME = "user_tags"; private static final String TEST_TABLE_NAME = "test_tb"; + private static final String USER_TAG_MV_NAME = "user_tags_mv"; @Override protected void beforeCreatingConnectContext() throws Exception { @@ -549,22 +557,7 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement createMv(createEmpsMVsql); String query = "select empid, deptno, salary from " + EMPS_TABLE_NAME + " e1 where empid = (select max(empid)" + " from " + EMPS_TABLE_NAME + " where deptno = e1.deptno);"; - PlanChecker.from(connectContext).checkPlannerResult(query, planner -> { - List scans = planner.getScanNodes(); - Assertions.assertEquals(2, scans.size()); - - ScanNode scanNode0 = scans.get(0); - Assertions.assertTrue(scanNode0 instanceof OlapScanNode); - OlapScanNode scan0 = (OlapScanNode) scanNode0; - Assertions.assertTrue(scan0.isPreAggregation()); - Assertions.assertEquals("emps_mv", scan0.getSelectedIndexName()); - - ScanNode scanNode1 = scans.get(1); - Assertions.assertTrue(scanNode1 instanceof OlapScanNode); - OlapScanNode scan1 = (OlapScanNode) scanNode1; - Assertions.assertTrue(scan1.isPreAggregation()); - Assertions.assertEquals("emps", scan1.getSelectedIndexName()); - }); + testMvWithTwoTable(query, "emps_mv", "emps"); } /** @@ -603,22 +596,7 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement String query = "select * from (select deptno, empid from " + EMPS_TABLE_NAME + " where deptno>100) A join " + "(select deptno, sum(salary) from " + EMPS_TABLE_NAME + " where deptno >200 group by deptno) B " + "on A.deptno=B.deptno"; - PlanChecker.from(connectContext).checkPlannerResult(query, planner -> { - List scans = planner.getScanNodes(); - Assertions.assertEquals(2, scans.size()); - - ScanNode scanNode0 = scans.get(0); - Assertions.assertTrue(scanNode0 instanceof OlapScanNode); - OlapScanNode scan0 = (OlapScanNode) scanNode0; - Assertions.assertTrue(scan0.isPreAggregation()); - Assertions.assertEquals("emp_mv_01", scan0.getSelectedIndexName()); - - ScanNode scanNode1 = scans.get(1); - Assertions.assertTrue(scanNode1 instanceof OlapScanNode); - OlapScanNode scan1 = (OlapScanNode) scanNode1; - Assertions.assertTrue(scan1.isPreAggregation()); - Assertions.assertEquals("emp_mv_02", scan1.getSelectedIndexName()); - }); + testMvWithTwoTable(query, "emp_mv_01", "emp_mv_02"); } @Test @@ -703,6 +681,7 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement + "distributed by hash(k1) buckets 3 properties('replication_num' = '1')"); createMv("create materialized view k1_k2 as select k1, k2 from t group by k1, k2"); singleTableTest("select k1, k2 from t group by k1, k2", "k1_k2", true); + dropTable("t", true); } @Test @@ -768,89 +747,112 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement } /** - * TODO: enable this when bitmap is supported. + * bitmap_union_count(to_bitmap()) -> bitmap_union_count without having */ - @Disabled + @Test + public void testBitmapUnionRewrite() throws Exception { + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + + " as select user_id, bitmap_union(to_bitmap(tag_id)) from " + + USER_TAG_TABLE_NAME + " group by user_id;"; + createMv(createUserTagMVSql); + String query = "select user_id, bitmap_union_count(to_bitmap(tag_id)) a from " + USER_TAG_TABLE_NAME + + " group by user_id"; + singleTableTest(query, USER_TAG_MV_NAME, true); + } + + /** + * bitmap_union_count(to_bitmap()) -> bitmap_union_count with having + */ + @Test public void testBitmapUnionInQuery() throws Exception { - // String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME - // + " as select user_id, bitmap_union(to_bitmap(tag_id)) from " - // + USER_TAG_TABLE_NAME + " group by user_id;"; - // dorisAssert.withMaterializedView(createUserTagMVSql); - // String query = "select user_id, bitmap_union_count(to_bitmap(tag_id)) a from " + USER_TAG_TABLE_NAME - // + " group by user_id having a>1 order by a;"; - // dorisAssert.query(query).explainContains(QUERY_USE_USER_TAG_MV); + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + + " as select user_id, bitmap_union(to_bitmap(tag_id)) from " + + USER_TAG_TABLE_NAME + " group by user_id;"; + createMv(createUserTagMVSql); + String query = "select user_id, bitmap_union_count(to_bitmap(tag_id)) a from " + USER_TAG_TABLE_NAME + + " group by user_id having a>1 order by a;"; + singleTableTest(query, USER_TAG_MV_NAME, true); } - /** - * TODO: enable this when bitmap is supported. - */ - @Disabled + @Test public void testBitmapUnionInSubquery() throws Exception { - // String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " - // + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; - // dorisAssert.withMaterializedView(createUserTagMVSql); - // String query = "select user_id from " + USER_TAG_TABLE_NAME + " where user_id in (select user_id from " - // + USER_TAG_TABLE_NAME + " group by user_id having bitmap_union_count(to_bitmap(tag_id)) >1 ) ;"; - // dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, USER_TAG_TABLE_NAME); + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + createMv(createUserTagMVSql); + String query = "select user_id from " + USER_TAG_TABLE_NAME + " where user_id in (select user_id from " + + USER_TAG_TABLE_NAME + " group by user_id having bitmap_union_count(to_bitmap(tag_id)) >1 ) ;"; + testMvWithTwoTable(query, "user_tags", "user_tags_mv"); } - /** - * TODO: enable this when bitmap is supported. - */ - @Disabled + @Test public void testIncorrectMVRewriteInQuery() throws Exception { - // String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " - // + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; - // dorisAssert.withMaterializedView(createUserTagMVSql); - // String createEmpMVSql = "create materialized view " + EMPS_MV_NAME + " as select name, deptno from " - // + EMPS_TABLE_NAME + ";"; - // dorisAssert.withMaterializedView(createEmpMVSql); - // String query = "select user_name, bitmap_union_count(to_bitmap(tag_id)) a from " + USER_TAG_TABLE_NAME + ", " - // + "(select name, deptno from " + EMPS_TABLE_NAME + ") a" + " where user_name=a.name group by " - // + "user_name having a>1 order by a;"; - // testMv(query, EMPS_MV_NAME); - // dorisAssert.query(query).explainWithout(QUERY_USE_USER_TAG_MV); + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + createMv(createUserTagMVSql); + String createEmpMVSql = "create materialized view " + EMPS_MV_NAME + " as select name, deptno from " + + EMPS_TABLE_NAME + ";"; + createMv(createEmpMVSql); + String query = "select user_name, bitmap_union_count(to_bitmap(tag_id)) a from " + USER_TAG_TABLE_NAME + ", " + + "(select name, deptno from " + EMPS_TABLE_NAME + ") a" + " where user_name=a.name group by " + + "user_name having a>1 order by a;"; + testMv(query, ImmutableMap.of("user_tags", "user_tags", "emps", "emps_mv")); } /** - * TODO: enable this when bitmap is supported. + * bitmap_union_count(to_bitmap(tag_id)) in subquery */ - @Disabled + @Test public void testIncorrectMVRewriteInSubquery() throws Exception { - // String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " - // + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; - // dorisAssert.withMaterializedView(createUserTagMVSql); - // String query = "select user_id, bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " where " - // + "user_name in (select user_name from " + USER_TAG_TABLE_NAME + " group by user_name having " - // + "bitmap_union_count(to_bitmap(tag_id)) >1 )" + " group by user_id;"; - // dorisAssert.query(query).explainContains(QUERY_USE_USER_TAG); + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + createMv(createUserTagMVSql); + String query = "select user_id, bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " where " + + "user_name in (select user_name from " + USER_TAG_TABLE_NAME + " group by user_name having " + + "bitmap_union_count(to_bitmap(tag_id)) >1 )" + " group by user_id;"; + // can't use mv index because it has no required column `user_name` + testMv(query, ImmutableMap.of("user_tags", "user_tags")); } - /** - * TODO: enable this when bitmap is supported. - */ - @Disabled + @Test public void testTwoTupleInQuery() throws Exception { - // String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " - // + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; - // dorisAssert.withMaterializedView(createUserTagMVSql); - // String query = "select * from (select user_id, bitmap_union_count(to_bitmap(tag_id)) x from " - // + USER_TAG_TABLE_NAME + " group by user_id) a, (select user_name, bitmap_union_count(to_bitmap(tag_id))" - // + "" + " y from " + USER_TAG_TABLE_NAME + " group by user_name) b where a.x=b.y;"; - // dorisAssert.query(query).explainContains(QUERY_USE_USER_TAG, QUERY_USE_USER_TAG_MV); + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + createMv(createUserTagMVSql); + String query = "select * from (select user_id, bitmap_union_count(to_bitmap(tag_id)) x from " + + USER_TAG_TABLE_NAME + " group by user_id) a, (select user_name, bitmap_union_count(to_bitmap(tag_id))" + + "" + " y from " + USER_TAG_TABLE_NAME + " group by user_name) b where a.x=b.y;"; + PlanChecker.from(connectContext) + .analyze(query) + .rewrite() + .matches(logicalJoin( + logicalAggregate( + logicalProject( + logicalOlapScan().when(scan -> "user_tags_mv".equals( + scan.getSelectedMaterializedIndexName().get())))), + logicalAggregate( + logicalProject( + logicalOlapScan().when(scan -> "user_tags".equals( + scan.getSelectedMaterializedIndexName().get())))))); + } /** - * TODO: enable this when bitmap is supported. + * count(distinct v) -> bitmap_union_count(v) without mv index. */ - @Disabled + @Test public void testAggTableCountDistinctInBitmapType() throws Exception { - // String aggTable = "CREATE TABLE " + TEST_TABLE_NAME + " (k1 int, v1 bitmap bitmap_union) Aggregate KEY (k1) " - // + "DISTRIBUTED BY HASH(k1) BUCKETS 3 PROPERTIES ('replication_num' = '1');"; - // dorisAssert.withTable(aggTable); - // String query = "select k1, count(distinct v1) from " + TEST_TABLE_NAME + " group by k1;"; - // dorisAssert.query(query).explainContains(TEST_TABLE_NAME, "bitmap_union_count"); - // dorisAssert.dropTable(TEST_TABLE_NAME, true); + String aggTable = "CREATE TABLE " + TEST_TABLE_NAME + " (k1 int, v1 bitmap bitmap_union) Aggregate KEY (k1) " + + "DISTRIBUTED BY HASH(k1) BUCKETS 3 PROPERTIES ('replication_num' = '1');"; + createTable(aggTable); + String query = "select k1, count(distinct v1) from " + TEST_TABLE_NAME + " group by k1;"; + PlanChecker.from(connectContext) + .analyze(query) + .rewrite() + .matches(logicalAggregate().when(agg -> { + assertOneAggFuncType(agg, BitmapUnionCount.class); + return true; + })); + dropTable(TEST_TABLE_NAME, true); } /** @@ -868,27 +870,38 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement } /** - * TODO: enable this when bitmap is supported. + * count distinct to bitmap_union_count in mv */ - @Disabled + @Test public void testCountDistinctToBitmap() throws Exception { - // String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " - // + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; - // dorisAssert.withMaterializedView(createUserTagMVSql); - // String query = "select count(distinct tag_id) from " + USER_TAG_TABLE_NAME + ";"; - // dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, "bitmap_union_count"); + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + createMv(createUserTagMVSql); + String query = "select count(distinct tag_id) from " + USER_TAG_TABLE_NAME + ";"; + PlanChecker.from(connectContext) + .analyze(query) + .rewrite() + .matches(logicalAggregate().when(agg -> { + assertOneAggFuncType(agg, BitmapUnionCount.class); + return true; + })); + testMv(query, USER_TAG_MV_NAME); } - /** - * TODO: enable this when bitmap is supported. - */ - @Disabled + @Test public void testIncorrectRewriteCountDistinct() throws Exception { - // String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " - // + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; - // dorisAssert.withMaterializedView(createUserTagMVSql); - // String query = "select user_name, count(distinct tag_id) from " + USER_TAG_TABLE_NAME + " group by user_name;"; - // dorisAssert.query(query).explainContains(USER_TAG_TABLE_NAME, FunctionSet.COUNT); + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + createMv(createUserTagMVSql); + String query = "select user_name, count(distinct tag_id) from " + USER_TAG_TABLE_NAME + " group by user_name;"; + PlanChecker.from(connectContext) + .analyze(query) + .rewrite() + .matches(logicalAggregate().when(agg -> { + assertOneAggFuncType(agg, Count.class); + return true; + })); + testMv(query, USER_TAG_TABLE_NAME); } /** @@ -982,10 +995,7 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement // dorisAssert.query(query).explainWithout(USER_TAG_MV_NAME); } - /** - * TODO: enable this when bitmap is supported. - */ - @Disabled + @Test public void testCreateMVBaseBitmapAggTable() throws Exception { String createTableSQL = "create table " + HR_DB_NAME + ".agg_table " + "(empid int, name varchar, salary bitmap " + FunctionSet.BITMAP_UNION + ") " @@ -1016,6 +1026,48 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement // dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, mvColumnName); } + @Test + public void selectBitmapMvWithProjectTest1() throws Exception { + createTable("create table t(\n" + + " a int, \n" + + " b int, \n" + + " c int\n" + + ")ENGINE=OLAP \n" + + "DISTRIBUTED BY HASH(a) BUCKETS 3\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\",\n" + + "\"disable_auto_compaction\" = \"false\"\n" + + ");"); + createMv("create materialized view mv as" + + " select a, bitmap_union(to_bitmap(b)) from t group by a;"); + + testMv("select a, count(distinct v) as cnt from (select a, b as v from t) t group by a", "mv"); + dropTable("t", true); + } + + @Test + public void selectBitmapMvWithProjectTest2() throws Exception { + createTable("create table t(\n" + + " a int, \n" + + " b int, \n" + + " c int\n" + + ")ENGINE=OLAP \n" + + "DISTRIBUTED BY HASH(a) BUCKETS 3\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\",\n" + + "\"disable_auto_compaction\" = \"false\"\n" + + ");"); + createMv("create materialized view mv as" + + " select a, bitmap_union(to_bitmap(b)) from t group by a;"); + + testMv("select a, bitmap_union_count(to_bitmap(v)) as cnt from (select a, b as v from t) t group by a", "mv"); + dropTable("t", true); + } + private void testMv(String sql, Map tableToIndex) { PlanChecker.from(connectContext).checkPlannerResult(sql, planner -> { List scans = planner.getScanNodes(); @@ -1032,4 +1084,34 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement private void testMv(String sql, String indexName) { singleTableTest(sql, indexName, true); } + + private void assertOneAggFuncType(LogicalAggregate agg, Class aggFuncType) { + Set aggFuncs = agg.getOutputExpressions() + .stream() + .flatMap(e -> e.>collect(AggregateFunction.class::isInstance) + .stream()) + .collect(Collectors.toSet()); + Assertions.assertEquals(1, aggFuncs.size()); + AggregateFunction aggFunc = aggFuncs.iterator().next(); + Assertions.assertTrue(aggFuncType.isInstance(aggFunc)); + } + + private void testMvWithTwoTable(String sql, String firstTableIndexName, String secondTableIndexName) { + PlanChecker.from(connectContext).checkPlannerResult(sql, planner -> { + List scans = planner.getScanNodes(); + Assertions.assertEquals(2, scans.size()); + + ScanNode scanNode0 = scans.get(0); + Assertions.assertTrue(scanNode0 instanceof OlapScanNode); + OlapScanNode scan0 = (OlapScanNode) scanNode0; + Assertions.assertTrue(scan0.isPreAggregation()); + Assertions.assertEquals(firstTableIndexName, scan0.getSelectedIndexName()); + + ScanNode scanNode1 = scans.get(1); + Assertions.assertTrue(scanNode1 instanceof OlapScanNode); + OlapScanNode scan1 = (OlapScanNode) scanNode1; + Assertions.assertTrue(scan1.isPreAggregation()); + Assertions.assertEquals(secondTableIndexName, scan1.getSelectedIndexName()); + }); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java index b9de02ec0a..157c449b6e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java @@ -81,7 +81,8 @@ public class PlanToStringTest { LogicalOlapScan plan = PlanConstructor.newLogicalOlapScan(0, "table", 0); Assertions.assertTrue( plan.toString().matches("LogicalOlapScan \\( qualified=db\\.table, " - + "output=\\[id#\\d+, name#\\d+], candidateIndexIds=\\[], selectedIndexId=-1, preAgg=ON \\)")); + + "output=\\[id#\\d+, name#\\d+], indexName=, " + + "selectedIndexId=-1, preAgg=ON \\)")); } @Test