database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Feature](Nereids) Support bitmap for materialized index. (#14863)

Browse Source 
This PR adds the rewriting and matching logic for the bitmap_union column in materialized index.

If a materialized index has bitmap_union column, we try to rewrite count distinct or bitmap_union_count to the bitmap_union column in materialized index.
This commit is contained in:
Shuo Wang
2022-12-22 14:40:25 +08:00
committed by GitHub
parent 0fa4c78e84
commit f8b368a85e
21 changed files with 911 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View File

@ -437,7 +437,10 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
@Override
public PlanFragment visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanTranslatorContext context) {
// Create OlapScanNode
List<Slot> slotList = olapScan.getOutput();
List<Slot> slotList = new ImmutableList.Builder<Slot>()
.addAll(olapScan.getOutput())
.addAll(olapScan.getNonUserVisibleOutput())
.build();
OlapTable olapTable = olapScan.getTable();
TupleDescriptor tupleDescriptor = generateTupleDesc(slotList, olapTable, context);
tupleDescriptor.setTable(olapTable);

2
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java
View File

@ -29,6 +29,7 @@ import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithAggregate;
import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithoutAggregate;
import org.apache.doris.nereids.rules.rewrite.logical.BuildAggForUnion;
import org.apache.doris.nereids.rules.rewrite.logical.ColumnPruning;
import org.apache.doris.nereids.rules.rewrite.logical.CountDistinctRewrite;
import org.apache.doris.nereids.rules.rewrite.logical.EliminateAggregate;
import org.apache.doris.nereids.rules.rewrite.logical.EliminateFilter;
import org.apache.doris.nereids.rules.rewrite.logical.EliminateGroupByConstant;
@ -95,6 +96,7 @@ public class NereidsRewriteJobExecutor extends BatchRulesJob {
.add(topDownBatch(ImmutableList.of(new EliminateLimit())))
.add(topDownBatch(ImmutableList.of(new EliminateFilter())))
.add(topDownBatch(ImmutableList.of(new PruneOlapScanPartition())))
.add(topDownBatch(ImmutableList.of(new CountDistinctRewrite())))
.add(topDownBatch(ImmutableList.of(new SelectMaterializedIndexWithAggregate())))
.add(topDownBatch(ImmutableList.of(new SelectMaterializedIndexWithoutAggregate())))
.add(topDownBatch(ImmutableList.of(new PruneOlapScanTablet())))

17
fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import java.util.HashSet;
@ -37,21 +38,29 @@ import java.util.stream.Collectors;
*/
public class LogicalProperties {
protected final Supplier<List<Slot>> outputSupplier;
protected final Supplier<List<Slot>> nonUserVisibleOutputSupplier;
protected final Supplier<List<Id>> outputExprIdsSupplier;
protected final Supplier<Set<Slot>> outputSetSupplier;
protected final Supplier<Set<ExprId>> outputExprIdSetSupplier;
private Integer hashCode = null;
public LogicalProperties(Supplier<List<Slot>> outputSupplier) {
this(outputSupplier, ImmutableList::of);
}
/**
* constructor of LogicalProperties.
*
* @param outputSupplier provide the output. Supplier can lazy compute output without
* throw exception for which children have UnboundRelation
*/
public LogicalProperties(Supplier<List<Slot>> outputSupplier) {
public LogicalProperties(Supplier<List<Slot>> outputSupplier, Supplier<List<Slot>> nonUserVisibleOutputSupplier) {
this.outputSupplier = Suppliers.memoize(
Objects.requireNonNull(outputSupplier, "outputSupplier can not be null")
);
this.nonUserVisibleOutputSupplier = Suppliers.memoize(
Objects.requireNonNull(nonUserVisibleOutputSupplier, "nonUserVisibleOutputSupplier can not be null")
);
this.outputExprIdsSupplier = Suppliers.memoize(
() -> this.outputSupplier.get().stream().map(NamedExpression::getExprId).map(Id.class::cast)
.collect(Collectors.toList())
@ -69,6 +78,10 @@ public class LogicalProperties {
return outputSupplier.get();
}
public List<Slot> getNonUserVisibleOutput() {
return nonUserVisibleOutputSupplier.get();
}
public Set<Slot> getOutputSet() {
return outputSetSupplier.get();
}
@ -82,7 +95,7 @@ public class LogicalProperties {
}
public LogicalProperties withOutput(List<Slot> output) {
return new LogicalProperties(Suppliers.ofInstance(output));
return new LogicalProperties(Suppliers.ofInstance(output), nonUserVisibleOutputSupplier);
}
@Override

1
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View File

@ -158,6 +158,7 @@ public enum RuleType {
HIDE_ONE_ROW_RELATION_UNDER_UNION(RuleTypeClass.REWRITE),
MERGE_SET_OPERATION(RuleTypeClass.REWRITE),
BUILD_AGG_FOR_UNION(RuleTypeClass.REWRITE),
COUNT_DISTINCT_REWRITE(RuleTypeClass.REWRITE),
REWRITE_SENTINEL(RuleTypeClass.REWRITE),
// limit push down

4
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
View File

@ -31,6 +31,7 @@ import org.apache.commons.lang.StringUtils;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* some check need to do after analyze whole plan.
@ -49,8 +50,7 @@ public class CheckAfterRewrite extends OneAnalysisRuleFactory {
.flatMap(expr -> expr.getInputSlots().stream())
.collect(Collectors.toSet());
Set<Slot> childrenOutput = plan.children().stream()
.map(Plan::getOutput)
.flatMap(List::stream)
.flatMap(child -> Stream.concat(child.getOutput().stream(), child.getNonUserVisibleOutput().stream()))
.collect(Collectors.toSet());
notFromChildren.removeAll(childrenOutput);
notFromChildren = removeValidVirtualSlots(notFromChildren, childrenOutput);

56
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/AbstractSelectMaterializedIndexRule.java
View File

@ -35,6 +35,7 @@ import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import org.apache.commons.collections.CollectionUtils;
import java.util.Comparator;
import java.util.List;
@ -61,21 +62,14 @@ public abstract class AbstractSelectMaterializedIndexRule {
}
}
/**
* 1. indexes have all the required columns.
* 2. find matching key prefix most.
* 3. sort by row count, column count and index id.
*/
protected List<Long> filterAndOrder(
Stream<MaterializedIndex> candidates,
protected boolean containAllRequiredColumns(
MaterializedIndex index,
LogicalOlapScan scan,
Set<Slot> requiredScanOutput,
List<Expression> predicates) {
Set<Slot> requiredScanOutput) {
OlapTable table = scan.getTable();
// Scan slot exprId -> slot name
Map<ExprId, String> exprIdToName = scan.getOutput()
.stream()
Map<ExprId, String> exprIdToName = Stream.concat(
scan.getOutput().stream(), scan.getNonUserVisibleOutput().stream())
.collect(Collectors.toMap(NamedExpression::getExprId, NamedExpression::getName));
// get required column names in metadata.
@ -84,22 +78,32 @@ public abstract class AbstractSelectMaterializedIndexRule {
.map(slot -> exprIdToName.get(slot.getExprId()))
.collect(Collectors.toSet());
// 1. filter index contains all the required columns by column name.
List<MaterializedIndex> containAllRequiredColumns = candidates
.filter(index -> table.getSchemaByIndexId(index.getId(), true)
.stream()
.map(Column::getName)
.collect(Collectors.toSet())
.containsAll(requiredColumnNames)
).collect(Collectors.toList());
return table.getSchemaByIndexId(index.getId(), true).stream()
.map(Column::getName)
.collect(Collectors.toSet())
.containsAll(requiredColumnNames);
}
// 2. find matching key prefix most.
List<MaterializedIndex> matchingKeyPrefixMost = matchPrefixMost(scan, containAllRequiredColumns, predicates,
exprIdToName);
/**
* 1. find matching key prefix most.
* 2. sort by row count, column count and index id.
*/
protected long selectBestIndex(
List<MaterializedIndex> candidates,
LogicalOlapScan scan,
List<Expression> predicates) {
OlapTable table = scan.getTable();
// Scan slot exprId -> slot name
Map<ExprId, String> exprIdToName = scan.getOutput()
.stream()
.collect(Collectors.toMap(NamedExpression::getExprId, NamedExpression::getName));
// find matching key prefix most.
List<MaterializedIndex> matchingKeyPrefixMost = matchPrefixMost(scan, candidates, predicates, exprIdToName);
List<Long> partitionIds = scan.getSelectedPartitionIds();
// 3. sort by row count, column count and index id.
return matchingKeyPrefixMost.stream()
// sort by row count, column count and index id.
List<Long> sortedIndexIds = matchingKeyPrefixMost.stream()
.map(MaterializedIndex::getId)
.sorted(Comparator
// compare by row count
@ -111,6 +115,8 @@ public abstract class AbstractSelectMaterializedIndexRule {
// compare by index id
.thenComparing(rid -> (Long) rid))
.collect(Collectors.toList());
return CollectionUtils.isEmpty(sortedIndexIds) ? scan.getTable().getBaseIndexId() : sortedIndexIds.get(0);
}
protected List<MaterializedIndex> matchPrefixMost(

472
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.nereids.rules.mv;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.MaterializedIndex;
@ -26,16 +27,21 @@ import org.apache.doris.nereids.annotation.Developing;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount;
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
import org.apache.doris.nereids.trees.expressions.functions.agg.Max;
import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmap;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PreAggStatus;
import org.apache.doris.nereids.trees.plans.algebra.Project;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
@ -48,6 +54,9 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.collect.Streams;
import java.util.List;
import java.util.Map;
@ -76,15 +85,25 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
// Aggregate(Scan)
logicalAggregate(logicalOlapScan().when(this::shouldSelectIndex)).then(agg -> {
LogicalOlapScan scan = agg.child();
Pair<PreAggStatus, List<Long>> result = select(
SelectResult result = select(
scan,
agg.getInputSlots(),
ImmutableList.of(),
extractAggFunctionAndReplaceSlot(agg, Optional.empty()),
agg.getGroupByExpressions());
return agg.withChildren(
scan.withMaterializedIndexSelected(result.key(), result.value())
);
if (result.exprRewriteMap.isEmpty()) {
return agg.withChildren(
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)
);
} else {
return new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.empty(), Optional.empty(), result.exprRewriteMap),
agg.isNormalized(),
agg.getSourceRepeat(),
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)
);
}
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_SCAN),
// filter could push down scan.
@ -98,16 +117,31 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.addAll(filter.getInputSlots())
.build();
Pair<PreAggStatus, List<Long>> result = select(
SelectResult result = select(
scan,
requiredSlots,
filter.getConjuncts(),
extractAggFunctionAndReplaceSlot(agg, Optional.empty()),
agg.getGroupByExpressions()
);
return agg.withChildren(filter.withChildren(
scan.withMaterializedIndexSelected(result.key(), result.value())
));
if (result.exprRewriteMap.isEmpty()) {
return agg.withChildren(filter.withChildren(
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)
));
} else {
return new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.empty(), Optional.empty(),
result.exprRewriteMap),
agg.isNormalized(),
agg.getSourceRepeat(),
// Not that no need to replace slots in the filter, because the slots to replace
// are value columns, which shouldn't appear in filters.
filter.withChildren(
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId))
);
}
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_FILTER_SCAN),
// column pruning or other projections such as alias, etc.
@ -116,7 +150,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.then(agg -> {
LogicalProject<LogicalOlapScan> project = agg.child();
LogicalOlapScan scan = project.child();
Pair<PreAggStatus, List<Long>> result = select(
SelectResult result = select(
scan,
project.getInputSlots(),
ImmutableList.of(),
@ -124,11 +158,28 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
Optional.of(project)),
agg.getGroupByExpressions()
);
return agg.withChildren(
project.withChildren(
scan.withMaterializedIndexSelected(result.key(), result.value())
)
);
if (result.exprRewriteMap.isEmpty()) {
return agg.withChildren(
project.withChildren(
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)
)
);
} else {
List<NamedExpression> newProjectList = replaceProjectList(project,
result.exprRewriteMap.projectExprMap);
LogicalProject<LogicalOlapScan> newProject = new LogicalProject<>(
newProjectList,
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId));
return new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.of(project), Optional.of(newProject),
result.exprRewriteMap),
agg.isNormalized(),
agg.getSourceRepeat(),
newProject
);
}
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_PROJECT_SCAN),
// filter could push down and project.
@ -141,7 +192,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
Set<Slot> requiredSlots = Stream.concat(
project.getInputSlots().stream(), filter.getInputSlots().stream())
.collect(Collectors.toSet());
Pair<PreAggStatus, List<Long>> result = select(
SelectResult result = select(
scan,
requiredSlots,
filter.getConjuncts(),
@ -149,9 +200,27 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
ExpressionUtils.replace(agg.getGroupByExpressions(),
project.getAliasToProducer())
);
return agg.withChildren(project.withChildren(filter.withChildren(
scan.withMaterializedIndexSelected(result.key(), result.value())
)));
if (result.exprRewriteMap.isEmpty()) {
return agg.withChildren(project.withChildren(filter.withChildren(
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)
)));
} else {
List<NamedExpression> newProjectList = replaceProjectList(project,
result.exprRewriteMap.projectExprMap);
LogicalProject<Plan> newProject = new LogicalProject<>(newProjectList,
filter.withChildren(scan.withMaterializedIndexSelected(result.preAggStatus,
result.indexId)));
return new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.of(project), Optional.of(newProject),
result.exprRewriteMap),
agg.isNormalized(),
agg.getSourceRepeat(),
newProject
);
}
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_PROJECT_FILTER_SCAN),
// filter can't push down
@ -161,7 +230,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
LogicalFilter<LogicalProject<LogicalOlapScan>> filter = agg.child();
LogicalProject<LogicalOlapScan> project = filter.child();
LogicalOlapScan scan = project.child();
Pair<PreAggStatus, List<Long>> result = select(
SelectResult result = select(
scan,
project.getInputSlots(),
ImmutableList.of(),
@ -169,9 +238,26 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
ExpressionUtils.replace(agg.getGroupByExpressions(),
project.getAliasToProducer())
);
return agg.withChildren(filter.withChildren(project.withChildren(
scan.withMaterializedIndexSelected(result.key(), result.value())
)));
if (result.exprRewriteMap.isEmpty()) {
return agg.withChildren(filter.withChildren(project.withChildren(
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)
)));
} else {
List<NamedExpression> newProjectList = replaceProjectList(project,
result.exprRewriteMap.projectExprMap);
LogicalProject<Plan> newProject = new LogicalProject<>(newProjectList,
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId));
return new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.of(project), Optional.of(newProject),
result.exprRewriteMap),
agg.isNormalized(),
agg.getSourceRepeat(),
filter.withChildren(newProject)
);
}
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_FILTER_PROJECT_SCAN)
);
}
@ -183,11 +269,12 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
/**
* Select materialized index ids.
* <p>
* 1. find candidate indexes by pre-agg status: checking input aggregate functions and group by expressions
* and pushdown predicates.
* 2. filter and order the candidate indexes.
* 1. find candidate indexes by pre-agg status:
* checking input aggregate functions and group by expressions and pushdown predicates.
* 2. filter indexes that have all the required columns.
* 3. select best index from all the candidate indexes that could use.
*/
private Pair<PreAggStatus, List<Long>> select(
private SelectResult select(
LogicalOlapScan scan,
Set<Slot> requiredScanOutput,
List<Expression> predicates,
@ -199,28 +286,32 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
OlapTable table = scan.getTable();
// 0. check pre-aggregation status.
final PreAggStatus preAggStatus;
final Stream<MaterializedIndex> checkPreAggResult;
switch (table.getKeysType()) {
case AGG_KEYS:
case UNIQUE_KEYS:
// Check pre-aggregation status by base index for aggregate-keys and unique-keys OLAP table.
preAggStatus = checkPreAggStatus(scan, table.getBaseIndexId(), predicates,
case UNIQUE_KEYS: {
// Only checking pre-aggregation status by base index is enough for aggregate-keys and
// unique-keys OLAP table.
// Because the schemas in non-base materialized index are subsets of the schema of base index.
PreAggStatus preAggStatus = checkPreAggStatus(scan, table.getBaseIndexId(), predicates,
aggregateFunctions, groupingExprs);
if (preAggStatus.isOff()) {
// return early if pre agg status if off.
return Pair.of(preAggStatus, ImmutableList.of(scan.getTable().getBaseIndexId()));
return new SelectResult(preAggStatus, scan.getTable().getBaseIndexId(), new ExprRewriteMap());
} else {
List<MaterializedIndex> rollupsWithAllRequiredCols = table.getVisibleIndex().stream()
.filter(index -> containAllRequiredColumns(index, scan, requiredScanOutput))
.collect(Collectors.toList());
return new SelectResult(preAggStatus, selectBestIndex(rollupsWithAllRequiredCols, scan, predicates),
new ExprRewriteMap());
}
checkPreAggResult = table.getVisibleIndex().stream();
break;
case DUP_KEYS:
}
case DUP_KEYS: {
Map<Boolean, List<MaterializedIndex>> indexesGroupByIsBaseOrNot = table.getVisibleIndex()
.stream()
.collect(Collectors.groupingBy(index -> index.getId() == table.getBaseIndexId()));
// Duplicate-keys table could use base index and indexes that pre-aggregation status is on.
checkPreAggResult = Stream.concat(
Stream<MaterializedIndex> checkPreAggResult = Stream.concat(
indexesGroupByIsBaseOrNot.get(true).stream(),
indexesGroupByIsBaseOrNot.getOrDefault(false, ImmutableList.of())
.stream()
@ -228,15 +319,51 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
aggregateFunctions, groupingExprs).isOn())
);
Set<MaterializedIndex> candidatesWithoutRewriting = checkPreAggResult.collect(Collectors.toSet());
// try to rewrite bitmap, hll by materialized index columns.
List<AggRewriteResult> candidatesWithRewriting = indexesGroupByIsBaseOrNot.getOrDefault(false,
ImmutableList.of())
.stream()
.filter(index -> !candidatesWithoutRewriting.contains(index))
.map(index -> rewriteAgg(index, scan, requiredScanOutput, predicates, aggregateFunctions,
groupingExprs))
.filter(result -> result.success)
.collect(Collectors.toList());
List<MaterializedIndex> haveAllRequiredColumns = Streams.concat(
candidatesWithoutRewriting.stream()
.filter(index -> containAllRequiredColumns(index, scan, requiredScanOutput)),
candidatesWithRewriting
.stream()
.filter(aggRewriteResult -> containAllRequiredColumns(aggRewriteResult.index, scan,
aggRewriteResult.requiredScanOutput))
.map(aggRewriteResult -> aggRewriteResult.index)
).collect(Collectors.toList());
long selectIndexId = selectBestIndex(haveAllRequiredColumns, scan, predicates);
Optional<AggRewriteResult> rewriteResultOpt = candidatesWithRewriting.stream()
.filter(aggRewriteResult -> aggRewriteResult.index.getId() == selectIndexId)
.findAny();
// Pre-aggregation is set to `on` by default for duplicate-keys table.
preAggStatus = PreAggStatus.on();
break;
return new SelectResult(PreAggStatus.on(), selectIndexId,
rewriteResultOpt.map(r -> r.exprRewriteMap).orElse(new ExprRewriteMap()));
}
default:
throw new RuntimeException("Not supported keys type: " + table.getKeysType());
}
}
List<Long> sortedIndexId = filterAndOrder(checkPreAggResult, scan, requiredScanOutput, predicates);
return Pair.of(preAggStatus, sortedIndexId);
private static class SelectResult {
public final PreAggStatus preAggStatus;
public final long indexId;
public ExprRewriteMap exprRewriteMap;
public SelectResult(PreAggStatus preAggStatus, long indexId, ExprRewriteMap exprRewriteMap) {
this.preAggStatus = preAggStatus;
this.indexId = indexId;
this.exprRewriteMap = exprRewriteMap;
}
}
/**
@ -271,6 +398,13 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.collect(Collectors.toList());
}
private static AggregateFunction replaceAggFuncInput(AggregateFunction aggFunc,
Optional<Map<Slot, Expression>> slotToProducerOpt) {
return slotToProducerOpt.map(
slotToExpressions -> (AggregateFunction) ExpressionUtils.replace(aggFunc, slotToExpressions))
.orElse(aggFunc);
}
///////////////////////////////////////////////////////////////////////////
// Set pre-aggregation status.
///////////////////////////////////////////////////////////////////////////
@ -338,7 +472,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
@Override
public PreAggStatus visitCount(Count count, CheckContext context) {
// Now count(distinct key_column) is only supported for aggregate-keys and unique-keys OLAP table.
if (count.isDistinct()) {
if (count.isDistinct() && count.arity() == 1) {
Optional<ExprId> exprIdOpt = extractSlotId(count.child(0));
if (exprIdOpt.isPresent() && context.exprIdToKeyColumn.containsKey(exprIdOpt.get())) {
return PreAggStatus.on();
@ -348,6 +482,16 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
"Count distinct is only valid for key columns, but meet %s.", count.toSql()));
}
@Override
public PreAggStatus visitBitmapUnionCount(BitmapUnionCount bitmapUnionCount, CheckContext context) {
Optional<Slot> slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(bitmapUnionCount.child());
if (slotOpt.isPresent() && context.exprIdToValueColumn.containsKey(slotOpt.get().getExprId())) {
return PreAggStatus.on();
} else {
return PreAggStatus.off("invalid bitmap_union_count: " + bitmapUnionCount.toSql());
}
}
private PreAggStatus checkAggFunc(
AggregateFunction aggFunc,
AggregateType matchingAggType,
@ -389,7 +533,10 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
public final Map<ExprId, Column> exprIdToKeyColumn;
public final Map<ExprId, Column> exprIdToValueColumn;
public final LogicalOlapScan scan;
public CheckContext(LogicalOlapScan scan, long indexId) {
this.scan = scan;
// map<is_key, map<column_name, column>>
Map<Boolean, Map<String, Column>> nameToColumnGroupingByIsKey
= scan.getTable().getSchemaByIndexId(indexId)
@ -400,8 +547,8 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
));
Map<String, Column> keyNameToColumn = nameToColumnGroupingByIsKey.get(true);
Map<String, Column> valueNameToColumn = nameToColumnGroupingByIsKey.getOrDefault(false, ImmutableMap.of());
Map<String, ExprId> nameToExprId = scan.getOutput()
.stream()
Map<String, ExprId> nameToExprId = Stream.concat(
scan.getOutput().stream(), scan.getNonUserVisibleOutput().stream())
.collect(Collectors.toMap(
NamedExpression::getName,
NamedExpression::getExprId)
@ -461,4 +608,241 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
exprToColumn.key().toSql(), exprToColumn.value().getName())))
.orElse(PreAggStatus.on());
}
/**
* rewrite for bitmap and hll
*/
private AggRewriteResult rewriteAgg(MaterializedIndex index,
LogicalOlapScan scan,
Set<Slot> requiredScanOutput,
List<Expression> predicates,
List<AggregateFunction> aggregateFunctions,
List<Expression> groupingExprs) {
ExprRewriteMap exprRewriteMap = new ExprRewriteMap();
RewriteContext context = new RewriteContext(new CheckContext(scan, index.getId()), exprRewriteMap);
aggregateFunctions.forEach(aggFun -> AggFuncRewriter.rewrite(aggFun, context));
// has rewritten agg functions
Map<Slot, Slot> slotMap = exprRewriteMap.slotMap;
if (!slotMap.isEmpty()) {
// Note that the slots in the rewritten agg functions shouldn't appear in filters or grouping expressions.
// For example: we have a duplicated-type table t(c1, c2) and a materialized index that has
// a bitmap_union column `mv_bitmap_union_c2` for the column c2.
// The query `select c1, count(distinct c2) from t where c2 > 0 group by c1` can't use the materialized
// index because we have a filter `c2 > 0` for the aggregated column c2.
Set<Slot> slotsToReplace = slotMap.keySet();
if (!isInputSlotsContainsAny(predicates, slotsToReplace)
&& !isInputSlotsContainsAny(groupingExprs, slotsToReplace)) {
ImmutableSet<Slot> newRequiredSlots = requiredScanOutput.stream()
.map(slot -> (Slot) ExpressionUtils.replace(slot, slotMap))
.collect(ImmutableSet.toImmutableSet());
return new AggRewriteResult(index, true, newRequiredSlots, exprRewriteMap);
}
}
return new AggRewriteResult(index, false, null, null);
}
private static class ExprRewriteMap {
/**
* Replace map for scan output slot.
*/
public final Map<Slot, Slot> slotMap;
/**
* Replace map for expressions in project.
*/
public final Map<Expression, Expression> projectExprMap;
/**
* Replace map for aggregate functions.
*/
public final Map<AggregateFunction, AggregateFunction> aggFuncMap;
public ExprRewriteMap() {
this.slotMap = Maps.newHashMap();
this.projectExprMap = Maps.newHashMap();
this.aggFuncMap = Maps.newHashMap();
}
public boolean isEmpty() {
return slotMap.isEmpty();
}
}
private static class AggRewriteResult {
public final MaterializedIndex index;
public final boolean success;
public final Set<Slot> requiredScanOutput;
public ExprRewriteMap exprRewriteMap;
public AggRewriteResult(MaterializedIndex index,
boolean success,
Set<Slot> requiredScanOutput,
ExprRewriteMap exprRewriteMap) {
this.index = index;
this.success = success;
this.requiredScanOutput = requiredScanOutput;
this.exprRewriteMap = exprRewriteMap;
}
}
private boolean isInputSlotsContainsAny(List<Expression> expressions, Set<Slot> slotsToCheck) {
Set<Slot> inputSlotSet = ExpressionUtils.getInputSlotSet(expressions);
return !Sets.intersection(inputSlotSet, slotsToCheck).isEmpty();
}
private static class RewriteContext {
public final CheckContext checkContext;
public final ExprRewriteMap exprRewriteMap;
public RewriteContext(CheckContext context, ExprRewriteMap exprRewriteMap) {
this.checkContext = context;
this.exprRewriteMap = exprRewriteMap;
}
}
private static class AggFuncRewriter extends DefaultExpressionRewriter<RewriteContext> {
public static final AggFuncRewriter INSTANCE = new AggFuncRewriter();
private static Expression rewrite(Expression expr, RewriteContext context) {
return expr.accept(INSTANCE, context);
}
/**
* count(distinct col) -> bitmap_union_count(mv_bitmap_union_col)
*/
@Override
public Expression visitCount(Count count, RewriteContext context) {
if (count.isDistinct() && count.arity() == 1) {
Optional<Slot> slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(count.child(0));
// count distinct a value column.
if (slotOpt.isPresent() && !context.checkContext.exprIdToKeyColumn.containsKey(
slotOpt.get().getExprId())) {
String bitmapUnionCountColumn = CreateMaterializedViewStmt
.mvColumnBuilder(AggregateType.BITMAP_UNION.name().toLowerCase(), slotOpt.get().getName());
Column mvColumn = context.checkContext.scan.getTable().getVisibleColumn(bitmapUnionCountColumn);
// has bitmap_union_count column
if (mvColumn != null && context.checkContext.exprIdToValueColumn.containsValue(mvColumn)) {
Slot bitmapUnionCountSlot = context.checkContext.scan.getNonUserVisibleOutput()
.stream()
.filter(s -> s.getName().equals(bitmapUnionCountColumn))
.findFirst()
.get();
context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionCountSlot);
context.exprRewriteMap.projectExprMap.put(slotOpt.get(), bitmapUnionCountSlot);
BitmapUnionCount bitmapUnionCount = new BitmapUnionCount(bitmapUnionCountSlot);
context.exprRewriteMap.aggFuncMap.put(count, bitmapUnionCount);
return bitmapUnionCount;
}
}
}
return count;
}
/**
* bitmap_union_count(to_bitmap(col)) -> bitmap_union_count(mv_bitmap_union_col)
*/
@Override
public Expression visitBitmapUnionCount(BitmapUnionCount bitmapUnionCount, RewriteContext context) {
if (bitmapUnionCount.child() instanceof ToBitmap) {
ToBitmap toBitmap = (ToBitmap) bitmapUnionCount.child();
Optional<Slot> slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(toBitmap.child());
if (slotOpt.isPresent()) {
String bitmapUnionCountColumn = CreateMaterializedViewStmt
.mvColumnBuilder(AggregateType.BITMAP_UNION.name().toLowerCase(), slotOpt.get().getName());
Column mvColumn = context.checkContext.scan.getTable().getVisibleColumn(bitmapUnionCountColumn);
// has bitmap_union_count column
if (mvColumn != null && context.checkContext.exprIdToValueColumn.containsValue(mvColumn)) {
Slot bitmapUnionCountSlot = context.checkContext.scan.getNonUserVisibleOutput()
.stream()
.filter(s -> s.getName().equals(bitmapUnionCountColumn))
.findFirst()
.get();
context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionCountSlot);
context.exprRewriteMap.projectExprMap.put(toBitmap, bitmapUnionCountSlot);
BitmapUnionCount newBitmapUnionCount = new BitmapUnionCount(bitmapUnionCountSlot);
context.exprRewriteMap.aggFuncMap.put(bitmapUnionCount, newBitmapUnionCount);
return newBitmapUnionCount;
}
}
}
return bitmapUnionCount;
}
}
private List<NamedExpression> replaceAggOutput(
LogicalAggregate<? extends Plan> agg,
Optional<Project> oldProjectOpt,
Optional<Project> newProjectOpt,
ExprRewriteMap exprRewriteMap) {
ResultAggFuncRewriteCtx ctx = new ResultAggFuncRewriteCtx(oldProjectOpt, newProjectOpt, exprRewriteMap);
return agg.getOutputExpressions()
.stream()
.map(expr -> (NamedExpression) ResultAggFuncRewriter.rewrite(expr, ctx))
.collect(ImmutableList.toImmutableList());
}
private static class ResultAggFuncRewriteCtx {
public final Optional<Map<Slot, Expression>> oldProjectSlotToProducerOpt;
public final Optional<Map<Expression, Slot>> newProjectExprMapOpt;
public final ExprRewriteMap exprRewriteMap;
public ResultAggFuncRewriteCtx(
Optional<Project> oldProject,
Optional<Project> newProject,
ExprRewriteMap exprRewriteMap) {
this.oldProjectSlotToProducerOpt = oldProject.map(Project::getAliasToProducer);
this.newProjectExprMapOpt = newProject.map(project -> project.getProjects()
.stream()
.filter(Alias.class::isInstance)
.collect(
Collectors.toMap(
// Avoid cast to alias, retrieving the first child expression.
alias -> alias.child(0),
NamedExpression::toSlot
)
));
this.exprRewriteMap = exprRewriteMap;
}
}
private static class ResultAggFuncRewriter extends DefaultExpressionRewriter<ResultAggFuncRewriteCtx> {
public static final ResultAggFuncRewriter INSTANCE = new ResultAggFuncRewriter();
public static Expression rewrite(Expression expr, ResultAggFuncRewriteCtx ctx) {
return expr.accept(INSTANCE, ctx);
}
@Override
public Expression visitAggregateFunction(AggregateFunction aggregateFunction,
ResultAggFuncRewriteCtx ctx) {
// normalize aggregate function to match the agg func replace map.
AggregateFunction aggFunc = replaceAggFuncInput(aggregateFunction, ctx.oldProjectSlotToProducerOpt);
Map<AggregateFunction, AggregateFunction> aggFuncMap = ctx.exprRewriteMap.aggFuncMap;
if (aggFuncMap.containsKey(aggFunc)) {
AggregateFunction replacedAggFunc = aggFuncMap.get(aggFunc);
// replace the input slot by new project expr mapping.
return ctx.newProjectExprMapOpt.map(map -> ExpressionUtils.replace(replacedAggFunc, map))
.orElse(replacedAggFunc);
} else {
return aggregateFunction;
}
}
}
private List<NamedExpression> replaceProjectList(
LogicalProject<? extends Plan> project,
Map<Expression, Expression> projectMap) {
return project.getProjects().stream()
.map(expr -> (NamedExpression) ExpressionUtils.replace(expr, projectMap))
.collect(ImmutableList.toImmutableList());
}
}

23
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.nereids.rules.mv;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.nereids.rules.Rule;
@ -124,24 +125,34 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater
List<MaterializedIndex> candidates = table.getVisibleIndex().stream()
.filter(index -> index.getId() == baseIndexId
|| table.getKeyColumnsByIndexId(index.getId()).size() == baseIndexKeySize)
.filter(index -> containAllRequiredColumns(index, scan, requiredScanOutputSupplier.get()))
.collect(Collectors.toList());
PreAggStatus preAgg = PreAggStatus.off("No aggregate on scan.");
if (candidates.size() == 1) {
// `candidates` only have base index.
return scan.withMaterializedIndexSelected(preAgg, ImmutableList.of(baseIndexId));
return scan.withMaterializedIndexSelected(preAgg, baseIndexId);
} else {
return scan.withMaterializedIndexSelected(preAgg,
filterAndOrder(candidates.stream(), scan, requiredScanOutputSupplier.get(),
predicatesSupplier.get()));
selectBestIndex(candidates, scan, predicatesSupplier.get()));
}
case DUP_KEYS:
// Set pre-aggregation to `on` to keep consistency with legacy logic.
List<MaterializedIndex> candidate = scan.getTable().getVisibleIndex().stream()
.filter(index -> !indexHasAggregate(index, scan))
.filter(index -> containAllRequiredColumns(index, scan,
requiredScanOutputSupplier.get()))
.collect(Collectors.toList());
return scan.withMaterializedIndexSelected(PreAggStatus.on(),
filterAndOrder(scan.getTable().getVisibleIndex().stream(), scan,
requiredScanOutputSupplier.get(),
predicatesSupplier.get()));
selectBestIndex(candidate, scan, predicatesSupplier.get()));
default:
throw new RuntimeException("Not supported keys type: " + scan.getTable().getKeysType());
}
}
private boolean indexHasAggregate(MaterializedIndex index, LogicalOlapScan scan) {
return scan.getTable().getSchemaByIndexId(index.getId())
.stream()
.anyMatch(Column::isAggregated);
}
}

70
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/CountDistinctRewrite.java Normal file
View File

@ -0,0 +1,70 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount;
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import com.google.common.collect.ImmutableList;
import java.util.List;
/**
* Rewrite count distinct for bitmap and hll type value.
* <p>
* count(distinct bitmap_col) -> bitmap_union_count(bitmap col)
* todo: add support for HLL type.
*/
public class CountDistinctRewrite extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalAggregate().then(agg -> {
List<NamedExpression> output = agg.getOutputExpressions()
.stream()
.map(CountDistinctRewriter::rewrite)
.map(NamedExpression.class::cast)
.collect(ImmutableList.toImmutableList());
return agg.withAggOutput(output);
}).toRule(RuleType.COUNT_DISTINCT_REWRITE);
}
private static class CountDistinctRewriter extends DefaultExpressionRewriter<Void> {
private static final CountDistinctRewriter INSTANCE = new CountDistinctRewriter();
public static Expression rewrite(Expression expr) {
return expr.accept(INSTANCE, null);
}
@Override
public Expression visitCount(Count count, Void context) {
if (count.isDistinct() && count.arity() == 1) {
Expression child = count.child(0);
if (child.getDataType().isBitmap()) {
return new BitmapUnionCount(child);
}
}
return count;
}
}
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/BitmapUnionCount.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BigIntType;
import org.apache.doris.nereids.types.BitmapType;
import org.apache.doris.nereids.types.DataType;
@ -62,4 +63,9 @@ public class BitmapUnionCount extends AggregateFunction
public List<FunctionSignature> getSignatures() {
return SIGNATURES;
}
@Override
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitBitmapUnionCount(this, context);
}
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.visitor;
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.Avg;
import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount;
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
import org.apache.doris.nereids.trees.expressions.functions.agg.GroupBitAnd;
import org.apache.doris.nereids.trees.expressions.functions.agg.GroupBitOr;
@ -72,4 +73,8 @@ public interface AggregateFunctionVisitor<R, C> {
default R visitSum(Sum sum, C context) {
return visitAggregateFunction(sum, context);
}
default R visitBitmapUnionCount(BitmapUnionCount bitmapUnionCount, C context) {
return visitAggregateFunction(bitmapUnionCount, context);
}
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java
View File

@ -139,6 +139,11 @@ public abstract class AbstractPlan extends AbstractTreeNode<Plan> implements Pla
return getLogicalProperties().getOutput();
}
@Override
public List<Slot> getNonUserVisibleOutput() {
return getLogicalProperties().getNonUserVisibleOutput();
}
@Override
public Set<Slot> getOutputSet() {
return getLogicalProperties().getOutputSet();

7
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java
View File

@ -23,6 +23,8 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@ -87,6 +89,11 @@ public class FakePlan implements Plan {
return new ArrayList<>();
}
@Override
public List<Slot> getNonUserVisibleOutput() {
return ImmutableList.of();
}
@Override
public String treeString() {
return "DUMMY";

6
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java
View File

@ -87,6 +87,8 @@ public interface Plan extends TreeNode<Plan> {
*/
List<Slot> getOutput();
List<Slot> getNonUserVisibleOutput();
/**
* Get output slot set of the plan.
*/
@ -114,6 +116,10 @@ public interface Plan extends TreeNode<Plan> {
throw new IllegalStateException("Not support compute output for " + getClass().getName());
}
default List<Slot> computeNonUserVisibleOutput() {
return ImmutableList.of();
}
String treeString();
default Plan withOutput(List<Slot> output) {

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java
View File

@ -83,6 +83,11 @@ public interface Command extends LogicalPlan {
throw new RuntimeException("Command do not implement getOutput");
}
@Override
default List<Slot> getNonUserVisibleOutput() {
throw new RuntimeException("Command do not implement getNonUserVisibleOutput");
}
@Override
default String treeString() {
throw new RuntimeException("Command do not implement treeString");

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java
View File

@ -51,7 +51,7 @@ public abstract class AbstractLogicalPlan extends AbstractPlan implements Logica
if (hasUnboundChild || hasUnboundExpression()) {
return UnboundLogicalProperties.INSTANCE;
} else {
return new LogicalProperties(this::computeOutput);
return new LogicalProperties(this::computeOutput, this::computeNonUserVisibleOutput);
}
}
}

111
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
@ -24,6 +25,8 @@ import org.apache.doris.catalog.Table;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.PreAggStatus;
@ -36,41 +39,75 @@ import org.apache.doris.nereids.util.Utils;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import org.apache.commons.collections.CollectionUtils;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Logical OlapScan.
*/
public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, OlapScan {
private final long selectedIndexId;
private final List<Long> selectedTabletIds;
private final boolean partitionPruned;
private final boolean tabletPruned;
///////////////////////////////////////////////////////////////////////////
// Members for materialized index.
///////////////////////////////////////////////////////////////////////////
private final List<Long> candidateIndexIds;
/**
* The select materialized index id to read data from.
*/
private final long selectedIndexId;
/**
* Status to indicate materialized index id is selected or not.
*/
private final boolean indexSelected;
/**
* Status to indicate using pre-aggregation or not.
*/
private final PreAggStatus preAggStatus;
///////////////////////////////////////////////////////////////////////////
// Members for tablet ids.
///////////////////////////////////////////////////////////////////////////
/**
* Selected tablet ids to read data from.
*/
private final ImmutableList<Long> selectedTabletIds;
/**
* Status to indicate tablets are pruned or not.
*/
private final boolean tabletPruned;
///////////////////////////////////////////////////////////////////////////
// Members for partition ids.
///////////////////////////////////////////////////////////////////////////
/**
* Status to indicate partitions are pruned or not.
* todo: should be pulled up to base class?
*/
private final boolean partitionPruned;
public LogicalOlapScan(RelationId id, OlapTable table) {
this(id, table, ImmutableList.of());
}
public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier) {
this(id, table, qualifier, Optional.empty(), Optional.empty(),
table.getPartitionIds(), false, ImmutableList.of(), false,
ImmutableList.of(), false, PreAggStatus.on());
table.getPartitionIds(), false,
ImmutableList.of(), false,
-1, false, PreAggStatus.on());
}
public LogicalOlapScan(RelationId id, Table table, List<String> qualifier) {
this(id, table, qualifier, Optional.empty(), Optional.empty(),
((OlapTable) table).getPartitionIds(), false, ImmutableList.of(), false,
ImmutableList.of(), false, PreAggStatus.on());
-1, false, PreAggStatus.on());
}
/**
@ -80,17 +117,13 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
List<Long> selectedPartitionIds, boolean partitionPruned,
List<Long> selectedTabletIds, boolean tabletPruned,
List<Long> candidateIndexIds, boolean indexSelected, PreAggStatus preAggStatus) {
long selectedIndexId, boolean indexSelected, PreAggStatus preAggStatus) {
super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier,
groupExpression, logicalProperties, selectedPartitionIds);
// TODO: use CBO manner to select best index id, according to index's statistics info,
// revisit this after rollup and materialized view selection are fully supported.
this.selectedIndexId = CollectionUtils.isEmpty(candidateIndexIds)
? getTable().getBaseIndexId() : candidateIndexIds.get(0);
this.selectedTabletIds = ImmutableList.copyOf(selectedTabletIds);
this.partitionPruned = partitionPruned;
this.tabletPruned = tabletPruned;
this.candidateIndexIds = ImmutableList.copyOf(candidateIndexIds);
this.selectedIndexId = selectedIndexId <= 0 ? getTable().getBaseIndexId() : selectedIndexId;
this.indexSelected = indexSelected;
this.preAggStatus = preAggStatus;
}
@ -113,7 +146,7 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation,
return Utils.toSqlString("LogicalOlapScan",
"qualified", qualifiedName(),
"output", getOutput(),
"candidateIndexIds", candidateIndexIds,
"indexName", getSelectedMaterializedIndexName().orElse("<index_not_selected>"),
"selectedIndexId", selectedIndexId,
"preAgg", preAggStatus
);
@ -127,46 +160,53 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation,
if (o == null || getClass() != o.getClass() || !super.equals(o)) {
return false;
}
return Objects.equals(selectedPartitionIds, ((LogicalOlapScan) o).selectedPartitionIds)
&& Objects.equals(candidateIndexIds, ((LogicalOlapScan) o).candidateIndexIds)
&& Objects.equals(selectedTabletIds, ((LogicalOlapScan) o).selectedTabletIds);
return Objects.equals(id, ((LogicalOlapScan) o).id)
&& Objects.equals(selectedPartitionIds, ((LogicalOlapScan) o).selectedPartitionIds)
&& Objects.equals(partitionPruned, ((LogicalOlapScan) o).partitionPruned)
&& Objects.equals(selectedIndexId, ((LogicalOlapScan) o).selectedIndexId)
&& Objects.equals(indexSelected, ((LogicalOlapScan) o).indexSelected)
&& Objects.equals(selectedTabletIds, ((LogicalOlapScan) o).selectedTabletIds)
&& Objects.equals(tabletPruned, ((LogicalOlapScan) o).tabletPruned);
}
@Override
public int hashCode() {
return Objects.hash(id, selectedPartitionIds, candidateIndexIds, selectedTabletIds);
return Objects.hash(id,
selectedPartitionIds, partitionPruned,
selectedIndexId, indexSelected,
selectedTabletIds, tabletPruned);
}
@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalOlapScan(id, table, qualifier, groupExpression, Optional.of(getLogicalProperties()),
selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned,
candidateIndexIds, indexSelected, preAggStatus);
selectedIndexId, indexSelected, preAggStatus);
}
@Override
public LogicalOlapScan withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
return new LogicalOlapScan(id, table, qualifier, Optional.empty(), logicalProperties,
selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned,
candidateIndexIds, indexSelected, preAggStatus);
selectedIndexId, indexSelected, preAggStatus);
}
public LogicalOlapScan withSelectedPartitionIds(List<Long> selectedPartitionIds) {
return new LogicalOlapScan(id, table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()),
selectedPartitionIds, true, selectedTabletIds, tabletPruned,
candidateIndexIds, indexSelected, preAggStatus);
selectedIndexId, indexSelected, preAggStatus);
}
public LogicalOlapScan withMaterializedIndexSelected(PreAggStatus preAgg, List<Long> candidateIndexIds) {
public LogicalOlapScan withMaterializedIndexSelected(PreAggStatus preAgg, long indexId) {
return new LogicalOlapScan(id, table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()),
selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned,
candidateIndexIds, true, preAgg);
indexId, true, preAgg);
}
public LogicalOlapScan withSelectedTabletIds(List<Long> selectedTabletIds) {
return new LogicalOlapScan(id, table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()),
selectedPartitionIds, partitionPruned, selectedTabletIds, true,
candidateIndexIds, indexSelected, preAggStatus);
selectedIndexId, indexSelected, preAggStatus);
}
@Override
@ -204,4 +244,23 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation,
return indexSelected ? Optional.ofNullable(((OlapTable) table).getIndexNameById(selectedIndexId))
: Optional.empty();
}
@Override
public List<Slot> computeNonUserVisibleOutput() {
Set<String> baseSchemaColNames = table.getBaseSchema().stream()
.map(Column::getName)
.collect(Collectors.toSet());
OlapTable olapTable = (OlapTable) table;
// extra columns in materialized index, such as `mv_bitmap_union_xxx`
return olapTable.getVisibleIndexIdToMeta().values()
.stream()
.filter(index -> index.getIndexId() != ((OlapTable) table).getBaseIndexId())
.flatMap(index -> index.getSchema()
.stream()
.filter(col -> !baseSchemaColNames.contains(col.getName()))
)
.map(col -> SlotReference.fromColumn(col, qualified()))
.collect(ImmutableList.toImmutableList());
}
}

4
fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
View File

@ -445,6 +445,10 @@ public abstract class DataType implements AbstractDataType {
return this instanceof DateTimeV2Type;
}
public boolean isBitmap() {
return this instanceof BitmapType;
}
public DataType promotion() {
if (PROMOTION_MAP.containsKey(this.getClass())) {
return PROMOTION_MAP.get(this.getClass()).get();

20
fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
View File

@ -44,6 +44,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -235,12 +236,20 @@ public class ExpressionUtils {
* Otherwise, return empty optional result.
*/
public static Optional<ExprId> isSlotOrCastOnSlot(Expression expr) {
return extractSlotOrCastOnSlot(expr).map(Slot::getExprId);
}
/**
* Check whether the input expression is a {@link org.apache.doris.nereids.trees.expressions.Slot}
* or at least one {@link Cast} on a {@link org.apache.doris.nereids.trees.expressions.Slot}
*/
public static Optional<Slot> extractSlotOrCastOnSlot(Expression expr) {
while (expr instanceof Cast) {
expr = expr.child(0);
}
if (expr instanceof SlotReference) {
return Optional.of(((SlotReference) expr).getExprId());
return Optional.of((Slot) expr);
} else {
return Optional.empty();
}
@ -425,4 +434,13 @@ public class ExpressionUtils {
// skip current expression
cubeToGroupingSets(cubeExpressions, activeIndex + 1, currentGroupingSet, groupingSets);
}
/**
* Get input slot set from list of expressions.
*/
public static Set<Slot> getInputSlotSet(Collection<? extends Expression> exprs) {
return exprs.stream()
.flatMap(expr -> expr.getInputSlots().stream())
.collect(ImmutableSet.toImmutableSet());
}
}

310
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java
View File

@ -19,6 +19,11 @@ package org.apache.doris.nereids.rules.mv;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.common.FeConstants;
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount;
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.util.PatternMatchSupported;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.planner.OlapScanNode;
@ -34,6 +39,8 @@ import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Tests ported from {@link org.apache.doris.planner.MaterializedViewFunctionTest}
@ -47,6 +54,7 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement
private static final String DEPTS_MV_NAME = "depts_mv";
private static final String USER_TAG_TABLE_NAME = "user_tags";
private static final String TEST_TABLE_NAME = "test_tb";
private static final String USER_TAG_MV_NAME = "user_tags_mv";
@Override
protected void beforeCreatingConnectContext() throws Exception {
@ -549,22 +557,7 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement
createMv(createEmpsMVsql);
String query = "select empid, deptno, salary from " + EMPS_TABLE_NAME + " e1 where empid = (select max(empid)"
+ " from " + EMPS_TABLE_NAME + " where deptno = e1.deptno);";
PlanChecker.from(connectContext).checkPlannerResult(query, planner -> {
List<ScanNode> scans = planner.getScanNodes();
Assertions.assertEquals(2, scans.size());
ScanNode scanNode0 = scans.get(0);
Assertions.assertTrue(scanNode0 instanceof OlapScanNode);
OlapScanNode scan0 = (OlapScanNode) scanNode0;
Assertions.assertTrue(scan0.isPreAggregation());
Assertions.assertEquals("emps_mv", scan0.getSelectedIndexName());
ScanNode scanNode1 = scans.get(1);
Assertions.assertTrue(scanNode1 instanceof OlapScanNode);
OlapScanNode scan1 = (OlapScanNode) scanNode1;
Assertions.assertTrue(scan1.isPreAggregation());
Assertions.assertEquals("emps", scan1.getSelectedIndexName());
});
testMvWithTwoTable(query, "emps_mv", "emps");
}
/**
@ -603,22 +596,7 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement
String query = "select * from (select deptno, empid from " + EMPS_TABLE_NAME + " where deptno>100) A join "
+ "(select deptno, sum(salary) from " + EMPS_TABLE_NAME + " where deptno >200 group by deptno) B "
+ "on A.deptno=B.deptno";
PlanChecker.from(connectContext).checkPlannerResult(query, planner -> {
List<ScanNode> scans = planner.getScanNodes();
Assertions.assertEquals(2, scans.size());
ScanNode scanNode0 = scans.get(0);
Assertions.assertTrue(scanNode0 instanceof OlapScanNode);
OlapScanNode scan0 = (OlapScanNode) scanNode0;
Assertions.assertTrue(scan0.isPreAggregation());
Assertions.assertEquals("emp_mv_01", scan0.getSelectedIndexName());
ScanNode scanNode1 = scans.get(1);
Assertions.assertTrue(scanNode1 instanceof OlapScanNode);
OlapScanNode scan1 = (OlapScanNode) scanNode1;
Assertions.assertTrue(scan1.isPreAggregation());
Assertions.assertEquals("emp_mv_02", scan1.getSelectedIndexName());
});
testMvWithTwoTable(query, "emp_mv_01", "emp_mv_02");
}
@Test
@ -703,6 +681,7 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement
+ "distributed by hash(k1) buckets 3 properties('replication_num' = '1')");
createMv("create materialized view k1_k2 as select k1, k2 from t group by k1, k2");
singleTableTest("select k1, k2 from t group by k1, k2", "k1_k2", true);
dropTable("t", true);
}
@Test
@ -768,89 +747,112 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement
}
/**
* TODO: enable this when bitmap is supported.
* bitmap_union_count(to_bitmap()) -> bitmap_union_count without having
*/
@Disabled
@Test
public void testBitmapUnionRewrite() throws Exception {
String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME
+ " as select user_id, bitmap_union(to_bitmap(tag_id)) from "
+ USER_TAG_TABLE_NAME + " group by user_id;";
createMv(createUserTagMVSql);
String query = "select user_id, bitmap_union_count(to_bitmap(tag_id)) a from " + USER_TAG_TABLE_NAME
+ " group by user_id";
singleTableTest(query, USER_TAG_MV_NAME, true);
}
/**
* bitmap_union_count(to_bitmap()) -> bitmap_union_count with having
*/
@Test
public void testBitmapUnionInQuery() throws Exception {
// String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME
// + " as select user_id, bitmap_union(to_bitmap(tag_id)) from "
// + USER_TAG_TABLE_NAME + " group by user_id;";
// dorisAssert.withMaterializedView(createUserTagMVSql);
// String query = "select user_id, bitmap_union_count(to_bitmap(tag_id)) a from " + USER_TAG_TABLE_NAME
// + " group by user_id having a>1 order by a;";
// dorisAssert.query(query).explainContains(QUERY_USE_USER_TAG_MV);
String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME
+ " as select user_id, bitmap_union(to_bitmap(tag_id)) from "
+ USER_TAG_TABLE_NAME + " group by user_id;";
createMv(createUserTagMVSql);
String query = "select user_id, bitmap_union_count(to_bitmap(tag_id)) a from " + USER_TAG_TABLE_NAME
+ " group by user_id having a>1 order by a;";
singleTableTest(query, USER_TAG_MV_NAME, true);
}
/**
* TODO: enable this when bitmap is supported.
*/
@Disabled
@Test
public void testBitmapUnionInSubquery() throws Exception {
// String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, "
// + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
// dorisAssert.withMaterializedView(createUserTagMVSql);
// String query = "select user_id from " + USER_TAG_TABLE_NAME + " where user_id in (select user_id from "
// + USER_TAG_TABLE_NAME + " group by user_id having bitmap_union_count(to_bitmap(tag_id)) >1 ) ;";
// dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, USER_TAG_TABLE_NAME);
String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, "
+ "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
createMv(createUserTagMVSql);
String query = "select user_id from " + USER_TAG_TABLE_NAME + " where user_id in (select user_id from "
+ USER_TAG_TABLE_NAME + " group by user_id having bitmap_union_count(to_bitmap(tag_id)) >1 ) ;";
testMvWithTwoTable(query, "user_tags", "user_tags_mv");
}
/**
* TODO: enable this when bitmap is supported.
*/
@Disabled
@Test
public void testIncorrectMVRewriteInQuery() throws Exception {
// String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, "
// + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
// dorisAssert.withMaterializedView(createUserTagMVSql);
// String createEmpMVSql = "create materialized view " + EMPS_MV_NAME + " as select name, deptno from "
// + EMPS_TABLE_NAME + ";";
// dorisAssert.withMaterializedView(createEmpMVSql);
// String query = "select user_name, bitmap_union_count(to_bitmap(tag_id)) a from " + USER_TAG_TABLE_NAME + ", "
// + "(select name, deptno from " + EMPS_TABLE_NAME + ") a" + " where user_name=a.name group by "
// + "user_name having a>1 order by a;";
// testMv(query, EMPS_MV_NAME);
// dorisAssert.query(query).explainWithout(QUERY_USE_USER_TAG_MV);
String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, "
+ "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
createMv(createUserTagMVSql);
String createEmpMVSql = "create materialized view " + EMPS_MV_NAME + " as select name, deptno from "
+ EMPS_TABLE_NAME + ";";
createMv(createEmpMVSql);
String query = "select user_name, bitmap_union_count(to_bitmap(tag_id)) a from " + USER_TAG_TABLE_NAME + ", "
+ "(select name, deptno from " + EMPS_TABLE_NAME + ") a" + " where user_name=a.name group by "
+ "user_name having a>1 order by a;";
testMv(query, ImmutableMap.of("user_tags", "user_tags", "emps", "emps_mv"));
}
/**
* TODO: enable this when bitmap is supported.
* bitmap_union_count(to_bitmap(tag_id)) in subquery
*/
@Disabled
@Test
public void testIncorrectMVRewriteInSubquery() throws Exception {
// String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, "
// + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
// dorisAssert.withMaterializedView(createUserTagMVSql);
// String query = "select user_id, bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " where "
// + "user_name in (select user_name from " + USER_TAG_TABLE_NAME + " group by user_name having "
// + "bitmap_union_count(to_bitmap(tag_id)) >1 )" + " group by user_id;";
// dorisAssert.query(query).explainContains(QUERY_USE_USER_TAG);
String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, "
+ "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
createMv(createUserTagMVSql);
String query = "select user_id, bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " where "
+ "user_name in (select user_name from " + USER_TAG_TABLE_NAME + " group by user_name having "
+ "bitmap_union_count(to_bitmap(tag_id)) >1 )" + " group by user_id;";
// can't use mv index because it has no required column `user_name`
testMv(query, ImmutableMap.of("user_tags", "user_tags"));
}
/**
* TODO: enable this when bitmap is supported.
*/
@Disabled
@Test
public void testTwoTupleInQuery() throws Exception {
// String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, "
// + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
// dorisAssert.withMaterializedView(createUserTagMVSql);
// String query = "select * from (select user_id, bitmap_union_count(to_bitmap(tag_id)) x from "
// + USER_TAG_TABLE_NAME + " group by user_id) a, (select user_name, bitmap_union_count(to_bitmap(tag_id))"
// + "" + " y from " + USER_TAG_TABLE_NAME + " group by user_name) b where a.x=b.y;";
// dorisAssert.query(query).explainContains(QUERY_USE_USER_TAG, QUERY_USE_USER_TAG_MV);
String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, "
+ "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
createMv(createUserTagMVSql);
String query = "select * from (select user_id, bitmap_union_count(to_bitmap(tag_id)) x from "
+ USER_TAG_TABLE_NAME + " group by user_id) a, (select user_name, bitmap_union_count(to_bitmap(tag_id))"
+ "" + " y from " + USER_TAG_TABLE_NAME + " group by user_name) b where a.x=b.y;";
PlanChecker.from(connectContext)
.analyze(query)
.rewrite()
.matches(logicalJoin(
logicalAggregate(
logicalProject(
logicalOlapScan().when(scan -> "user_tags_mv".equals(
scan.getSelectedMaterializedIndexName().get())))),
logicalAggregate(
logicalProject(
logicalOlapScan().when(scan -> "user_tags".equals(
scan.getSelectedMaterializedIndexName().get()))))));
}
/**
* TODO: enable this when bitmap is supported.
* count(distinct v) -> bitmap_union_count(v) without mv index.
*/
@Disabled
@Test
public void testAggTableCountDistinctInBitmapType() throws Exception {
// String aggTable = "CREATE TABLE " + TEST_TABLE_NAME + " (k1 int, v1 bitmap bitmap_union) Aggregate KEY (k1) "
// + "DISTRIBUTED BY HASH(k1) BUCKETS 3 PROPERTIES ('replication_num' = '1');";
// dorisAssert.withTable(aggTable);
// String query = "select k1, count(distinct v1) from " + TEST_TABLE_NAME + " group by k1;";
// dorisAssert.query(query).explainContains(TEST_TABLE_NAME, "bitmap_union_count");
// dorisAssert.dropTable(TEST_TABLE_NAME, true);
String aggTable = "CREATE TABLE " + TEST_TABLE_NAME + " (k1 int, v1 bitmap bitmap_union) Aggregate KEY (k1) "
+ "DISTRIBUTED BY HASH(k1) BUCKETS 3 PROPERTIES ('replication_num' = '1');";
createTable(aggTable);
String query = "select k1, count(distinct v1) from " + TEST_TABLE_NAME + " group by k1;";
PlanChecker.from(connectContext)
.analyze(query)
.rewrite()
.matches(logicalAggregate().when(agg -> {
assertOneAggFuncType(agg, BitmapUnionCount.class);
return true;
}));
dropTable(TEST_TABLE_NAME, true);
}
/**
@ -868,27 +870,38 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement
}
/**
* TODO: enable this when bitmap is supported.
* count distinct to bitmap_union_count in mv
*/
@Disabled
@Test
public void testCountDistinctToBitmap() throws Exception {
// String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, "
// + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
// dorisAssert.withMaterializedView(createUserTagMVSql);
// String query = "select count(distinct tag_id) from " + USER_TAG_TABLE_NAME + ";";
// dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, "bitmap_union_count");
String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, "
+ "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
createMv(createUserTagMVSql);
String query = "select count(distinct tag_id) from " + USER_TAG_TABLE_NAME + ";";
PlanChecker.from(connectContext)
.analyze(query)
.rewrite()
.matches(logicalAggregate().when(agg -> {
assertOneAggFuncType(agg, BitmapUnionCount.class);
return true;
}));
testMv(query, USER_TAG_MV_NAME);
}
/**
* TODO: enable this when bitmap is supported.
*/
@Disabled
@Test
public void testIncorrectRewriteCountDistinct() throws Exception {
// String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, "
// + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
// dorisAssert.withMaterializedView(createUserTagMVSql);
// String query = "select user_name, count(distinct tag_id) from " + USER_TAG_TABLE_NAME + " group by user_name;";
// dorisAssert.query(query).explainContains(USER_TAG_TABLE_NAME, FunctionSet.COUNT);
String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, "
+ "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;";
createMv(createUserTagMVSql);
String query = "select user_name, count(distinct tag_id) from " + USER_TAG_TABLE_NAME + " group by user_name;";
PlanChecker.from(connectContext)
.analyze(query)
.rewrite()
.matches(logicalAggregate().when(agg -> {
assertOneAggFuncType(agg, Count.class);
return true;
}));
testMv(query, USER_TAG_TABLE_NAME);
}
/**
@ -982,10 +995,7 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement
// dorisAssert.query(query).explainWithout(USER_TAG_MV_NAME);
}
/**
* TODO: enable this when bitmap is supported.
*/
@Disabled
@Test
public void testCreateMVBaseBitmapAggTable() throws Exception {
String createTableSQL = "create table " + HR_DB_NAME + ".agg_table "
+ "(empid int, name varchar, salary bitmap " + FunctionSet.BITMAP_UNION + ") "
@ -1016,6 +1026,48 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement
// dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, mvColumnName);
}
@Test
public void selectBitmapMvWithProjectTest1() throws Exception {
createTable("create table t(\n"
+ " a int, \n"
+ " b int, \n"
+ " c int\n"
+ ")ENGINE=OLAP \n"
+ "DISTRIBUTED BY HASH(a) BUCKETS 3\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"disable_auto_compaction\" = \"false\"\n"
+ ");");
createMv("create materialized view mv as"
+ " select a, bitmap_union(to_bitmap(b)) from t group by a;");
testMv("select a, count(distinct v) as cnt from (select a, b as v from t) t group by a", "mv");
dropTable("t", true);
}
@Test
public void selectBitmapMvWithProjectTest2() throws Exception {
createTable("create table t(\n"
+ " a int, \n"
+ " b int, \n"
+ " c int\n"
+ ")ENGINE=OLAP \n"
+ "DISTRIBUTED BY HASH(a) BUCKETS 3\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"disable_auto_compaction\" = \"false\"\n"
+ ");");
createMv("create materialized view mv as"
+ " select a, bitmap_union(to_bitmap(b)) from t group by a;");
testMv("select a, bitmap_union_count(to_bitmap(v)) as cnt from (select a, b as v from t) t group by a", "mv");
dropTable("t", true);
}
private void testMv(String sql, Map<String, String> tableToIndex) {
PlanChecker.from(connectContext).checkPlannerResult(sql, planner -> {
List<ScanNode> scans = planner.getScanNodes();
@ -1032,4 +1084,34 @@ public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implement
private void testMv(String sql, String indexName) {
singleTableTest(sql, indexName, true);
}
private void assertOneAggFuncType(LogicalAggregate<? extends Plan> agg, Class<?> aggFuncType) {
Set<AggregateFunction> aggFuncs = agg.getOutputExpressions()
.stream()
.flatMap(e -> e.<Set<AggregateFunction>>collect(AggregateFunction.class::isInstance)
.stream())
.collect(Collectors.toSet());
Assertions.assertEquals(1, aggFuncs.size());
AggregateFunction aggFunc = aggFuncs.iterator().next();
Assertions.assertTrue(aggFuncType.isInstance(aggFunc));
}
private void testMvWithTwoTable(String sql, String firstTableIndexName, String secondTableIndexName) {
PlanChecker.from(connectContext).checkPlannerResult(sql, planner -> {
List<ScanNode> scans = planner.getScanNodes();
Assertions.assertEquals(2, scans.size());
ScanNode scanNode0 = scans.get(0);
Assertions.assertTrue(scanNode0 instanceof OlapScanNode);
OlapScanNode scan0 = (OlapScanNode) scanNode0;
Assertions.assertTrue(scan0.isPreAggregation());
Assertions.assertEquals(firstTableIndexName, scan0.getSelectedIndexName());
ScanNode scanNode1 = scans.get(1);
Assertions.assertTrue(scanNode1 instanceof OlapScanNode);
OlapScanNode scan1 = (OlapScanNode) scanNode1;
Assertions.assertTrue(scan1.isPreAggregation());
Assertions.assertEquals(secondTableIndexName, scan1.getSelectedIndexName());
});
}
}

3
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java
View File

@ -81,7 +81,8 @@ public class PlanToStringTest {
LogicalOlapScan plan = PlanConstructor.newLogicalOlapScan(0, "table", 0);
Assertions.assertTrue(
plan.toString().matches("LogicalOlapScan \\( qualified=db\\.table, "
+ "output=\\[id#\\d+, name#\\d+], candidateIndexIds=\\[], selectedIndexId=-1, preAgg=ON \\)"));
+ "output=\\[id#\\d+, name#\\d+], indexName=<index_not_selected>, "
+ "selectedIndexId=-1, preAgg=ON \\)"));
}
@Test