database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](statistics nereids)Nereids support select mv. (#30267)

Browse Source
This commit is contained in:
Jibing-Li
2024-01-24 19:27:15 +08:00
committed by yiguolei
parent 67051318a2
commit 86d7a8be44
9 changed files with 215 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
View File

@ -353,7 +353,7 @@ public class ExportJob implements Writable {
List<String> partitions, List<NamedExpression> selectLists) {
// UnboundRelation
LogicalPlan plan = new UnboundRelation(StatementScopeIdGenerator.newRelationId(), qualifiedTableName,
partitions, false, tabletIds, ImmutableList.of(), Optional.empty());
partitions, false, tabletIds, ImmutableList.of(), Optional.empty(), Optional.empty());
// LogicalCheckPolicy
plan = new LogicalCheckPolicy<>(plan);
// LogicalFilter

20
fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java
View File

@ -52,21 +52,22 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu
private final boolean isTempPart;
private final List<String> hints;
private final Optional<TableSample> tableSample;
private final Optional<String> indexName;
public UnboundRelation(RelationId id, List<String> nameParts) {
this(id, nameParts, Optional.empty(), Optional.empty(), ImmutableList.of(), false, ImmutableList.of(),
ImmutableList.of(), Optional.empty());
ImmutableList.of(), Optional.empty(), Optional.empty());
}
public UnboundRelation(RelationId id, List<String> nameParts, List<String> partNames, boolean isTempPart) {
this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, ImmutableList.of(),
ImmutableList.of(), Optional.empty());
ImmutableList.of(), Optional.empty(), Optional.empty());
}
public UnboundRelation(RelationId id, List<String> nameParts, List<String> partNames, boolean isTempPart,
List<Long> tabletIds, List<String> hints, Optional<TableSample> tableSample) {
List<Long> tabletIds, List<String> hints, Optional<TableSample> tableSample, Optional<String> indexName) {
this(id, nameParts, Optional.empty(), Optional.empty(),
partNames, isTempPart, tabletIds, hints, tableSample);
partNames, isTempPart, tabletIds, hints, tableSample, indexName);
}
/**
@ -74,7 +75,7 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu
*/
public UnboundRelation(RelationId id, List<String> nameParts, Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<String> partNames, boolean isTempPart,
List<Long> tabletIds, List<String> hints, Optional<TableSample> tableSample) {
List<Long> tabletIds, List<String> hints, Optional<TableSample> tableSample, Optional<String> indexName) {
super(id, PlanType.LOGICAL_UNBOUND_RELATION, groupExpression, logicalProperties);
this.nameParts = ImmutableList.copyOf(Objects.requireNonNull(nameParts, "nameParts should not null"));
this.partNames = ImmutableList.copyOf(Objects.requireNonNull(partNames, "partNames should not null"));
@ -82,6 +83,7 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu
this.isTempPart = isTempPart;
this.hints = ImmutableList.copyOf(Objects.requireNonNull(hints, "hints should not be null."));
this.tableSample = tableSample;
this.indexName = indexName;
}
public List<String> getNameParts() {
@ -102,14 +104,14 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new UnboundRelation(relationId, nameParts,
groupExpression, Optional.of(getLogicalProperties()),
partNames, isTempPart, tabletIds, hints, tableSample);
partNames, isTempPart, tabletIds, hints, tableSample, indexName);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new UnboundRelation(relationId, nameParts, groupExpression, logicalProperties, partNames,
isTempPart, tabletIds, hints, tableSample);
isTempPart, tabletIds, hints, tableSample, indexName);
}
@Override
@ -152,6 +154,10 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu
return tabletIds;
}
public Optional<String> getIndexName() {
return indexName;
}
public List<String> getHints() {
return hints;
}

7
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -1248,6 +1248,11 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
}
}
Optional<String> indexName = Optional.empty();
if (ctx.materializedViewName() != null) {
indexName = Optional.ofNullable(ctx.materializedViewName().indexName.getText());
}
List<Long> tabletIdLists = new ArrayList<>();
if (ctx.tabletList() != null) {
ctx.tabletList().tabletIdList.stream().forEach(tabletToken -> {
@ -1266,7 +1271,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
LogicalPlan checkedRelation = LogicalPlanBuilderAssistant.withCheckPolicy(
new UnboundRelation(StatementScopeIdGenerator.newRelationId(),
tableId, partitionNames, isTempPart, tabletIdLists, relationHints,
Optional.ofNullable(tableSample)));
Optional.ofNullable(tableSample), indexName));
LogicalPlan plan = withTableAlias(checkedRelation, ctx.tableAlias());
for (LateralViewContext lateralViewContext : ctx.lateralView()) {
plan = withGenerate(plan, lateralViewContext);

19
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
View File

@ -189,13 +189,26 @@ public class BindRelation extends OneAnalysisRuleFactory {
(OlapTable) table, ImmutableList.of(tableQualifier.get(1)), partIds,
tabletIds, unboundRelation.getHints(), unboundRelation.getTableSample());
} else {
scan = new LogicalOlapScan(unboundRelation.getRelationId(),
Optional<String> indexName = unboundRelation.getIndexName();
if (indexName.isPresent()) {
OlapTable olapTable = (OlapTable) table;
Long indexId = olapTable.getIndexIdByName(indexName.get());
if (indexId == null) {
throw new AnalysisException("Table " + olapTable.getName()
+ " doesn't have materialized view " + indexName.get());
}
scan = new LogicalOlapScan(unboundRelation.getRelationId(),
(OlapTable) table, ImmutableList.of(tableQualifier.get(1)), tabletIds, indexId,
unboundRelation.getHints(), unboundRelation.getTableSample());
} else {
scan = new LogicalOlapScan(unboundRelation.getRelationId(),
(OlapTable) table, ImmutableList.of(tableQualifier.get(1)), tabletIds, unboundRelation.getHints(),
unboundRelation.getTableSample());
}
}
if (!Util.showHiddenColumns() && scan.getTable().hasDeleteSign()
&& !ConnectContext.get().getSessionVariable()
.skipDeleteSign()) {
&& !ConnectContext.get().getSessionVariable().skipDeleteSign()
&& !scan.isDirectMvScan()) {
// table qualifier is catalog.db.table, we make db.table.column
Slot deleteSlot = null;
for (Slot slot : scan.getOutput()) {

6
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
View File

@ -515,10 +515,14 @@ public class AggregateStrategies implements ImplementationRuleFactory {
return canNotPush;
}
if (logicalScan instanceof LogicalOlapScan) {
KeysType keysType = ((LogicalOlapScan) logicalScan).getTable().getKeysType();
LogicalOlapScan logicalOlapScan = (LogicalOlapScan) logicalScan;
KeysType keysType = logicalOlapScan.getTable().getKeysType();
if (functionClasses.contains(Count.class) && keysType != KeysType.DUP_KEYS) {
return canNotPush;
}
if (functionClasses.contains(Count.class) && logicalOlapScan.isDirectMvScan()) {
return canNotPush;
}
}
if (aggregateFunctions.stream().anyMatch(fun -> fun.arity() > 1)) {
return canNotPush;

35
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
View File

@ -108,6 +108,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
private final Optional<TableSample> tableSample;
private final boolean directMvScan;
public LogicalOlapScan(RelationId id, OlapTable table) {
this(id, table, ImmutableList.of());
}
@ -117,7 +119,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
table.getPartitionIds(), false,
ImmutableList.of(),
-1, false, PreAggStatus.on(), ImmutableList.of(), ImmutableList.of(),
Maps.newHashMap(), Optional.empty());
Maps.newHashMap(), Optional.empty(), false);
}
public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> tabletIds,
@ -125,7 +127,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
this(id, table, qualifier, Optional.empty(), Optional.empty(),
table.getPartitionIds(), false, tabletIds,
-1, false, PreAggStatus.on(), ImmutableList.of(), hints, Maps.newHashMap(),
tableSample);
tableSample, false);
}
public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> specifiedPartitions,
@ -134,7 +136,15 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
// must use specifiedPartitions here for prune partition by sql like 'select * from t partition p1'
specifiedPartitions, false, tabletIds,
-1, false, PreAggStatus.on(), specifiedPartitions, hints, Maps.newHashMap(),
tableSample);
tableSample, false);
}
public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> tabletIds,
long selectedIndexId, List<String> hints, Optional<TableSample> tableSample) {
this(id, table, qualifier, Optional.empty(), Optional.empty(),
table.getPartitionIds(), false, tabletIds,
selectedIndexId, true, PreAggStatus.off("For direct index scan."),
ImmutableList.of(), hints, Maps.newHashMap(), tableSample, true);
}
/**
@ -146,7 +156,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
List<Long> selectedTabletIds, long selectedIndexId, boolean indexSelected,
PreAggStatus preAggStatus, List<Long> specifiedPartitions,
List<String> hints, Map<Pair<Long, String>, Slot> cacheSlotWithSlotName,
Optional<TableSample> tableSample) {
Optional<TableSample> tableSample, boolean directMvScan) {
super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier,
groupExpression, logicalProperties);
Preconditions.checkArgument(selectedPartitionIds != null,
@ -164,6 +174,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
this.cacheSlotWithSlotName = Objects.requireNonNull(cacheSlotWithSlotName,
"mvNameToSlot can not be null");
this.tableSample = tableSample;
this.directMvScan = directMvScan;
}
public List<Long> getSelectedPartitionIds() {
@ -220,7 +231,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
groupExpression, Optional.of(getLogicalProperties()),
selectedPartitionIds, partitionPruned, selectedTabletIds,
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, tableSample);
hints, cacheSlotWithSlotName, tableSample, directMvScan);
}
@Override
@ -229,7 +240,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
return new LogicalOlapScan(relationId, (Table) table, qualifier, groupExpression, logicalProperties,
selectedPartitionIds, partitionPruned, selectedTabletIds,
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, tableSample);
hints, cacheSlotWithSlotName, tableSample, directMvScan);
}
public LogicalOlapScan withSelectedPartitionIds(List<Long> selectedPartitionIds) {
@ -237,7 +248,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
Optional.empty(), Optional.of(getLogicalProperties()),
selectedPartitionIds, true, selectedTabletIds,
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, tableSample);
hints, cacheSlotWithSlotName, tableSample, directMvScan);
}
public LogicalOlapScan withMaterializedIndexSelected(PreAggStatus preAgg, long indexId) {
@ -245,7 +256,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
Optional.empty(), Optional.of(getLogicalProperties()),
selectedPartitionIds, partitionPruned, selectedTabletIds,
indexId, true, preAgg, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName,
tableSample);
tableSample, directMvScan);
}
public LogicalOlapScan withSelectedTabletIds(List<Long> selectedTabletIds) {
@ -253,7 +264,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
Optional.empty(), Optional.of(getLogicalProperties()),
selectedPartitionIds, partitionPruned, selectedTabletIds,
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, tableSample);
hints, cacheSlotWithSlotName, tableSample, directMvScan);
}
public LogicalOlapScan withPreAggStatus(PreAggStatus preAggStatus) {
@ -261,7 +272,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
Optional.empty(), Optional.of(getLogicalProperties()),
selectedPartitionIds, partitionPruned, selectedTabletIds,
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, tableSample);
hints, cacheSlotWithSlotName, tableSample, directMvScan);
}
@Override
@ -359,4 +370,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
public Optional<TableSample> getTableSample() {
return tableSample;
}
public boolean isDirectMvScan() {
return directMvScan;
}
}