database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Feature](Nereids) Add hint to enable pre-aggregation when scan OLAP table. (#15614)

Browse Source 
This pr added support for the pre-aggregation hint. Users could use /*+PREAGGOPEN*/ to enable pre-preaggregation for OLAP table.
For example:
Let's say we have an aggregate-keys table t (k1 int, k2 int, v1 int sum, v2 int sum). Pre-aggregation could be enabled by query with a hint: select k1, v1 from t /*+PREAGGOPEN*/.
This commit is contained in:
Shuo Wang
2023-02-07 11:59:10 +08:00
committed by GitHub
parent 27216dc7e0
commit bed1ab7c19
9 changed files with 183 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
View File

@ -127,8 +127,13 @@ joinRelation
// Just like `opt_plan_hints` in legacy CUP parser.
joinHint
: LEFT_BRACKET identifier RIGHT_BRACKET #bracketStyleHint
| HINT_START identifier HINT_END #commentStyleHint
: LEFT_BRACKET identifier RIGHT_BRACKET #bracketJoinHint
| HINT_START identifier HINT_END #commentJoinHint
;
relationHint
: LEFT_BRACKET identifier (COMMA identifier)* RIGHT_BRACKET #bracketRelationHint
| HINT_START identifier (COMMA identifier)* HINT_END #commentRelationHint
;
aggClause
@ -208,11 +213,11 @@ identifierSeq
;
relationPrimary
: multipartIdentifier specifiedPartition? tableAlias lateralView* #tableName
| LEFT_PAREN query RIGHT_PAREN tableAlias lateralView* #aliasedQuery
: multipartIdentifier specifiedPartition? tableAlias relationHint? lateralView* #tableName
| LEFT_PAREN query RIGHT_PAREN tableAlias lateralView* #aliasedQuery
| tvfName=identifier LEFT_PAREN
(properties+=tvfProperty (COMMA properties+=tvfProperty)*)?
RIGHT_PAREN tableAlias #tableValuedFunction
RIGHT_PAREN tableAlias #tableValuedFunction
;
tvfProperty

34
fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java
View File

@ -32,6 +32,8 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.List;
@ -46,21 +48,29 @@ public class UnboundRelation extends LogicalRelation implements Unbound {
private final List<String> nameParts;
private final List<String> partNames;
private final boolean isTempPart;
private final List<String> hints;
public UnboundRelation(RelationId id, List<String> nameParts) {
this(id, nameParts, Optional.empty(), Optional.empty(), ImmutableList.of(), false);
this(id, nameParts, Optional.empty(), Optional.empty(), ImmutableList.of(), false, ImmutableList.of());
}
public UnboundRelation(RelationId id, List<String> nameParts, List<String> partNames, boolean isTempPart) {
this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart);
this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, ImmutableList.of());
}
public UnboundRelation(RelationId id, List<String> nameParts, List<String> partNames, boolean isTempPart,
List<String> hints) {
this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, hints);
}
public UnboundRelation(RelationId id, List<String> nameParts, Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<String> partNames, boolean isTempPart) {
Optional<LogicalProperties> logicalProperties, List<String> partNames, boolean isTempPart,
List<String> hints) {
super(id, PlanType.LOGICAL_UNBOUND_RELATION, groupExpression, logicalProperties);
this.nameParts = ImmutableList.copyOf(Objects.requireNonNull(nameParts, "nameParts should not null"));
this.partNames = ImmutableList.copyOf(Objects.requireNonNull(partNames, "partNames should not null"));
this.isTempPart = isTempPart;
this.hints = ImmutableList.copyOf(Objects.requireNonNull(hints, "hints should not be null."));
}
@Override
@ -84,13 +94,14 @@ public class UnboundRelation extends LogicalRelation implements Unbound {
@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new UnboundRelation(id, nameParts, groupExpression, Optional.of(getLogicalProperties()),
partNames, isTempPart);
return new UnboundRelation(id, nameParts, groupExpression, Optional.of(getLogicalProperties()), partNames,
isTempPart, hints);
}
@Override
public Plan withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
return new UnboundRelation(id, nameParts, Optional.empty(), logicalProperties, partNames, isTempPart);
return new UnboundRelation(id, nameParts, Optional.empty(), logicalProperties, partNames,
isTempPart, hints);
}
@Override
@ -100,10 +111,15 @@ public class UnboundRelation extends LogicalRelation implements Unbound {
@Override
public String toString() {
return Utils.toSqlString("UnboundRelation",
List<Object> args = Lists.newArrayList(
"id", id,
"nameParts", StringUtils.join(nameParts, ".")
);
if (CollectionUtils.isNotEmpty(hints)) {
args.add("hints");
args.add(StringUtils.join(hints, ", "));
}
return Utils.toSqlString("UnboundRelation", args.toArray());
}
@Override
@ -147,4 +163,8 @@ public class UnboundRelation extends LogicalRelation implements Unbound {
public boolean isTempPart() {
return isTempPart;
}
public List<String> getHints() {
return hints;
}
}

34
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -29,9 +29,11 @@ import org.apache.doris.nereids.DorisParser.ArithmeticBinaryContext;
import org.apache.doris.nereids.DorisParser.ArithmeticUnaryContext;
import org.apache.doris.nereids.DorisParser.BitOperationContext;
import org.apache.doris.nereids.DorisParser.BooleanLiteralContext;
import org.apache.doris.nereids.DorisParser.BracketStyleHintContext;
import org.apache.doris.nereids.DorisParser.BracketJoinHintContext;
import org.apache.doris.nereids.DorisParser.BracketRelationHintContext;
import org.apache.doris.nereids.DorisParser.ColumnReferenceContext;
import org.apache.doris.nereids.DorisParser.CommentStyleHintContext;
import org.apache.doris.nereids.DorisParser.CommentJoinHintContext;
import org.apache.doris.nereids.DorisParser.CommentRelationHintContext;
import org.apache.doris.nereids.DorisParser.ComparisonContext;
import org.apache.doris.nereids.DorisParser.CreateRowPolicyContext;
import org.apache.doris.nereids.DorisParser.CteContext;
@ -460,8 +462,16 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
partitionNames.addAll(visitIdentifierList(ctx.specifiedPartition().identifierList()));
}
}
final List<String> relationHints;
if (ctx.relationHint() != null) {
relationHints = typedVisit(ctx.relationHint());
} else {
relationHints = ImmutableList.of();
}
LogicalPlan checkedRelation = withCheckPolicy(
new UnboundRelation(RelationUtil.newRelationId(), tableId, partitionNames, isTempPart));
new UnboundRelation(RelationUtil.newRelationId(), tableId, partitionNames, isTempPart, relationHints));
LogicalPlan plan = withTableAlias(checkedRelation, ctx.tableAlias());
for (LateralViewContext lateralViewContext : ctx.lateralView()) {
plan = withGenerate(plan, lateralViewContext);
@ -1412,15 +1422,29 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
}
@Override
public String visitBracketStyleHint(BracketStyleHintContext ctx) {
public String visitBracketJoinHint(BracketJoinHintContext ctx) {
return ctx.identifier().getText();
}
@Override
public Object visitCommentStyleHint(CommentStyleHintContext ctx) {
public String visitCommentJoinHint(CommentJoinHintContext ctx) {
return ctx.identifier().getText();
}
@Override
public List<String> visitBracketRelationHint(BracketRelationHintContext ctx) {
return ctx.identifier().stream()
.map(RuleContext::getText)
.collect(ImmutableList.toImmutableList());
}
@Override
public Object visitCommentRelationHint(CommentRelationHintContext ctx) {
return ctx.identifier().stream()
.map(RuleContext::getText)
.collect(ImmutableList.toImmutableList());
}
private LogicalPlan withProjection(LogicalPlan input, SelectColumnClauseContext selectCtx,
Optional<AggClauseContext> aggCtx, boolean isDistinct) {
return ParserUtils.withOrigin(selectCtx, () -> {

4
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
View File

@ -161,10 +161,10 @@ public class BindRelation extends OneAnalysisRuleFactory {
List<Long> partIds = getPartitionIds(table, unboundRelation);
if (!CollectionUtils.isEmpty(partIds)) {
scan = new LogicalOlapScan(RelationUtil.newRelationId(),
(OlapTable) table, ImmutableList.of(tableQualifier.get(1)), partIds);
(OlapTable) table, ImmutableList.of(tableQualifier.get(1)), partIds, unboundRelation.getHints());
} else {
scan = new LogicalOlapScan(RelationUtil.newRelationId(),
(OlapTable) table, ImmutableList.of(tableQualifier.get(1)));
(OlapTable) table, ImmutableList.of(tableQualifier.get(1)), unboundRelation.getHints());
}
if (!Util.showHiddenColumns() && scan.getTable().hasDeleteSign()
&& !ConnectContext.get().getSessionVariable()

4
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/AbstractSelectMaterializedIndexRule.java
View File

@ -282,4 +282,8 @@ public abstract class AbstractSelectMaterializedIndexRule {
}
return matchCount;
}
protected boolean preAggEnabledByHint(LogicalOlapScan olapScan) {
return olapScan.getHints().stream().anyMatch("PREAGGOPEN"::equalsIgnoreCase);
}
}

16
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java
View File

@ -493,11 +493,17 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
switch (table.getKeysType()) {
case AGG_KEYS:
case UNIQUE_KEYS: {
// Only checking pre-aggregation status by base index is enough for aggregate-keys and
// unique-keys OLAP table.
// Because the schemas in non-base materialized index are subsets of the schema of base index.
PreAggStatus preAggStatus = checkPreAggStatus(scan, table.getBaseIndexId(), predicates,
aggregateFunctions, groupingExprs);
final PreAggStatus preAggStatus;
if (preAggEnabledByHint(scan)) {
// PreAggStatus could be enabled by pre-aggregation hint for agg-keys and unique-keys.
preAggStatus = PreAggStatus.on();
} else {
// Only checking pre-aggregation status by base index is enough for aggregate-keys and
// unique-keys OLAP table.
// Because the schemas in non-base materialized index are subsets of the schema of base index.
preAggStatus = checkPreAggStatus(scan, table.getBaseIndexId(), predicates,
aggregateFunctions, groupingExprs);
}
if (preAggStatus.isOff()) {
// return early if pre agg status if off.
return new SelectResult(preAggStatus, scan.getTable().getBaseIndexId(), new ExprRewriteMap());

12
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java
View File

@ -128,12 +128,18 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater
.filter(index -> containAllRequiredColumns(index, scan, requiredScanOutputSupplier.get()))
.collect(Collectors.toList());
PreAggStatus preAgg = PreAggStatus.off("No aggregate on scan.");
final PreAggStatus preAggStatus;
if (preAggEnabledByHint(scan)) {
// PreAggStatus could be enabled by pre-aggregation hint for agg-keys and unique-keys.
preAggStatus = PreAggStatus.on();
} else {
preAggStatus = PreAggStatus.off("No aggregate on scan.");
}
if (candidates.size() == 1) {
// `candidates` only have base index.
return scan.withMaterializedIndexSelected(preAgg, baseIndexId);
return scan.withMaterializedIndexSelected(preAggStatus, baseIndexId);
} else {
return scan.withMaterializedIndexSelected(preAgg,
return scan.withMaterializedIndexSelected(preAggStatus,
selectBestIndex(candidates, scan, predicatesSupplier.get()));
}
case DUP_KEYS:

44
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
View File

@ -99,6 +99,11 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation,
private final List<Long> selectedPartitionIds;
///////////////////////////////////////////////////////////////////////////
// Members for hints.
///////////////////////////////////////////////////////////////////////////
private final List<String> hints;
public LogicalOlapScan(RelationId id, OlapTable table) {
this(id, table, ImmutableList.of());
}
@ -107,19 +112,27 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation,
this(id, table, qualifier, Optional.empty(), Optional.empty(),
table.getPartitionIds(), false,
ImmutableList.of(), false,
-1, false, PreAggStatus.on(), ImmutableList.of());
-1, false, PreAggStatus.on(), ImmutableList.of(), ImmutableList.of());
}
public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> specifiedPartitions) {
public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<String> hints) {
this(id, table, qualifier, Optional.empty(), Optional.empty(),
table.getPartitionIds(), false,
ImmutableList.of(), false,
-1, false, PreAggStatus.on(), ImmutableList.of(), hints);
}
public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> specifiedPartitions,
List<String> hints) {
this(id, table, qualifier, Optional.empty(), Optional.empty(),
specifiedPartitions, false, ImmutableList.of(), false,
-1, false, PreAggStatus.on(), specifiedPartitions);
-1, false, PreAggStatus.on(), specifiedPartitions, hints);
}
public LogicalOlapScan(RelationId id, Table table, List<String> qualifier) {
this(id, table, qualifier, Optional.empty(), Optional.empty(),
((OlapTable) table).getPartitionIds(), false, ImmutableList.of(), false,
-1, false, PreAggStatus.on(), ImmutableList.of());
-1, false, PreAggStatus.on(), ImmutableList.of(), ImmutableList.of());
}
/**
@ -129,7 +142,8 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
List<Long> selectedPartitionIds, boolean partitionPruned,
List<Long> selectedTabletIds, boolean tabletPruned,
long selectedIndexId, boolean indexSelected, PreAggStatus preAggStatus, List<Long> partitions) {
long selectedIndexId, boolean indexSelected, PreAggStatus preAggStatus, List<Long> partitions,
List<String> hints) {
super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier,
groupExpression, logicalProperties);
@ -142,6 +156,7 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation,
this.manuallySpecifiedPartitions = ImmutableList.copyOf(partitions);
this.selectedPartitionIds = ImmutableList.copyOf(
Objects.requireNonNull(selectedPartitionIds, "selectedPartitionIds can not be null"));
this.hints = Objects.requireNonNull(hints, "hints can not be null");
}
public List<Long> getSelectedPartitionIds() {
@ -186,7 +201,8 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation,
&& Objects.equals(selectedIndexId, ((LogicalOlapScan) o).selectedIndexId)
&& Objects.equals(indexSelected, ((LogicalOlapScan) o).indexSelected)
&& Objects.equals(selectedTabletIds, ((LogicalOlapScan) o).selectedTabletIds)
&& Objects.equals(tabletPruned, ((LogicalOlapScan) o).tabletPruned);
&& Objects.equals(tabletPruned, ((LogicalOlapScan) o).tabletPruned)
&& Objects.equals(hints, ((LogicalOlapScan) o).hints);
}
@Override
@ -194,39 +210,40 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation,
return Objects.hash(id,
selectedPartitionIds, partitionPruned,
selectedIndexId, indexSelected,
selectedTabletIds, tabletPruned);
selectedTabletIds, tabletPruned,
hints);
}
@Override
public LogicalOlapScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalOlapScan(id, (Table) table, qualifier, groupExpression, Optional.of(getLogicalProperties()),
selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned,
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions);
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints);
}
@Override
public LogicalOlapScan withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), logicalProperties,
selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned,
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions);
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints);
}
public LogicalOlapScan withSelectedPartitionIds(List<Long> selectedPartitionIds) {
return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()),
selectedPartitionIds, true, selectedTabletIds, tabletPruned,
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions);
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints);
}
public LogicalOlapScan withMaterializedIndexSelected(PreAggStatus preAgg, long indexId) {
return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()),
selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned,
indexId, true, preAgg, manuallySpecifiedPartitions);
indexId, true, preAgg, manuallySpecifiedPartitions, hints);
}
public LogicalOlapScan withSelectedTabletIds(List<Long> selectedTabletIds) {
return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()),
selectedPartitionIds, partitionPruned, selectedTabletIds, true,
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions);
selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints);
}
@Override
@ -301,4 +318,7 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation,
return manuallySpecifiedPartitions;
}
public List<String> getHints() {
return hints;
}
}

59
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java
View File

@ -379,4 +379,63 @@ class SelectRollupIndexTest extends BaseMaterializedIndexSelectTest implements P
public void testOnlyValueColumn2() throws Exception {
singleTableTest("select v1 from t", "t", false);
}
@Test
public void testPreAggHint() throws Exception {
createTable(" CREATE TABLE `test_preagg_hint` (\n"
+ " `k1` int(11) NULL,\n"
+ " `k2` int(11) NULL,\n"
+ " `v1` int(11) SUM NULL,\n"
+ " `v2` int(11) SUM NULL\n"
+ ") ENGINE=OLAP\n"
+ "AGGREGATE KEY(`k1`, `k2`)\n"
+ "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 3\n"
+ "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n"
+ "\"disable_auto_compaction\" = \"false\"\n"
+ ");");
addRollup("alter table test_preagg_hint add rollup r1(k1, k2, v1)");
// no pre-agg hint
String queryWithoutHint = "select k1, v1 from test_preagg_hint";
// legacy planner
Assertions.assertTrue(getSQLPlanOrErrorMsg(queryWithoutHint).contains(
"TABLE: default_cluster:test.test_preagg_hint(r1), PREAGGREGATION: OFF. Reason: No AggregateInfo"));
// nereids planner
PlanChecker.from(connectContext)
.analyze(queryWithoutHint)
.rewrite()
.matches(logicalOlapScan().when(scan -> {
Assertions.assertTrue(scan.getHints().isEmpty());
Assertions.assertEquals("r1", scan.getSelectedMaterializedIndexName().get());
PreAggStatus preAggStatus = scan.getPreAggStatus();
Assertions.assertTrue(preAggStatus.isOff());
Assertions.assertEquals("No aggregate on scan.", preAggStatus.getOffReason());
return true;
}));
// has pre-agg hint
String queryWithHint = "select k1, v1 from test_preagg_hint /*+ PREAGGOPEN*/";
// legacy planner
Assertions.assertTrue(getSQLPlanOrErrorMsg(queryWithHint).contains(
"TABLE: default_cluster:test.test_preagg_hint(r1), PREAGGREGATION: ON"));
// nereids planner
PlanChecker.from(connectContext)
.analyze(queryWithHint)
.rewrite()
.matches(logicalOlapScan().when(scan -> {
Assertions.assertEquals(1, scan.getHints().size());
Assertions.assertEquals("PREAGGOPEN", scan.getHints().get(0));
Assertions.assertEquals("r1", scan.getSelectedMaterializedIndexName().get());
PreAggStatus preAggStatus = scan.getPreAggStatus();
Assertions.assertTrue(preAggStatus.isOn());
return true;
}));
dropTable("test_preagg_hint", true);
}
}