From 940e26f341b49184216b569400b96f9dd9d0b606 Mon Sep 17 00:00:00 2001 From: Ashin Gau Date: Mon, 3 Jun 2024 20:24:11 +0800 Subject: [PATCH] [feat](nereids) support Iceberg time travel syntax (#35812) backport: #34681 Co-authored-by: Butao Zhang --- .../org/apache/doris/nereids/DorisParser.g4 | 7 +++- .../doris/datasource/FileQueryScanNode.java | 4 +++ .../iceberg/source/IcebergScanNode.java | 7 ++++ .../nereids/analyzer/UnboundRelation.java | 35 +++++++++++++------ .../translator/PhysicalPlanTranslator.java | 4 +++ .../nereids/parser/LogicalPlanBuilder.java | 23 ++++++++++-- .../nereids/rules/analysis/BindRelation.java | 6 ++-- .../LogicalFileScanToPhysicalFileScan.java | 3 +- .../trees/plans/logical/LogicalFileScan.java | 24 ++++++++----- .../plans/physical/PhysicalFileScan.java | 19 +++++++--- 10 files changed, 102 insertions(+), 30 deletions(-) diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index a7b4c4788c..f7938f9cac 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -530,7 +530,7 @@ optScanParams relationPrimary : multipartIdentifier optScanParams? materializedViewName? specifiedPartition? - tabletList? tableAlias sample? relationHint? lateralView* #tableName + tabletList? tableAlias sample? tableSnapshot? relationHint? lateralView* #tableName | LEFT_PAREN query RIGHT_PAREN tableAlias lateralView* #aliasedQuery | tvfName=identifier LEFT_PAREN (properties=propertyItemList)? @@ -971,6 +971,11 @@ sampleMethod | INTEGER_VALUE ROWS #sampleByRows ; +tableSnapshot + : FOR VERSION AS OF version=INTEGER_VALUE + | FOR TIME AS OF time=STRING_LITERAL + ; + // this rule is used for explicitly capturing wrong identifiers such as test-table, which should actually be `test-table` // replace identifier with errorCapturingIdentifier where the immediate follow symbol is not an expression, otherwise // valid expressions such as "a-b" can be recognized as an identifier diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index ff25464b4c..4399ebaf0b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TableSample; +import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; @@ -96,6 +97,9 @@ public abstract class FileQueryScanNode extends FileScanNode { protected String brokerName; + @Getter + protected TableSnapshot tableSnapshot; + /** * External file scan node for Query hms table * needCheckColumnPriv: Some of ExternalFileScanNode do not need to check column priv diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index ab8b889fdc..486e1242d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -277,6 +277,9 @@ public class IcebergScanNode extends FileQueryScanNode { public Long getSpecifiedSnapshot() throws UserException { TableSnapshot tableSnapshot = source.getDesc().getRef().getTableSnapshot(); + if (tableSnapshot == null) { + tableSnapshot = this.tableSnapshot; + } if (tableSnapshot != null) { TableSnapshot.VersionType type = tableSnapshot.getType(); try { @@ -455,4 +458,8 @@ public class IcebergScanNode extends FileQueryScanNode { return super.getNodeExplainString(prefix, detailLevel) + String.format("%sicebergPredicatePushdown=\n%s\n", prefix, sb); } + + public void setTableSnapshot(TableSnapshot tableSnapshot) { + this.tableSnapshot = tableSnapshot; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index 2dc81e0144..8180fd8518 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.analyzer; import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.common.Pair; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.memo.GroupExpression; @@ -59,41 +60,47 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu // the start and end position of the sql substring(e.g. "t1", "db1.t1", "ctl1.db1.t1") private final Optional> indexInSqlString; + private final Optional tableSnapshot; + public UnboundRelation(RelationId id, List nameParts) { this(id, nameParts, Optional.empty(), Optional.empty(), ImmutableList.of(), false, ImmutableList.of(), - ImmutableList.of(), Optional.empty(), Optional.empty(), null, Optional.empty()); + ImmutableList.of(), Optional.empty(), Optional.empty(), null, Optional.empty(), Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart) { this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, ImmutableList.of(), - ImmutableList.of(), Optional.empty(), Optional.empty(), null, Optional.empty()); + ImmutableList.of(), Optional.empty(), Optional.empty(), null, Optional.empty(), Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, null, Optional.empty()); + partNames, isTempPart, tabletIds, hints, tableSample, indexName, null, Optional.empty(), + Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName, - TableScanParams scanParams) { + TableScanParams scanParams, Optional tableSnapshot) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, Optional.empty()); + partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, Optional.empty(), + tableSnapshot); } public UnboundRelation(RelationId id, List nameParts, Optional groupExpression, Optional logicalProperties, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName) { this(id, nameParts, groupExpression, logicalProperties, partNames, - isTempPart, tabletIds, hints, tableSample, indexName, null, Optional.empty()); + isTempPart, tabletIds, hints, tableSample, indexName, null, Optional.empty(), Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName, - TableScanParams scanParams, Optional> indexInSqlString) { + TableScanParams scanParams, Optional> indexInSqlString, + Optional tableSnapshot) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, indexInSqlString); + partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, indexInSqlString, + tableSnapshot); } /** @@ -102,7 +109,8 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu public UnboundRelation(RelationId id, List nameParts, Optional groupExpression, Optional logicalProperties, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName, - TableScanParams scanParams, Optional> indexInSqlString) { + TableScanParams scanParams, Optional> indexInSqlString, + Optional tableSnapshot) { super(id, PlanType.LOGICAL_UNBOUND_RELATION, groupExpression, logicalProperties); this.nameParts = ImmutableList.copyOf(Objects.requireNonNull(nameParts, "nameParts should not null")); this.partNames = ImmutableList.copyOf(Objects.requireNonNull(partNames, "partNames should not null")); @@ -113,6 +121,7 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu this.indexName = indexName; this.scanParams = scanParams; this.indexInSqlString = indexInSqlString; + this.tableSnapshot = tableSnapshot; } public List getNameParts() { @@ -133,14 +142,14 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu public Plan withGroupExpression(Optional groupExpression) { return new UnboundRelation(relationId, nameParts, groupExpression, Optional.of(getLogicalProperties()), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, null, indexInSqlString); + partNames, isTempPart, tabletIds, hints, tableSample, indexName, null, indexInSqlString, tableSnapshot); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new UnboundRelation(relationId, nameParts, groupExpression, logicalProperties, partNames, - isTempPart, tabletIds, hints, tableSample, indexName, null, indexInSqlString); + isTempPart, tabletIds, hints, tableSample, indexName, null, indexInSqlString, tableSnapshot); } @Override @@ -207,4 +216,8 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu public Optional> getIndexInSqlString() { return indexInSqlString; } + + public Optional getTableSnapshot() { + return tableSnapshot; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index d3bf9e4d73..e58e6c697c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -571,6 +571,10 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor { scanParams = new TableScanParams(ctx.optScanParams().funcName.getText(), map); } + TableSnapshot tableSnapshot = null; + if (ctx.tableSnapshot() != null) { + if (ctx.tableSnapshot().TIME() != null) { + tableSnapshot = new TableSnapshot(stripQuotes(ctx.tableSnapshot().time.getText())); + } else { + tableSnapshot = new TableSnapshot(Long.parseLong(ctx.tableSnapshot().version.getText())); + } + } + MultipartIdentifierContext identifier = ctx.multipartIdentifier(); TableSample tableSample = ctx.sample() == null ? null : (TableSample) visit(ctx.sample()); UnboundRelation relation = forCreateView ? new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableId, partitionNames, isTempPart, tabletIdLists, relationHints, Optional.ofNullable(tableSample), indexName, scanParams, - Optional.of(Pair.of(identifier.start.getStartIndex(), identifier.stop.getStopIndex()))) : + Optional.of(Pair.of(identifier.start.getStartIndex(), identifier.stop.getStopIndex())), + Optional.ofNullable(tableSnapshot)) : new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableId, partitionNames, isTempPart, tabletIdLists, relationHints, - Optional.ofNullable(tableSample), indexName, scanParams); + Optional.ofNullable(tableSample), indexName, scanParams, Optional.ofNullable(tableSnapshot)); LogicalPlan checkedRelation = LogicalPlanBuilderAssistant.withCheckPolicy(relation); LogicalPlan plan = withTableAlias(checkedRelation, ctx.tableAlias()); for (LateralViewContext lateralViewContext : ctx.lateralView()) { @@ -1387,6 +1398,14 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { return plan; } + public static String stripQuotes(String str) { + if ((str.charAt(0) == '\'' && str.charAt(str.length() - 1) == '\'') + || (str.charAt(0) == '\"' && str.charAt(str.length() - 1) == '\"')) { + str = str.substring(1, str.length() - 1); + } + return str; + } + @Override public LogicalPlan visitAliasedQuery(AliasedQueryContext ctx) { if (ctx.tableAlias().getText().equals("")) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 8ae260f6f0..b243cb5f26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -269,12 +269,14 @@ public class BindRelation extends OneAnalysisRuleFactory { } hmsTable.setScanParams(unboundRelation.getScanParams()); return new LogicalFileScan(unboundRelation.getRelationId(), (HMSExternalTable) table, - qualifierWithoutTableName, unboundRelation.getTableSample()); + qualifierWithoutTableName, unboundRelation.getTableSample(), + unboundRelation.getTableSnapshot()); case ICEBERG_EXTERNAL_TABLE: case PAIMON_EXTERNAL_TABLE: case MAX_COMPUTE_EXTERNAL_TABLE: return new LogicalFileScan(unboundRelation.getRelationId(), (ExternalTable) table, - qualifierWithoutTableName, unboundRelation.getTableSample()); + qualifierWithoutTableName, unboundRelation.getTableSample(), + unboundRelation.getTableSnapshot()); case SCHEMA: return new LogicalSchemaScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); case JDBC_EXTERNAL_TABLE: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java index d86e1d1667..8edb683151 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java @@ -40,7 +40,8 @@ public class LogicalFileScanToPhysicalFileScan extends OneImplementationRuleFact fileScan.getLogicalProperties(), fileScan.getConjuncts(), fileScan.getSelectedPartitions(), - fileScan.getTableSample()) + fileScan.getTableSample(), + fileScan.getTableSnapshot()) ).toRule(RuleType.LOGICAL_FILE_SCAN_TO_PHYSICAL_FILE_SCAN_RULE); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java index f7bdae5c2f..06d349fe2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.logical; +import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.nereids.memo.GroupExpression; @@ -46,22 +47,25 @@ public class LogicalFileScan extends LogicalExternalRelation { private final SelectedPartitions selectedPartitions; private final Optional tableSample; + private final Optional tableSnapshot; /** * Constructor for LogicalFileScan. */ public LogicalFileScan(RelationId id, ExternalTable table, List qualifier, Optional groupExpression, Optional logicalProperties, - Set conjuncts, SelectedPartitions selectedPartitions, Optional tableSample) { + Set conjuncts, SelectedPartitions selectedPartitions, Optional tableSample, + Optional tableSnapshot) { super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties); this.selectedPartitions = selectedPartitions; this.tableSample = tableSample; + this.tableSnapshot = tableSnapshot; } public LogicalFileScan(RelationId id, ExternalTable table, List qualifier, - Optional tableSample) { + Optional tableSample, Optional tableSnapshot) { this(id, table, qualifier, Optional.empty(), Optional.empty(), - Sets.newHashSet(), SelectedPartitions.NOT_PRUNED, tableSample); + Sets.newHashSet(), SelectedPartitions.NOT_PRUNED, tableSample, tableSnapshot); } public SelectedPartitions getSelectedPartitions() { @@ -72,6 +76,10 @@ public class LogicalFileScan extends LogicalExternalRelation { return tableSample; } + public Optional getTableSnapshot() { + return tableSnapshot; + } + @Override public ExternalTable getTable() { Preconditions.checkArgument(table instanceof ExternalTable, @@ -90,31 +98,31 @@ public class LogicalFileScan extends LogicalExternalRelation { @Override public LogicalFileScan withGroupExpression(Optional groupExpression) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, groupExpression, - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample); + Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, - groupExpression, logicalProperties, conjuncts, selectedPartitions, tableSample); + groupExpression, logicalProperties, conjuncts, selectedPartitions, tableSample, tableSnapshot); } @Override public LogicalFileScan withConjuncts(Set conjuncts) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample); + Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot); } public LogicalFileScan withSelectedPartitions(SelectedPartitions selectedPartitions) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample); + Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot); } @Override public LogicalFileScan withRelationId(RelationId relationId) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.empty(), conjuncts, selectedPartitions, tableSample); + Optional.empty(), conjuncts, selectedPartitions, tableSample, tableSnapshot); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java index 843a03f69f..8706db65f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.physical; +import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DistributionSpec; @@ -45,6 +46,7 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { private final Set conjuncts; private final SelectedPartitions selectedPartitions; private final Optional tableSample; + private final Optional tableSnapshot; /** * Constructor for PhysicalFileScan. @@ -52,12 +54,14 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { public PhysicalFileScan(RelationId id, ExternalTable table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, LogicalProperties logicalProperties, Set conjuncts, - SelectedPartitions selectedPartitions, Optional tableSample) { + SelectedPartitions selectedPartitions, Optional tableSample, + Optional tableSnapshot) { super(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, groupExpression, logicalProperties); this.distributionSpec = distributionSpec; this.conjuncts = conjuncts; this.selectedPartitions = selectedPartitions; this.tableSample = tableSample; + this.tableSnapshot = tableSnapshot; } /** @@ -67,13 +71,14 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { DistributionSpec distributionSpec, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics, Set conjuncts, SelectedPartitions selectedPartitions, - Optional tableSample) { + Optional tableSample, Optional tableSnapshot) { super(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, groupExpression, logicalProperties, physicalProperties, statistics); this.distributionSpec = distributionSpec; this.conjuncts = conjuncts; this.selectedPartitions = selectedPartitions; this.tableSample = tableSample; + this.tableSnapshot = tableSnapshot; } public DistributionSpec getDistributionSpec() { @@ -92,6 +97,10 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { return tableSample; } + public Optional getTableSnapshot() { + return tableSnapshot; + } + @Override public String toString() { return Utils.toSqlString("PhysicalFileScan", @@ -112,14 +121,14 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { @Override public PhysicalFileScan withGroupExpression(Optional groupExpression) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, getLogicalProperties(), conjuncts, selectedPartitions, tableSample); + groupExpression, getLogicalProperties(), conjuncts, selectedPartitions, tableSample, tableSnapshot); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, logicalProperties.get(), conjuncts, selectedPartitions, tableSample); + groupExpression, logicalProperties.get(), conjuncts, selectedPartitions, tableSample, tableSnapshot); } @Override @@ -132,6 +141,6 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { Statistics statistics) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, groupExpression, getLogicalProperties(), physicalProperties, statistics, conjuncts, - selectedPartitions, tableSample); + selectedPartitions, tableSample, tableSnapshot); } }