From 79eb575d7cc6292f96d9d68c4668ca77f0e7c8fb Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Wed, 3 Jan 2024 12:51:07 +0800 Subject: [PATCH] [Improvement](nereids)Support ODBC table for new planner. (#29129) --- .../doris/nereids/cost/CostModelV1.java | 7 ++ .../doris/nereids/cost/CostModelV2.java | 6 + .../translator/PhysicalPlanTranslator.java | 26 +++++ .../doris/nereids/jobs/executor/Rewriter.java | 2 + .../ChildOutputPropertyDeriver.java | 6 + .../apache/doris/nereids/rules/RuleSet.java | 2 + .../apache/doris/nereids/rules/RuleType.java | 2 + .../nereids/rules/analysis/BindRelation.java | 3 + .../LogicalOdbcScanToPhysicalOdbcScan.java | 42 +++++++ .../rewrite/PushConjunctsIntoOdbcScan.java | 39 +++++++ .../doris/nereids/stats/StatsCalculator.java | 13 +++ .../trees/copier/LogicalPlanDeepCopier.java | 13 +++ .../doris/nereids/trees/plans/PlanType.java | 2 + .../trees/plans/logical/LogicalOdbcScan.java | 104 ++++++++++++++++++ .../plans/physical/PhysicalOdbcScan.java | 102 +++++++++++++++++ .../trees/plans/visitor/RelationVisitor.java | 10 ++ .../planner/external/odbc/OdbcScanNode.java | 44 +++++++- 17 files changed, 417 insertions(+), 6 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java index b2fe0f9457..011afb46fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java @@ -35,6 +35,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; @@ -137,6 +138,12 @@ class CostModelV1 extends PlanVisitor { return CostV1.ofCpu(context.getSessionVariable(), statistics.getRowCount()); } + @Override + public Cost visitPhysicalOdbcScan(PhysicalOdbcScan physicalOdbcScan, PlanContext context) { + Statistics statistics = context.getStatisticsWithCheck(); + return CostV1.ofCpu(context.getSessionVariable(), statistics.getRowCount()); + } + @Override public Cost visitPhysicalEsScan(PhysicalEsScan physicalEsScan, PlanContext context) { Statistics statistics = context.getStatisticsWithCheck(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java index 7f1c6a6c5a..96afa5123c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit; import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; @@ -142,6 +143,11 @@ class CostModelV2 extends PlanVisitor { return calculateScanWithoutRF(context.getStatisticsWithCheck()); } + @Override + public Cost visitPhysicalOdbcScan(PhysicalOdbcScan physicalOdbcScan, PlanContext context) { + return calculateScanWithoutRF(context.getStatisticsWithCheck()); + } + @Override public Cost visitPhysicalEsScan(PhysicalEsScan physicalEsScan, PlanContext context) { return calculateScanWithoutRF(context.getStatisticsWithCheck()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index f8b2393531..da0a58f4f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -41,6 +41,7 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Function.NullableMode; +import org.apache.doris.catalog.OdbcTable; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf; @@ -112,6 +113,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect; import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit; import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; @@ -176,6 +178,7 @@ import org.apache.doris.planner.external.MaxComputeScanNode; import org.apache.doris.planner.external.hudi.HudiScanNode; import org.apache.doris.planner.external.iceberg.IcebergScanNode; import org.apache.doris.planner.external.jdbc.JdbcScanNode; +import org.apache.doris.planner.external.odbc.OdbcScanNode; import org.apache.doris.planner.external.paimon.PaimonScanNode; import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.StatisticConstants; @@ -577,6 +580,29 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor slots = odbcScan.getOutput(); + TableIf table = odbcScan.getTable(); + TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table, context); + OdbcScanNode odbcScanNode = new OdbcScanNode(odbcScan.translatePlanNodeId(), tupleDescriptor, + (OdbcTable) table); + odbcScanNode.addConjuncts(translateToLegacyConjuncts(odbcScan.getConjuncts())); + Utils.execWithUncheckedException(odbcScanNode::init); + context.addScanNode(odbcScanNode); + context.getRuntimeTranslator().ifPresent( + runtimeFilterGenerator -> runtimeFilterGenerator.getContext().getTargetListByScan(odbcScan).forEach( + expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, odbcScanNode, context) + ) + ); + Utils.execWithUncheckedException(odbcScanNode::finalizeForNereids); + DataPartition dataPartition = DataPartition.RANDOM; + PlanFragment planFragment = new PlanFragment(context.nextFragmentId(), odbcScanNode, dataPartition); + context.addPlanFragment(planFragment); + updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), odbcScan); + return planFragment; + } + @Override public PlanFragment visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanTranslatorContext context) { List slots = olapScan.getOutput(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index a97bd61e5c..e02c299e71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -95,6 +95,7 @@ import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderLimit; import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderTopN; import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoEsScan; import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoJdbcScan; +import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoOdbcScan; import org.apache.doris.nereids.rules.rewrite.PushDownCountThroughJoin; import org.apache.doris.nereids.rules.rewrite.PushDownCountThroughJoinOneSide; import org.apache.doris.nereids.rules.rewrite.PushDownDistinctThroughJoin; @@ -344,6 +345,7 @@ public class Rewriter extends AbstractBatchJobExecutor { new PruneEmptyPartition(), new PruneFileScanPartition(), new PushConjunctsIntoJdbcScan(), + new PushConjunctsIntoOdbcScan(), new PushConjunctsIntoEsScan() ) ), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java index 3b07f2bbe9..bed9f4fe2e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java @@ -47,6 +47,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit; import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN; @@ -149,6 +150,11 @@ public class ChildOutputPropertyDeriver extends PlanVisitor + new PhysicalOdbcScan( + odbcScan.getRelationId(), + odbcScan.getTable(), + odbcScan.getQualifier(), + Optional.empty(), + odbcScan.getLogicalProperties(), + odbcScan.getConjuncts()) + ).toRule(RuleType.LOGICAL_ODBC_SCAN_TO_PHYSICAL_ODBC_SCAN_RULE); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java new file mode 100644 index 0000000000..8a1ba4d1b6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan; + +/** + * Rewrite odbc plan to set the conjuncts. + */ +public class PushConjunctsIntoOdbcScan extends OneRewriteRuleFactory { + + @Override + public Rule build() { + return logicalFilter(logicalOdbcScan()).thenApply(ctx -> { + LogicalFilter filter = ctx.root; + LogicalOdbcScan scan = filter.child(); + LogicalOdbcScan rewrittenScan = scan.withConjuncts(filter.getConjuncts()); + return new LogicalFilter<>(filter.getConjuncts(), rewrittenScan); + }).toRule(RuleType.PUSH_CONJUNCTS_INTO_ODBC_SCAN); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index 8495411745..a475d716fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -67,6 +67,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; +import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN; @@ -98,6 +99,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect; import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit; import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN; @@ -317,6 +319,12 @@ public class StatsCalculator extends DefaultPlanVisitor { return computeCatalogRelation(jdbcScan); } + @Override + public Statistics visitLogicalOdbcScan(LogicalOdbcScan odbcScan, Void context) { + odbcScan.getExpressions(); + return computeCatalogRelation(odbcScan); + } + @Override public Statistics visitLogicalEsScan(LogicalEsScan esScan, Void context) { esScan.getExpressions(); @@ -460,6 +468,11 @@ public class StatsCalculator extends DefaultPlanVisitor { return computeCatalogRelation(jdbcScan); } + @Override + public Statistics visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, Void context) { + return computeCatalogRelation(odbcScan); + } + @Override public Statistics visitPhysicalEsScan(PhysicalEsScan esScan, Void context) { return computeCatalogRelation(esScan); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java index 1135422187..3ae47c3a3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java @@ -49,6 +49,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; +import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN; @@ -238,6 +239,18 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter conjuncts; + + public LogicalOdbcScan(RelationId id, TableIf table, List qualifier, + Optional groupExpression, + Optional logicalProperties, + Set conjuncts) { + super(id, PlanType.LOGICAL_ODBC_SCAN, table, qualifier, + groupExpression, logicalProperties); + this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null")); + } + + public LogicalOdbcScan(RelationId id, TableIf table, List qualifier) { + this(id, table, qualifier, Optional.empty(), Optional.empty(), ImmutableSet.of()); + } + + @Override + public TableIf getTable() { + Preconditions.checkArgument(table instanceof OdbcTable, + String.format("Table %s is not OdbcTable", table.getName())); + return table; + } + + @Override + public String toString() { + return Utils.toSqlString("LogicalOdbcScan", + "qualified", qualifiedName(), + "output", getOutput() + ); + } + + @Override + public LogicalOdbcScan withGroupExpression(Optional groupExpression) { + return new LogicalOdbcScan(relationId, table, qualifier, groupExpression, + Optional.of(getLogicalProperties()), conjuncts); + } + + public LogicalOdbcScan withConjuncts(Set conjuncts) { + return new LogicalOdbcScan(relationId, table, qualifier, groupExpression, + Optional.of(getLogicalProperties()), conjuncts); + } + + @Override + public Plan withGroupExprLogicalPropChildren(Optional groupExpression, + Optional logicalProperties, List children) { + return new LogicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties, conjuncts); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitLogicalOdbcScan(this, context); + } + + @Override + public boolean equals(Object o) { + return super.equals(o) && Objects.equals(conjuncts, ((LogicalOdbcScan) o).conjuncts); + } + + public Set getConjuncts() { + return this.conjuncts; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java new file mode 100644 index 0000000000..f63b5a8d42 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.physical; + +import org.apache.doris.catalog.TableIf; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; +import org.apache.doris.statistics.Statistics; + +import com.google.common.collect.ImmutableSet; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * Physical odbc scan for odbc table. + */ +public class PhysicalOdbcScan extends PhysicalCatalogRelation { + + private final Set conjuncts; + + /** + * Constructor for PhysicalOdbcScan. + */ + public PhysicalOdbcScan(RelationId id, TableIf table, List qualifier, + Optional groupExpression, LogicalProperties logicalProperties, Set conjuncts) { + this(id, table, qualifier, groupExpression, logicalProperties, + null, null, conjuncts); + } + + /** + * Constructor for PhysicalOdbcScan. + */ + public PhysicalOdbcScan(RelationId id, TableIf table, List qualifier, + Optional groupExpression, + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics, + Set conjuncts) { + super(id, PlanType.PHYSICAL_ODBC_SCAN, table, qualifier, groupExpression, + logicalProperties, physicalProperties, statistics); + this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null")); + } + + @Override + public String toString() { + return Utils.toSqlString("PhysicalOdbcScan", + "qualified", Utils.qualifiedName(qualifier, table.getName()), + "output", getOutput(), + "stats", statistics + ); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitPhysicalOdbcScan(this, context); + } + + @Override + public PhysicalOdbcScan withGroupExpression(Optional groupExpression) { + return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, getLogicalProperties(), conjuncts); + } + + @Override + public Plan withGroupExprLogicalPropChildren(Optional groupExpression, + Optional logicalProperties, List children) { + return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties.get(), conjuncts); + } + + @Override + public PhysicalOdbcScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, + Statistics statistics) { + return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, + getLogicalProperties(), physicalProperties, statistics, conjuncts); + } + + public Set getConjuncts() { + return this.conjuncts; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java index 65a0350502..adfe499166 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; @@ -35,6 +36,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalEsScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; @@ -90,6 +92,10 @@ public interface RelationVisitor { return visitLogicalRelation(jdbcScan, context); } + default R visitLogicalOdbcScan(LogicalOdbcScan odbcScan, C context) { + return visitLogicalRelation(odbcScan, context); + } + default R visitLogicalOlapScan(LogicalOlapScan olapScan, C context) { return visitLogicalRelation(olapScan, context); } @@ -131,6 +137,10 @@ public interface RelationVisitor { return visitPhysicalRelation(jdbcScan, context); } + default R visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, C context) { + return visitPhysicalRelation(odbcScan, context); + } + default R visitPhysicalOlapScan(PhysicalOlapScan olapScan, C context) { return visitPhysicalRelation(olapScan, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java index 68950ebab8..d827ec4a4e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ExprSubstitutionMap; import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.SlotDescriptor; +import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; @@ -31,6 +32,7 @@ import org.apache.doris.catalog.OdbcTable; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.UserException; +import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.external.ExternalScanNode; import org.apache.doris.planner.external.jdbc.JdbcScanNode; @@ -52,6 +54,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.List; +import java.util.Set; /** * Full scan of an ODBC table. @@ -78,6 +81,22 @@ public class OdbcScanNode extends ExternalScanNode { this.tbl = tbl; } + @Override + public void init(Analyzer analyzer) throws UserException { + super.init(analyzer); + } + + /** + * Used for Nereids. Should NOT use this function in anywhere else. + */ + @Override + public void init() throws UserException { + super.init(); + numNodes = numNodes <= 0 ? 1 : numNodes; + StatsRecursiveDerive.getStatsRecursiveDerive().statsRecursiveDerive(this); + cardinality = (long) statsDeriveResult.getRowCount(); + } + @Override protected String debugString() { MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this); @@ -87,11 +106,24 @@ public class OdbcScanNode extends ExternalScanNode { @Override public void finalize(Analyzer analyzer) throws UserException { // Convert predicates to Odbc columns and filters. - createOdbcColumns(analyzer); - createOdbcFilters(analyzer); + createOdbcColumns(); + createOdbcFilters(); createScanRangeLocations(); } + @Override + public void finalizeForNereids() throws UserException { + createOdbcColumns(); + createOdbcFilters(); + createScanRangeLocations(); + } + + @Override + public void updateRequiredSlots(PlanTranslatorContext context, Set requiredByProjectSlotIdSet) + throws UserException { + createOdbcColumns(); + } + @Override protected void createScanRangeLocations() throws UserException { scanRangeLocations = Lists.newArrayList(createSingleScanRangeLocations(backendPolicy)); @@ -152,7 +184,8 @@ public class OdbcScanNode extends ExternalScanNode { return sql.toString(); } - private void createOdbcColumns(Analyzer analyzer) { + private void createOdbcColumns() { + columns.clear(); for (SlotDescriptor slot : desc.getSlots()) { if (!slot.isMaterialized()) { continue; @@ -161,16 +194,15 @@ public class OdbcScanNode extends ExternalScanNode { columns.add(JdbcTable.databaseProperName(odbcType, col.getName())); } // this happens when count(*) - if (0 == columns.size()) { + if (columns.isEmpty()) { columns.add("*"); } } // We convert predicates of the form op to Odbc filters - private void createOdbcFilters(Analyzer analyzer) { + private void createOdbcFilters() { if (conjuncts.isEmpty()) { return; - } List slotRefs = Lists.newArrayList(); Expr.collectList(conjuncts, SlotRef.class, slotRefs);