database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Improvement](nereids)Support ODBC table for new planner. (#29129)

Browse Source
This commit is contained in:
zy-kkk
2024-01-03 12:51:07 +08:00
committed by GitHub
parent 1fbbff32b2
commit 79eb575d7c
17 changed files with 417 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
View File

@ -35,6 +35,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
@ -137,6 +138,12 @@ class CostModelV1 extends PlanVisitor<Cost, PlanContext> {
return CostV1.ofCpu(context.getSessionVariable(), statistics.getRowCount());
}
@Override
public Cost visitPhysicalOdbcScan(PhysicalOdbcScan physicalOdbcScan, PlanContext context) {
Statistics statistics = context.getStatisticsWithCheck();
return CostV1.ofCpu(context.getSessionVariable(), statistics.getRowCount());
}
@Override
public Cost visitPhysicalEsScan(PhysicalEsScan physicalEsScan, PlanContext context) {
Statistics statistics = context.getStatisticsWithCheck();

6
fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java
View File

@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
@ -142,6 +143,11 @@ class CostModelV2 extends PlanVisitor<Cost, PlanContext> {
return calculateScanWithoutRF(context.getStatisticsWithCheck());
}
@Override
public Cost visitPhysicalOdbcScan(PhysicalOdbcScan physicalOdbcScan, PlanContext context) {
return calculateScanWithoutRF(context.getStatisticsWithCheck());
}
@Override
public Cost visitPhysicalEsScan(PhysicalEsScan physicalEsScan, PlanContext context) {
return calculateScanWithoutRF(context.getStatisticsWithCheck());

26
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View File

@ -41,6 +41,7 @@ import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Function.NullableMode;
import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
@ -112,6 +113,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
@ -176,6 +178,7 @@ import org.apache.doris.planner.external.MaxComputeScanNode;
import org.apache.doris.planner.external.hudi.HudiScanNode;
import org.apache.doris.planner.external.iceberg.IcebergScanNode;
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
import org.apache.doris.planner.external.odbc.OdbcScanNode;
import org.apache.doris.planner.external.paimon.PaimonScanNode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.statistics.StatisticConstants;
@ -577,6 +580,29 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
return planFragment;
}
@Override
public PlanFragment visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, PlanTranslatorContext context) {
List<Slot> slots = odbcScan.getOutput();
TableIf table = odbcScan.getTable();
TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table, context);
OdbcScanNode odbcScanNode = new OdbcScanNode(odbcScan.translatePlanNodeId(), tupleDescriptor,
(OdbcTable) table);
odbcScanNode.addConjuncts(translateToLegacyConjuncts(odbcScan.getConjuncts()));
Utils.execWithUncheckedException(odbcScanNode::init);
context.addScanNode(odbcScanNode);
context.getRuntimeTranslator().ifPresent(
runtimeFilterGenerator -> runtimeFilterGenerator.getContext().getTargetListByScan(odbcScan).forEach(
expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, odbcScanNode, context)
)
);
Utils.execWithUncheckedException(odbcScanNode::finalizeForNereids);
DataPartition dataPartition = DataPartition.RANDOM;
PlanFragment planFragment = new PlanFragment(context.nextFragmentId(), odbcScanNode, dataPartition);
context.addPlanFragment(planFragment);
updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), odbcScan);
return planFragment;
}
@Override
public PlanFragment visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanTranslatorContext context) {
List<Slot> slots = olapScan.getOutput();

2
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
View File

@ -95,6 +95,7 @@ import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderLimit;
import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderTopN;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoEsScan;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoJdbcScan;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoOdbcScan;
import org.apache.doris.nereids.rules.rewrite.PushDownCountThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushDownCountThroughJoinOneSide;
import org.apache.doris.nereids.rules.rewrite.PushDownDistinctThroughJoin;
@ -344,6 +345,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
new PruneEmptyPartition(),
new PruneFileScanPartition(),
new PushConjunctsIntoJdbcScan(),
new PushConjunctsIntoOdbcScan(),
new PushConjunctsIntoEsScan()
)
),

6
fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
View File

@ -47,6 +47,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
@ -149,6 +150,11 @@ public class ChildOutputPropertyDeriver extends PlanVisitor<PhysicalProperties,
return PhysicalProperties.STORAGE_ANY;
}
@Override
public PhysicalProperties visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, PlanContext context) {
return PhysicalProperties.STORAGE_ANY;
}
@Override
public PhysicalProperties visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanContext context) {
return new PhysicalProperties(olapScan.getDistributionSpec());

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
View File

@ -69,6 +69,7 @@ import org.apache.doris.nereids.rules.implementation.LogicalJdbcScanToPhysicalJd
import org.apache.doris.nereids.rules.implementation.LogicalJoinToHashJoin;
import org.apache.doris.nereids.rules.implementation.LogicalJoinToNestedLoopJoin;
import org.apache.doris.nereids.rules.implementation.LogicalLimitToPhysicalLimit;
import org.apache.doris.nereids.rules.implementation.LogicalOdbcScanToPhysicalOdbcScan;
import org.apache.doris.nereids.rules.implementation.LogicalOlapScanToPhysicalOlapScan;
import org.apache.doris.nereids.rules.implementation.LogicalOlapTableSinkToPhysicalOlapTableSink;
import org.apache.doris.nereids.rules.implementation.LogicalOneRowRelationToPhysicalOneRowRelation;
@ -165,6 +166,7 @@ public class RuleSet {
.add(new LogicalSchemaScanToPhysicalSchemaScan())
.add(new LogicalFileScanToPhysicalFileScan())
.add(new LogicalJdbcScanToPhysicalJdbcScan())
.add(new LogicalOdbcScanToPhysicalOdbcScan())
.add(new LogicalEsScanToPhysicalEsScan())
.add(new LogicalProjectToPhysicalProject())
.add(new LogicalLimitToPhysicalLimit())

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View File

@ -233,6 +233,7 @@ public enum RuleType {
OLAP_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
FILE_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
PUSH_CONJUNCTS_INTO_JDBC_SCAN(RuleTypeClass.REWRITE),
PUSH_CONJUNCTS_INTO_ODBC_SCAN(RuleTypeClass.REWRITE),
PUSH_CONJUNCTS_INTO_ES_SCAN(RuleTypeClass.REWRITE),
OLAP_SCAN_TABLET_PRUNE(RuleTypeClass.REWRITE),
PUSH_AGGREGATE_TO_OLAP_SCAN(RuleTypeClass.REWRITE),
@ -368,6 +369,7 @@ public enum RuleType {
LOGICAL_SCHEMA_SCAN_TO_PHYSICAL_SCHEMA_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_FILE_SCAN_TO_PHYSICAL_FILE_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_JDBC_SCAN_TO_PHYSICAL_JDBC_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_ODBC_SCAN_TO_PHYSICAL_ODBC_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_ES_SCAN_TO_PHYSICAL_ES_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_OLAP_TABLE_SINK_TO_PHYSICAL_OLAP_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_RESULT_SINK_TO_PHYSICAL_RESULT_SINK_RULE(RuleTypeClass.IMPLEMENTATION),

3
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
View File

@ -53,6 +53,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
@ -242,6 +243,8 @@ public class BindRelation extends OneAnalysisRuleFactory {
case JDBC_EXTERNAL_TABLE:
case JDBC:
return new LogicalJdbcScan(unboundRelation.getRelationId(), table, tableQualifier);
case ODBC:
return new LogicalOdbcScan(unboundRelation.getRelationId(), table, tableQualifier);
case ES_EXTERNAL_TABLE:
return new LogicalEsScan(unboundRelation.getRelationId(), (EsExternalTable) table, tableQualifier);
default:

42
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java Normal file
View File

@ -0,0 +1,42 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.implementation;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import java.util.Optional;
/**
* Implementation rule that convert logical OdbcScan to physical OdbcScan.
*/
public class LogicalOdbcScanToPhysicalOdbcScan extends OneImplementationRuleFactory {
@Override
public Rule build() {
return logicalOdbcScan().then(odbcScan ->
new PhysicalOdbcScan(
odbcScan.getRelationId(),
odbcScan.getTable(),
odbcScan.getQualifier(),
Optional.empty(),
odbcScan.getLogicalProperties(),
odbcScan.getConjuncts())
).toRule(RuleType.LOGICAL_ODBC_SCAN_TO_PHYSICAL_ODBC_SCAN_RULE);
}
}

39
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java Normal file
View File

@ -0,0 +1,39 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
/**
* Rewrite odbc plan to set the conjuncts.
*/
public class PushConjunctsIntoOdbcScan extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalFilter(logicalOdbcScan()).thenApply(ctx -> {
LogicalFilter<LogicalOdbcScan> filter = ctx.root;
LogicalOdbcScan scan = filter.child();
LogicalOdbcScan rewrittenScan = scan.withConjuncts(filter.getConjuncts());
return new LogicalFilter<>(filter.getConjuncts(), rewrittenScan);
}).toRule(RuleType.PUSH_CONJUNCTS_INTO_ODBC_SCAN);
}
}

13
fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
View File

@ -67,6 +67,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
@ -98,6 +99,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
@ -317,6 +319,12 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
return computeCatalogRelation(jdbcScan);
}
@Override
public Statistics visitLogicalOdbcScan(LogicalOdbcScan odbcScan, Void context) {
odbcScan.getExpressions();
return computeCatalogRelation(odbcScan);
}
@Override
public Statistics visitLogicalEsScan(LogicalEsScan esScan, Void context) {
esScan.getExpressions();
@ -460,6 +468,11 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
return computeCatalogRelation(jdbcScan);
}
@Override
public Statistics visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, Void context) {
return computeCatalogRelation(odbcScan);
}
@Override
public Statistics visitPhysicalEsScan(PhysicalEsScan esScan, Void context) {
return computeCatalogRelation(esScan);

13
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
View File

@ -49,6 +49,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
@ -238,6 +239,18 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext
return newJdbcScan;
}
@Override
public Plan visitLogicalOdbcScan(LogicalOdbcScan odbcScan, DeepCopierContext context) {
if (context.getRelationReplaceMap().containsKey(odbcScan.getRelationId())) {
return context.getRelationReplaceMap().get(odbcScan.getRelationId());
}
LogicalOdbcScan newOdbcScan = new LogicalOdbcScan(StatementScopeIdGenerator.newRelationId(),
odbcScan.getTable(), odbcScan.getQualifier());
updateReplaceMapWithOutput(odbcScan, newOdbcScan, context.exprIdReplaceMap);
context.putRelation(odbcScan.getRelationId(), newOdbcScan);
return newOdbcScan;
}
@Override
public Plan visitLogicalEsScan(LogicalEsScan esScan, DeepCopierContext context) {
if (context.getRelationReplaceMap().containsKey(esScan.getRelationId())) {

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
View File

@ -33,6 +33,7 @@ public enum PlanType {
LOGICAL_EMPTY_RELATION,
LOGICAL_ES_SCAN,
LOGICAL_JDBC_SCAN,
LOGICAL_ODBC_SCAN,
LOGICAL_OLAP_SCAN,
LOGICAL_ONE_ROW_RELATION,
LOGICAL_SCHEMA_SCAN,
@ -83,6 +84,7 @@ public enum PlanType {
PHYSICAL_ES_SCAN,
PHYSICAL_FILE_SCAN,
PHYSICAL_JDBC_SCAN,
PHYSICAL_ODBC_SCAN,
PHYSICAL_ONE_ROW_RELATION,
PHYSICAL_OLAP_SCAN,
PHYSICAL_SCHEMA_SCAN,

104
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java Normal file
View File

@ -0,0 +1,104 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
/**
* Logical scan for external odbc table.
*/
public class LogicalOdbcScan extends LogicalCatalogRelation {
private final Set<Expression> conjuncts;
public LogicalOdbcScan(RelationId id, TableIf table, List<String> qualifier,
Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties,
Set<Expression> conjuncts) {
super(id, PlanType.LOGICAL_ODBC_SCAN, table, qualifier,
groupExpression, logicalProperties);
this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null"));
}
public LogicalOdbcScan(RelationId id, TableIf table, List<String> qualifier) {
this(id, table, qualifier, Optional.empty(), Optional.empty(), ImmutableSet.of());
}
@Override
public TableIf getTable() {
Preconditions.checkArgument(table instanceof OdbcTable,
String.format("Table %s is not OdbcTable", table.getName()));
return table;
}
@Override
public String toString() {
return Utils.toSqlString("LogicalOdbcScan",
"qualified", qualifiedName(),
"output", getOutput()
);
}
@Override
public LogicalOdbcScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalOdbcScan(relationId, table, qualifier, groupExpression,
Optional.of(getLogicalProperties()), conjuncts);
}
public LogicalOdbcScan withConjuncts(Set<Expression> conjuncts) {
return new LogicalOdbcScan(relationId, table, qualifier, groupExpression,
Optional.of(getLogicalProperties()), conjuncts);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new LogicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties, conjuncts);
}
@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitLogicalOdbcScan(this, context);
}
@Override
public boolean equals(Object o) {
return super.equals(o) && Objects.equals(conjuncts, ((LogicalOdbcScan) o).conjuncts);
}
public Set<Expression> getConjuncts() {
return this.conjuncts;
}
}

102
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java Normal file
View File

@ -0,0 +1,102 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.plans.physical;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.statistics.Statistics;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
/**
* Physical odbc scan for odbc table.
*/
public class PhysicalOdbcScan extends PhysicalCatalogRelation {
private final Set<Expression> conjuncts;
/**
* Constructor for PhysicalOdbcScan.
*/
public PhysicalOdbcScan(RelationId id, TableIf table, List<String> qualifier,
Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties, Set<Expression> conjuncts) {
this(id, table, qualifier, groupExpression, logicalProperties,
null, null, conjuncts);
}
/**
* Constructor for PhysicalOdbcScan.
*/
public PhysicalOdbcScan(RelationId id, TableIf table, List<String> qualifier,
Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics,
Set<Expression> conjuncts) {
super(id, PlanType.PHYSICAL_ODBC_SCAN, table, qualifier, groupExpression,
logicalProperties, physicalProperties, statistics);
this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null"));
}
@Override
public String toString() {
return Utils.toSqlString("PhysicalOdbcScan",
"qualified", Utils.qualifiedName(qualifier, table.getName()),
"output", getOutput(),
"stats", statistics
);
}
@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitPhysicalOdbcScan(this, context);
}
@Override
public PhysicalOdbcScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, getLogicalProperties(), conjuncts);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties.get(), conjuncts);
}
@Override
public PhysicalOdbcScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties,
Statistics statistics) {
return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression,
getLogicalProperties(), physicalProperties, statistics, conjuncts);
}
public Set<Expression> getConjuncts() {
return this.conjuncts;
}
}

10
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
View File

@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
@ -35,6 +36,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalEsScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
@ -90,6 +92,10 @@ public interface RelationVisitor<R, C> {
return visitLogicalRelation(jdbcScan, context);
}
default R visitLogicalOdbcScan(LogicalOdbcScan odbcScan, C context) {
return visitLogicalRelation(odbcScan, context);
}
default R visitLogicalOlapScan(LogicalOlapScan olapScan, C context) {
return visitLogicalRelation(olapScan, context);
}
@ -131,6 +137,10 @@ public interface RelationVisitor<R, C> {
return visitPhysicalRelation(jdbcScan, context);
}
default R visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, C context) {
return visitPhysicalRelation(odbcScan, context);
}
default R visitPhysicalOlapScan(PhysicalOlapScan olapScan, C context) {
return visitPhysicalRelation(olapScan, context);
}

44
fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java vendored
View File

@ -22,6 +22,7 @@ import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.ExprSubstitutionMap;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.SlotDescriptor;
import org.apache.doris.analysis.SlotId;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Column;
@ -31,6 +32,7 @@ import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.planner.external.ExternalScanNode;
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
@ -52,6 +54,7 @@ import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
/**
* Full scan of an ODBC table.
@ -78,6 +81,22 @@ public class OdbcScanNode extends ExternalScanNode {
this.tbl = tbl;
}
@Override
public void init(Analyzer analyzer) throws UserException {
super.init(analyzer);
}
/**
* Used for Nereids. Should NOT use this function in anywhere else.
*/
@Override
public void init() throws UserException {
super.init();
numNodes = numNodes <= 0 ? 1 : numNodes;
StatsRecursiveDerive.getStatsRecursiveDerive().statsRecursiveDerive(this);
cardinality = (long) statsDeriveResult.getRowCount();
}
@Override
protected String debugString() {
MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this);
@ -87,11 +106,24 @@ public class OdbcScanNode extends ExternalScanNode {
@Override
public void finalize(Analyzer analyzer) throws UserException {
// Convert predicates to Odbc columns and filters.
createOdbcColumns(analyzer);
createOdbcFilters(analyzer);
createOdbcColumns();
createOdbcFilters();
createScanRangeLocations();
}
@Override
public void finalizeForNereids() throws UserException {
createOdbcColumns();
createOdbcFilters();
createScanRangeLocations();
}
@Override
public void updateRequiredSlots(PlanTranslatorContext context, Set<SlotId> requiredByProjectSlotIdSet)
throws UserException {
createOdbcColumns();
}
@Override
protected void createScanRangeLocations() throws UserException {
scanRangeLocations = Lists.newArrayList(createSingleScanRangeLocations(backendPolicy));
@ -152,7 +184,8 @@ public class OdbcScanNode extends ExternalScanNode {
return sql.toString();
}
private void createOdbcColumns(Analyzer analyzer) {
private void createOdbcColumns() {
columns.clear();
for (SlotDescriptor slot : desc.getSlots()) {
if (!slot.isMaterialized()) {
continue;
@ -161,16 +194,15 @@ public class OdbcScanNode extends ExternalScanNode {
columns.add(JdbcTable.databaseProperName(odbcType, col.getName()));
}
// this happens when count(*)
if (0 == columns.size()) {
if (columns.isEmpty()) {
columns.add("*");
}
}
// We convert predicates of the form <slotref> op <constant> to Odbc filters
private void createOdbcFilters(Analyzer analyzer) {
private void createOdbcFilters() {
if (conjuncts.isEmpty()) {
return;
}
List<SlotRef> slotRefs = Lists.newArrayList();
Expr.collectList(conjuncts, SlotRef.class, slotRefs);