database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[2.1][opt](Catalog) Remove unnecessary conjuncts handling on External Scan (#42261)

Browse Source 
pick (#41218)

In the previous FileScanNode, some parts that used conjuncts for
predicate conversion were placed in the init phase. However, for the
Nereids planner, pushing the filter down to the scan happens in the
Translator, which means that the ScanNode can only get the complete
conjuncts in the finalized phase. Therefore, in this PR, I have removed
all conjuncts variables in External for the Nereids planner. They no
longer need to store conjuncts themselves or add them to the ScanNode.
Instead, all places in the ScanNode that use conjuncts should be moved
to the finalized phase.

This refactor also fix a performance issue introduced from #40176 After
introducing the change of generating SelectNode for consecutive projects
or filters, FileScan still adds conjuncts too early in the init phase,
resulting in the discovery of consecutive filters when the upper layer
continues to translate, a selectnode was unexpectedly generated on the
scannode, causing the project to be unable to prune the scannode
columns. However, the Project node trims columns of SelectNode and
ScanNode differently, which causes ScanNode to scan unnecessary columns.

My modification removes the addition of conjuncts in the scannode step,
so that we can keep the structure from ScanNode to Project and achieve
correct column trimming.
This commit is contained in:
zy-kkk
2024-10-22 21:42:55 +08:00
committed by GitHub
parent 25d7d0b255
commit 56c2a0f523
35 changed files with 169 additions and 544 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
View File

@ -144,7 +144,6 @@ public abstract class FileQueryScanNode extends FileScanNode {
table.getName()));
}
}
computeColumnsFilter();
initBackendPolicy();
initSchemaParams();
}
@ -213,6 +212,7 @@ public abstract class FileQueryScanNode extends FileScanNode {
if (ConnectContext.get().getExecutor() != null) {
ConnectContext.get().getExecutor().getSummaryProfile().setFinalizeScanNodeStartTime();
}
convertPredicate();
createScanRangeLocations();
updateRequiredSlots();
if (ConnectContext.get().getExecutor() != null) {
@ -220,6 +220,15 @@ public abstract class FileQueryScanNode extends FileScanNode {
}
}
/**
* Used as a predicate to convert conjuncts into corresponding data sources.
* All predicate conversions from different data sources should override this method.
* and this method must be called in finalize,
* because in nereids planner, conjuncts are only generated in the finalize stage.
*/
protected void convertPredicate() {
}
private void setColumnPositionMapping()
throws UserException {
TableIf tbl = getTargetTable();

2
fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java
View File

@ -114,11 +114,13 @@ public class EsScanNode extends ExternalScanNode {
@Override
public void finalize(Analyzer analyzer) throws UserException {
buildQuery();
doFinalize();
}
@Override
public void finalizeForNereids() throws UserException {
buildQuery();
doFinalize();
}

16
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
View File

@ -37,6 +37,7 @@ import org.apache.doris.mtmv.MTMVRelatedTableIf;
import org.apache.doris.mtmv.MTMVSnapshotIf;
import org.apache.doris.mtmv.MTMVTimestampSnapshot;
import org.apache.doris.nereids.exceptions.NotSupportedException;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
import org.apache.doris.qe.GlobalVariable;
import org.apache.doris.statistics.AnalysisInfo;
import org.apache.doris.statistics.BaseAnalysisTask;
@ -288,6 +289,21 @@ public class HMSExternalTable extends ExternalTable implements MTMVRelatedTableI
.orElse(Collections.emptyList());
}
public SelectedPartitions getAllPartitions() {
if (CollectionUtils.isEmpty(this.getPartitionColumns())) {
return SelectedPartitions.NOT_PRUNED;
}
HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
.getMetaStoreCache((HMSExternalCatalog) this.getCatalog());
List<Type> partitionColumnTypes = this.getPartitionColumnTypes();
HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues(
this.getDbName(), this.getName(), partitionColumnTypes);
Map<Long, PartitionItem> idToPartitionItem = hivePartitionValues.getIdToPartitionItem();
return new SelectedPartitions(idToPartitionItem.size(), idToPartitionItem, false);
}
public boolean isHiveTransactionalTable() {
return dlaType == DLAType.HIVE && AcidUtils.isTransactionalTable(remoteTable)
&& isSupportedTransactionalFileFormat();

39
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
View File

@ -42,7 +42,6 @@ import org.apache.doris.datasource.hive.HiveProperties;
import org.apache.doris.datasource.hive.HiveTransaction;
import org.apache.doris.datasource.hive.source.HiveSplit.HiveSplitCreator;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
import org.apache.doris.planner.ListPartitionPrunerV2;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.spi.Split;
@ -123,46 +122,16 @@ public class HiveScanNode extends FileQueryScanNode {
protected List<HivePartition> getPartitions() throws AnalysisException {
List<HivePartition> resPartitions = Lists.newArrayList();
long start = System.currentTimeMillis();
HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
.getMetaStoreCache((HMSExternalCatalog) hmsTable.getCatalog());
List<Type> partitionColumnTypes = hmsTable.getPartitionColumnTypes();
if (!partitionColumnTypes.isEmpty()) {
// partitioned table
boolean isPartitionPruned = selectedPartitions != null && selectedPartitions.isPruned;
Collection<PartitionItem> partitionItems;
if (!isPartitionPruned) {
// partitionItems is null means that the partition is not pruned by Nereids,
// so need to prune partitions here by legacy ListPartitionPrunerV2.
HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues(
hmsTable.getDbName(), hmsTable.getName(), partitionColumnTypes);
Map<Long, PartitionItem> idToPartitionItem = hivePartitionValues.getIdToPartitionItem();
this.totalPartitionNum = idToPartitionItem.size();
if (!conjuncts.isEmpty()) {
ListPartitionPrunerV2 pruner = new ListPartitionPrunerV2(idToPartitionItem,
hmsTable.getPartitionColumns(), columnNameToRange,
hivePartitionValues.getUidToPartitionRange(),
hivePartitionValues.getRangeToId(),
hivePartitionValues.getSingleColumnRangeMap(),
true);
Collection<Long> filteredPartitionIds = pruner.prune();
if (LOG.isDebugEnabled()) {
LOG.debug("hive partition fetch and prune for table {}.{} cost: {} ms",
hmsTable.getDbName(), hmsTable.getName(), (System.currentTimeMillis() - start));
}
partitionItems = Lists.newArrayListWithCapacity(filteredPartitionIds.size());
for (Long id : filteredPartitionIds) {
partitionItems.add(idToPartitionItem.get(id));
}
} else {
partitionItems = idToPartitionItem.values();
}
} else {
// partitions has benn pruned by Nereids, in PruneFileScanPartition,
// so just use the selected partitions.
this.totalPartitionNum = selectedPartitions.totalPartitionNum;
partitionItems = selectedPartitions.selectedPartitions.values();
}
// partitions has benn pruned by Nereids, in PruneFileScanPartition,
// so just use the selected partitions.
this.totalPartitionNum = selectedPartitions.totalPartitionNum;
partitionItems = selectedPartitions.selectedPartitions.values();
Preconditions.checkNotNull(partitionItems);
this.selectedPartitionNum = partitionItems.size();

14
fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
View File

@ -82,6 +82,7 @@ public class MaxComputeScanNode extends FileQueryScanNode {
private final MaxComputeExternalTable table;
TableBatchReadSession tableBatchReadSession;
private Predicate filterPredicate;
public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) {
this(id, desc, "MCScanNode", StatisticalType.MAX_COMPUTE_SCAN_NODE, needCheckColumnPriv);
@ -115,8 +116,6 @@ public class MaxComputeScanNode extends FileQueryScanNode {
}
void createTableBatchReadSession() throws UserException {
Predicate filterPredicate = convertPredicate();
List<String> requiredPartitionColumns = new ArrayList<>();
List<String> orderedRequiredDataColumns = new ArrayList<>();
@ -164,9 +163,10 @@ public class MaxComputeScanNode extends FileQueryScanNode {
}
protected Predicate convertPredicate() {
@Override
protected void convertPredicate() {
if (conjuncts.isEmpty()) {
return Predicate.NO_PREDICATE;
this.filterPredicate = Predicate.NO_PREDICATE;
}
List<Predicate> odpsPredicates = new ArrayList<>();
@ -180,9 +180,9 @@ public class MaxComputeScanNode extends FileQueryScanNode {
}
if (odpsPredicates.isEmpty()) {
return Predicate.NO_PREDICATE;
this.filterPredicate = Predicate.NO_PREDICATE;
} else if (odpsPredicates.size() == 1) {
return odpsPredicates.get(0);
this.filterPredicate = odpsPredicates.get(0);
} else {
com.aliyun.odps.table.optimizer.predicate.CompoundPredicate
filterPredicate = new com.aliyun.odps.table.optimizer.predicate.CompoundPredicate(
@ -191,7 +191,7 @@ public class MaxComputeScanNode extends FileQueryScanNode {
for (Predicate odpsPredicate : odpsPredicates) {
filterPredicate.addPredicate(odpsPredicate);
}
return filterPredicate;
this.filterPredicate = filterPredicate;
}
}

4
fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
View File

@ -111,6 +111,10 @@ public class PaimonScanNode extends FileQueryScanNode {
super.doInitialize();
source = new PaimonSource(desc);
Preconditions.checkNotNull(source);
}
@Override
protected void convertPredicate() {
PaimonPredicateConverter paimonPredicateConverter = new PaimonPredicateConverter(
source.getPaimonTable().rowType());
predicates = paimonPredicateConverter.convertToPaimonExpr(conjuncts);

4
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View File

@ -649,7 +649,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
EsScanNode esScanNode = new EsScanNode(context.nextPlanNodeId(), tupleDescriptor,
table instanceof EsExternalTable);
esScanNode.setNereidsId(esScan.getId());
esScanNode.addConjuncts(translateToLegacyConjuncts(esScan.getConjuncts()));
Utils.execWithUncheckedException(esScanNode::init);
context.addScanNode(esScanNode);
context.getRuntimeTranslator().ifPresent(
@ -690,7 +689,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
ScanNode scanNode,
ExternalTable table, TupleDescriptor tupleDescriptor) {
scanNode.setNereidsId(fileScan.getId());
scanNode.addConjuncts(translateToLegacyConjuncts(fileScan.getConjuncts()));
scanNode.setPushDownAggNoGrouping(context.getRelationPushAggOp(fileScan.getRelationId()));
TableName tableName = new TableName(null, "", "");
@ -724,7 +722,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
JdbcScanNode jdbcScanNode = new JdbcScanNode(context.nextPlanNodeId(), tupleDescriptor,
table instanceof JdbcExternalTable);
jdbcScanNode.setNereidsId(jdbcScan.getId());
jdbcScanNode.addConjuncts(translateToLegacyConjuncts(jdbcScan.getConjuncts()));
Utils.execWithUncheckedException(jdbcScanNode::init);
context.addScanNode(jdbcScanNode);
context.getRuntimeTranslator().ifPresent(
@ -747,7 +744,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
OdbcScanNode odbcScanNode = new OdbcScanNode(context.nextPlanNodeId(), tupleDescriptor,
(OdbcTable) table);
odbcScanNode.setNereidsId(odbcScan.getId());
odbcScanNode.addConjuncts(translateToLegacyConjuncts(odbcScan.getConjuncts()));
Utils.execWithUncheckedException(odbcScanNode::init);
context.addScanNode(odbcScanNode);
context.getRuntimeTranslator().ifPresent(

8
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
View File

@ -104,9 +104,6 @@ import org.apache.doris.nereids.rules.rewrite.PullUpJoinFromUnionAll;
import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderApply;
import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderLimit;
import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderTopN;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoEsScan;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoJdbcScan;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoOdbcScan;
import org.apache.doris.nereids.rules.rewrite.PushDownAggThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushDownAggThroughJoinOneSide;
import org.apache.doris.nereids.rules.rewrite.PushDownDistinctThroughJoin;
@ -390,10 +387,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
topDown(
new PruneOlapScanPartition(),
new PruneEmptyPartition(),
new PruneFileScanPartition(),
new PushConjunctsIntoJdbcScan(),
new PushConjunctsIntoOdbcScan(),
new PushConjunctsIntoEsScan()
new PruneFileScanPartition()
)
),
topic("MV optimization",

4
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
View File

@ -418,7 +418,9 @@ public class BindRelation extends OneAnalysisRuleFactory {
return hudiScan;
} else {
return new LogicalFileScan(unboundRelation.getRelationId(), (HMSExternalTable) table,
qualifierWithoutTableName, unboundRelation.getTableSample(),
qualifierWithoutTableName,
((HMSExternalTable) table).getAllPartitions(),
unboundRelation.getTableSample(),
unboundRelation.getTableSnapshot());
}
case ICEBERG_EXTERNAL_TABLE:

3
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEsScanToPhysicalEsScan.java
View File

@ -37,8 +37,7 @@ public class LogicalEsScanToPhysicalEsScan extends OneImplementationRuleFactory
esScan.getQualifier(),
DistributionSpecAny.INSTANCE,
Optional.empty(),
esScan.getLogicalProperties(),
esScan.getConjuncts())
esScan.getLogicalProperties())
).toRule(RuleType.LOGICAL_ES_SCAN_TO_PHYSICAL_ES_SCAN_RULE);
}
}

1
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java
View File

@ -39,7 +39,6 @@ public class LogicalFileScanToPhysicalFileScan extends OneImplementationRuleFact
DistributionSpecAny.INSTANCE,
Optional.empty(),
fileScan.getLogicalProperties(),
fileScan.getConjuncts(),
fileScan.getSelectedPartitions(),
fileScan.getTableSample(),
fileScan.getTableSnapshot())

1
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalHudiScanToPhysicalHudiScan.java
View File

@ -38,7 +38,6 @@ public class LogicalHudiScanToPhysicalHudiScan extends OneImplementationRuleFact
DistributionSpecAny.INSTANCE,
Optional.empty(),
fileScan.getLogicalProperties(),
fileScan.getConjuncts(),
fileScan.getSelectedPartitions(),
fileScan.getTableSample(),
fileScan.getTableSnapshot(),

3
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJdbcScanToPhysicalJdbcScan.java
View File

@ -35,8 +35,7 @@ public class LogicalJdbcScanToPhysicalJdbcScan extends OneImplementationRuleFact
jdbcScan.getTable(),
jdbcScan.getQualifier(),
Optional.empty(),
jdbcScan.getLogicalProperties(),
jdbcScan.getConjuncts())
jdbcScan.getLogicalProperties())
).toRule(RuleType.LOGICAL_JDBC_SCAN_TO_PHYSICAL_JDBC_SCAN_RULE);
}
}

3
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java
View File

@ -35,8 +35,7 @@ public class LogicalOdbcScanToPhysicalOdbcScan extends OneImplementationRuleFact
odbcScan.getTable(),
odbcScan.getQualifier(),
Optional.empty(),
odbcScan.getLogicalProperties(),
odbcScan.getConjuncts())
odbcScan.getLogicalProperties())
).toRule(RuleType.LOGICAL_ODBC_SCAN_TO_PHYSICAL_ODBC_SCAN_RULE);
}
}

12
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
View File

@ -31,7 +31,6 @@ import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewri
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
import org.apache.doris.nereids.trees.plans.logical.LogicalExternalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
@ -277,17 +276,6 @@ public class AdjustNullable extends DefaultPlanRewriter<Map<ExprId, Slot>> imple
return cteConsumer.withTwoMaps(consumerToProducerOutputMap, producerToConsumerOutputMap);
}
@Override
public Plan visitLogicalExternalRelation(LogicalExternalRelation relation, Map<ExprId, Slot> replaceMap) {
if (!relation.getConjuncts().isEmpty()) {
relation.getOutputSet().forEach(s -> replaceMap.put(s.getExprId(), s));
Set<Expression> conjuncts = updateExpressions(relation.getConjuncts(), replaceMap);
return relation.withConjuncts(conjuncts).recomputeLogicalProperties();
} else {
return relation;
}
}
private <T extends Expression> T updateExpression(T input, Map<ExprId, Slot> replaceMap) {
return (T) input.rewriteDownShortCircuit(e -> e.accept(SlotReferenceReplacer.INSTANCE, replaceMap));
}

12
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
View File

@ -17,13 +17,10 @@
package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable.DLAType;
import org.apache.doris.datasource.hive.HiveMetaStoreCache;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
@ -72,8 +69,7 @@ public class PruneFileScanPartition extends OneRewriteRuleFactory {
selectedPartitions = new SelectedPartitions(0, ImmutableMap.of(), true);
}
LogicalFileScan rewrittenScan = scan.withConjuncts(filter.getConjuncts())
.withSelectedPartitions(selectedPartitions);
LogicalFileScan rewrittenScan = scan.withSelectedPartitions(selectedPartitions);
return new LogicalFilter<>(filter.getConjuncts(), rewrittenScan);
}).toRule(RuleType.FILE_SCAN_PARTITION_PRUNE);
}
@ -95,11 +91,7 @@ public class PruneFileScanPartition extends OneRewriteRuleFactory {
.map(column -> scanOutput.get(column.getName().toLowerCase()))
.collect(Collectors.toList());
HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
.getMetaStoreCache((HMSExternalCatalog) hiveTbl.getCatalog());
HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues(
hiveTbl.getDbName(), hiveTbl.getName(), hiveTbl.getPartitionColumnTypes());
Map<Long, PartitionItem> idToPartitionItem = hivePartitionValues.getIdToPartitionItem();
Map<Long, PartitionItem> idToPartitionItem = scan.getSelectedPartitions().selectedPartitions;
List<Long> prunedPartitions = new ArrayList<>(PartitionPruner.prune(
partitionSlots, filter.getPredicate(), idToPartitionItem, ctx, PartitionTableType.HIVE));

39
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoEsScan.java
View File

@ -1,39 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
/**
* Rewrite es plan to set the conjuncts.
*/
public class PushConjunctsIntoEsScan extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalFilter(logicalEsScan()).thenApply(ctx -> {
LogicalFilter<LogicalEsScan> filter = ctx.root;
LogicalEsScan scan = filter.child();
LogicalEsScan rewrittenScan = scan.withConjuncts(filter.getConjuncts());
return rewrittenScan;
}).toRule(RuleType.PUSH_CONJUNCTS_INTO_ES_SCAN);
}
}

39
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoJdbcScan.java
View File

@ -1,39 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
/**
* Rewrite jdbc plan to set the conjuncts.
*/
public class PushConjunctsIntoJdbcScan extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalFilter(logicalJdbcScan()).thenApply(ctx -> {
LogicalFilter<LogicalJdbcScan> filter = ctx.root;
LogicalJdbcScan scan = filter.child();
LogicalJdbcScan rewrittenScan = scan.withConjuncts(filter.getConjuncts());
return new LogicalFilter<>(filter.getConjuncts(), rewrittenScan);
}).toRule(RuleType.PUSH_CONJUNCTS_INTO_JDBC_SCAN);
}
}

39
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java
View File

@ -1,39 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
/**
* Rewrite odbc plan to set the conjuncts.
*/
public class PushConjunctsIntoOdbcScan extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalFilter(logicalOdbcScan()).thenApply(ctx -> {
LogicalFilter<LogicalOdbcScan> filter = ctx.root;
LogicalOdbcScan scan = filter.child();
LogicalOdbcScan rewrittenScan = scan.withConjuncts(filter.getConjuncts());
return new LogicalFilter<>(filter.getConjuncts(), rewrittenScan);
}).toRule(RuleType.PUSH_CONJUNCTS_INTO_ODBC_SCAN);
}
}

17
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
View File

@ -40,7 +40,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapS
import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
import org.apache.doris.nereids.trees.plans.logical.LogicalExternalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate;
import org.apache.doris.nereids.trees.plans.logical.LogicalHaving;
@ -189,22 +188,6 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext
return new LogicalDeferMaterializeOlapScan(newScan, newSlotIds, newRowId);
}
@Override
public Plan visitLogicalExternalRelation(LogicalExternalRelation relation,
DeepCopierContext context) {
if (context.getRelationReplaceMap().containsKey(relation.getRelationId())) {
return context.getRelationReplaceMap().get(relation.getRelationId());
}
LogicalExternalRelation newRelation = relation.withRelationId(StatementScopeIdGenerator.newRelationId());
updateReplaceMapWithOutput(relation, newRelation, context.exprIdReplaceMap);
Set<Expression> conjuncts = relation.getConjuncts().stream()
.map(p -> ExpressionDeepCopier.INSTANCE.deepCopy(p, context))
.collect(ImmutableSet.toImmutableSet());
newRelation = newRelation.withConjuncts(conjuncts);
context.putRelation(relation.getRelationId(), newRelation);
return newRelation;
}
@Override
public Plan visitLogicalProject(LogicalProject<? extends Plan> project, DeepCopierContext context) {
Plan child = project.child().accept(this, context);

26
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java
View File

@ -20,35 +20,31 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
/**
* Logical scan for external es catalog.
*/
public class LogicalEsScan extends LogicalExternalRelation {
public class LogicalEsScan extends LogicalCatalogRelation {
/**
* Constructor for LogicalEsScan.
*/
public LogicalEsScan(RelationId id, TableIf table, List<String> qualifier,
Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, Set<Expression> conjuncts) {
super(id, PlanType.LOGICAL_ES_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties);
Optional<LogicalProperties> logicalProperties) {
super(id, PlanType.LOGICAL_ES_SCAN, table, qualifier, groupExpression, logicalProperties);
}
public LogicalEsScan(RelationId id, TableIf table, List<String> qualifier) {
this(id, table, qualifier, Optional.empty(), Optional.empty(), ImmutableSet.of());
this(id, table, qualifier, Optional.empty(), Optional.empty());
}
@Override
@ -62,26 +58,18 @@ public class LogicalEsScan extends LogicalExternalRelation {
@Override
public LogicalEsScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalEsScan(relationId, table, qualifier, groupExpression,
Optional.of(getLogicalProperties()), conjuncts);
Optional.of(getLogicalProperties()));
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new LogicalEsScan(relationId, table, qualifier, groupExpression, logicalProperties,
conjuncts);
}
@Override
public LogicalEsScan withConjuncts(Set<Expression> conjuncts) {
return new LogicalEsScan(relationId, table, qualifier, Optional.empty(),
Optional.of(getLogicalProperties()), conjuncts);
return new LogicalEsScan(relationId, table, qualifier, groupExpression, logicalProperties);
}
@Override
public LogicalEsScan withRelationId(RelationId relationId) {
return new LogicalEsScan(relationId, table, qualifier, Optional.empty(),
Optional.empty(), conjuncts);
return new LogicalEsScan(relationId, table, qualifier, Optional.empty(), Optional.empty());
}
@Override

68
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRelation.java
View File

@ -1,68 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
/**
* abstract class catalog relation for logical relation
*/
public abstract class LogicalExternalRelation extends LogicalCatalogRelation {
// TODO remove this conjuncts when old planner is removed
protected final Set<Expression> conjuncts;
public LogicalExternalRelation(RelationId relationId, PlanType type, TableIf table, List<String> qualifier,
Set<Expression> conjuncts,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties) {
super(relationId, type, table, qualifier, groupExpression, logicalProperties);
this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null"));
}
public abstract LogicalExternalRelation withConjuncts(Set<Expression> conjuncts);
@Override
public abstract LogicalExternalRelation withRelationId(RelationId relationId);
public Set<Expression> getConjuncts() {
return conjuncts;
}
@Override
public boolean equals(Object o) {
return super.equals(o) && Objects.equals(conjuncts, ((LogicalExternalRelation) o).conjuncts);
}
@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitLogicalExternalRelation(this, context);
}
}

32
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
View File

@ -23,7 +23,6 @@ import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.TableSample;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
@ -32,18 +31,16 @@ import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
/**
* Logical file scan for external catalog.
*/
public class LogicalFileScan extends LogicalExternalRelation {
public class LogicalFileScan extends LogicalCatalogRelation {
protected final SelectedPartitions selectedPartitions;
protected final Optional<TableSample> tableSample;
@ -54,18 +51,25 @@ public class LogicalFileScan extends LogicalExternalRelation {
*/
protected LogicalFileScan(RelationId id, ExternalTable table, List<String> qualifier,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
Set<Expression> conjuncts, SelectedPartitions selectedPartitions, Optional<TableSample> tableSample,
SelectedPartitions selectedPartitions, Optional<TableSample> tableSample,
Optional<TableSnapshot> tableSnapshot) {
super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties);
super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, groupExpression, logicalProperties);
this.selectedPartitions = selectedPartitions;
this.tableSample = tableSample;
this.tableSnapshot = tableSnapshot;
}
public LogicalFileScan(RelationId id, ExternalTable table, List<String> qualifier,
SelectedPartitions selectedPartitions,
Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot) {
this(id, table, qualifier, Optional.empty(), Optional.empty(),
selectedPartitions, tableSample, tableSnapshot);
}
public LogicalFileScan(RelationId id, ExternalTable table, List<String> qualifier,
Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot) {
this(id, table, qualifier, Optional.empty(), Optional.empty(),
Sets.newHashSet(), SelectedPartitions.NOT_PRUNED, tableSample, tableSnapshot);
SelectedPartitions.NOT_PRUNED, tableSample, tableSnapshot);
}
public SelectedPartitions getSelectedPartitions() {
@ -98,31 +102,25 @@ public class LogicalFileScan extends LogicalExternalRelation {
@Override
public LogicalFileScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, groupExpression,
Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot);
Optional.of(getLogicalProperties()), selectedPartitions, tableSample, tableSnapshot);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
groupExpression, logicalProperties, conjuncts, selectedPartitions, tableSample, tableSnapshot);
}
@Override
public LogicalFileScan withConjuncts(Set<Expression> conjuncts) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, Optional.empty(),
Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot);
groupExpression, logicalProperties, selectedPartitions, tableSample, tableSnapshot);
}
public LogicalFileScan withSelectedPartitions(SelectedPartitions selectedPartitions) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, Optional.empty(),
Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot);
Optional.of(getLogicalProperties()), selectedPartitions, tableSample, tableSnapshot);
}
@Override
public LogicalFileScan withRelationId(RelationId relationId) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, Optional.empty(),
Optional.empty(), conjuncts, selectedPartitions, tableSample, tableSnapshot);
Optional.empty(), selectedPartitions, tableSample, tableSnapshot);
}
@Override

24
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
View File

@ -44,7 +44,6 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -71,10 +70,10 @@ public class LogicalHudiScan extends LogicalFileScan {
*/
protected LogicalHudiScan(RelationId id, ExternalTable table, List<String> qualifier,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
Set<Expression> conjuncts, SelectedPartitions selectedPartitions, Optional<TableSample> tableSample,
SelectedPartitions selectedPartitions, Optional<TableSample> tableSample,
Optional<TableSnapshot> tableSnapshot,
Optional<TableScanParams> scanParams, Optional<IncrementalRelation> incrementalRelation) {
super(id, table, qualifier, groupExpression, logicalProperties, conjuncts,
super(id, table, qualifier, groupExpression, logicalProperties,
selectedPartitions, tableSample, tableSnapshot);
Objects.requireNonNull(scanParams, "scanParams should not null");
Objects.requireNonNull(incrementalRelation, "incrementalRelation should not null");
@ -85,7 +84,7 @@ public class LogicalHudiScan extends LogicalFileScan {
public LogicalHudiScan(RelationId id, ExternalTable table, List<String> qualifier,
Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot) {
this(id, table, qualifier, Optional.empty(), Optional.empty(),
Sets.newHashSet(), SelectedPartitions.NOT_PRUNED, tableSample, tableSnapshot,
SelectedPartitions.NOT_PRUNED, tableSample, tableSnapshot,
Optional.empty(), Optional.empty());
}
@ -136,7 +135,7 @@ public class LogicalHudiScan extends LogicalFileScan {
@Override
public LogicalHudiScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, groupExpression,
Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot,
Optional.of(getLogicalProperties()), selectedPartitions, tableSample, tableSnapshot,
scanParams, incrementalRelation);
}
@ -144,27 +143,20 @@ public class LogicalHudiScan extends LogicalFileScan {
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
groupExpression, logicalProperties, conjuncts, selectedPartitions, tableSample, tableSnapshot,
scanParams, incrementalRelation);
}
@Override
public LogicalHudiScan withConjuncts(Set<Expression> conjuncts) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, Optional.empty(),
Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot,
groupExpression, logicalProperties, selectedPartitions, tableSample, tableSnapshot,
scanParams, incrementalRelation);
}
public LogicalHudiScan withSelectedPartitions(SelectedPartitions selectedPartitions) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, Optional.empty(),
Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot,
Optional.of(getLogicalProperties()), selectedPartitions, tableSample, tableSnapshot,
scanParams, incrementalRelation);
}
@Override
public LogicalHudiScan withRelationId(RelationId relationId) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, Optional.empty(),
Optional.empty(), conjuncts, selectedPartitions, tableSample, tableSnapshot,
Optional.empty(), selectedPartitions, tableSample, tableSnapshot,
scanParams, incrementalRelation);
}
@ -223,7 +215,7 @@ public class LogicalHudiScan extends LogicalFileScan {
}
newScanParams = Optional.ofNullable(scanParams);
return new LogicalHudiScan(relationId, table, qualifier, Optional.empty(),
Optional.empty(), conjuncts, selectedPartitions, tableSample, tableSnapshot,
Optional.empty(), selectedPartitions, tableSample, tableSnapshot,
newScanParams, newIncrementalRelation);
}
}

24
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java
View File

@ -22,7 +22,6 @@ import org.apache.doris.catalog.TableIf;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
@ -30,29 +29,26 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
/**
* Logical scan for external jdbc catalog and jdbc table.
*/
public class LogicalJdbcScan extends LogicalExternalRelation {
public class LogicalJdbcScan extends LogicalCatalogRelation {
/**
* Constructor for LogicalJdbcScan.
*/
public LogicalJdbcScan(RelationId id, TableIf table, List<String> qualifier,
Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties,
Set<Expression> conjuncts) {
super(id, PlanType.LOGICAL_JDBC_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties);
Optional<LogicalProperties> logicalProperties) {
super(id, PlanType.LOGICAL_JDBC_SCAN, table, qualifier, groupExpression, logicalProperties);
}
public LogicalJdbcScan(RelationId id, TableIf table, List<String> qualifier) {
this(id, table, qualifier, Optional.empty(), Optional.empty(), ImmutableSet.of());
this(id, table, qualifier, Optional.empty(), Optional.empty());
}
@Override
@ -73,24 +69,18 @@ public class LogicalJdbcScan extends LogicalExternalRelation {
@Override
public LogicalJdbcScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalJdbcScan(relationId, table, qualifier, groupExpression,
Optional.of(getLogicalProperties()), conjuncts);
}
@Override
public LogicalJdbcScan withConjuncts(Set<Expression> conjuncts) {
return new LogicalJdbcScan(relationId, table, qualifier, Optional.empty(),
Optional.of(getLogicalProperties()), conjuncts);
Optional.of(getLogicalProperties()));
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new LogicalJdbcScan(relationId, table, qualifier, groupExpression, logicalProperties, conjuncts);
return new LogicalJdbcScan(relationId, table, qualifier, groupExpression, logicalProperties);
}
@Override
public LogicalJdbcScan withRelationId(RelationId relationId) {
return new LogicalJdbcScan(relationId, table, qualifier, Optional.empty(), Optional.empty(), conjuncts);
return new LogicalJdbcScan(relationId, table, qualifier, Optional.empty(), Optional.empty());
}
@Override

24
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java
View File

@ -21,7 +21,6 @@ import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
@ -29,26 +28,23 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
/**
* Logical scan for external odbc table.
*/
public class LogicalOdbcScan extends LogicalExternalRelation {
public class LogicalOdbcScan extends LogicalCatalogRelation {
public LogicalOdbcScan(RelationId id, TableIf table, List<String> qualifier,
Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties,
Set<Expression> conjuncts) {
super(id, PlanType.LOGICAL_ODBC_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties);
Optional<LogicalProperties> logicalProperties) {
super(id, PlanType.LOGICAL_ODBC_SCAN, table, qualifier, groupExpression, logicalProperties);
}
public LogicalOdbcScan(RelationId id, TableIf table, List<String> qualifier) {
this(id, table, qualifier, Optional.empty(), Optional.empty(), ImmutableSet.of());
this(id, table, qualifier, Optional.empty(), Optional.empty());
}
@Override
@ -69,24 +65,18 @@ public class LogicalOdbcScan extends LogicalExternalRelation {
@Override
public LogicalOdbcScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalOdbcScan(relationId, table, qualifier, groupExpression,
Optional.of(getLogicalProperties()), conjuncts);
}
@Override
public LogicalOdbcScan withConjuncts(Set<Expression> conjuncts) {
return new LogicalOdbcScan(relationId, table, qualifier, Optional.empty(),
Optional.of(getLogicalProperties()), conjuncts);
Optional.of(getLogicalProperties()));
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new LogicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties, conjuncts);
return new LogicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties);
}
@Override
public LogicalOdbcScan withRelationId(RelationId relationId) {
return new LogicalOdbcScan(relationId, table, qualifier, Optional.empty(), Optional.empty(), conjuncts);
return new LogicalOdbcScan(relationId, table, qualifier, Optional.empty(), Optional.empty());
}
@Override

23
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java
View File

@ -22,7 +22,6 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.DistributionSpec;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
@ -30,12 +29,8 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.statistics.Statistics;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
/**
* Physical es scan for external catalog.
@ -43,17 +38,15 @@ import java.util.Set;
public class PhysicalEsScan extends PhysicalCatalogRelation {
private final DistributionSpec distributionSpec;
private final Set<Expression> conjuncts;
/**
* Constructor for PhysicalEsScan.
*/
public PhysicalEsScan(RelationId id, TableIf table, List<String> qualifier,
DistributionSpec distributionSpec, Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties, Set<Expression> conjuncts) {
LogicalProperties logicalProperties) {
super(id, PlanType.PHYSICAL_ES_SCAN, table, qualifier, groupExpression, logicalProperties);
this.distributionSpec = distributionSpec;
this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null"));
}
/**
@ -61,12 +54,10 @@ public class PhysicalEsScan extends PhysicalCatalogRelation {
*/
public PhysicalEsScan(RelationId id, TableIf table, List<String> qualifier,
DistributionSpec distributionSpec, Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics,
Set<Expression> conjuncts) {
LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) {
super(id, PlanType.PHYSICAL_ES_SCAN, table, qualifier, groupExpression, logicalProperties,
physicalProperties, statistics);
this.distributionSpec = distributionSpec;
this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null"));
}
@Override
@ -86,24 +77,20 @@ public class PhysicalEsScan extends PhysicalCatalogRelation {
@Override
public PhysicalEsScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new PhysicalEsScan(relationId, getTable(), qualifier, distributionSpec,
groupExpression, getLogicalProperties(), conjuncts);
groupExpression, getLogicalProperties());
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new PhysicalEsScan(relationId, getTable(), qualifier, distributionSpec,
groupExpression, logicalProperties.get(), conjuncts);
groupExpression, logicalProperties.get());
}
@Override
public PhysicalEsScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties,
Statistics statsDeriveResult) {
return new PhysicalEsScan(relationId, getTable(), qualifier, distributionSpec,
groupExpression, getLogicalProperties(), physicalProperties, statsDeriveResult, conjuncts);
}
public Set<Expression> getConjuncts() {
return this.conjuncts;
groupExpression, getLogicalProperties(), physicalProperties, statsDeriveResult);
}
}

29
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java
View File

@ -24,7 +24,6 @@ import org.apache.doris.nereids.properties.DistributionSpec;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.TableSample;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
@ -35,7 +34,6 @@ import org.apache.doris.statistics.Statistics;
import java.util.List;
import java.util.Optional;
import java.util.Set;
/**
* Physical file scan for external catalog.
@ -43,7 +41,6 @@ import java.util.Set;
public class PhysicalFileScan extends PhysicalCatalogRelation {
protected final DistributionSpec distributionSpec;
protected final Set<Expression> conjuncts;
protected final SelectedPartitions selectedPartitions;
protected final Optional<TableSample> tableSample;
protected final Optional<TableSnapshot> tableSnapshot;
@ -53,11 +50,11 @@ public class PhysicalFileScan extends PhysicalCatalogRelation {
*/
public PhysicalFileScan(RelationId id, ExternalTable table, List<String> qualifier,
DistributionSpec distributionSpec, Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties, Set<Expression> conjuncts,
LogicalProperties logicalProperties,
SelectedPartitions selectedPartitions, Optional<TableSample> tableSample,
Optional<TableSnapshot> tableSnapshot) {
this(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, distributionSpec, groupExpression,
logicalProperties, conjuncts, selectedPartitions, tableSample, tableSnapshot);
logicalProperties, selectedPartitions, tableSample, tableSnapshot);
}
/**
@ -66,11 +63,10 @@ public class PhysicalFileScan extends PhysicalCatalogRelation {
public PhysicalFileScan(RelationId id, ExternalTable table, List<String> qualifier,
DistributionSpec distributionSpec, Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties, PhysicalProperties physicalProperties,
Statistics statistics, Set<Expression> conjuncts, SelectedPartitions selectedPartitions,
Statistics statistics, SelectedPartitions selectedPartitions,
Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot) {
this(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, distributionSpec, groupExpression,
logicalProperties, physicalProperties, statistics, conjuncts, selectedPartitions, tableSample,
tableSnapshot);
logicalProperties, physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot);
}
/**
@ -78,12 +74,11 @@ public class PhysicalFileScan extends PhysicalCatalogRelation {
*/
protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, List<String> qualifier,
DistributionSpec distributionSpec, Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties, Set<Expression> conjuncts,
LogicalProperties logicalProperties,
SelectedPartitions selectedPartitions, Optional<TableSample> tableSample,
Optional<TableSnapshot> tableSnapshot) {
super(id, type, table, qualifier, groupExpression, logicalProperties);
this.distributionSpec = distributionSpec;
this.conjuncts = conjuncts;
this.selectedPartitions = selectedPartitions;
this.tableSample = tableSample;
this.tableSnapshot = tableSnapshot;
@ -92,12 +87,11 @@ public class PhysicalFileScan extends PhysicalCatalogRelation {
protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, List<String> qualifier,
DistributionSpec distributionSpec, Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties, PhysicalProperties physicalProperties,
Statistics statistics, Set<Expression> conjuncts, SelectedPartitions selectedPartitions,
Statistics statistics, SelectedPartitions selectedPartitions,
Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot) {
super(id, type, table, qualifier, groupExpression, logicalProperties,
physicalProperties, statistics);
this.distributionSpec = distributionSpec;
this.conjuncts = conjuncts;
this.selectedPartitions = selectedPartitions;
this.tableSample = tableSample;
this.tableSnapshot = tableSnapshot;
@ -107,10 +101,6 @@ public class PhysicalFileScan extends PhysicalCatalogRelation {
return distributionSpec;
}
public Set<Expression> getConjuncts() {
return conjuncts;
}
public SelectedPartitions getSelectedPartitions() {
return selectedPartitions;
}
@ -129,7 +119,6 @@ public class PhysicalFileScan extends PhysicalCatalogRelation {
"qualified", Utils.qualifiedName(qualifier, table.getName()),
"output", getOutput(),
"stats", statistics,
"conjuncts", conjuncts,
"selected partitions num",
selectedPartitions.isPruned ? selectedPartitions.selectedPartitions.size() : "unknown"
);
@ -143,14 +132,14 @@ public class PhysicalFileScan extends PhysicalCatalogRelation {
@Override
public PhysicalFileScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec,
groupExpression, getLogicalProperties(), conjuncts, selectedPartitions, tableSample, tableSnapshot);
groupExpression, getLogicalProperties(), selectedPartitions, tableSample, tableSnapshot);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec,
groupExpression, logicalProperties.get(), conjuncts, selectedPartitions, tableSample, tableSnapshot);
groupExpression, logicalProperties.get(), selectedPartitions, tableSample, tableSnapshot);
}
@Override
@ -162,7 +151,7 @@ public class PhysicalFileScan extends PhysicalCatalogRelation {
public PhysicalFileScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties,
Statistics statistics) {
return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec,
groupExpression, getLogicalProperties(), physicalProperties, statistics, conjuncts,
groupExpression, getLogicalProperties(), physicalProperties, statistics,
selectedPartitions, tableSample, tableSnapshot);
}
}

17
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHudiScan.java
View File

@ -26,7 +26,6 @@ import org.apache.doris.nereids.properties.DistributionSpec;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.TableSample;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
@ -38,7 +37,6 @@ import org.apache.doris.statistics.Statistics;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
/**
* Physical Hudi scan for Hudi table.
@ -54,12 +52,12 @@ public class PhysicalHudiScan extends PhysicalFileScan {
*/
public PhysicalHudiScan(RelationId id, ExternalTable table, List<String> qualifier,
DistributionSpec distributionSpec, Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties, Set<Expression> conjuncts,
LogicalProperties logicalProperties,
SelectedPartitions selectedPartitions, Optional<TableSample> tableSample,
Optional<TableSnapshot> tableSnapshot,
Optional<TableScanParams> scanParams, Optional<IncrementalRelation> incrementalRelation) {
super(id, PlanType.PHYSICAL_HUDI_SCAN, table, qualifier, distributionSpec, groupExpression, logicalProperties,
conjuncts, selectedPartitions, tableSample, tableSnapshot);
selectedPartitions, tableSample, tableSnapshot);
Objects.requireNonNull(scanParams, "scanParams should not null");
Objects.requireNonNull(incrementalRelation, "incrementalRelation should not null");
this.scanParams = scanParams;
@ -72,11 +70,11 @@ public class PhysicalHudiScan extends PhysicalFileScan {
public PhysicalHudiScan(RelationId id, ExternalTable table, List<String> qualifier,
DistributionSpec distributionSpec, Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties, PhysicalProperties physicalProperties,
Statistics statistics, Set<Expression> conjuncts, SelectedPartitions selectedPartitions,
Statistics statistics, SelectedPartitions selectedPartitions,
Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot,
Optional<TableScanParams> scanParams, Optional<IncrementalRelation> incrementalRelation) {
super(id, PlanType.PHYSICAL_HUDI_SCAN, table, qualifier, distributionSpec, groupExpression, logicalProperties,
physicalProperties, statistics, conjuncts, selectedPartitions, tableSample, tableSnapshot);
physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot);
this.scanParams = scanParams;
this.incrementalRelation = incrementalRelation;
}
@ -92,7 +90,7 @@ public class PhysicalHudiScan extends PhysicalFileScan {
@Override
public PhysicalHudiScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new PhysicalHudiScan(relationId, getTable(), qualifier, distributionSpec,
groupExpression, getLogicalProperties(), conjuncts, selectedPartitions, tableSample, tableSnapshot,
groupExpression, getLogicalProperties(), selectedPartitions, tableSample, tableSnapshot,
scanParams, incrementalRelation);
}
@ -100,7 +98,7 @@ public class PhysicalHudiScan extends PhysicalFileScan {
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new PhysicalHudiScan(relationId, getTable(), qualifier, distributionSpec,
groupExpression, logicalProperties.get(), conjuncts, selectedPartitions, tableSample, tableSnapshot,
groupExpression, logicalProperties.get(), selectedPartitions, tableSample, tableSnapshot,
scanParams, incrementalRelation);
}
@ -108,7 +106,7 @@ public class PhysicalHudiScan extends PhysicalFileScan {
public PhysicalHudiScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties,
Statistics statistics) {
return new PhysicalHudiScan(relationId, getTable(), qualifier, distributionSpec,
groupExpression, getLogicalProperties(), physicalProperties, statistics, conjuncts,
groupExpression, getLogicalProperties(), physicalProperties, statistics,
selectedPartitions, tableSample, tableSnapshot,
scanParams, incrementalRelation);
}
@ -124,7 +122,6 @@ public class PhysicalHudiScan extends PhysicalFileScan {
"qualified", Utils.qualifiedName(qualifier, table.getName()),
"output", getOutput(),
"stats", statistics,
"conjuncts", conjuncts,
"selected partitions num",
selectedPartitions.isPruned ? selectedPartitions.selectedPartitions.size() : "unknown",
"isIncremental", incrementalRelation.isPresent()

25
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalJdbcScan.java
View File

@ -21,7 +21,6 @@ import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
@ -29,27 +28,21 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.statistics.Statistics;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
/**
* Physical jdbc scan for external catalog.
*/
public class PhysicalJdbcScan extends PhysicalCatalogRelation {
private final Set<Expression> conjuncts;
/**
* Constructor for PhysicalJdbcScan.
*/
public PhysicalJdbcScan(RelationId id, TableIf table, List<String> qualifier,
Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties, Set<Expression> conjuncts) {
Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties) {
this(id, table, qualifier, groupExpression, logicalProperties,
null, null, conjuncts);
null, null);
}
/**
@ -57,11 +50,9 @@ public class PhysicalJdbcScan extends PhysicalCatalogRelation {
*/
public PhysicalJdbcScan(RelationId id, TableIf table, List<String> qualifier,
Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics,
Set<Expression> conjuncts) {
LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) {
super(id, PlanType.PHYSICAL_JDBC_SCAN, table, qualifier, groupExpression,
logicalProperties, physicalProperties, statistics);
this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null"));
}
@Override
@ -80,23 +71,19 @@ public class PhysicalJdbcScan extends PhysicalCatalogRelation {
@Override
public PhysicalJdbcScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, getLogicalProperties(), conjuncts);
return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, getLogicalProperties());
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, logicalProperties.get(), conjuncts);
return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, logicalProperties.get());
}
@Override
public PhysicalJdbcScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties,
Statistics statistics) {
return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression,
getLogicalProperties(), physicalProperties, statistics, conjuncts);
}
public Set<Expression> getConjuncts() {
return this.conjuncts;
getLogicalProperties(), physicalProperties, statistics);
}
}

26
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java
View File

@ -21,7 +21,6 @@ import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
@ -29,27 +28,20 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.statistics.Statistics;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
/**
* Physical odbc scan for odbc table.
*/
public class PhysicalOdbcScan extends PhysicalCatalogRelation {
private final Set<Expression> conjuncts;
/**
* Constructor for PhysicalOdbcScan.
*/
public PhysicalOdbcScan(RelationId id, TableIf table, List<String> qualifier,
Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties, Set<Expression> conjuncts) {
Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties) {
this(id, table, qualifier, groupExpression, logicalProperties,
null, null, conjuncts);
null, null);
}
/**
@ -57,11 +49,9 @@ public class PhysicalOdbcScan extends PhysicalCatalogRelation {
*/
public PhysicalOdbcScan(RelationId id, TableIf table, List<String> qualifier,
Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics,
Set<Expression> conjuncts) {
LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) {
super(id, PlanType.PHYSICAL_ODBC_SCAN, table, qualifier, groupExpression,
logicalProperties, physicalProperties, statistics);
this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null"));
}
@Override
@ -80,23 +70,19 @@ public class PhysicalOdbcScan extends PhysicalCatalogRelation {
@Override
public PhysicalOdbcScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, getLogicalProperties(), conjuncts);
return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, getLogicalProperties());
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties.get(), conjuncts);
return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties.get());
}
@Override
public PhysicalOdbcScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties,
Statistics statistics) {
return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression,
getLogicalProperties(), physicalProperties, statistics, conjuncts);
}
public Set<Expression> getConjuncts() {
return this.conjuncts;
getLogicalProperties(), physicalProperties, statistics);
}
}

13
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
View File

@ -24,7 +24,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalExternalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalHudiScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
@ -95,16 +94,12 @@ public interface RelationVisitor<R, C> {
return visitLogicalRelation(emptyRelation, context);
}
default R visitLogicalExternalRelation(LogicalExternalRelation relation, C context) {
return visitLogicalCatalogRelation(relation, context);
}
default R visitLogicalEsScan(LogicalEsScan esScan, C context) {
return visitLogicalExternalRelation(esScan, context);
return visitLogicalCatalogRelation(esScan, context);
}
default R visitLogicalFileScan(LogicalFileScan fileScan, C context) {
return visitLogicalExternalRelation(fileScan, context);
return visitLogicalCatalogRelation(fileScan, context);
}
default R visitLogicalHudiScan(LogicalHudiScan fileScan, C context) {
@ -112,11 +107,11 @@ public interface RelationVisitor<R, C> {
}
default R visitLogicalJdbcScan(LogicalJdbcScan jdbcScan, C context) {
return visitLogicalExternalRelation(jdbcScan, context);
return visitLogicalCatalogRelation(jdbcScan, context);
}
default R visitLogicalOdbcScan(LogicalOdbcScan odbcScan, C context) {
return visitLogicalExternalRelation(odbcScan, context);
return visitLogicalCatalogRelation(odbcScan, context);
}
default R visitLogicalOlapScan(LogicalOlapScan olapScan, C context) {

30
fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java vendored
View File

@ -49,6 +49,7 @@ import java.util.Optional;
public class HmsCatalogTest extends AnalyzeCheckTestBase {
private static final String HMS_CATALOG = "hms_ctl";
private static final long NOW = System.currentTimeMillis();
private Env env;
private CatalogMgr mgr;
@ -101,6 +102,11 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase {
Deencapsulation.setField(db, "initialized", true);
Deencapsulation.setField(tbl, "objectCreated", true);
Deencapsulation.setField(tbl, "schemaUpdateTime", NOW);
Deencapsulation.setField(tbl, "eventUpdateTime", 0);
Deencapsulation.setField(tbl, "catalog", hmsCatalog);
Deencapsulation.setField(tbl, "dbName", "hms_db");
Deencapsulation.setField(tbl, "name", "hms_tbl");
new Expectations(tbl) {
{
tbl.getId();
@ -131,6 +137,10 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase {
minTimes = 0;
result = TableIf.TableType.HMS_EXTERNAL_TABLE;
// mock initSchemaAndUpdateTime and do nothing
tbl.initSchemaAndUpdateTime();
minTimes = 0;
tbl.getDatabase();
minTimes = 0;
result = db;
@ -142,6 +152,11 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase {
};
Deencapsulation.setField(view1, "objectCreated", true);
Deencapsulation.setField(view1, "schemaUpdateTime", NOW);
Deencapsulation.setField(view1, "eventUpdateTime", 0);
Deencapsulation.setField(view1, "catalog", hmsCatalog);
Deencapsulation.setField(view1, "dbName", "hms_db");
Deencapsulation.setField(view1, "name", "hms_view1");
new Expectations(view1) {
{
@ -188,6 +203,11 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase {
};
Deencapsulation.setField(view2, "objectCreated", true);
Deencapsulation.setField(view2, "schemaUpdateTime", NOW);
Deencapsulation.setField(view2, "eventUpdateTime", 0);
Deencapsulation.setField(view2, "catalog", hmsCatalog);
Deencapsulation.setField(view2, "dbName", "hms_db");
Deencapsulation.setField(view2, "name", "hms_view2");
new Expectations(view2) {
{
@ -234,6 +254,11 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase {
};
Deencapsulation.setField(view3, "objectCreated", true);
Deencapsulation.setField(view3, "schemaUpdateTime", NOW);
Deencapsulation.setField(view3, "eventUpdateTime", 0);
Deencapsulation.setField(view3, "catalog", hmsCatalog);
Deencapsulation.setField(view3, "dbName", "hms_db");
Deencapsulation.setField(view3, "name", "hms_view3");
new Expectations(view3) {
{
@ -280,6 +305,11 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase {
};
Deencapsulation.setField(view4, "objectCreated", true);
Deencapsulation.setField(view4, "schemaUpdateTime", NOW);
Deencapsulation.setField(view4, "eventUpdateTime", 0);
Deencapsulation.setField(view4, "catalog", hmsCatalog);
Deencapsulation.setField(view4, "dbName", "hms_db");
Deencapsulation.setField(view4, "name", "hms_view4");
new Expectations(view4) {
{

75
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/AdjustNullableTest.java
View File

@ -1,75 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
import org.apache.doris.nereids.util.PlanConstructor;
import mockit.Mock;
import mockit.MockUp;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
/**
* Tests for {@link AdjustNullableTest}.
*/
class AdjustNullableTest implements MemoPatternMatchSupported {
private final LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0);
@Test
void testLogicalExternalRelation() {
new MockUp<LogicalJdbcScan>() {
@Mock
public Set<Slot> getOutputSet() {
Set<Slot> output = new HashSet<>();
output.add(new SlotReference(new ExprId(1), "id", IntegerType.INSTANCE, false,
new ArrayList<>()));
return output;
}
};
GreaterThan gt = new GreaterThan(new SlotReference(new ExprId(1), "id",
IntegerType.INSTANCE, true, new ArrayList<>()), Literal.of("1"));
Set<Expression> conjuncts = new HashSet<>();
conjuncts.add(gt);
Assertions.assertTrue(conjuncts.iterator().next().nullable());
LogicalJdbcScan jdbcScan =
new LogicalJdbcScan(new RelationId(1), PlanConstructor.newOlapTable(0, "t1", 0),
new ArrayList<>(), Optional.empty(), Optional.empty(), conjuncts);
AdjustNullable adjustNullable = new AdjustNullable();
LogicalJdbcScan newJdbcScan = (LogicalJdbcScan) adjustNullable.rewriteRoot(jdbcScan, null);
conjuncts = newJdbcScan.getConjuncts();
Assertions.assertFalse(conjuncts.iterator().next().nullable());
}
}

6
fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java
View File

@ -121,6 +121,9 @@ public class HmsQueryCacheTest extends AnalyzeCheckTestBase {
Deencapsulation.setField(tbl, "objectCreated", true);
Deencapsulation.setField(tbl, "schemaUpdateTime", NOW);
Deencapsulation.setField(tbl, "eventUpdateTime", 0);
Deencapsulation.setField(tbl, "catalog", hmsCatalog);
Deencapsulation.setField(tbl, "dbName", "hms_db");
Deencapsulation.setField(tbl, "name", "hms_tbl");
new Expectations(tbl) {
{
tbl.getId();
@ -168,6 +171,9 @@ public class HmsQueryCacheTest extends AnalyzeCheckTestBase {
Deencapsulation.setField(tbl2, "objectCreated", true);
Deencapsulation.setField(tbl2, "schemaUpdateTime", NOW);
Deencapsulation.setField(tbl2, "eventUpdateTime", 0);
Deencapsulation.setField(tbl2, "catalog", hmsCatalog);
Deencapsulation.setField(tbl2, "dbName", "hms_db");
Deencapsulation.setField(tbl2, "name", "hms_tbl2");
new Expectations(tbl2) {
{
tbl2.getId();