From 56c2a0f523f403b3a57cbba554a415c3a25310cc Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Tue, 22 Oct 2024 21:42:55 +0800 Subject: [PATCH] [2.1][opt](Catalog) Remove unnecessary conjuncts handling on External Scan (#42261) pick (#41218) In the previous FileScanNode, some parts that used conjuncts for predicate conversion were placed in the init phase. However, for the Nereids planner, pushing the filter down to the scan happens in the Translator, which means that the ScanNode can only get the complete conjuncts in the finalized phase. Therefore, in this PR, I have removed all conjuncts variables in External for the Nereids planner. They no longer need to store conjuncts themselves or add them to the ScanNode. Instead, all places in the ScanNode that use conjuncts should be moved to the finalized phase. This refactor also fix a performance issue introduced from #40176 After introducing the change of generating SelectNode for consecutive projects or filters, FileScan still adds conjuncts too early in the init phase, resulting in the discovery of consecutive filters when the upper layer continues to translate, a selectnode was unexpectedly generated on the scannode, causing the project to be unable to prune the scannode columns. However, the Project node trims columns of SelectNode and ScanNode differently, which causes ScanNode to scan unnecessary columns. My modification removes the addition of conjuncts in the scannode step, so that we can keep the structure from ScanNode to Project and achieve correct column trimming. --- .../doris/datasource/FileQueryScanNode.java | 11 ++- .../datasource/es/source/EsScanNode.java | 2 + .../datasource/hive/HMSExternalTable.java | 16 ++++ .../datasource/hive/source/HiveScanNode.java | 39 +--------- .../maxcompute/source/MaxComputeScanNode.java | 14 ++-- .../paimon/source/PaimonScanNode.java | 4 + .../translator/PhysicalPlanTranslator.java | 4 - .../doris/nereids/jobs/executor/Rewriter.java | 8 +- .../nereids/rules/analysis/BindRelation.java | 4 +- .../LogicalEsScanToPhysicalEsScan.java | 3 +- .../LogicalFileScanToPhysicalFileScan.java | 1 - .../LogicalHudiScanToPhysicalHudiScan.java | 1 - .../LogicalJdbcScanToPhysicalJdbcScan.java | 3 +- .../LogicalOdbcScanToPhysicalOdbcScan.java | 3 +- .../nereids/rules/rewrite/AdjustNullable.java | 12 --- .../rules/rewrite/PruneFileScanPartition.java | 12 +-- .../rewrite/PushConjunctsIntoEsScan.java | 39 ---------- .../rewrite/PushConjunctsIntoJdbcScan.java | 39 ---------- .../rewrite/PushConjunctsIntoOdbcScan.java | 39 ---------- .../trees/copier/LogicalPlanDeepCopier.java | 17 ----- .../trees/plans/logical/LogicalEsScan.java | 26 ++----- .../logical/LogicalExternalRelation.java | 68 ----------------- .../trees/plans/logical/LogicalFileScan.java | 32 ++++---- .../trees/plans/logical/LogicalHudiScan.java | 24 ++---- .../trees/plans/logical/LogicalJdbcScan.java | 24 ++---- .../trees/plans/logical/LogicalOdbcScan.java | 24 ++---- .../trees/plans/physical/PhysicalEsScan.java | 23 ++---- .../plans/physical/PhysicalFileScan.java | 29 +++---- .../plans/physical/PhysicalHudiScan.java | 17 ++--- .../plans/physical/PhysicalJdbcScan.java | 25 ++----- .../plans/physical/PhysicalOdbcScan.java | 26 ++----- .../trees/plans/visitor/RelationVisitor.java | 13 +--- .../doris/external/hms/HmsCatalogTest.java | 30 ++++++++ .../rules/rewrite/AdjustNullableTest.java | 75 ------------------- .../apache/doris/qe/HmsQueryCacheTest.java | 6 ++ 35 files changed, 169 insertions(+), 544 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoEsScan.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoJdbcScan.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRelation.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/AdjustNullableTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index b4a31611a7..116f0a795f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -144,7 +144,6 @@ public abstract class FileQueryScanNode extends FileScanNode { table.getName())); } } - computeColumnsFilter(); initBackendPolicy(); initSchemaParams(); } @@ -213,6 +212,7 @@ public abstract class FileQueryScanNode extends FileScanNode { if (ConnectContext.get().getExecutor() != null) { ConnectContext.get().getExecutor().getSummaryProfile().setFinalizeScanNodeStartTime(); } + convertPredicate(); createScanRangeLocations(); updateRequiredSlots(); if (ConnectContext.get().getExecutor() != null) { @@ -220,6 +220,15 @@ public abstract class FileQueryScanNode extends FileScanNode { } } + /** + * Used as a predicate to convert conjuncts into corresponding data sources. + * All predicate conversions from different data sources should override this method. + * and this method must be called in finalize, + * because in nereids planner, conjuncts are only generated in the finalize stage. + */ + protected void convertPredicate() { + } + private void setColumnPositionMapping() throws UserException { TableIf tbl = getTargetTable(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java index 3b0d2f08b2..c91fd86a08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java @@ -114,11 +114,13 @@ public class EsScanNode extends ExternalScanNode { @Override public void finalize(Analyzer analyzer) throws UserException { + buildQuery(); doFinalize(); } @Override public void finalizeForNereids() throws UserException { + buildQuery(); doFinalize(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index e6a7828a18..8f10af4bc2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -37,6 +37,7 @@ import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.mtmv.MTMVSnapshotIf; import org.apache.doris.mtmv.MTMVTimestampSnapshot; import org.apache.doris.nereids.exceptions.NotSupportedException; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; import org.apache.doris.qe.GlobalVariable; import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.BaseAnalysisTask; @@ -288,6 +289,21 @@ public class HMSExternalTable extends ExternalTable implements MTMVRelatedTableI .orElse(Collections.emptyList()); } + public SelectedPartitions getAllPartitions() { + if (CollectionUtils.isEmpty(this.getPartitionColumns())) { + return SelectedPartitions.NOT_PRUNED; + } + + HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() + .getMetaStoreCache((HMSExternalCatalog) this.getCatalog()); + List partitionColumnTypes = this.getPartitionColumnTypes(); + HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( + this.getDbName(), this.getName(), partitionColumnTypes); + Map idToPartitionItem = hivePartitionValues.getIdToPartitionItem(); + + return new SelectedPartitions(idToPartitionItem.size(), idToPartitionItem, false); + } + public boolean isHiveTransactionalTable() { return dlaType == DLAType.HIVE && AcidUtils.isTransactionalTable(remoteTable) && isSupportedTransactionalFileFormat(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java index 435967cef0..1dbcd5064f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java @@ -42,7 +42,6 @@ import org.apache.doris.datasource.hive.HiveProperties; import org.apache.doris.datasource.hive.HiveTransaction; import org.apache.doris.datasource.hive.source.HiveSplit.HiveSplitCreator; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; -import org.apache.doris.planner.ListPartitionPrunerV2; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.qe.ConnectContext; import org.apache.doris.spi.Split; @@ -123,46 +122,16 @@ public class HiveScanNode extends FileQueryScanNode { protected List getPartitions() throws AnalysisException { List resPartitions = Lists.newArrayList(); - long start = System.currentTimeMillis(); HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() .getMetaStoreCache((HMSExternalCatalog) hmsTable.getCatalog()); List partitionColumnTypes = hmsTable.getPartitionColumnTypes(); if (!partitionColumnTypes.isEmpty()) { // partitioned table - boolean isPartitionPruned = selectedPartitions != null && selectedPartitions.isPruned; Collection partitionItems; - if (!isPartitionPruned) { - // partitionItems is null means that the partition is not pruned by Nereids, - // so need to prune partitions here by legacy ListPartitionPrunerV2. - HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( - hmsTable.getDbName(), hmsTable.getName(), partitionColumnTypes); - Map idToPartitionItem = hivePartitionValues.getIdToPartitionItem(); - this.totalPartitionNum = idToPartitionItem.size(); - if (!conjuncts.isEmpty()) { - ListPartitionPrunerV2 pruner = new ListPartitionPrunerV2(idToPartitionItem, - hmsTable.getPartitionColumns(), columnNameToRange, - hivePartitionValues.getUidToPartitionRange(), - hivePartitionValues.getRangeToId(), - hivePartitionValues.getSingleColumnRangeMap(), - true); - Collection filteredPartitionIds = pruner.prune(); - if (LOG.isDebugEnabled()) { - LOG.debug("hive partition fetch and prune for table {}.{} cost: {} ms", - hmsTable.getDbName(), hmsTable.getName(), (System.currentTimeMillis() - start)); - } - partitionItems = Lists.newArrayListWithCapacity(filteredPartitionIds.size()); - for (Long id : filteredPartitionIds) { - partitionItems.add(idToPartitionItem.get(id)); - } - } else { - partitionItems = idToPartitionItem.values(); - } - } else { - // partitions has benn pruned by Nereids, in PruneFileScanPartition, - // so just use the selected partitions. - this.totalPartitionNum = selectedPartitions.totalPartitionNum; - partitionItems = selectedPartitions.selectedPartitions.values(); - } + // partitions has benn pruned by Nereids, in PruneFileScanPartition, + // so just use the selected partitions. + this.totalPartitionNum = selectedPartitions.totalPartitionNum; + partitionItems = selectedPartitions.selectedPartitions.values(); Preconditions.checkNotNull(partitionItems); this.selectedPartitionNum = partitionItems.size(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java index 856bcfff16..4ec4319c22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java @@ -82,6 +82,7 @@ public class MaxComputeScanNode extends FileQueryScanNode { private final MaxComputeExternalTable table; TableBatchReadSession tableBatchReadSession; + private Predicate filterPredicate; public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) { this(id, desc, "MCScanNode", StatisticalType.MAX_COMPUTE_SCAN_NODE, needCheckColumnPriv); @@ -115,8 +116,6 @@ public class MaxComputeScanNode extends FileQueryScanNode { } void createTableBatchReadSession() throws UserException { - Predicate filterPredicate = convertPredicate(); - List requiredPartitionColumns = new ArrayList<>(); List orderedRequiredDataColumns = new ArrayList<>(); @@ -164,9 +163,10 @@ public class MaxComputeScanNode extends FileQueryScanNode { } - protected Predicate convertPredicate() { + @Override + protected void convertPredicate() { if (conjuncts.isEmpty()) { - return Predicate.NO_PREDICATE; + this.filterPredicate = Predicate.NO_PREDICATE; } List odpsPredicates = new ArrayList<>(); @@ -180,9 +180,9 @@ public class MaxComputeScanNode extends FileQueryScanNode { } if (odpsPredicates.isEmpty()) { - return Predicate.NO_PREDICATE; + this.filterPredicate = Predicate.NO_PREDICATE; } else if (odpsPredicates.size() == 1) { - return odpsPredicates.get(0); + this.filterPredicate = odpsPredicates.get(0); } else { com.aliyun.odps.table.optimizer.predicate.CompoundPredicate filterPredicate = new com.aliyun.odps.table.optimizer.predicate.CompoundPredicate( @@ -191,7 +191,7 @@ public class MaxComputeScanNode extends FileQueryScanNode { for (Predicate odpsPredicate : odpsPredicates) { filterPredicate.addPredicate(odpsPredicate); } - return filterPredicate; + this.filterPredicate = filterPredicate; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index e0d0f9a3ea..02f831ba37 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -111,6 +111,10 @@ public class PaimonScanNode extends FileQueryScanNode { super.doInitialize(); source = new PaimonSource(desc); Preconditions.checkNotNull(source); + } + + @Override + protected void convertPredicate() { PaimonPredicateConverter paimonPredicateConverter = new PaimonPredicateConverter( source.getPaimonTable().rowType()); predicates = paimonPredicateConverter.convertToPaimonExpr(conjuncts); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index b9a6c169c2..0bc8b28461 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -649,7 +649,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor> imple return cteConsumer.withTwoMaps(consumerToProducerOutputMap, producerToConsumerOutputMap); } - @Override - public Plan visitLogicalExternalRelation(LogicalExternalRelation relation, Map replaceMap) { - if (!relation.getConjuncts().isEmpty()) { - relation.getOutputSet().forEach(s -> replaceMap.put(s.getExprId(), s)); - Set conjuncts = updateExpressions(relation.getConjuncts(), replaceMap); - return relation.withConjuncts(conjuncts).recomputeLogicalProperties(); - } else { - return relation; - } - } - private T updateExpression(T input, Map replaceMap) { return (T) input.rewriteDownShortCircuit(e -> e.accept(SlotReferenceReplacer.INSTANCE, replaceMap)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java index 9b2f499901..2de4efab2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java @@ -17,13 +17,10 @@ package org.apache.doris.nereids.rules.rewrite; -import org.apache.doris.catalog.Env; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.datasource.ExternalTable; -import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; -import org.apache.doris.datasource.hive.HiveMetaStoreCache; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; @@ -72,8 +69,7 @@ public class PruneFileScanPartition extends OneRewriteRuleFactory { selectedPartitions = new SelectedPartitions(0, ImmutableMap.of(), true); } - LogicalFileScan rewrittenScan = scan.withConjuncts(filter.getConjuncts()) - .withSelectedPartitions(selectedPartitions); + LogicalFileScan rewrittenScan = scan.withSelectedPartitions(selectedPartitions); return new LogicalFilter<>(filter.getConjuncts(), rewrittenScan); }).toRule(RuleType.FILE_SCAN_PARTITION_PRUNE); } @@ -95,11 +91,7 @@ public class PruneFileScanPartition extends OneRewriteRuleFactory { .map(column -> scanOutput.get(column.getName().toLowerCase())) .collect(Collectors.toList()); - HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() - .getMetaStoreCache((HMSExternalCatalog) hiveTbl.getCatalog()); - HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( - hiveTbl.getDbName(), hiveTbl.getName(), hiveTbl.getPartitionColumnTypes()); - Map idToPartitionItem = hivePartitionValues.getIdToPartitionItem(); + Map idToPartitionItem = scan.getSelectedPartitions().selectedPartitions; List prunedPartitions = new ArrayList<>(PartitionPruner.prune( partitionSlots, filter.getPredicate(), idToPartitionItem, ctx, PartitionTableType.HIVE)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoEsScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoEsScan.java deleted file mode 100644 index 9bd485716d..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoEsScan.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; - -/** - * Rewrite es plan to set the conjuncts. - */ -public class PushConjunctsIntoEsScan extends OneRewriteRuleFactory { - - @Override - public Rule build() { - return logicalFilter(logicalEsScan()).thenApply(ctx -> { - LogicalFilter filter = ctx.root; - LogicalEsScan scan = filter.child(); - LogicalEsScan rewrittenScan = scan.withConjuncts(filter.getConjuncts()); - return rewrittenScan; - }).toRule(RuleType.PUSH_CONJUNCTS_INTO_ES_SCAN); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoJdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoJdbcScan.java deleted file mode 100644 index 21d723f956..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoJdbcScan.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan; - -/** - * Rewrite jdbc plan to set the conjuncts. - */ -public class PushConjunctsIntoJdbcScan extends OneRewriteRuleFactory { - - @Override - public Rule build() { - return logicalFilter(logicalJdbcScan()).thenApply(ctx -> { - LogicalFilter filter = ctx.root; - LogicalJdbcScan scan = filter.child(); - LogicalJdbcScan rewrittenScan = scan.withConjuncts(filter.getConjuncts()); - return new LogicalFilter<>(filter.getConjuncts(), rewrittenScan); - }).toRule(RuleType.PUSH_CONJUNCTS_INTO_JDBC_SCAN); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java deleted file mode 100644 index 8a1ba4d1b6..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan; - -/** - * Rewrite odbc plan to set the conjuncts. - */ -public class PushConjunctsIntoOdbcScan extends OneRewriteRuleFactory { - - @Override - public Rule build() { - return logicalFilter(logicalOdbcScan()).thenApply(ctx -> { - LogicalFilter filter = ctx.root; - LogicalOdbcScan scan = filter.child(); - LogicalOdbcScan rewrittenScan = scan.withConjuncts(filter.getConjuncts()); - return new LogicalFilter<>(filter.getConjuncts(), rewrittenScan); - }).toRule(RuleType.PUSH_CONJUNCTS_INTO_ODBC_SCAN); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java index 7b7c95dd81..20e72b81aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java @@ -40,7 +40,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapS import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; -import org.apache.doris.nereids.trees.plans.logical.LogicalExternalRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; @@ -189,22 +188,6 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter conjuncts = relation.getConjuncts().stream() - .map(p -> ExpressionDeepCopier.INSTANCE.deepCopy(p, context)) - .collect(ImmutableSet.toImmutableSet()); - newRelation = newRelation.withConjuncts(conjuncts); - context.putRelation(relation.getRelationId(), newRelation); - return newRelation; - } - @Override public Plan visitLogicalProject(LogicalProject project, DeepCopierContext context) { Plan child = project.child().accept(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java index 5895469442..8fb916b8a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java @@ -20,35 +20,31 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; -import com.google.common.collect.ImmutableSet; - import java.util.List; import java.util.Optional; -import java.util.Set; /** * Logical scan for external es catalog. */ -public class LogicalEsScan extends LogicalExternalRelation { +public class LogicalEsScan extends LogicalCatalogRelation { /** * Constructor for LogicalEsScan. */ public LogicalEsScan(RelationId id, TableIf table, List qualifier, Optional groupExpression, - Optional logicalProperties, Set conjuncts) { - super(id, PlanType.LOGICAL_ES_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties); + Optional logicalProperties) { + super(id, PlanType.LOGICAL_ES_SCAN, table, qualifier, groupExpression, logicalProperties); } public LogicalEsScan(RelationId id, TableIf table, List qualifier) { - this(id, table, qualifier, Optional.empty(), Optional.empty(), ImmutableSet.of()); + this(id, table, qualifier, Optional.empty(), Optional.empty()); } @Override @@ -62,26 +58,18 @@ public class LogicalEsScan extends LogicalExternalRelation { @Override public LogicalEsScan withGroupExpression(Optional groupExpression) { return new LogicalEsScan(relationId, table, qualifier, groupExpression, - Optional.of(getLogicalProperties()), conjuncts); + Optional.of(getLogicalProperties())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalEsScan(relationId, table, qualifier, groupExpression, logicalProperties, - conjuncts); - } - - @Override - public LogicalEsScan withConjuncts(Set conjuncts) { - return new LogicalEsScan(relationId, table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts); + return new LogicalEsScan(relationId, table, qualifier, groupExpression, logicalProperties); } @Override public LogicalEsScan withRelationId(RelationId relationId) { - return new LogicalEsScan(relationId, table, qualifier, Optional.empty(), - Optional.empty(), conjuncts); + return new LogicalEsScan(relationId, table, qualifier, Optional.empty(), Optional.empty()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRelation.java deleted file mode 100644 index bc6313f529..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRelation.java +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.logical; - -import org.apache.doris.catalog.TableIf; -import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.plans.PlanType; -import org.apache.doris.nereids.trees.plans.RelationId; -import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; - -import com.google.common.collect.ImmutableSet; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -/** - * abstract class catalog relation for logical relation - */ -public abstract class LogicalExternalRelation extends LogicalCatalogRelation { - - // TODO remove this conjuncts when old planner is removed - protected final Set conjuncts; - - public LogicalExternalRelation(RelationId relationId, PlanType type, TableIf table, List qualifier, - Set conjuncts, - Optional groupExpression, Optional logicalProperties) { - super(relationId, type, table, qualifier, groupExpression, logicalProperties); - this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null")); - } - - public abstract LogicalExternalRelation withConjuncts(Set conjuncts); - - @Override - public abstract LogicalExternalRelation withRelationId(RelationId relationId); - - public Set getConjuncts() { - return conjuncts; - } - - @Override - public boolean equals(Object o) { - return super.equals(o) && Objects.equals(conjuncts, ((LogicalExternalRelation) o).conjuncts); - } - - @Override - public R accept(PlanVisitor visitor, C context) { - return visitor.visitLogicalExternalRelation(this, context); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java index 8dd47c44b1..0a2c69b68c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java @@ -23,7 +23,6 @@ import org.apache.doris.datasource.ExternalTable; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.TableSample; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -32,18 +31,16 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Sets; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; /** * Logical file scan for external catalog. */ -public class LogicalFileScan extends LogicalExternalRelation { +public class LogicalFileScan extends LogicalCatalogRelation { protected final SelectedPartitions selectedPartitions; protected final Optional tableSample; @@ -54,18 +51,25 @@ public class LogicalFileScan extends LogicalExternalRelation { */ protected LogicalFileScan(RelationId id, ExternalTable table, List qualifier, Optional groupExpression, Optional logicalProperties, - Set conjuncts, SelectedPartitions selectedPartitions, Optional tableSample, + SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot) { - super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties); + super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, groupExpression, logicalProperties); this.selectedPartitions = selectedPartitions; this.tableSample = tableSample; this.tableSnapshot = tableSnapshot; } + public LogicalFileScan(RelationId id, ExternalTable table, List qualifier, + SelectedPartitions selectedPartitions, + Optional tableSample, Optional tableSnapshot) { + this(id, table, qualifier, Optional.empty(), Optional.empty(), + selectedPartitions, tableSample, tableSnapshot); + } + public LogicalFileScan(RelationId id, ExternalTable table, List qualifier, Optional tableSample, Optional tableSnapshot) { this(id, table, qualifier, Optional.empty(), Optional.empty(), - Sets.newHashSet(), SelectedPartitions.NOT_PRUNED, tableSample, tableSnapshot); + SelectedPartitions.NOT_PRUNED, tableSample, tableSnapshot); } public SelectedPartitions getSelectedPartitions() { @@ -98,31 +102,25 @@ public class LogicalFileScan extends LogicalExternalRelation { @Override public LogicalFileScan withGroupExpression(Optional groupExpression) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, groupExpression, - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot); + Optional.of(getLogicalProperties()), selectedPartitions, tableSample, tableSnapshot); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, - groupExpression, logicalProperties, conjuncts, selectedPartitions, tableSample, tableSnapshot); - } - - @Override - public LogicalFileScan withConjuncts(Set conjuncts) { - return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot); + groupExpression, logicalProperties, selectedPartitions, tableSample, tableSnapshot); } public LogicalFileScan withSelectedPartitions(SelectedPartitions selectedPartitions) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot); + Optional.of(getLogicalProperties()), selectedPartitions, tableSample, tableSnapshot); } @Override public LogicalFileScan withRelationId(RelationId relationId) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.empty(), conjuncts, selectedPartitions, tableSample, tableSnapshot); + Optional.empty(), selectedPartitions, tableSample, tableSnapshot); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java index deeca65efc..6296908894 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java @@ -44,7 +44,6 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -71,10 +70,10 @@ public class LogicalHudiScan extends LogicalFileScan { */ protected LogicalHudiScan(RelationId id, ExternalTable table, List qualifier, Optional groupExpression, Optional logicalProperties, - Set conjuncts, SelectedPartitions selectedPartitions, Optional tableSample, + SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot, Optional scanParams, Optional incrementalRelation) { - super(id, table, qualifier, groupExpression, logicalProperties, conjuncts, + super(id, table, qualifier, groupExpression, logicalProperties, selectedPartitions, tableSample, tableSnapshot); Objects.requireNonNull(scanParams, "scanParams should not null"); Objects.requireNonNull(incrementalRelation, "incrementalRelation should not null"); @@ -85,7 +84,7 @@ public class LogicalHudiScan extends LogicalFileScan { public LogicalHudiScan(RelationId id, ExternalTable table, List qualifier, Optional tableSample, Optional tableSnapshot) { this(id, table, qualifier, Optional.empty(), Optional.empty(), - Sets.newHashSet(), SelectedPartitions.NOT_PRUNED, tableSample, tableSnapshot, + SelectedPartitions.NOT_PRUNED, tableSample, tableSnapshot, Optional.empty(), Optional.empty()); } @@ -136,7 +135,7 @@ public class LogicalHudiScan extends LogicalFileScan { @Override public LogicalHudiScan withGroupExpression(Optional groupExpression) { return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, groupExpression, - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot, + Optional.of(getLogicalProperties()), selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation); } @@ -144,27 +143,20 @@ public class LogicalHudiScan extends LogicalFileScan { public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, - groupExpression, logicalProperties, conjuncts, selectedPartitions, tableSample, tableSnapshot, - scanParams, incrementalRelation); - } - - @Override - public LogicalHudiScan withConjuncts(Set conjuncts) { - return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot, + groupExpression, logicalProperties, selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation); } public LogicalHudiScan withSelectedPartitions(SelectedPartitions selectedPartitions) { return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot, + Optional.of(getLogicalProperties()), selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation); } @Override public LogicalHudiScan withRelationId(RelationId relationId) { return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.empty(), conjuncts, selectedPartitions, tableSample, tableSnapshot, + Optional.empty(), selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation); } @@ -223,7 +215,7 @@ public class LogicalHudiScan extends LogicalFileScan { } newScanParams = Optional.ofNullable(scanParams); return new LogicalHudiScan(relationId, table, qualifier, Optional.empty(), - Optional.empty(), conjuncts, selectedPartitions, tableSample, tableSnapshot, + Optional.empty(), selectedPartitions, tableSample, tableSnapshot, newScanParams, newIncrementalRelation); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java index cde2b6be24..53176f72c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java @@ -22,7 +22,6 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -30,29 +29,26 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Optional; -import java.util.Set; /** * Logical scan for external jdbc catalog and jdbc table. */ -public class LogicalJdbcScan extends LogicalExternalRelation { +public class LogicalJdbcScan extends LogicalCatalogRelation { /** * Constructor for LogicalJdbcScan. */ public LogicalJdbcScan(RelationId id, TableIf table, List qualifier, Optional groupExpression, - Optional logicalProperties, - Set conjuncts) { - super(id, PlanType.LOGICAL_JDBC_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties); + Optional logicalProperties) { + super(id, PlanType.LOGICAL_JDBC_SCAN, table, qualifier, groupExpression, logicalProperties); } public LogicalJdbcScan(RelationId id, TableIf table, List qualifier) { - this(id, table, qualifier, Optional.empty(), Optional.empty(), ImmutableSet.of()); + this(id, table, qualifier, Optional.empty(), Optional.empty()); } @Override @@ -73,24 +69,18 @@ public class LogicalJdbcScan extends LogicalExternalRelation { @Override public LogicalJdbcScan withGroupExpression(Optional groupExpression) { return new LogicalJdbcScan(relationId, table, qualifier, groupExpression, - Optional.of(getLogicalProperties()), conjuncts); - } - - @Override - public LogicalJdbcScan withConjuncts(Set conjuncts) { - return new LogicalJdbcScan(relationId, table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts); + Optional.of(getLogicalProperties())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalJdbcScan(relationId, table, qualifier, groupExpression, logicalProperties, conjuncts); + return new LogicalJdbcScan(relationId, table, qualifier, groupExpression, logicalProperties); } @Override public LogicalJdbcScan withRelationId(RelationId relationId) { - return new LogicalJdbcScan(relationId, table, qualifier, Optional.empty(), Optional.empty(), conjuncts); + return new LogicalJdbcScan(relationId, table, qualifier, Optional.empty(), Optional.empty()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java index 414cb335af..3ce9b8d740 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java @@ -21,7 +21,6 @@ import org.apache.doris.catalog.OdbcTable; import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -29,26 +28,23 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Optional; -import java.util.Set; /** * Logical scan for external odbc table. */ -public class LogicalOdbcScan extends LogicalExternalRelation { +public class LogicalOdbcScan extends LogicalCatalogRelation { public LogicalOdbcScan(RelationId id, TableIf table, List qualifier, Optional groupExpression, - Optional logicalProperties, - Set conjuncts) { - super(id, PlanType.LOGICAL_ODBC_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties); + Optional logicalProperties) { + super(id, PlanType.LOGICAL_ODBC_SCAN, table, qualifier, groupExpression, logicalProperties); } public LogicalOdbcScan(RelationId id, TableIf table, List qualifier) { - this(id, table, qualifier, Optional.empty(), Optional.empty(), ImmutableSet.of()); + this(id, table, qualifier, Optional.empty(), Optional.empty()); } @Override @@ -69,24 +65,18 @@ public class LogicalOdbcScan extends LogicalExternalRelation { @Override public LogicalOdbcScan withGroupExpression(Optional groupExpression) { return new LogicalOdbcScan(relationId, table, qualifier, groupExpression, - Optional.of(getLogicalProperties()), conjuncts); - } - - @Override - public LogicalOdbcScan withConjuncts(Set conjuncts) { - return new LogicalOdbcScan(relationId, table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts); + Optional.of(getLogicalProperties())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties, conjuncts); + return new LogicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties); } @Override public LogicalOdbcScan withRelationId(RelationId relationId) { - return new LogicalOdbcScan(relationId, table, qualifier, Optional.empty(), Optional.empty(), conjuncts); + return new LogicalOdbcScan(relationId, table, qualifier, Optional.empty(), Optional.empty()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java index 474436f1db..8c6d348e6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java @@ -22,7 +22,6 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -30,12 +29,8 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import org.apache.doris.statistics.Statistics; -import com.google.common.collect.ImmutableSet; - import java.util.List; -import java.util.Objects; import java.util.Optional; -import java.util.Set; /** * Physical es scan for external catalog. @@ -43,17 +38,15 @@ import java.util.Set; public class PhysicalEsScan extends PhysicalCatalogRelation { private final DistributionSpec distributionSpec; - private final Set conjuncts; /** * Constructor for PhysicalEsScan. */ public PhysicalEsScan(RelationId id, TableIf table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, - LogicalProperties logicalProperties, Set conjuncts) { + LogicalProperties logicalProperties) { super(id, PlanType.PHYSICAL_ES_SCAN, table, qualifier, groupExpression, logicalProperties); this.distributionSpec = distributionSpec; - this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null")); } /** @@ -61,12 +54,10 @@ public class PhysicalEsScan extends PhysicalCatalogRelation { */ public PhysicalEsScan(RelationId id, TableIf table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, - LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics, - Set conjuncts) { + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { super(id, PlanType.PHYSICAL_ES_SCAN, table, qualifier, groupExpression, logicalProperties, physicalProperties, statistics); this.distributionSpec = distributionSpec; - this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null")); } @Override @@ -86,24 +77,20 @@ public class PhysicalEsScan extends PhysicalCatalogRelation { @Override public PhysicalEsScan withGroupExpression(Optional groupExpression) { return new PhysicalEsScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, getLogicalProperties(), conjuncts); + groupExpression, getLogicalProperties()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new PhysicalEsScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, logicalProperties.get(), conjuncts); + groupExpression, logicalProperties.get()); } @Override public PhysicalEsScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statsDeriveResult) { return new PhysicalEsScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, getLogicalProperties(), physicalProperties, statsDeriveResult, conjuncts); - } - - public Set getConjuncts() { - return this.conjuncts; + groupExpression, getLogicalProperties(), physicalProperties, statsDeriveResult); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java index 639e026949..3f77c39f5f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.TableSample; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -35,7 +34,6 @@ import org.apache.doris.statistics.Statistics; import java.util.List; import java.util.Optional; -import java.util.Set; /** * Physical file scan for external catalog. @@ -43,7 +41,6 @@ import java.util.Set; public class PhysicalFileScan extends PhysicalCatalogRelation { protected final DistributionSpec distributionSpec; - protected final Set conjuncts; protected final SelectedPartitions selectedPartitions; protected final Optional tableSample; protected final Optional tableSnapshot; @@ -53,11 +50,11 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { */ public PhysicalFileScan(RelationId id, ExternalTable table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, - LogicalProperties logicalProperties, Set conjuncts, + LogicalProperties logicalProperties, SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot) { this(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, distributionSpec, groupExpression, - logicalProperties, conjuncts, selectedPartitions, tableSample, tableSnapshot); + logicalProperties, selectedPartitions, tableSample, tableSnapshot); } /** @@ -66,11 +63,10 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { public PhysicalFileScan(RelationId id, ExternalTable table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, - Statistics statistics, Set conjuncts, SelectedPartitions selectedPartitions, + Statistics statistics, SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot) { this(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, distributionSpec, groupExpression, - logicalProperties, physicalProperties, statistics, conjuncts, selectedPartitions, tableSample, - tableSnapshot); + logicalProperties, physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot); } /** @@ -78,12 +74,11 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { */ protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, - LogicalProperties logicalProperties, Set conjuncts, + LogicalProperties logicalProperties, SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot) { super(id, type, table, qualifier, groupExpression, logicalProperties); this.distributionSpec = distributionSpec; - this.conjuncts = conjuncts; this.selectedPartitions = selectedPartitions; this.tableSample = tableSample; this.tableSnapshot = tableSnapshot; @@ -92,12 +87,11 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, - Statistics statistics, Set conjuncts, SelectedPartitions selectedPartitions, + Statistics statistics, SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot) { super(id, type, table, qualifier, groupExpression, logicalProperties, physicalProperties, statistics); this.distributionSpec = distributionSpec; - this.conjuncts = conjuncts; this.selectedPartitions = selectedPartitions; this.tableSample = tableSample; this.tableSnapshot = tableSnapshot; @@ -107,10 +101,6 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { return distributionSpec; } - public Set getConjuncts() { - return conjuncts; - } - public SelectedPartitions getSelectedPartitions() { return selectedPartitions; } @@ -129,7 +119,6 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { "qualified", Utils.qualifiedName(qualifier, table.getName()), "output", getOutput(), "stats", statistics, - "conjuncts", conjuncts, "selected partitions num", selectedPartitions.isPruned ? selectedPartitions.selectedPartitions.size() : "unknown" ); @@ -143,14 +132,14 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { @Override public PhysicalFileScan withGroupExpression(Optional groupExpression) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, getLogicalProperties(), conjuncts, selectedPartitions, tableSample, tableSnapshot); + groupExpression, getLogicalProperties(), selectedPartitions, tableSample, tableSnapshot); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, logicalProperties.get(), conjuncts, selectedPartitions, tableSample, tableSnapshot); + groupExpression, logicalProperties.get(), selectedPartitions, tableSample, tableSnapshot); } @Override @@ -162,7 +151,7 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { public PhysicalFileScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, getLogicalProperties(), physicalProperties, statistics, conjuncts, + groupExpression, getLogicalProperties(), physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHudiScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHudiScan.java index d5bc299a2b..451c572339 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHudiScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHudiScan.java @@ -26,7 +26,6 @@ import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.TableSample; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -38,7 +37,6 @@ import org.apache.doris.statistics.Statistics; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.Set; /** * Physical Hudi scan for Hudi table. @@ -54,12 +52,12 @@ public class PhysicalHudiScan extends PhysicalFileScan { */ public PhysicalHudiScan(RelationId id, ExternalTable table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, - LogicalProperties logicalProperties, Set conjuncts, + LogicalProperties logicalProperties, SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot, Optional scanParams, Optional incrementalRelation) { super(id, PlanType.PHYSICAL_HUDI_SCAN, table, qualifier, distributionSpec, groupExpression, logicalProperties, - conjuncts, selectedPartitions, tableSample, tableSnapshot); + selectedPartitions, tableSample, tableSnapshot); Objects.requireNonNull(scanParams, "scanParams should not null"); Objects.requireNonNull(incrementalRelation, "incrementalRelation should not null"); this.scanParams = scanParams; @@ -72,11 +70,11 @@ public class PhysicalHudiScan extends PhysicalFileScan { public PhysicalHudiScan(RelationId id, ExternalTable table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, - Statistics statistics, Set conjuncts, SelectedPartitions selectedPartitions, + Statistics statistics, SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot, Optional scanParams, Optional incrementalRelation) { super(id, PlanType.PHYSICAL_HUDI_SCAN, table, qualifier, distributionSpec, groupExpression, logicalProperties, - physicalProperties, statistics, conjuncts, selectedPartitions, tableSample, tableSnapshot); + physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot); this.scanParams = scanParams; this.incrementalRelation = incrementalRelation; } @@ -92,7 +90,7 @@ public class PhysicalHudiScan extends PhysicalFileScan { @Override public PhysicalHudiScan withGroupExpression(Optional groupExpression) { return new PhysicalHudiScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, getLogicalProperties(), conjuncts, selectedPartitions, tableSample, tableSnapshot, + groupExpression, getLogicalProperties(), selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation); } @@ -100,7 +98,7 @@ public class PhysicalHudiScan extends PhysicalFileScan { public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new PhysicalHudiScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, logicalProperties.get(), conjuncts, selectedPartitions, tableSample, tableSnapshot, + groupExpression, logicalProperties.get(), selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation); } @@ -108,7 +106,7 @@ public class PhysicalHudiScan extends PhysicalFileScan { public PhysicalHudiScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalHudiScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, getLogicalProperties(), physicalProperties, statistics, conjuncts, + groupExpression, getLogicalProperties(), physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation); } @@ -124,7 +122,6 @@ public class PhysicalHudiScan extends PhysicalFileScan { "qualified", Utils.qualifiedName(qualifier, table.getName()), "output", getOutput(), "stats", statistics, - "conjuncts", conjuncts, "selected partitions num", selectedPartitions.isPruned ? selectedPartitions.selectedPartitions.size() : "unknown", "isIncremental", incrementalRelation.isPresent() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalJdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalJdbcScan.java index d68a459bd3..7923c8af9f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalJdbcScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalJdbcScan.java @@ -21,7 +21,6 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -29,27 +28,21 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import org.apache.doris.statistics.Statistics; -import com.google.common.collect.ImmutableSet; - import java.util.List; -import java.util.Objects; import java.util.Optional; -import java.util.Set; /** * Physical jdbc scan for external catalog. */ public class PhysicalJdbcScan extends PhysicalCatalogRelation { - private final Set conjuncts; - /** * Constructor for PhysicalJdbcScan. */ public PhysicalJdbcScan(RelationId id, TableIf table, List qualifier, - Optional groupExpression, LogicalProperties logicalProperties, Set conjuncts) { + Optional groupExpression, LogicalProperties logicalProperties) { this(id, table, qualifier, groupExpression, logicalProperties, - null, null, conjuncts); + null, null); } /** @@ -57,11 +50,9 @@ public class PhysicalJdbcScan extends PhysicalCatalogRelation { */ public PhysicalJdbcScan(RelationId id, TableIf table, List qualifier, Optional groupExpression, - LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics, - Set conjuncts) { + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { super(id, PlanType.PHYSICAL_JDBC_SCAN, table, qualifier, groupExpression, logicalProperties, physicalProperties, statistics); - this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null")); } @Override @@ -80,23 +71,19 @@ public class PhysicalJdbcScan extends PhysicalCatalogRelation { @Override public PhysicalJdbcScan withGroupExpression(Optional groupExpression) { - return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, getLogicalProperties(), conjuncts); + return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, getLogicalProperties()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, logicalProperties.get(), conjuncts); + return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, logicalProperties.get()); } @Override public PhysicalJdbcScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, - getLogicalProperties(), physicalProperties, statistics, conjuncts); - } - - public Set getConjuncts() { - return this.conjuncts; + getLogicalProperties(), physicalProperties, statistics); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java index f63b5a8d42..6ae6b93112 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java @@ -21,7 +21,6 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -29,27 +28,20 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; import org.apache.doris.statistics.Statistics; -import com.google.common.collect.ImmutableSet; - import java.util.List; -import java.util.Objects; import java.util.Optional; -import java.util.Set; /** * Physical odbc scan for odbc table. */ public class PhysicalOdbcScan extends PhysicalCatalogRelation { - - private final Set conjuncts; - /** * Constructor for PhysicalOdbcScan. */ public PhysicalOdbcScan(RelationId id, TableIf table, List qualifier, - Optional groupExpression, LogicalProperties logicalProperties, Set conjuncts) { + Optional groupExpression, LogicalProperties logicalProperties) { this(id, table, qualifier, groupExpression, logicalProperties, - null, null, conjuncts); + null, null); } /** @@ -57,11 +49,9 @@ public class PhysicalOdbcScan extends PhysicalCatalogRelation { */ public PhysicalOdbcScan(RelationId id, TableIf table, List qualifier, Optional groupExpression, - LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics, - Set conjuncts) { + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { super(id, PlanType.PHYSICAL_ODBC_SCAN, table, qualifier, groupExpression, logicalProperties, physicalProperties, statistics); - this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null")); } @Override @@ -80,23 +70,19 @@ public class PhysicalOdbcScan extends PhysicalCatalogRelation { @Override public PhysicalOdbcScan withGroupExpression(Optional groupExpression) { - return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, getLogicalProperties(), conjuncts); + return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, getLogicalProperties()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties.get(), conjuncts); + return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties.get()); } @Override public PhysicalOdbcScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, - getLogicalProperties(), physicalProperties, statistics, conjuncts); - } - - public Set getConjuncts() { - return this.conjuncts; + getLogicalProperties(), physicalProperties, statistics); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java index 0871e3dca3..fef94ff52f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalExternalRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; import org.apache.doris.nereids.trees.plans.logical.LogicalHudiScan; import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan; @@ -95,16 +94,12 @@ public interface RelationVisitor { return visitLogicalRelation(emptyRelation, context); } - default R visitLogicalExternalRelation(LogicalExternalRelation relation, C context) { - return visitLogicalCatalogRelation(relation, context); - } - default R visitLogicalEsScan(LogicalEsScan esScan, C context) { - return visitLogicalExternalRelation(esScan, context); + return visitLogicalCatalogRelation(esScan, context); } default R visitLogicalFileScan(LogicalFileScan fileScan, C context) { - return visitLogicalExternalRelation(fileScan, context); + return visitLogicalCatalogRelation(fileScan, context); } default R visitLogicalHudiScan(LogicalHudiScan fileScan, C context) { @@ -112,11 +107,11 @@ public interface RelationVisitor { } default R visitLogicalJdbcScan(LogicalJdbcScan jdbcScan, C context) { - return visitLogicalExternalRelation(jdbcScan, context); + return visitLogicalCatalogRelation(jdbcScan, context); } default R visitLogicalOdbcScan(LogicalOdbcScan odbcScan, C context) { - return visitLogicalExternalRelation(odbcScan, context); + return visitLogicalCatalogRelation(odbcScan, context); } default R visitLogicalOlapScan(LogicalOlapScan olapScan, C context) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java index b968d27b5b..d15c11e55c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java @@ -49,6 +49,7 @@ import java.util.Optional; public class HmsCatalogTest extends AnalyzeCheckTestBase { private static final String HMS_CATALOG = "hms_ctl"; + private static final long NOW = System.currentTimeMillis(); private Env env; private CatalogMgr mgr; @@ -101,6 +102,11 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase { Deencapsulation.setField(db, "initialized", true); Deencapsulation.setField(tbl, "objectCreated", true); + Deencapsulation.setField(tbl, "schemaUpdateTime", NOW); + Deencapsulation.setField(tbl, "eventUpdateTime", 0); + Deencapsulation.setField(tbl, "catalog", hmsCatalog); + Deencapsulation.setField(tbl, "dbName", "hms_db"); + Deencapsulation.setField(tbl, "name", "hms_tbl"); new Expectations(tbl) { { tbl.getId(); @@ -131,6 +137,10 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase { minTimes = 0; result = TableIf.TableType.HMS_EXTERNAL_TABLE; + // mock initSchemaAndUpdateTime and do nothing + tbl.initSchemaAndUpdateTime(); + minTimes = 0; + tbl.getDatabase(); minTimes = 0; result = db; @@ -142,6 +152,11 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase { }; Deencapsulation.setField(view1, "objectCreated", true); + Deencapsulation.setField(view1, "schemaUpdateTime", NOW); + Deencapsulation.setField(view1, "eventUpdateTime", 0); + Deencapsulation.setField(view1, "catalog", hmsCatalog); + Deencapsulation.setField(view1, "dbName", "hms_db"); + Deencapsulation.setField(view1, "name", "hms_view1"); new Expectations(view1) { { @@ -188,6 +203,11 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase { }; Deencapsulation.setField(view2, "objectCreated", true); + Deencapsulation.setField(view2, "schemaUpdateTime", NOW); + Deencapsulation.setField(view2, "eventUpdateTime", 0); + Deencapsulation.setField(view2, "catalog", hmsCatalog); + Deencapsulation.setField(view2, "dbName", "hms_db"); + Deencapsulation.setField(view2, "name", "hms_view2"); new Expectations(view2) { { @@ -234,6 +254,11 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase { }; Deencapsulation.setField(view3, "objectCreated", true); + Deencapsulation.setField(view3, "schemaUpdateTime", NOW); + Deencapsulation.setField(view3, "eventUpdateTime", 0); + Deencapsulation.setField(view3, "catalog", hmsCatalog); + Deencapsulation.setField(view3, "dbName", "hms_db"); + Deencapsulation.setField(view3, "name", "hms_view3"); new Expectations(view3) { { @@ -280,6 +305,11 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase { }; Deencapsulation.setField(view4, "objectCreated", true); + Deencapsulation.setField(view4, "schemaUpdateTime", NOW); + Deencapsulation.setField(view4, "eventUpdateTime", 0); + Deencapsulation.setField(view4, "catalog", hmsCatalog); + Deencapsulation.setField(view4, "dbName", "hms_db"); + Deencapsulation.setField(view4, "name", "hms_view4"); new Expectations(view4) { { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/AdjustNullableTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/AdjustNullableTest.java deleted file mode 100644 index 5a6b7ac1b4..0000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/AdjustNullableTest.java +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.GreaterThan; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.nereids.trees.plans.RelationId; -import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; -import org.apache.doris.nereids.types.IntegerType; -import org.apache.doris.nereids.util.MemoPatternMatchSupported; -import org.apache.doris.nereids.util.PlanConstructor; - -import mockit.Mock; -import mockit.MockUp; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; - -/** - * Tests for {@link AdjustNullableTest}. - */ -class AdjustNullableTest implements MemoPatternMatchSupported { - private final LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); - - @Test - void testLogicalExternalRelation() { - new MockUp() { - @Mock - public Set getOutputSet() { - Set output = new HashSet<>(); - output.add(new SlotReference(new ExprId(1), "id", IntegerType.INSTANCE, false, - new ArrayList<>())); - return output; - } - - }; - - GreaterThan gt = new GreaterThan(new SlotReference(new ExprId(1), "id", - IntegerType.INSTANCE, true, new ArrayList<>()), Literal.of("1")); - Set conjuncts = new HashSet<>(); - conjuncts.add(gt); - Assertions.assertTrue(conjuncts.iterator().next().nullable()); - LogicalJdbcScan jdbcScan = - new LogicalJdbcScan(new RelationId(1), PlanConstructor.newOlapTable(0, "t1", 0), - new ArrayList<>(), Optional.empty(), Optional.empty(), conjuncts); - AdjustNullable adjustNullable = new AdjustNullable(); - LogicalJdbcScan newJdbcScan = (LogicalJdbcScan) adjustNullable.rewriteRoot(jdbcScan, null); - conjuncts = newJdbcScan.getConjuncts(); - Assertions.assertFalse(conjuncts.iterator().next().nullable()); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java index f342ca697b..358254cd2b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java @@ -121,6 +121,9 @@ public class HmsQueryCacheTest extends AnalyzeCheckTestBase { Deencapsulation.setField(tbl, "objectCreated", true); Deencapsulation.setField(tbl, "schemaUpdateTime", NOW); Deencapsulation.setField(tbl, "eventUpdateTime", 0); + Deencapsulation.setField(tbl, "catalog", hmsCatalog); + Deencapsulation.setField(tbl, "dbName", "hms_db"); + Deencapsulation.setField(tbl, "name", "hms_tbl"); new Expectations(tbl) { { tbl.getId(); @@ -168,6 +171,9 @@ public class HmsQueryCacheTest extends AnalyzeCheckTestBase { Deencapsulation.setField(tbl2, "objectCreated", true); Deencapsulation.setField(tbl2, "schemaUpdateTime", NOW); Deencapsulation.setField(tbl2, "eventUpdateTime", 0); + Deencapsulation.setField(tbl2, "catalog", hmsCatalog); + Deencapsulation.setField(tbl2, "dbName", "hms_db"); + Deencapsulation.setField(tbl2, "name", "hms_tbl2"); new Expectations(tbl2) { { tbl2.getId();