diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/TableIdentifier.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/TableIdentifier.java index 2688fd5784..8e510ec7a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/TableIdentifier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/constraint/TableIdentifier.java @@ -40,8 +40,9 @@ public class TableIdentifier { Preconditions.checkArgument(tableIf != null, "Table can not be null in constraint"); tableId = tableIf.getId(); - databaseId = tableIf.getDatabase().getId(); - catalogId = tableIf.getDatabase().getCatalog().getId(); + databaseId = tableIf.getDatabase() == null ? 0L : tableIf.getDatabase().getId(); + catalogId = tableIf.getDatabase() == null || tableIf.getDatabase().getCatalog() == null + ? 0L : tableIf.getDatabase().getCatalog().getId(); } public TableIf toTableIf() { @@ -69,13 +70,14 @@ public class TableIdentifier { return false; } TableIdentifier that = (TableIdentifier) o; - return databaseId == that.databaseId + return catalogId == that.catalogId + && databaseId == that.databaseId && tableId == that.tableId; } @Override public int hashCode() { - return Objects.hash(databaseId, tableId); + return Objects.hash(catalogId, databaseId, tableId); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 2a354eb9c4..21c950c40c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -494,10 +494,17 @@ public class NereidsPlanner extends Planner { plan = super.getExplainString(explainOptions) + MaterializationContext.toSummaryString(cascadesContext.getMaterializationContexts(), this.getPhysicalPlan()); + if (statementContext != null) { + if (statementContext.isHasUnknownColStats()) { + plan += "\n\nStatistics\n planed with unknown column statistics\n"; + } + } } - if (statementContext != null && !statementContext.getHints().isEmpty()) { - String hint = getHintExplainString(statementContext.getHints()); - return plan + hint; + if (statementContext != null) { + if (!statementContext.getHints().isEmpty()) { + String hint = getHintExplainString(statementContext.getHints()); + return plan + hint; + } } return plan; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 3274233c16..6e4b3c23fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids; import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; import org.apache.doris.nereids.hint.Hint; @@ -29,8 +30,10 @@ import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.TableId; import org.apache.doris.nereids.trees.plans.algebra.Relation; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -55,6 +58,7 @@ import java.util.BitSet; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -99,6 +103,7 @@ public class StatementContext implements Closeable { private final IdGenerator objectIdGenerator = ObjectId.createGenerator(); private final IdGenerator relationIdGenerator = RelationId.createGenerator(); private final IdGenerator cteIdGenerator = CTEId.createGenerator(); + private final IdGenerator talbeIdGenerator = TableId.createGenerator(); private final Map> cteIdToConsumers = new HashMap<>(); private final Map> cteIdToOutputIds = new HashMap<>(); @@ -138,6 +143,9 @@ public class StatementContext implements Closeable { // and value is the new string used for replacement. private final TreeMap, String> indexInSqlToString = new TreeMap<>(new Pair.PairComparator<>()); + // Record table id mapping, the key is the hash code of union catalogId, databaseId, tableId + // the value is the auto-increment id in the cascades context + private final Map tableIdMapping = new LinkedHashMap<>(); public StatementContext() { this(ConnectContext.get(), null, 0); @@ -290,6 +298,10 @@ public class StatementContext implements Closeable { return relationIdGenerator.getNextId(); } + public TableId getNextTableId() { + return talbeIdGenerator.getNextId(); + } + public void setParsedStatement(StatementBase parsedStatement) { this.parsedStatement = parsedStatement; } @@ -485,4 +497,16 @@ public class StatementContext implements Closeable { + ",\n sql:\n" + sql + "\n}"; } } + + /** Get table id with lazy */ + public TableId getTableId(TableIf tableIf) { + TableIdentifier tableIdentifier = new TableIdentifier(tableIf); + TableId tableId = this.tableIdMapping.get(tableIdentifier); + if (tableId != null) { + return tableId; + } + tableId = StatementScopeIdGenerator.newTableId(); + this.tableIdMapping.put(tableIdentifier, tableId); + return tableId; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index df41eb3eb4..0c11b5fbb2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.memo; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.exploration.mv.StructInfo; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; @@ -51,20 +52,21 @@ public class StructInfoMap { * @param group the group that the mv matched * @return struct info or null if not found */ - public @Nullable StructInfo getStructInfo(Memo memo, BitSet tableMap, Group group, Plan originPlan) { + public @Nullable StructInfo getStructInfo(CascadesContext cascadesContext, BitSet tableMap, Group group, + Plan originPlan) { StructInfo structInfo = infoMap.get(tableMap); if (structInfo != null) { return structInfo; } if (groupExpressionMap.isEmpty() || !groupExpressionMap.containsKey(tableMap)) { - refresh(group, memo.getRefreshVersion()); - group.getstructInfoMap().setRefreshVersion(memo.getRefreshVersion()); + refresh(group, cascadesContext); + group.getstructInfoMap().setRefreshVersion(cascadesContext.getMemo().getRefreshVersion()); } if (groupExpressionMap.containsKey(tableMap)) { Pair> groupExpressionBitSetPair = getGroupExpressionWithChildren( tableMap); - structInfo = constructStructInfo(groupExpressionBitSetPair.first, - groupExpressionBitSetPair.second, tableMap, originPlan); + structInfo = constructStructInfo(groupExpressionBitSetPair.first, groupExpressionBitSetPair.second, + tableMap, originPlan, cascadesContext); infoMap.put(tableMap, structInfo); } return structInfo; @@ -87,10 +89,11 @@ public class StructInfoMap { } private StructInfo constructStructInfo(GroupExpression groupExpression, List children, - BitSet tableMap, Plan originPlan) { + BitSet tableMap, Plan originPlan, CascadesContext cascadesContext) { // this plan is not origin plan, should record origin plan in struct info Plan plan = constructPlan(groupExpression, children, tableMap); - return originPlan == null ? StructInfo.of(plan) : StructInfo.of(plan, originPlan); + return originPlan == null ? StructInfo.of(plan, cascadesContext) + : StructInfo.of(plan, originPlan, cascadesContext); } private Plan constructPlan(GroupExpression groupExpression, List children, BitSet tableMap) { @@ -112,8 +115,9 @@ public class StructInfoMap { * @param group the root group * */ - public void refresh(Group group, long memoVersion) { + public void refresh(Group group, CascadesContext cascadesContext) { StructInfoMap structInfoMap = group.getstructInfoMap(); + long memoVersion = cascadesContext.getMemo().getRefreshVersion(); if (!structInfoMap.getTableMaps().isEmpty() && memoVersion == structInfoMap.refreshVersion) { return; } @@ -121,14 +125,14 @@ public class StructInfoMap { for (GroupExpression groupExpression : group.getLogicalExpressions()) { List> childrenTableMap = new LinkedList<>(); if (groupExpression.children().isEmpty()) { - BitSet leaf = constructLeaf(groupExpression); + BitSet leaf = constructLeaf(groupExpression, cascadesContext); groupExpressionMap.put(leaf, Pair.of(groupExpression, new LinkedList<>())); continue; } for (Group child : groupExpression.children()) { StructInfoMap childStructInfoMap = child.getstructInfoMap(); if (!refreshedGroup.contains(child.getGroupId().asInt())) { - childStructInfoMap.refresh(child, memoVersion); + childStructInfoMap.refresh(child, cascadesContext); childStructInfoMap.setRefreshVersion(memoVersion); } refreshedGroup.add(child.getGroupId().asInt()); @@ -156,12 +160,12 @@ public class StructInfoMap { } } - private BitSet constructLeaf(GroupExpression groupExpression) { + private BitSet constructLeaf(GroupExpression groupExpression, CascadesContext cascadesContext) { Plan plan = groupExpression.getPlan(); BitSet tableMap = new BitSet(); if (plan instanceof LogicalCatalogRelation) { - // TODO: Bitset is not compatible with long, use tree map instead - tableMap.set((int) ((LogicalCatalogRelation) plan).getTable().getId()); + tableMap.set(cascadesContext.getStatementContext() + .getTableId(((LogicalCatalogRelation) plan).getTable()).asInt()); } // one row relation / CTE consumer return tableMap; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index c2c2448b24..ebb231c3c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -107,13 +107,21 @@ public abstract class MaterializationContext { // mv output expression shuttle, this will be used to expression rewrite this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions, this.mvScanPlan.getOutput()); - // copy the plan from cache, which the plan in cache may change - List viewStructInfos = MaterializedViewUtils.extractStructInfo( - mvPlan, cascadesContext, new BitSet()); - if (viewStructInfos.size() > 1) { - // view struct info should only have one, log error and use the first struct info - LOG.warn(String.format("view strut info is more than one, materialization name is %s, mv plan is %s", - getMaterializationQualifier(), getMvPlan().treeString())); + // Construct mv struct info, catch exception which may cause planner roll back + List viewStructInfos; + try { + viewStructInfos = MaterializedViewUtils.extractStructInfo(mvPlan, cascadesContext, new BitSet()); + if (viewStructInfos.size() > 1) { + // view struct info should only have one, log error and use the first struct info + LOG.warn(String.format("view strut info is more than one, materialization name is %s, mv plan is %s", + getMaterializationQualifier(), getMvPlan().treeString())); + } + } catch (Exception exception) { + LOG.warn(String.format("construct mv struct info fail, materialization name is %s, mv plan is %s", + getMaterializationQualifier(), getMvPlan().treeString()), exception); + this.available = false; + this.structInfo = null; + return; } this.structInfo = viewStructInfos.get(0); } @@ -276,9 +284,8 @@ public abstract class MaterializationContext { // rewrite success and chosen builder.append("\nMaterializedViewRewriteSuccessAndChose:\n"); if (!chosenMaterializationQualifiers.isEmpty()) { - builder.append(" Names: "); chosenMaterializationQualifiers.forEach(materializationQualifier -> - builder.append(generateQualifierName(materializationQualifier)).append(", ")); + builder.append(generateQualifierName(materializationQualifier)).append(", \n")); } // rewrite success but not chosen builder.append("\nMaterializedViewRewriteSuccessButNotChose:\n"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index f1a90fb940..e4dc120c20 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -149,7 +149,7 @@ public class MaterializedViewUtils { Group ownerGroup = plan.getGroupExpression().get().getOwnerGroup(); StructInfoMap structInfoMap = ownerGroup.getstructInfoMap(); // Refresh struct info in current level plan from top to bottom - structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion()); + structInfoMap.refresh(ownerGroup, cascadesContext); structInfoMap.setRefreshVersion(cascadesContext.getMemo().getRefreshVersion()); Set queryTableSets = structInfoMap.getTableMaps(); @@ -161,7 +161,7 @@ public class MaterializedViewUtils { && !materializedViewTableSet.equals(queryTableSet)) { continue; } - StructInfo structInfo = structInfoMap.getStructInfo(cascadesContext.getMemo(), + StructInfo structInfo = structInfoMap.getStructInfo(cascadesContext, queryTableSet, ownerGroup, plan); if (structInfo != null) { structInfosBuilder.add(structInfo); @@ -171,7 +171,7 @@ public class MaterializedViewUtils { } } // if plan doesn't belong to any group, construct it directly - return ImmutableList.of(StructInfo.of(plan)); + return ImmutableList.of(StructInfo.of(plan, cascadesContext)); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 55649c658b..fec6626588 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -96,7 +96,7 @@ public class StructInfo { // bottom plan which top plan only contain join or scan. this is needed by hyper graph private final Plan bottomPlan; private final List relations; - private final BitSet tableBitSet = new BitSet(); + private final BitSet tableBitSet; // this is for LogicalCompatibilityContext later private final Map relationIdStructInfoNodeMap; // this recorde the predicates which can pull up, not shuttled @@ -113,12 +113,13 @@ public class StructInfo { /** * The construct method for StructInfo */ - public StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGraph, boolean valid, Plan topPlan, + private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGraph, boolean valid, Plan topPlan, Plan bottomPlan, List relations, Map relationIdStructInfoNodeMap, @Nullable Predicates predicates, Map> shuttledExpressionsToExpressionsMap, - Map namedExprIdAndExprMapping) { + Map namedExprIdAndExprMapping, + BitSet talbeIdSet) { this.originalPlan = originalPlan; this.originalPlanId = originalPlanId; this.hyperGraph = hyperGraph; @@ -127,7 +128,7 @@ public class StructInfo { this.topPlan = topPlan; this.bottomPlan = bottomPlan; this.relations = relations; - relations.forEach(relation -> this.tableBitSet.set((int) (relation.getTable().getId()))); + this.tableBitSet = talbeIdSet; this.relationIdStructInfoNodeMap = relationIdStructInfoNodeMap; this.predicates = predicates; if (predicates == null) { @@ -150,7 +151,7 @@ public class StructInfo { public StructInfo withPredicates(Predicates predicates) { return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan, this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, predicates, - this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping); + this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping, this.tableBitSet); } private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, @@ -265,15 +266,15 @@ public class StructInfo { * Build Struct info from plan. * Maybe return multi structInfo when original plan already be rewritten by mv */ - public static StructInfo of(Plan originalPlan) { - return of(originalPlan, originalPlan); + public static StructInfo of(Plan originalPlan, CascadesContext cascadesContext) { + return of(originalPlan, originalPlan, cascadesContext); } /** * Build Struct info from plan. * Maybe return multi structInfo when original plan already be rewritten by mv */ - public static StructInfo of(Plan derivedPlan, Plan originalPlan) { + public static StructInfo of(Plan derivedPlan, Plan originalPlan, CascadesContext cascadesContext) { // Split plan by the boundary which contains multi child LinkedHashSet> set = Sets.newLinkedHashSet(); set.add(LogicalJoin.class); @@ -281,14 +282,15 @@ public class StructInfo { // if single table without join, the bottom is derivedPlan.accept(PLAN_SPLITTER, planSplitContext); return StructInfo.of(originalPlan, planSplitContext.getTopPlan(), planSplitContext.getBottomPlan(), - HyperGraph.builderForMv(planSplitContext.getBottomPlan()).build()); + HyperGraph.builderForMv(planSplitContext.getBottomPlan()).build(), cascadesContext); } /** * The construct method for init StructInfo */ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable Plan bottomPlan, - HyperGraph hyperGraph) { + HyperGraph hyperGraph, + CascadesContext cascadesContext) { ObjectId originalPlanId = originalPlan.getGroupExpression() .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1)); // if any of topPlan or bottomPlan is null, split the top plan to two parts by join node @@ -310,9 +312,14 @@ public class StructInfo { namedExprIdAndExprMapping, relationList, relationIdStructInfoNodeMap); + // Get mapped table id in relation and set + BitSet tableBitSet = new BitSet(); + for (CatalogRelation relation : relationList) { + tableBitSet.set(cascadesContext.getStatementContext().getTableId(relation.getTable()).asInt()); + } return new StructInfo(originalPlan, originalPlanId, hyperGraph, valid, topPlan, bottomPlan, relationList, relationIdStructInfoNodeMap, null, shuttledHashConjunctsToConjunctsMap, - namedExprIdAndExprMapping); + namedExprIdAndExprMapping, tableBitSet); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java index eb53923da5..a6f68d047b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.exploration.mv.mapping; import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.common.Pair; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; @@ -62,22 +63,22 @@ public class RelationMapping extends Mapping { */ public static List generate(List sources, List targets) { // Construct tmp map, key is the table qualifier, value is the corresponding catalog relations - HashMultimap sourceTableRelationIdMap = HashMultimap.create(); + HashMultimap sourceTableRelationIdMap = HashMultimap.create(); for (CatalogRelation relation : sources) { - sourceTableRelationIdMap.put(getTableQualifier(relation.getTable()), + sourceTableRelationIdMap.put(getTableIdentifier(relation.getTable()), MappedRelation.of(relation.getRelationId(), relation)); } - HashMultimap targetTableRelationIdMap = HashMultimap.create(); + HashMultimap targetTableRelationIdMap = HashMultimap.create(); for (CatalogRelation relation : targets) { - targetTableRelationIdMap.put(getTableQualifier(relation.getTable()), + targetTableRelationIdMap.put(getTableIdentifier(relation.getTable()), MappedRelation.of(relation.getRelationId(), relation)); } - Set sourceTableKeySet = sourceTableRelationIdMap.keySet(); + Set sourceTableKeySet = sourceTableRelationIdMap.keySet(); List>> mappedRelations = new ArrayList<>(); - for (Long sourceTableId : sourceTableKeySet) { - Set sourceMappedRelations = sourceTableRelationIdMap.get(sourceTableId); - Set targetMappedRelations = targetTableRelationIdMap.get(sourceTableId); + for (TableIdentifier tableIdentifier : sourceTableKeySet) { + Set sourceMappedRelations = sourceTableRelationIdMap.get(tableIdentifier); + Set targetMappedRelations = targetTableRelationIdMap.get(tableIdentifier); if (targetMappedRelations.isEmpty()) { continue; } @@ -141,8 +142,8 @@ public class RelationMapping extends Mapping { return RelationMapping.of(mappingBuilder.build()); } - private static Long getTableQualifier(TableIf tableIf) { - return tableIf.getId(); + private static TableIdentifier getTableIdentifier(TableIf tableIf) { + return new TableIdentifier(tableIf); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java index 170c7cd5c7..df7ef2ab69 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java @@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.expressions; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.TableId; import org.apache.doris.qe.ConnectContext; import com.google.common.annotations.VisibleForTesting; @@ -65,6 +66,13 @@ public class StatementScopeIdGenerator { return ConnectContext.get().getStatementContext().getNextCTEId(); } + public static TableId newTableId() { + if (ConnectContext.get() == null || ConnectContext.get().getStatementContext() == null) { + return statementContext.getNextTableId(); + } + return ConnectContext.get().getStatementContext().getNextTableId(); + } + /** * Reset Id Generator */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/TableId.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/TableId.java new file mode 100644 index 0000000000..03e0ec450a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/TableId.java @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans; + +import org.apache.doris.common.Id; +import org.apache.doris.common.IdGenerator; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; + +/** + * Table id + */ +public class TableId extends Id { + + public TableId(int id) { + super(id); + } + + /** + * Should be only called by {@link StatementScopeIdGenerator}. + */ + public static IdGenerator createGenerator() { + return new IdGenerator() { + @Override + public TableId getNextId() { + return new TableId(nextId++); + } + }; + } + + @Override + public String toString() { + return "TableId#" + id; + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java index ca543a0fd5..490090a43f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java @@ -21,11 +21,6 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult; import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator; -import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext; -import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; -import org.apache.doris.nereids.rules.exploration.mv.StructInfo; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; -import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.JoinType; @@ -37,7 +32,6 @@ import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.BitSet; import java.util.List; class CompareOuterJoinTest extends SqlTestBase { @@ -68,7 +62,7 @@ class CompareOuterJoinTest extends SqlTestBase { HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); Assertions.assertFalse( - HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)).isInvalid()); + HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)).isInvalid()); } @Test @@ -76,9 +70,10 @@ class CompareOuterJoinTest extends SqlTestBase { connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); Plan p1 = new HyperGraphBuilder(Sets.newHashSet(JoinType.INNER_JOIN)) .randomBuildPlanWith(3, 3); - p1 = PlanChecker.from(connectContext, p1) + PlanChecker planChecker = PlanChecker.from(connectContext, p1) .analyze() - .rewrite() + .rewrite(); + p1 = planChecker .getPlan(); Plan p2 = PlanChecker.from(connectContext, p1) .analyze() @@ -88,7 +83,8 @@ class CompareOuterJoinTest extends SqlTestBase { HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); Assertions.assertFalse( - HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)).isInvalid()); + HyperGraphComparator.isLogicCompatible(h1, h2, + constructContext(p1, p2, planChecker.getCascadesContext())).isInvalid()); } @Test @@ -113,7 +109,7 @@ class CompareOuterJoinTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertEquals(1, res.getQueryExpressions().size()); Assertions.assertEquals("(id = 0)", res.getQueryExpressions().get(0).toSql()); } @@ -140,7 +136,8 @@ class CompareOuterJoinTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - List exprList = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)).getQueryExpressions(); + List exprList = HyperGraphComparator.isLogicCompatible(h1, h2, + constructContext(p1, p2, c1)).getQueryExpressions(); Assertions.assertEquals(0, exprList.size()); } @@ -166,7 +163,7 @@ class CompareOuterJoinTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertEquals(1, res.getQueryExpressions().size()); Assertions.assertEquals("(id = 0)", res.getQueryExpressions().get(0).toSql()); } @@ -193,17 +190,7 @@ class CompareOuterJoinTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(res.isInvalid()); } - - LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { - StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null, new BitSet()).get(0); - StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null, new BitSet()).get(0); - RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); - SlotMapping sm = SlotMapping.generate(rm); - return LogicalCompatibilityContext.from(rm, sm, st1, st2); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java index c7d93a363e..026c354d78 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java @@ -21,11 +21,6 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult; import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator; -import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext; -import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; -import org.apache.doris.nereids.rules.exploration.mv.StructInfo; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; -import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.plans.Plan; @@ -35,7 +30,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import java.util.BitSet; import java.util.stream.Collectors; class InferJoinTest extends SqlTestBase { @@ -61,7 +55,7 @@ class InferJoinTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertFalse(res.isInvalid()); Assertions.assertEquals(1, res.getViewNoNullableSlot().size()); Assertions.assertEquals("[id, score]", @@ -90,7 +84,7 @@ class InferJoinTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertFalse(res.isInvalid()); Assertions.assertEquals(1, res.getViewNoNullableSlot().size()); Assertions.assertEquals("[id, score]", @@ -127,7 +121,7 @@ class InferJoinTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertFalse(res.isInvalid()); Assertions.assertEquals(1, res.getViewNoNullableSlot().size()); Assertions.assertEquals("[id, score]", @@ -157,17 +151,7 @@ class InferJoinTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(res.isInvalid()); } - - LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { - StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null, new BitSet()).get(0); - StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null, new BitSet()).get(0); - RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); - SlotMapping sm = SlotMapping.generate(rm); - return LogicalCompatibilityContext.from(rm, sm, st1, st2); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java index b6f3fbb45b..acc4c90b67 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java @@ -21,11 +21,6 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult; import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator; -import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext; -import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; -import org.apache.doris.nereids.rules.exploration.mv.StructInfo; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; -import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.PlanChecker; @@ -33,8 +28,6 @@ import org.apache.doris.nereids.util.PlanChecker; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.BitSet; - class InferPredicateTest extends SqlTestBase { @Test void testPullUpQueryFilter() { @@ -58,18 +51,8 @@ class InferPredicateTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertFalse(res.isInvalid()); Assertions.assertEquals("(id = 1)", res.getQueryExpressions().get(0).toSql()); } - - LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { - StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null, new BitSet()).get(0); - StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null, new BitSet()).get(0); - RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); - SlotMapping sm = SlotMapping.generate(rm); - return LogicalCompatibilityContext.from(rm, sm, st1, st2); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java index def3281932..26daa649ec 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java @@ -21,11 +21,6 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult; import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator; -import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext; -import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; -import org.apache.doris.nereids.rules.exploration.mv.StructInfo; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; -import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.PlanChecker; @@ -33,8 +28,6 @@ import org.apache.doris.nereids.util.PlanChecker; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.BitSet; - class PullupExpressionTest extends SqlTestBase { @Test void testPullUpQueryFilter() { @@ -58,7 +51,7 @@ class PullupExpressionTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertEquals(1, res.getQueryExpressions().size()); Assertions.assertEquals("(id = 1)", res.getQueryExpressions().get(0).toSql()); } @@ -85,7 +78,7 @@ class PullupExpressionTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertEquals(1, res.getQueryExpressions().size()); Assertions.assertEquals("(score = score)", res.getQueryExpressions().get(0).toSql()); } @@ -112,7 +105,7 @@ class PullupExpressionTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertEquals(2, res.getViewExpressions().size()); Assertions.assertEquals("(id = 1)", res.getViewExpressions().get(0).toSql()); Assertions.assertEquals("(id = 1)", res.getViewExpressions().get(1).toSql()); @@ -140,18 +133,8 @@ class PullupExpressionTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertEquals(1, res.getViewExpressions().size()); Assertions.assertEquals("(score = score)", res.getViewExpressions().get(0).toSql()); } - - LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { - StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null, new BitSet()).get(0); - StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null, new BitSet()).get(0); - RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); - SlotMapping sm = SlotMapping.generate(rm); - return LogicalCompatibilityContext.from(rm, sm, st1, st2); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java index 80670dc366..0645cf8515 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java @@ -22,6 +22,7 @@ import org.apache.doris.mtmv.MTMVRelationManager; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.exploration.mv.StructInfo; import org.apache.doris.nereids.sqltest.SqlTestBase; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; @@ -59,7 +60,7 @@ class StructInfoMapTest extends SqlTestBase { Group root = c1.getMemo().getRoot(); Set tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertTrue(tableMaps.isEmpty()); - root.getstructInfoMap().refresh(root, 1); + root.getstructInfoMap().refresh(root, c1); Assertions.assertEquals(1, tableMaps.size()); new MockUp() { @Mock @@ -87,7 +88,7 @@ class StructInfoMapTest extends SqlTestBase { .optimize() .printlnBestPlanTree(); root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root, 1); + root.getstructInfoMap().refresh(root, c1); tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertEquals(2, tableMaps.size()); dropMvByNereids("drop materialized view mv1"); @@ -116,8 +117,8 @@ class StructInfoMapTest extends SqlTestBase { Group root = c1.getMemo().getRoot(); Set tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertTrue(tableMaps.isEmpty()); - root.getstructInfoMap().refresh(root, 1); - root.getstructInfoMap().refresh(root, 1); + root.getstructInfoMap().refresh(root, c1); + root.getstructInfoMap().refresh(root, c1); Assertions.assertEquals(1, tableMaps.size()); new MockUp() { @Mock @@ -144,8 +145,8 @@ class StructInfoMapTest extends SqlTestBase { .optimize() .printlnBestPlanTree(); root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root, 1); - root.getstructInfoMap().refresh(root, 1); + root.getstructInfoMap().refresh(root, c1); + root.getstructInfoMap().refresh(root, c1); tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertEquals(2, tableMaps.size()); dropMvByNereids("drop materialized view mv1"); @@ -191,16 +192,18 @@ class StructInfoMapTest extends SqlTestBase { .rewrite() .optimize(); Group root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root, 1); + root.getstructInfoMap().refresh(root, c1); StructInfoMap structInfoMap = root.getstructInfoMap(); Assertions.assertEquals(2, structInfoMap.getTableMaps().size()); BitSet mvMap = structInfoMap.getTableMaps().stream() .filter(b -> b.cardinality() == 2) .collect(Collectors.toList()).get(0); - StructInfo structInfo = structInfoMap.getStructInfo(c1.getMemo(), mvMap, root, null); + StructInfo structInfo = structInfoMap.getStructInfo(c1, mvMap, root, null); System.out.println(structInfo.getOriginalPlan().treeString()); BitSet bitSet = new BitSet(); - structInfo.getRelations().forEach(r -> bitSet.set((int) r.getTable().getId())); + for (CatalogRelation relation : structInfo.getRelations()) { + bitSet.set(c1.getStatementContext().getTableId(relation.getTable()).asInt()); + } Assertions.assertEquals(bitSet, mvMap); dropMvByNereids("drop materialized view mv1"); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java index 1756107b28..2bec767d46 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java @@ -20,8 +20,6 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; -import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.PlanChecker; @@ -30,8 +28,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import java.util.BitSet; - class EliminateJoinTest extends SqlTestBase { @Test void testLOJWithGroupBy() { @@ -67,9 +63,10 @@ class EliminateJoinTest extends SqlTestBase { HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); HyperGraph h3 = HyperGraph.builderForMv(p3).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); - Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, constructContext(p1, p2)).isInvalid()); + Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, + constructContext(p1, p2, c1)).isInvalid()); Assertions.assertTrue(res.getViewExpressions().isEmpty()); } @@ -97,7 +94,7 @@ class EliminateJoinTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertTrue(res.getViewExpressions().isEmpty()); dropConstraint("alter table T2 drop constraint uk"); @@ -139,10 +136,10 @@ class EliminateJoinTest extends SqlTestBase { HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); HyperGraph h3 = HyperGraph.builderForMv(p3).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertTrue(res.getViewExpressions().isEmpty()); - Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, constructContext(p1, p2)).isInvalid()); + Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, constructContext(p1, p2, c1)).isInvalid()); dropConstraint("alter table T2 drop constraint pk"); } @@ -173,7 +170,7 @@ class EliminateJoinTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertTrue(res.getViewExpressions().isEmpty()); dropConstraint("alter table T2 drop constraint pk"); @@ -206,20 +203,10 @@ class EliminateJoinTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertTrue(res.getViewExpressions().isEmpty()); dropConstraint("alter table T2 drop constraint pk"); dropConstraint("alter table T3 drop constraint uk"); } - - LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { - StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null, new BitSet()).get(0); - StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null, new BitSet()).get(0); - RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); - SlotMapping sm = SlotMapping.generate(rm); - return LogicalCompatibilityContext.from(rm, sm, st1, st2); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java index 360f1310e5..6d752cd6ca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java @@ -20,8 +20,6 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; -import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.PlanChecker; @@ -30,8 +28,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import java.util.BitSet; - class HyperGraphComparatorTest extends SqlTestBase { @Test void testInnerJoinAndLOJ() { @@ -59,7 +55,7 @@ class HyperGraphComparatorTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); } @@ -90,7 +86,7 @@ class HyperGraphComparatorTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); } @@ -122,7 +118,7 @@ class HyperGraphComparatorTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); } @@ -157,18 +153,8 @@ class HyperGraphComparatorTest extends SqlTestBase { .getAllPlan().get(0).child(0); HyperGraph h1 = HyperGraph.builderForMv(p1).build(); HyperGraph h2 = HyperGraph.builderForMv(p2).build(); - ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); + ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2, c1)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); } - - LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { - StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null, new BitSet()).get(0); - StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null, new BitSet()).get(0); - RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); - SlotMapping sm = SlotMapping.generate(rm); - return LogicalCompatibilityContext.from(rm, sm, st1, st2); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java index 5c7aaed36e..e879f46101 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java @@ -17,10 +17,19 @@ package org.apache.doris.nereids.sqltest; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo; +import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.utframe.TestWithFeService; +import java.util.BitSet; + public abstract class SqlTestBase extends TestWithFeService implements MemoPatternMatchSupported { @Override protected void runBeforeAll() throws Exception { @@ -82,4 +91,14 @@ public abstract class SqlTestBase extends TestWithFeService implements MemoPatte protected void runBeforeEach() throws Exception { StatementScopeIdGenerator.clear(); } + + protected LogicalCompatibilityContext constructContext(Plan p1, Plan p2, CascadesContext context) { + StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, + context, new BitSet()).get(0); + StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, + context, new BitSet()).get(0); + RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); + SlotMapping sm = SlotMapping.generate(rm); + return LogicalCompatibilityContext.from(rm, sm, st1, st2); + } }