From 95f10aa8a15625ac0e01ff2925c83597f3ab114c Mon Sep 17 00:00:00 2001 From: zhangdong Date: Sat, 28 Jun 2025 10:54:44 +0800 Subject: [PATCH] branch-2.1:[fix](mtmv) fix when compatible fail, will throw NPE (#49875) (#52273) pick: https://github.com/apache/doris/pull/49875 --- .../java/org/apache/doris/alter/Alter.java | 5 ++ .../java/org/apache/doris/catalog/MTMV.java | 17 ++++++ .../org/apache/doris/mtmv/BaseTableInfo.java | 16 ++++- .../apache/doris/mtmv/MTMVPartitionInfo.java | 2 +- .../mtmv/MTMVRefreshPartitionSnapshot.java | 61 ++++++++++--------- .../doris/mtmv/MTMVRefreshSnapshot.java | 2 +- .../org/apache/doris/mtmv/MTMVRelation.java | 4 +- .../doris/mtmv/MTMVRelationManager.java | 43 ++++++------- .../apache/doris/mtmv/MTMVRewriteUtil.java | 5 +- .../org/apache/doris/mtmv/MTMVStatus.java | 6 ++ .../doris/nereids/StatementContext.java | 6 ++ .../rules/analysis/CollectRelation.java | 8 ++- ...tConsistentMaterializationContextHook.java | 8 +-- .../mv/InitMaterializationContextHook.java | 7 +-- .../org/apache/doris/mtmv/AlterMTMVTest.java | 2 +- .../doris/mtmv/MTMVRewriteUtilTest.java | 12 ++-- .../doris/nereids/memo/StructInfoMapTest.java | 18 ++++++ .../doris/nereids/mv/IdStatisticsMapTest.java | 6 ++ .../doris/nereids/mv/MvTableIdIsLongTest.java | 6 ++ .../mv/OptimizeGetAvailableMvsTest.java | 20 ++++++ .../doris/nereids/util/PlanChecker.java | 11 ++++ .../insert/dml_insert_and_overwrite.groovy | 9 +-- .../mv/dml/outfile/dml_into_outfile.groovy | 5 +- 23 files changed, 191 insertions(+), 88 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index 3b5c64b7ff..95e9d92aa4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -1009,6 +1009,11 @@ public class Alter { case ADD_TASK: mtmv.addTaskResult(alterMTMV.getTask(), alterMTMV.getRelation(), alterMTMV.getPartitionSnapshots(), isReplay); + // If it is not a replay thread, it means that the current service is already a new version + // and does not require compatibility + if (isReplay) { + mtmv.compatible(Env.getCurrentEnv().getCatalogMgr()); + } break; default: throw new RuntimeException("Unknown type value: " + alterMTMV.getOpType()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index ea4e94d5c9..c82c749120 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -480,6 +480,10 @@ public class MTMV extends OlapTable { this.refreshSnapshot = refreshSnapshot; } + public boolean canBeCandidate() { + return getStatus().canBeCandidate(); + } + public void readMvLock() { this.mvRwLock.readLock().lock(); } @@ -556,6 +560,19 @@ public class MTMV extends OlapTable { * The logic here is to be compatible with older versions by converting ID to name */ public void compatible(CatalogMgr catalogMgr) { + try { + compatibleInternal(catalogMgr); + Env.getCurrentEnv().getMtmvService().unregisterMTMV(this); + Env.getCurrentEnv().getMtmvService().registerMTMV(this, this.getDatabase().getId()); + } catch (Throwable e) { + LOG.warn("MTMV compatible failed, dbName: {}, mvName: {}, errMsg: {}", getQualifiedDbName(), name, + e.getMessage()); + status.setState(MTMVState.SCHEMA_CHANGE); + status.setSchemaChangeDetail("compatible failed, please refresh or recreate it, reason: " + e.getMessage()); + } + } + + private void compatibleInternal(CatalogMgr catalogMgr) throws Exception { if (mvPartitionInfo != null) { mvPartitionInfo.compatible(catalogMgr); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java index 2f0810c7a7..49f0d4f948 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java @@ -159,10 +159,18 @@ public class BaseTableInfo { + '}'; } - public void compatible(CatalogMgr catalogMgr) { + public void compatible(CatalogMgr catalogMgr) throws Exception { if (!StringUtils.isEmpty(ctlName)) { return; } + // should not get meta from external catalog when replay, because the timeout period may be very long + if (ctlId != InternalCatalog.INTERNAL_CATALOG_ID) { + String msg = String.format( + "Can not compatibility external table, ctlId: %s, dbId: %s, tableId: %s", + ctlId, dbId, tableId); + LOG.warn(msg); + throw new Exception(msg); + } try { CatalogIf catalog = catalogMgr.getCatalogOrAnalysisException(ctlId); DatabaseIf db = catalog.getDbOrAnalysisException(dbId); @@ -171,7 +179,11 @@ public class BaseTableInfo { this.dbName = db.getFullName(); this.tableName = table.getName(); } catch (AnalysisException e) { - LOG.warn("MTMV compatible failed, ctlId: {}, dbId: {}, tableId: {}", ctlId, dbId, tableId, e); + String msg = String.format( + "Failed to get name based on id during compatibility process, ctlId: %s, dbId: %s, tableId: %s", + ctlId, dbId, tableId); + LOG.warn(msg, e); + throw new Exception(msg); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java index 7eae44db0a..682273a93c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java @@ -152,7 +152,7 @@ public class MTMVPartitionInfo { } } - public void compatible(CatalogMgr catalogMgr) { + public void compatible(CatalogMgr catalogMgr) throws Exception { if (relatedTable == null) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java index a8de5b6597..2601ace5ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java @@ -20,8 +20,9 @@ package org.apache.doris.mtmv; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; @@ -77,29 +78,28 @@ public class MTMVRefreshPartitionSnapshot { + '}'; } - public void compatible(MTMV mtmv) { - try { - // snapshot add partitionId resolve problem of insert overwrite - compatiblePartitions(mtmv); - } catch (Throwable e) { - LOG.warn("MTMV compatiblePartitions failed, mtmv: {}", mtmv.getName(), e); - } - try { - // change table id to BaseTableInfo - compatibleTables(mtmv); - } catch (Throwable e) { - LOG.warn("MTMV compatibleTables failed, mtmv: {}", mtmv.getName(), e); - } - - try { - // snapshot add tableId resolve problem of recreate table - compatibleTablesSnapshot(); - } catch (Throwable e) { - LOG.warn("MTMV compatibleTables failed, mtmv: {}", mtmv.getName(), e); - } + public void compatible(MTMV mtmv) throws Exception { + // snapshot add partitionId resolve problem of insert overwrite + compatiblePartitions(mtmv); + // change table id to BaseTableInfo + compatibleTables(mtmv); + // snapshot add tableId resolve problem of recreate table + compatibleTablesSnapshot(); } private void compatiblePartitions(MTMV mtmv) throws AnalysisException { + if (mtmv.getMvPartitionInfo().getPartitionType().equals(MTMVPartitionType.SELF_MANAGE)) { + return; + } + // Only olapTable has historical data issues that require compatibility + if (mtmv.getMvPartitionInfo().getRelatedTableInfo().getCtlId() != InternalCatalog.INTERNAL_CATALOG_ID) { + return; + } + MTMVRelatedTableIf relatedTableIf = mtmv.getMvPartitionInfo().getRelatedTable(); + // Only olapTable has historical data issues that require compatibility + if (!(relatedTableIf instanceof OlapTable)) { + return; + } if (!checkHasDataWithoutPartitionId()) { return; } @@ -108,6 +108,8 @@ public class MTMVRefreshPartitionSnapshot { MTMVVersionSnapshot versionSnapshot = (MTMVVersionSnapshot) entry.getValue(); if (versionSnapshot.getId() == 0) { Partition partition = relatedTable.getPartition(entry.getKey()); + // if not find partition, may be partition has been dropped, + // the impact is that MTMV will consider this partition to be async if (partition != null) { (versionSnapshot).setId(partition.getId()); } @@ -131,12 +133,7 @@ public class MTMVRefreshPartitionSnapshot { for (Entry entry : tablesInfo.entrySet()) { MTMVVersionSnapshot versionSnapshot = (MTMVVersionSnapshot) entry.getValue(); if (versionSnapshot.getId() == 0) { - try { - TableIf table = MTMVUtil.getTable(entry.getKey()); - versionSnapshot.setId(table.getId()); - } catch (AnalysisException e) { - LOG.warn("MTMV compatibleTablesSnapshot failed, can not get table by: {}", entry.getKey()); - } + versionSnapshot.setId(entry.getKey().getTableId()); } } } @@ -150,7 +147,7 @@ public class MTMVRefreshPartitionSnapshot { return false; } - private void compatibleTables(MTMV mtmv) { + private void compatibleTables(MTMV mtmv) throws Exception { if (tables.size() == tablesInfo.size()) { return; } @@ -164,8 +161,12 @@ public class MTMVRefreshPartitionSnapshot { if (tableInfo.isPresent()) { tablesInfo.put(tableInfo.get(), entry.getValue()); } else { - LOG.warn("MTMV compatibleTables failed, tableId: {}, relationTables: {}", entry.getKey(), - relation.getBaseTablesOneLevel()); + String msg = String.format( + "Failed to get table info based on id during compatibility process, " + + "tableId: %s, relationTables: %s", + entry.getKey(), relation.getBaseTablesOneLevel()); + LOG.warn(msg); + throw new Exception(msg); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java index 74fc3cc1c5..0d9665cb44 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java @@ -91,7 +91,7 @@ public class MTMVRefreshSnapshot { + '}'; } - public void compatible(MTMV mtmv) { + public void compatible(MTMV mtmv) throws Exception { if (MapUtils.isEmpty(partitionSnapshots)) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java index 87a0199f12..148d2d0088 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java @@ -65,13 +65,13 @@ public class MTMVRelation { + '}'; } - public void compatible(CatalogMgr catalogMgr) { + public void compatible(CatalogMgr catalogMgr) throws Exception { compatible(catalogMgr, baseTables); compatible(catalogMgr, baseViews); compatible(catalogMgr, baseTablesOneLevel); } - private void compatible(CatalogMgr catalogMgr, Set infos) { + private void compatible(CatalogMgr catalogMgr, Set infos) throws Exception { if (CollectionUtils.isEmpty(infos)) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java index c45558ec8c..ba498636d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java @@ -78,48 +78,43 @@ public class MTMVRelationManager implements MTMVHookService { /** * if At least one partition is available, return this mtmv * - * @param tableInfos + * @param candidateMTMVs * @param ctx * @return */ - public Set getAvailableMTMVs(List tableInfos, ConnectContext ctx, + public Set getAvailableMTMVs(Set candidateMTMVs, ConnectContext ctx, boolean forceConsistent, BiPredicate predicate) { Set res = Sets.newLinkedHashSet(); - Set mvInfos = getMTMVInfos(tableInfos); Map, Set> queryUsedPartitions = PartitionCompensator.getQueryUsedPartitions( ctx.getStatementContext(), new BitSet()); - - for (BaseTableInfo tableInfo : mvInfos) { - try { - MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo); - if (predicate.test(ctx, mtmv)) { - continue; - } - if (!mtmv.isUseForRewrite()) { - continue; - } - BaseTableInfo relatedTableInfo = mtmv.getMvPartitionInfo().getRelatedTableInfo(); - if (isMVPartitionValid(mtmv, ctx, forceConsistent, - relatedTableInfo == null ? null : queryUsedPartitions.get(relatedTableInfo.toList()))) { - res.add(mtmv); - } - } catch (Exception e) { - // not throw exception to client, just ignore it - LOG.warn("getTable failed: {}", tableInfo.toString(), e); + for (MTMV mtmv : candidateMTMVs) { + if (predicate.test(ctx, mtmv)) { + continue; + } + if (!mtmv.isUseForRewrite()) { + continue; + } + BaseTableInfo relatedTableInfo = mtmv.getMvPartitionInfo().getRelatedTableInfo(); + if (isMVPartitionValid(mtmv, ctx, forceConsistent, + relatedTableInfo == null ? null : queryUsedPartitions.get(relatedTableInfo.toList()))) { + res.add(mtmv); } } return res; } /** - * get all mtmv related to tableInfos. + * get candidate mtmv related to tableInfos. */ - public Set getAllMTMVs(List tableInfos) { + public Set getCandidateMTMVs(List tableInfos) { Set mtmvs = Sets.newLinkedHashSet(); Set mvInfos = getMTMVInfos(tableInfos); for (BaseTableInfo tableInfo : mvInfos) { try { - mtmvs.add((MTMV) MTMVUtil.getTable(tableInfo)); + MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo); + if (mtmv.canBeCandidate()) { + mtmvs.add(mtmv); + } } catch (Exception e) { // not throw exception to client, just ignore it LOG.warn("getTable failed: {}", tableInfo.toString(), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index afaad55a34..58b2a37d50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -20,8 +20,6 @@ package org.apache.doris.mtmv; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.Partition; import org.apache.doris.common.AnalysisException; -import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVRefreshState; -import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.Lists; @@ -56,8 +54,7 @@ public class MTMVRewriteUtil { return res; } // check mv is normal - MTMVStatus mtmvStatus = mtmv.getStatus(); - if (mtmvStatus.getState() != MTMVState.NORMAL || mtmvStatus.getRefreshState() == MTMVRefreshState.INIT) { + if (!mtmv.canBeCandidate()) { return res; } // if relatedPartitions is empty but not null, which means query no partitions diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java index b1761b9e97..aa058e628c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java @@ -82,6 +82,12 @@ public class MTMVStatus { return this; } + public boolean canBeCandidate() { + // MTMVRefreshState.FAIL also can be candidate, because may have some sync partitions + return getState() == MTMVState.NORMAL + && getRefreshState() != MTMVRefreshState.INIT; + } + @Override public String toString() { return "MTMVStatus{" diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 009c51595b..cc04721ab8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids; import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.View; @@ -189,6 +190,7 @@ public class StatementContext implements Closeable { // if query is: select * from t2 join t5 // mtmvRelatedTables is mv1, mv2, mv3, t1, t2, t3, t4, t5 private final Map, TableIf> mtmvRelatedTables = Maps.newHashMap(); + private final Set candidateMTMVs = Sets.newHashSet(); // insert into target tables private final Map, TableIf> insertTargetTables = Maps.newHashMap(); // save view's def to avoid them change before lock @@ -306,6 +308,10 @@ public class StatementContext implements Closeable { return mtmvRelatedTables; } + public Set getCandidateMTMVs() { + return candidateMTMVs; + } + public Map, TableIf> getTables() { return tables; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java index 410028ff73..c529cfb580 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java @@ -207,9 +207,12 @@ public class CollectRelation implements AnalysisRuleFactory { } if (shouldCollect) { Set mtmvSet = Env.getCurrentEnv().getMtmvService().getRelationManager() - .getAllMTMVs(Lists.newArrayList(new BaseTableInfo(table))); - LOG.info("table {} related mv set is {}", new BaseTableInfo(table), mtmvSet); + .getCandidateMTMVs(Lists.newArrayList(new BaseTableInfo(table))); + if (LOG.isDebugEnabled()) { + LOG.debug("table {} related mv set is {}", new BaseTableInfo(table), mtmvSet); + } for (MTMV mtmv : mtmvSet) { + cascadesContext.getStatementContext().getCandidateMTMVs().add(mtmv); cascadesContext.getStatementContext().getMtmvRelatedTables().put(mtmv.getFullQualifiers(), mtmv); mtmv.readMvLock(); try { @@ -221,6 +224,7 @@ public class CollectRelation implements AnalysisRuleFactory { LOG.debug("mtmv {} related base table include {}", new BaseTableInfo(mtmv), baseTableInfo); } try { + // Collect all base tables and lock them before querying cascadesContext.getStatementContext().getAndCacheTable(baseTableInfo.toList(), TableFrom.MTMV); } catch (AnalysisException exception) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java index fbcf4726a1..e86cca263d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java @@ -20,16 +20,13 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.TableIf; -import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.PlannerHook; import com.google.common.annotations.VisibleForTesting; -import java.util.List; import java.util.Set; -import java.util.stream.Collectors; /** * If enable query rewrite with mv in dml, should init consistent materialization context after analyze @@ -49,10 +46,9 @@ public class InitConsistentMaterializationContextHook extends InitMaterializatio } protected Set getAvailableMTMVs(Set usedTables, CascadesContext cascadesContext) { - List usedBaseTables = - usedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList()); return Env.getCurrentEnv().getMtmvService().getRelationManager() - .getAvailableMTMVs(usedBaseTables, cascadesContext.getConnectContext(), + .getAvailableMTMVs(cascadesContext.getStatementContext().getCandidateMTMVs(), + cascadesContext.getConnectContext(), true, ((connectContext, mtmv) -> { return MTMVUtil.mtmvContainsExternalTable(mtmv) && (!connectContext.getSessionVariable() .isEnableDmlMaterializedViewRewriteWhenBaseTableUnawareness()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index 914ae3a519..ba3afad0fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -20,7 +20,6 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.TableIf; -import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.nereids.CascadesContext; @@ -39,7 +38,6 @@ import java.util.ArrayList; import java.util.BitSet; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; /** * If enable query rewrite with mv, should init materialization context after analyze @@ -93,10 +91,9 @@ public class InitMaterializationContextHook implements PlannerHook { } protected Set getAvailableMTMVs(Set usedTables, CascadesContext cascadesContext) { - List usedBaseTables = - usedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList()); return Env.getCurrentEnv().getMtmvService().getRelationManager() - .getAvailableMTMVs(usedBaseTables, cascadesContext.getConnectContext(), + .getAvailableMTMVs(cascadesContext.getStatementContext().getCandidateMTMVs(), + cascadesContext.getConnectContext(), false, ((connectContext, mtmv) -> { return MTMVUtil.mtmvContainsExternalTable(mtmv) && (!connectContext.getSessionVariable() .isEnableMaterializedViewRewriteWhenBaseTableUnawareness()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java index 17ec145f58..342f9fd60c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java @@ -51,7 +51,7 @@ public class AlterMTMVTest extends TestWithFeService { MTMVRelationManager relationManager = Env.getCurrentEnv().getMtmvService().getRelationManager(); Table table = Env.getCurrentInternalCatalog().getDb("test").get().getTableOrMetaException("stu"); - Set allMTMVs = relationManager.getAllMTMVs(Lists.newArrayList(new BaseTableInfo(table))); + Set allMTMVs = relationManager.getCandidateMTMVs(Lists.newArrayList(new BaseTableInfo(table))); boolean hasMvA = false; boolean hasMvB = false; for (MTMV mtmv : allMTMVs) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java index 82c7eaac63..e4788c1840 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java @@ -122,6 +122,10 @@ public class MTMVRewriteUtilTest { MTMVUtil.mtmvContainsExternalTable((MTMV) any); minTimes = 0; result = false; + + mtmv.canBeCandidate(); + minTimes = 0; + result = true; } }; } @@ -279,9 +283,9 @@ public class MTMVRewriteUtilTest { public void testGetMTMVCanRewritePartitionsStateAbnormal() { new Expectations() { { - status.getState(); + mtmv.canBeCandidate(); minTimes = 0; - result = MTMVState.SCHEMA_CHANGE; + result = false; } }; Collection mtmvCanRewritePartitions = MTMVRewriteUtil @@ -309,9 +313,9 @@ public class MTMVRewriteUtilTest { public void testGetMTMVCanRewritePartitionsRefreshStateInit() { new Expectations() { { - status.getRefreshState(); + mtmv.canBeCandidate(); minTimes = 0; - result = MTMVRefreshState.INIT; + result = false; } }; Collection mtmvCanRewritePartitions = MTMVRewriteUtil diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java index 19d1efdbbd..0d3181d15e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java @@ -70,6 +70,12 @@ class StructInfoMapTest extends SqlTestBase { return true; } }; + new MockUp() { + @Mock + public boolean canBeCandidate() { + return true; + } + }; connectContext.getSessionVariable().enableMaterializedViewRewrite = true; connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; @@ -129,6 +135,12 @@ class StructInfoMapTest extends SqlTestBase { return true; } }; + new MockUp() { + @Mock + public boolean canBeCandidate() { + return true; + } + }; connectContext.getSessionVariable().enableMaterializedViewRewrite = true; connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" @@ -177,6 +189,12 @@ class StructInfoMapTest extends SqlTestBase { return true; } }; + new MockUp() { + @Mock + public boolean canBeCandidate() { + return true; + } + }; connectContext.getSessionVariable().enableMaterializedViewRewrite = true; connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java index 1403a9fee5..3e93919ed2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java @@ -59,6 +59,12 @@ public class IdStatisticsMapTest extends SqlTestBase { return true; } }; + new MockUp() { + @Mock + public boolean canBeCandidate() { + return true; + } + }; connectContext.getSessionVariable().enableMaterializedViewRewrite = true; connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; createMvByNereids("create materialized view mv100 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java index 4fa0a68e77..5c9fb3c039 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java @@ -55,6 +55,12 @@ public class MvTableIdIsLongTest extends SqlTestBase { return true; } }; + new MockUp() { + @Mock + public boolean canBeCandidate() { + return true; + } + }; connectContext.getSessionVariable().enableMaterializedViewRewrite = true; connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java index da327b2ba6..ab70b9dac4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java @@ -18,13 +18,16 @@ package org.apache.doris.nereids.mv; import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.MaterializedIndex.IndexState; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.common.Pair; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVRelationManager; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.expression.rules.PartitionPruner; import org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionTableType; @@ -48,6 +51,7 @@ import java.util.BitSet; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -127,6 +131,14 @@ public class OptimizeGetAvailableMvsTest extends SqlTestBase { + "inner join T3 on T4.id = T3.id", connectContext ); + CatalogIf internal = getCatalog("internal"); + Optional table = internal.getDbOrAnalysisException("test").getTable("mv1"); + new MockUp() { + @Mock + public Set getCandidateMTMVs(List baseTableInfos) { + return Sets.newHashSet((MTMV) table.get()); + } + }; PlanChecker.from(c1) .analyze() .rewrite() @@ -238,6 +250,14 @@ public class OptimizeGetAvailableMvsTest extends SqlTestBase { + "where T4.id > 0", connectContext ); + CatalogIf internal = getCatalog("internal"); + Optional table = internal.getDbOrAnalysisException("test").getTable("mv2"); + new MockUp() { + @Mock + public Set getCandidateMTMVs(List baseTableInfos) { + return Sets.newHashSet((MTMV) table.get()); + } + }; PlanChecker.from(c1) .analyze() .rewrite() diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index e535e3a8ac..71d0f0101b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.util; import org.apache.doris.analysis.ExplainOptions; +import org.apache.doris.nereids.CTEContext; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.PlanProcess; @@ -49,6 +50,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleFactory; import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.mv.InitConsistentMaterializationContextHook; import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; @@ -120,6 +122,9 @@ public class PlanChecker { } public PlanChecker analyze() { + this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE); + this.cascadesContext.newTableCollector().collect(); + this.cascadesContext.setCteContext(new CTEContext()); this.cascadesContext.newAnalyzer().analyze(); this.cascadesContext.toMemo(); return this; @@ -127,6 +132,9 @@ public class PlanChecker { public PlanChecker analyze(Plan plan) { this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); + this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE); + this.cascadesContext.newTableCollector().collect(); + this.cascadesContext.setCteContext(new CTEContext()); Set originDisableRules = connectContext.getSessionVariable().getDisableNereidsRuleNames(); Set disableRuleWithAuth = Sets.newHashSet(originDisableRules); disableRuleWithAuth.add(RuleType.RELATION_AUTHENTICATION.name()); @@ -140,6 +148,9 @@ public class PlanChecker { public PlanChecker analyze(String sql) { this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, sql); + this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE); + this.cascadesContext.newTableCollector().collect(); + this.cascadesContext.setCteContext(new CTEContext()); this.cascadesContext.newAnalyzer().analyze(); this.cascadesContext.toMemo(); return this; diff --git a/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy b/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy index 54e9b57f7e..753217f922 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy @@ -106,7 +106,8 @@ suite("dml_insert_and_overwrite") { ps_comment;""") // disable query rewrite by mv - sql "set enable_materialized_view_rewrite=false"; + // todo: Temporarily turn off, otherwise usable materialized views will not be collected and will need to be changed back in the future + sql "set enable_materialized_view_rewrite=true"; // enable dml rewrite by mv sql "set enable_dml_materialized_view_rewrite=true"; @@ -155,7 +156,7 @@ suite("dml_insert_and_overwrite") { ps_comment;""") // disable query rewrite by mv - sql "set enable_materialized_view_rewrite=false"; + sql "set enable_materialized_view_rewrite=true"; // enable dml rewrite by mv sql "set enable_dml_materialized_view_rewrite=true"; @@ -203,7 +204,7 @@ suite("dml_insert_and_overwrite") { ps_comment;""") // disable query rewrite by mv - sql "set enable_materialized_view_rewrite=false"; + sql "set enable_materialized_view_rewrite=true"; // enable dml rewrite by mv sql "set enable_dml_materialized_view_rewrite=true"; @@ -249,7 +250,7 @@ suite("dml_insert_and_overwrite") { ps_comment;""") // disable query rewrite by mv - sql "set enable_materialized_view_rewrite=false"; + sql "set enable_materialized_view_rewrite=true"; // enable dml rewrite by mv sql "set enable_dml_materialized_view_rewrite=true"; diff --git a/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy b/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy index 350e49057d..dc23130b99 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy @@ -122,7 +122,8 @@ suite("dml_into_outfile", "p0") { ps_comment;""") // disable query rewrite by mv - sql "set enable_materialized_view_rewrite=false"; + // todo: Temporarily turn off, otherwise usable materialized views will not be collected and will need to be changed back in the future + sql "set enable_materialized_view_rewrite=true"; // enable dml rewrite by mv sql "set enable_dml_materialized_view_rewrite=true"; @@ -185,7 +186,7 @@ suite("dml_into_outfile", "p0") { ps_comment;""") // disable query rewrite by mv - sql "set enable_materialized_view_rewrite=false"; + sql "set enable_materialized_view_rewrite=true"; // enable dml rewrite by mv sql "set enable_dml_materialized_view_rewrite=true";