database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

branch-2.1:[fix](mtmv) fix when compatible fail, will throw NPE (#49875) (#52273)

Browse Source 
pick: https://github.com/apache/doris/pull/49875
This commit is contained in:
zhangdong
2025-06-28 10:54:44 +08:00
committed by GitHub
parent af070ac853
commit 95f10aa8a1
23 changed files with 191 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
View File

@ -1009,6 +1009,11 @@ public class Alter {
case ADD_TASK:
mtmv.addTaskResult(alterMTMV.getTask(), alterMTMV.getRelation(), alterMTMV.getPartitionSnapshots(),
isReplay);
// If it is not a replay thread, it means that the current service is already a new version
// and does not require compatibility
if (isReplay) {
mtmv.compatible(Env.getCurrentEnv().getCatalogMgr());
}
break;
default:
throw new RuntimeException("Unknown type value: " + alterMTMV.getOpType());

17
fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
View File

@ -480,6 +480,10 @@ public class MTMV extends OlapTable {
this.refreshSnapshot = refreshSnapshot;
}
public boolean canBeCandidate() {
return getStatus().canBeCandidate();
}
public void readMvLock() {
this.mvRwLock.readLock().lock();
}
@ -556,6 +560,19 @@ public class MTMV extends OlapTable {
* The logic here is to be compatible with older versions by converting ID to name
*/
public void compatible(CatalogMgr catalogMgr) {
try {
compatibleInternal(catalogMgr);
Env.getCurrentEnv().getMtmvService().unregisterMTMV(this);
Env.getCurrentEnv().getMtmvService().registerMTMV(this, this.getDatabase().getId());
} catch (Throwable e) {
LOG.warn("MTMV compatible failed, dbName: {}, mvName: {}, errMsg: {}", getQualifiedDbName(), name,
e.getMessage());
status.setState(MTMVState.SCHEMA_CHANGE);
status.setSchemaChangeDetail("compatible failed, please refresh or recreate it, reason: " + e.getMessage());
}
}
private void compatibleInternal(CatalogMgr catalogMgr) throws Exception {
if (mvPartitionInfo != null) {
mvPartitionInfo.compatible(catalogMgr);
}

16
fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java
View File

@ -159,10 +159,18 @@ public class BaseTableInfo {
+ '}';
}
public void compatible(CatalogMgr catalogMgr) {
public void compatible(CatalogMgr catalogMgr) throws Exception {
if (!StringUtils.isEmpty(ctlName)) {
return;
}
// should not get meta from external catalog when replay, because the timeout period may be very long
if (ctlId != InternalCatalog.INTERNAL_CATALOG_ID) {
String msg = String.format(
"Can not compatibility external table, ctlId: %s, dbId: %s, tableId: %s",
ctlId, dbId, tableId);
LOG.warn(msg);
throw new Exception(msg);
}
try {
CatalogIf catalog = catalogMgr.getCatalogOrAnalysisException(ctlId);
DatabaseIf db = catalog.getDbOrAnalysisException(dbId);
@ -171,7 +179,11 @@ public class BaseTableInfo {
this.dbName = db.getFullName();
this.tableName = table.getName();
} catch (AnalysisException e) {
LOG.warn("MTMV compatible failed, ctlId: {}, dbId: {}, tableId: {}", ctlId, dbId, tableId, e);
String msg = String.format(
"Failed to get name based on id during compatibility process, ctlId: %s, dbId: %s, tableId: %s",
ctlId, dbId, tableId);
LOG.warn(msg, e);
throw new Exception(msg);
}
}

2
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
View File

@ -152,7 +152,7 @@ public class MTMVPartitionInfo {
}
}
public void compatible(CatalogMgr catalogMgr) {
public void compatible(CatalogMgr catalogMgr) throws Exception {
if (relatedTable == null) {
return;
}

61
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
View File

@ -20,8 +20,9 @@ package org.apache.doris.mtmv;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
@ -77,29 +78,28 @@ public class MTMVRefreshPartitionSnapshot {
+ '}';
}
public void compatible(MTMV mtmv) {
try {
// snapshot add partitionId resolve problem of insert overwrite
compatiblePartitions(mtmv);
} catch (Throwable e) {
LOG.warn("MTMV compatiblePartitions failed, mtmv: {}", mtmv.getName(), e);
}
try {
// change table id to BaseTableInfo
compatibleTables(mtmv);
} catch (Throwable e) {
LOG.warn("MTMV compatibleTables failed, mtmv: {}", mtmv.getName(), e);
}
try {
// snapshot add tableId resolve problem of recreate table
compatibleTablesSnapshot();
} catch (Throwable e) {
LOG.warn("MTMV compatibleTables failed, mtmv: {}", mtmv.getName(), e);
}
public void compatible(MTMV mtmv) throws Exception {
// snapshot add partitionId resolve problem of insert overwrite
compatiblePartitions(mtmv);
// change table id to BaseTableInfo
compatibleTables(mtmv);
// snapshot add tableId resolve problem of recreate table
compatibleTablesSnapshot();
}
private void compatiblePartitions(MTMV mtmv) throws AnalysisException {
if (mtmv.getMvPartitionInfo().getPartitionType().equals(MTMVPartitionType.SELF_MANAGE)) {
return;
}
// Only olapTable has historical data issues that require compatibility
if (mtmv.getMvPartitionInfo().getRelatedTableInfo().getCtlId() != InternalCatalog.INTERNAL_CATALOG_ID) {
return;
}
MTMVRelatedTableIf relatedTableIf = mtmv.getMvPartitionInfo().getRelatedTable();
// Only olapTable has historical data issues that require compatibility
if (!(relatedTableIf instanceof OlapTable)) {
return;
}
if (!checkHasDataWithoutPartitionId()) {
return;
}
@ -108,6 +108,8 @@ public class MTMVRefreshPartitionSnapshot {
MTMVVersionSnapshot versionSnapshot = (MTMVVersionSnapshot) entry.getValue();
if (versionSnapshot.getId() == 0) {
Partition partition = relatedTable.getPartition(entry.getKey());
// if not find partition, may be partition has been dropped,
// the impact is that MTMV will consider this partition to be async
if (partition != null) {
(versionSnapshot).setId(partition.getId());
}
@ -131,12 +133,7 @@ public class MTMVRefreshPartitionSnapshot {
for (Entry<BaseTableInfo, MTMVSnapshotIf> entry : tablesInfo.entrySet()) {
MTMVVersionSnapshot versionSnapshot = (MTMVVersionSnapshot) entry.getValue();
if (versionSnapshot.getId() == 0) {
try {
TableIf table = MTMVUtil.getTable(entry.getKey());
versionSnapshot.setId(table.getId());
} catch (AnalysisException e) {
LOG.warn("MTMV compatibleTablesSnapshot failed, can not get table by: {}", entry.getKey());
}
versionSnapshot.setId(entry.getKey().getTableId());
}
}
}
@ -150,7 +147,7 @@ public class MTMVRefreshPartitionSnapshot {
return false;
}
private void compatibleTables(MTMV mtmv) {
private void compatibleTables(MTMV mtmv) throws Exception {
if (tables.size() == tablesInfo.size()) {
return;
}
@ -164,8 +161,12 @@ public class MTMVRefreshPartitionSnapshot {
if (tableInfo.isPresent()) {
tablesInfo.put(tableInfo.get(), entry.getValue());
} else {
LOG.warn("MTMV compatibleTables failed, tableId: {}, relationTables: {}", entry.getKey(),
relation.getBaseTablesOneLevel());
String msg = String.format(
"Failed to get table info based on id during compatibility process, "
+ "tableId: %s, relationTables: %s",
entry.getKey(), relation.getBaseTablesOneLevel());
LOG.warn(msg);
throw new Exception(msg);
}
}
}

2
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java
View File

@ -91,7 +91,7 @@ public class MTMVRefreshSnapshot {
+ '}';
}
public void compatible(MTMV mtmv) {
public void compatible(MTMV mtmv) throws Exception {
if (MapUtils.isEmpty(partitionSnapshots)) {
return;
}

4
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
View File

@ -65,13 +65,13 @@ public class MTMVRelation {
+ '}';
}
public void compatible(CatalogMgr catalogMgr) {
public void compatible(CatalogMgr catalogMgr) throws Exception {
compatible(catalogMgr, baseTables);
compatible(catalogMgr, baseViews);
compatible(catalogMgr, baseTablesOneLevel);
}
private void compatible(CatalogMgr catalogMgr, Set<BaseTableInfo> infos) {
private void compatible(CatalogMgr catalogMgr, Set<BaseTableInfo> infos) throws Exception {
if (CollectionUtils.isEmpty(infos)) {
return;
}

43
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
View File

@ -78,48 +78,43 @@ public class MTMVRelationManager implements MTMVHookService {
/**
* if At least one partition is available, return this mtmv
*
* @param tableInfos
* @param candidateMTMVs
* @param ctx
* @return
*/
public Set<MTMV> getAvailableMTMVs(List<BaseTableInfo> tableInfos, ConnectContext ctx,
public Set<MTMV> getAvailableMTMVs(Set<MTMV> candidateMTMVs, ConnectContext ctx,
boolean forceConsistent, BiPredicate<ConnectContext, MTMV> predicate) {
Set<MTMV> res = Sets.newLinkedHashSet();
Set<BaseTableInfo> mvInfos = getMTMVInfos(tableInfos);
Map<List<String>, Set<String>> queryUsedPartitions = PartitionCompensator.getQueryUsedPartitions(
ctx.getStatementContext(), new BitSet());
for (BaseTableInfo tableInfo : mvInfos) {
try {
MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo);
if (predicate.test(ctx, mtmv)) {
continue;
}
if (!mtmv.isUseForRewrite()) {
continue;
}
BaseTableInfo relatedTableInfo = mtmv.getMvPartitionInfo().getRelatedTableInfo();
if (isMVPartitionValid(mtmv, ctx, forceConsistent,
relatedTableInfo == null ? null : queryUsedPartitions.get(relatedTableInfo.toList()))) {
res.add(mtmv);
}
} catch (Exception e) {
// not throw exception to client, just ignore it
LOG.warn("getTable failed: {}", tableInfo.toString(), e);
for (MTMV mtmv : candidateMTMVs) {
if (predicate.test(ctx, mtmv)) {
continue;
}
if (!mtmv.isUseForRewrite()) {
continue;
}
BaseTableInfo relatedTableInfo = mtmv.getMvPartitionInfo().getRelatedTableInfo();
if (isMVPartitionValid(mtmv, ctx, forceConsistent,
relatedTableInfo == null ? null : queryUsedPartitions.get(relatedTableInfo.toList()))) {
res.add(mtmv);
}
}
return res;
}
/**
* get all mtmv related to tableInfos.
* get candidate mtmv related to tableInfos.
*/
public Set<MTMV> getAllMTMVs(List<BaseTableInfo> tableInfos) {
public Set<MTMV> getCandidateMTMVs(List<BaseTableInfo> tableInfos) {
Set<MTMV> mtmvs = Sets.newLinkedHashSet();
Set<BaseTableInfo> mvInfos = getMTMVInfos(tableInfos);
for (BaseTableInfo tableInfo : mvInfos) {
try {
mtmvs.add((MTMV) MTMVUtil.getTable(tableInfo));
MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo);
if (mtmv.canBeCandidate()) {
mtmvs.add(mtmv);
}
} catch (Exception e) {
// not throw exception to client, just ignore it
LOG.warn("getTable failed: {}", tableInfo.toString(), e);

5
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
View File

@ -20,8 +20,6 @@ package org.apache.doris.mtmv;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.Partition;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVRefreshState;
import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Lists;
@ -56,8 +54,7 @@ public class MTMVRewriteUtil {
return res;
}
// check mv is normal
MTMVStatus mtmvStatus = mtmv.getStatus();
if (mtmvStatus.getState() != MTMVState.NORMAL || mtmvStatus.getRefreshState() == MTMVRefreshState.INIT) {
if (!mtmv.canBeCandidate()) {
return res;
}
// if relatedPartitions is empty but not null, which means query no partitions

6
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java
View File

@ -82,6 +82,12 @@ public class MTMVStatus {
return this;
}
public boolean canBeCandidate() {
// MTMVRefreshState.FAIL also can be candidate, because may have some sync partitions
return getState() == MTMVState.NORMAL
&& getRefreshState() != MTMVRefreshState.INIT;
}
@Override
public String toString() {
return "MTMVStatus{"

6
fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.nereids;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.View;
@ -189,6 +190,7 @@ public class StatementContext implements Closeable {
// if query is: select * from t2 join t5
// mtmvRelatedTables is mv1, mv2, mv3, t1, t2, t3, t4, t5
private final Map<List<String>, TableIf> mtmvRelatedTables = Maps.newHashMap();
private final Set<MTMV> candidateMTMVs = Sets.newHashSet();
// insert into target tables
private final Map<List<String>, TableIf> insertTargetTables = Maps.newHashMap();
// save view's def to avoid them change before lock
@ -306,6 +308,10 @@ public class StatementContext implements Closeable {
return mtmvRelatedTables;
}
public Set<MTMV> getCandidateMTMVs() {
return candidateMTMVs;
}
public Map<List<String>, TableIf> getTables() {
return tables;
}

8
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
View File

@ -207,9 +207,12 @@ public class CollectRelation implements AnalysisRuleFactory {
}
if (shouldCollect) {
Set<MTMV> mtmvSet = Env.getCurrentEnv().getMtmvService().getRelationManager()
.getAllMTMVs(Lists.newArrayList(new BaseTableInfo(table)));
LOG.info("table {} related mv set is {}", new BaseTableInfo(table), mtmvSet);
.getCandidateMTMVs(Lists.newArrayList(new BaseTableInfo(table)));
if (LOG.isDebugEnabled()) {
LOG.debug("table {} related mv set is {}", new BaseTableInfo(table), mtmvSet);
}
for (MTMV mtmv : mtmvSet) {
cascadesContext.getStatementContext().getCandidateMTMVs().add(mtmv);
cascadesContext.getStatementContext().getMtmvRelatedTables().put(mtmv.getFullQualifiers(), mtmv);
mtmv.readMvLock();
try {
@ -221,6 +224,7 @@ public class CollectRelation implements AnalysisRuleFactory {
LOG.debug("mtmv {} related base table include {}", new BaseTableInfo(mtmv), baseTableInfo);
}
try {
// Collect all base tables and lock them before querying
cascadesContext.getStatementContext().getAndCacheTable(baseTableInfo.toList(),
TableFrom.MTMV);
} catch (AnalysisException exception) {

8
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java
View File

@ -20,16 +20,13 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVUtil;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.PlannerHook;
import com.google.common.annotations.VisibleForTesting;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* If enable query rewrite with mv in dml, should init consistent materialization context after analyze
@ -49,10 +46,9 @@ public class InitConsistentMaterializationContextHook extends InitMaterializatio
}
protected Set<MTMV> getAvailableMTMVs(Set<TableIf> usedTables, CascadesContext cascadesContext) {
List<BaseTableInfo> usedBaseTables =
usedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList());
return Env.getCurrentEnv().getMtmvService().getRelationManager()
.getAvailableMTMVs(usedBaseTables, cascadesContext.getConnectContext(),
.getAvailableMTMVs(cascadesContext.getStatementContext().getCandidateMTMVs(),
cascadesContext.getConnectContext(),
true, ((connectContext, mtmv) -> {
return MTMVUtil.mtmvContainsExternalTable(mtmv) && (!connectContext.getSessionVariable()
.isEnableDmlMaterializedViewRewriteWhenBaseTableUnawareness());

7
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
View File

@ -20,7 +20,6 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVCache;
import org.apache.doris.mtmv.MTMVUtil;
import org.apache.doris.nereids.CascadesContext;
@ -39,7 +38,6 @@ import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* If enable query rewrite with mv, should init materialization context after analyze
@ -93,10 +91,9 @@ public class InitMaterializationContextHook implements PlannerHook {
}
protected Set<MTMV> getAvailableMTMVs(Set<TableIf> usedTables, CascadesContext cascadesContext) {
List<BaseTableInfo> usedBaseTables =
usedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList());
return Env.getCurrentEnv().getMtmvService().getRelationManager()
.getAvailableMTMVs(usedBaseTables, cascadesContext.getConnectContext(),
.getAvailableMTMVs(cascadesContext.getStatementContext().getCandidateMTMVs(),
cascadesContext.getConnectContext(),
false, ((connectContext, mtmv) -> {
return MTMVUtil.mtmvContainsExternalTable(mtmv) && (!connectContext.getSessionVariable()
.isEnableMaterializedViewRewriteWhenBaseTableUnawareness());

2
fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java
View File

@ -51,7 +51,7 @@ public class AlterMTMVTest extends TestWithFeService {
MTMVRelationManager relationManager = Env.getCurrentEnv().getMtmvService().getRelationManager();
Table table = Env.getCurrentInternalCatalog().getDb("test").get().getTableOrMetaException("stu");
Set<MTMV> allMTMVs = relationManager.getAllMTMVs(Lists.newArrayList(new BaseTableInfo(table)));
Set<MTMV> allMTMVs = relationManager.getCandidateMTMVs(Lists.newArrayList(new BaseTableInfo(table)));
boolean hasMvA = false;
boolean hasMvB = false;
for (MTMV mtmv : allMTMVs) {

12
fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
View File

@ -122,6 +122,10 @@ public class MTMVRewriteUtilTest {
MTMVUtil.mtmvContainsExternalTable((MTMV) any);
minTimes = 0;
result = false;
mtmv.canBeCandidate();
minTimes = 0;
result = true;
}
};
}
@ -279,9 +283,9 @@ public class MTMVRewriteUtilTest {
public void testGetMTMVCanRewritePartitionsStateAbnormal() {
new Expectations() {
{
status.getState();
mtmv.canBeCandidate();
minTimes = 0;
result = MTMVState.SCHEMA_CHANGE;
result = false;
}
};
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
@ -309,9 +313,9 @@ public class MTMVRewriteUtilTest {
public void testGetMTMVCanRewritePartitionsRefreshStateInit() {
new Expectations() {
{
status.getRefreshState();
mtmv.canBeCandidate();
minTimes = 0;
result = MTMVRefreshState.INIT;
result = false;
}
};
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil

18
fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
View File

@ -70,6 +70,12 @@ class StructInfoMapTest extends SqlTestBase {
return true;
}
};
new MockUp<MTMV>() {
@Mock
public boolean canBeCandidate() {
return true;
}
};
connectContext.getSessionVariable().enableMaterializedViewRewrite = true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true;
@ -129,6 +135,12 @@ class StructInfoMapTest extends SqlTestBase {
return true;
}
};
new MockUp<MTMV>() {
@Mock
public boolean canBeCandidate() {
return true;
}
};
connectContext.getSessionVariable().enableMaterializedViewRewrite = true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true;
createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n"
@ -177,6 +189,12 @@ class StructInfoMapTest extends SqlTestBase {
return true;
}
};
new MockUp<MTMV>() {
@Mock
public boolean canBeCandidate() {
return true;
}
};
connectContext.getSessionVariable().enableMaterializedViewRewrite = true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true;
createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n"

6
fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java
View File

@ -59,6 +59,12 @@ public class IdStatisticsMapTest extends SqlTestBase {
return true;
}
};
new MockUp<MTMV>() {
@Mock
public boolean canBeCandidate() {
return true;
}
};
connectContext.getSessionVariable().enableMaterializedViewRewrite = true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true;
createMvByNereids("create materialized view mv100 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n"

6
fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java
View File

@ -55,6 +55,12 @@ public class MvTableIdIsLongTest extends SqlTestBase {
return true;
}
};
new MockUp<MTMV>() {
@Mock
public boolean canBeCandidate() {
return true;
}
};
connectContext.getSessionVariable().enableMaterializedViewRewrite = true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true;
createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n"

20
fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java
View File

@ -18,13 +18,16 @@
package org.apache.doris.nereids.mv;
import org.apache.doris.catalog.DistributionInfo;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.MaterializedIndex.IndexState;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.common.Pair;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVRelationManager;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.expression.rules.PartitionPruner;
import org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionTableType;
@ -48,6 +51,7 @@ import java.util.BitSet;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
/**
@ -127,6 +131,14 @@ public class OptimizeGetAvailableMvsTest extends SqlTestBase {
+ "inner join T3 on T4.id = T3.id",
connectContext
);
CatalogIf internal = getCatalog("internal");
Optional table = internal.getDbOrAnalysisException("test").getTable("mv1");
new MockUp<MTMVRelationManager>() {
@Mock
public Set<MTMV> getCandidateMTMVs(List<BaseTableInfo> baseTableInfos) {
return Sets.newHashSet((MTMV) table.get());
}
};
PlanChecker.from(c1)
.analyze()
.rewrite()
@ -238,6 +250,14 @@ public class OptimizeGetAvailableMvsTest extends SqlTestBase {
+ "where T4.id > 0",
connectContext
);
CatalogIf internal = getCatalog("internal");
Optional table = internal.getDbOrAnalysisException("test").getTable("mv2");
new MockUp<MTMVRelationManager>() {
@Mock
public Set<MTMV> getCandidateMTMVs(List<BaseTableInfo> baseTableInfos) {
return Sets.newHashSet((MTMV) table.get());
}
};
PlanChecker.from(c1)
.analyze()
.rewrite()

11
fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
View File

@ -18,6 +18,7 @@
package org.apache.doris.nereids.util;
import org.apache.doris.analysis.ExplainOptions;
import org.apache.doris.nereids.CTEContext;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.PlanProcess;
@ -49,6 +50,7 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleFactory;
import org.apache.doris.nereids.rules.RuleSet;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.exploration.mv.InitConsistentMaterializationContextHook;
import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
@ -120,6 +122,9 @@ public class PlanChecker {
}
public PlanChecker analyze() {
this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE);
this.cascadesContext.newTableCollector().collect();
this.cascadesContext.setCteContext(new CTEContext());
this.cascadesContext.newAnalyzer().analyze();
this.cascadesContext.toMemo();
return this;
@ -127,6 +132,9 @@ public class PlanChecker {
public PlanChecker analyze(Plan plan) {
this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan);
this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE);
this.cascadesContext.newTableCollector().collect();
this.cascadesContext.setCteContext(new CTEContext());
Set<String> originDisableRules = connectContext.getSessionVariable().getDisableNereidsRuleNames();
Set<String> disableRuleWithAuth = Sets.newHashSet(originDisableRules);
disableRuleWithAuth.add(RuleType.RELATION_AUTHENTICATION.name());
@ -140,6 +148,9 @@ public class PlanChecker {
public PlanChecker analyze(String sql) {
this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, sql);
this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE);
this.cascadesContext.newTableCollector().collect();
this.cascadesContext.setCteContext(new CTEContext());
this.cascadesContext.newAnalyzer().analyze();
this.cascadesContext.toMemo();
return this;