database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

branch-2.1: [enhance](mtmv)Exclude trigger table support db and catalog #49961 (#50554)

Browse Source 
Cherry-picked from #49961

Co-authored-by: zhangdong <zhangdong@selectdb.com>
This commit is contained in:
github-actions[bot]
2025-05-09 09:22:31 +08:00
committed by GitHub
parent 523681d58e
commit 12e6a02158
10 changed files with 381 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
fe/fe-core/src/main/java/org/apache/doris/analysis/TableName.java
View File

@ -20,19 +20,23 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.persist.gson.GsonUtils;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang3.StringUtils;
import java.io.DataInput;
import java.io.DataOutput;
@ -76,6 +80,27 @@ public class TableName implements Writable {
this.tbl = tbl;
}
public TableName(TableIf tableIf) throws AnalysisException {
String tableName = tableIf.getName();
if (StringUtils.isEmpty(tableName)) {
throw new AnalysisException("tableName is empty");
}
DatabaseIf db = tableIf.getDatabase();
if (db == null) {
throw new AnalysisException("db is null, tableName: " + tableName);
}
CatalogIf catalog = db.getCatalog();
if (catalog == null) {
throw new AnalysisException("catalog is null, dbName: " + db.getFullName());
}
if (Env.isStoredTableNamesLowerCase()) {
tableName = tableName.toLowerCase();
}
this.ctl = catalog.getName();
this.db = db.getFullName();
this.tbl = tableName;
}
public void analyze(Analyzer analyzer) throws AnalysisException {
if (Strings.isNullOrEmpty(ctl)) {
ctl = analyzer.getDefaultCatalog();

11
fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
View File

@ -18,6 +18,7 @@
package org.apache.doris.catalog;
import org.apache.doris.analysis.PartitionKeyDesc;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.OlapTableFactory.MTMVParams;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
@ -290,14 +291,18 @@ public class MTMV extends OlapTable {
}
}
public Set<String> getExcludedTriggerTables() {
public Set<TableName> getExcludedTriggerTables() {
Set<TableName> res = Sets.newHashSet();
readMvLock();
try {
if (StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES))) {
return Sets.newHashSet();
return res;
}
String[] split = mvProperties.get(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES).split(",");
return Sets.newHashSet(split);
for (String alias : split) {
res.add(new TableName(alias));
}
return res;
} finally {
readMvUnlock();
}

51
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.analysis.AllPartitionDesc;
import org.apache.doris.analysis.DropPartitionClause;
import org.apache.doris.analysis.PartitionKeyDesc;
import org.apache.doris.analysis.SinglePartitionDesc;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
@ -41,6 +42,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -84,14 +86,14 @@ public class MTMVPartitionUtil {
*/
public static boolean isMTMVPartitionSync(MTMVRefreshContext refreshContext, String partitionName,
Set<BaseTableInfo> tables,
Set<String> excludedTriggerTables) throws AnalysisException {
Set<TableName> excludedTriggerTables) throws AnalysisException {
MTMV mtmv = refreshContext.getMtmv();
Set<String> relatedPartitionNames = refreshContext.getPartitionMappings().get(partitionName);
boolean isSyncWithPartition = true;
if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE) {
MTMVRelatedTableIf relatedTable = mtmv.getMvPartitionInfo().getRelatedTable();
// if follow base table, not need compare with related table, only should compare with related partition
excludedTriggerTables.add(relatedTable.getName());
excludedTriggerTables.add(new TableName(relatedTable));
if (CollectionUtils.isEmpty(relatedPartitionNames)) {
LOG.warn("can not found related partition, partitionId: {}, mtmvName: {}, relatedTableName: {}",
partitionName, mtmv.getName(), relatedTable.getName());
@ -209,7 +211,8 @@ public class MTMVPartitionUtil {
* @return
* @throws AnalysisException
*/
public static boolean isMTMVSync(MTMVRefreshContext context, Set<BaseTableInfo> tables, Set<String> excludeTables)
public static boolean isMTMVSync(MTMVRefreshContext context, Set<BaseTableInfo> tables,
Set<TableName> excludeTables)
throws AnalysisException {
MTMV mtmv = context.getMtmv();
Set<String> partitionNames = mtmv.getPartitionNames();
@ -405,7 +408,7 @@ public class MTMVPartitionUtil {
*/
private static boolean isSyncWithAllBaseTables(MTMVRefreshContext context, String mtmvPartitionName,
Set<BaseTableInfo> tables,
Set<String> excludedTriggerTables) throws AnalysisException {
Set<TableName> excludedTriggerTables) throws AnalysisException {
for (BaseTableInfo baseTableInfo : tables) {
TableIf table = null;
try {
@ -414,7 +417,7 @@ public class MTMVPartitionUtil {
LOG.warn("get table failed, {}", baseTableInfo, e);
return false;
}
if (excludedTriggerTables.contains(table.getName())) {
if (isTableExcluded(excludedTriggerTables, new TableName(table))) {
continue;
}
boolean syncWithBaseTable = isSyncWithBaseTable(context, mtmvPartitionName, baseTableInfo);
@ -425,6 +428,44 @@ public class MTMVPartitionUtil {
return true;
}
public static boolean isTableExcluded(Set<TableName> excludedTriggerTables, TableName tableNameToCheck) {
for (TableName tableName : excludedTriggerTables) {
if (isTableNamelike(tableName, tableNameToCheck)) {
return true;
}
}
return false;
}
/**
* if excludedTriggerTable.field is empty, we think they are like,otherwise they must equal to tableNameToCheck's
*
* @param excludedTriggerTable User-configured tables to excluded,
* where dbName and ctlName are not mandatory fields and may therefore be empty.
* @param tableNameToCheck The table used to create an MTMV, must have non-empty tableName, dbName, and ctlName.
* @return
*/
public static boolean isTableNamelike(TableName excludedTriggerTable, TableName tableNameToCheck) {
Objects.requireNonNull(excludedTriggerTable, "excludedTriggerTable can not be null");
Objects.requireNonNull(tableNameToCheck, "tableNameToCheck can not be null");
String excludedCtl = excludedTriggerTable.getCtl();
String excludedDb = excludedTriggerTable.getDb();
String excludedTbl = excludedTriggerTable.getTbl();
String checkCtl = tableNameToCheck.getCtl();
String checkDb = tableNameToCheck.getDb();
String checkTbl = tableNameToCheck.getTbl();
Objects.requireNonNull(excludedTbl, "excludedTbl can not be null");
Objects.requireNonNull(checkCtl, "checkCtl can not be null");
Objects.requireNonNull(checkDb, "checkDb can not be null");
Objects.requireNonNull(checkTbl, "checkTbl can not be null");
return (excludedTbl.equals(checkTbl))
&& (StringUtils.isEmpty(excludedDb) || excludedDb.equals(checkDb))
&& (StringUtils.isEmpty(excludedCtl) || excludedCtl.equals(checkCtl));
}
private static boolean isSyncWithBaseTable(MTMVRefreshContext context, String mtmvPartitionName,
BaseTableInfo baseTableInfo)
throws AnalysisException {

13
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.mtmv;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.Table;
@ -207,8 +208,16 @@ public class MTMVService implements EventListener {
}
}
private boolean canRefresh(MTMV mtmv, TableIf table) {
if (mtmv.getExcludedTriggerTables().contains(table.getName())) {
private boolean canRefresh(MTMV mtmv, TableIf table) {
TableName tableName = null;
try {
tableName = new TableName(table);
} catch (AnalysisException e) {
LOG.warn("skip refresh mtmv: {}, because get TableName failed: {}",
mtmv.getName(), table.getName());
return false;
}
if (MTMVPartitionUtil.isTableExcluded(mtmv.getExcludedTriggerTables(), tableName)) {
LOG.info("skip refresh mtmv: {}, because exclude trigger table: {}",
mtmv.getName(), table.getName());
return false;

78
fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
View File

@ -19,10 +19,13 @@ package org.apache.doris.mtmv;
import org.apache.doris.analysis.PartitionKeyDesc;
import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
import com.google.common.collect.Lists;
@ -52,6 +55,10 @@ public class MTMVPartitionUtilTest {
@Mocked
private OlapTable baseOlapTable;
@Mocked
private DatabaseIf databaseIf;
@Mocked
private CatalogIf catalogIf;
@Mocked
private MTMVSnapshotIf baseSnapshotIf;
@Mocked
private MTMVRefreshSnapshot refreshSnapshot;
@ -144,6 +151,30 @@ public class MTMVPartitionUtilTest {
refreshSnapshot.getSnapshotPartitions(anyString);
minTimes = 0;
result = Sets.newHashSet("name2");
baseOlapTable.getName();
minTimes = 0;
result = "t1";
baseOlapTable.getDatabase();
minTimes = 0;
result = databaseIf;
databaseIf.getFullName();
minTimes = 0;
result = "db1";
databaseIf.getCatalog();
minTimes = 0;
result = catalogIf;
databaseIf.getCatalog();
minTimes = 0;
result = catalogIf;
catalogIf.getName();
minTimes = 0;
result = "ctl1";
}
};
}
@ -218,4 +249,51 @@ public class MTMVPartitionUtilTest {
String rangeName = MTMVPartitionUtil.generatePartitionName(rangeDesc);
Assert.assertEquals("p_1_2", rangeName);
}
@Test
public void testIsTableExcluded() {
Set<TableName> excludedTriggerTables = Sets.newHashSet(new TableName("table1"));
Assert.assertTrue(
MTMVPartitionUtil.isTableExcluded(excludedTriggerTables, new TableName("ctl1", "db1", "table1")));
Assert.assertTrue(
MTMVPartitionUtil.isTableExcluded(excludedTriggerTables, new TableName("ctl1", "db2", "table1")));
Assert.assertTrue(
MTMVPartitionUtil.isTableExcluded(excludedTriggerTables, new TableName("ctl2", "db1", "table1")));
Assert.assertFalse(
MTMVPartitionUtil.isTableExcluded(excludedTriggerTables, new TableName("ctl1", "db1", "table2")));
excludedTriggerTables = Sets.newHashSet(new TableName("db1.table1"));
Assert.assertTrue(
MTMVPartitionUtil.isTableExcluded(excludedTriggerTables, new TableName("ctl1", "db1", "table1")));
Assert.assertFalse(
MTMVPartitionUtil.isTableExcluded(excludedTriggerTables, new TableName("ctl1", "db2", "table1")));
Assert.assertTrue(
MTMVPartitionUtil.isTableExcluded(excludedTriggerTables, new TableName("ctl2", "db1", "table1")));
Assert.assertFalse(
MTMVPartitionUtil.isTableExcluded(excludedTriggerTables, new TableName("ctl1", "db1", "table2")));
excludedTriggerTables = Sets.newHashSet(new TableName("ctl1.db1.table1"));
Assert.assertTrue(
MTMVPartitionUtil.isTableExcluded(excludedTriggerTables, new TableName("ctl1", "db1", "table1")));
Assert.assertFalse(
MTMVPartitionUtil.isTableExcluded(excludedTriggerTables, new TableName("ctl1", "db2", "table1")));
Assert.assertFalse(
MTMVPartitionUtil.isTableExcluded(excludedTriggerTables, new TableName("ctl2", "db1", "table1")));
Assert.assertFalse(
MTMVPartitionUtil.isTableExcluded(excludedTriggerTables, new TableName("ctl1", "db1", "table2")));
}
@Test
public void testIsTableNamelike() {
TableName tableNameToCheck = new TableName("ctl1", "db1", "table1");
Assert.assertTrue(MTMVPartitionUtil.isTableNamelike(new TableName("table1"), tableNameToCheck));
Assert.assertTrue(MTMVPartitionUtil.isTableNamelike(new TableName("db1.table1"), tableNameToCheck));
Assert.assertTrue(MTMVPartitionUtil.isTableNamelike(new TableName("ctl1.db1.table1"), tableNameToCheck));
Assert.assertFalse(MTMVPartitionUtil.isTableNamelike(new TableName("ctl1.table1"), tableNameToCheck));
Assert.assertFalse(MTMVPartitionUtil.isTableNamelike(new TableName("ctl1.db2.table1"), tableNameToCheck));
Assert.assertFalse(MTMVPartitionUtil.isTableNamelike(new TableName("ctl1.db1.table2"), tableNameToCheck));
Assert.assertFalse(MTMVPartitionUtil.isTableNamelike(new TableName("ctl2.db1.table1"), tableNameToCheck));
Assert.assertFalse(MTMVPartitionUtil.isTableNamelike(new TableName("db1"), tableNameToCheck));
Assert.assertFalse(MTMVPartitionUtil.isTableNamelike(new TableName("ctl1"), tableNameToCheck));
}
}

11
fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.mtmv;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.Partition;
import org.apache.doris.common.AnalysisException;
@ -105,7 +106,7 @@ public class MTMVRewriteUtilTest {
MTMVPartitionUtil.isMTMVPartitionSync((MTMVRefreshContext) any, anyString,
(Set<BaseTableInfo>) any,
(Set<String>) any);
(Set<TableName>) any);
minTimes = 0;
result = true;
@ -126,7 +127,7 @@ public class MTMVRewriteUtilTest {
MTMVPartitionUtil.isMTMVPartitionSync((MTMVRefreshContext) any, anyString,
(Set<BaseTableInfo>) any,
(Set<String>) any);
(Set<TableName>) any);
minTimes = 0;
result = false;
}
@ -156,7 +157,7 @@ public class MTMVRewriteUtilTest {
MTMVPartitionUtil.isMTMVPartitionSync((MTMVRefreshContext) any, anyString,
(Set<BaseTableInfo>) any,
(Set<String>) any);
(Set<TableName>) any);
minTimes = 0;
result = false;
}
@ -177,7 +178,7 @@ public class MTMVRewriteUtilTest {
MTMVPartitionUtil.isMTMVPartitionSync((MTMVRefreshContext) any, anyString,
(Set<BaseTableInfo>) any,
(Set<String>) any);
(Set<TableName>) any);
minTimes = 0;
result = false;
}
@ -210,7 +211,7 @@ public class MTMVRewriteUtilTest {
{
MTMVPartitionUtil.isMTMVPartitionSync((MTMVRefreshContext) any, anyString,
(Set<BaseTableInfo>) any,
(Set<String>) any);
(Set<TableName>) any);
minTimes = 0;
result = false;
}

7
fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.mtmv;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
@ -82,7 +83,7 @@ public class MTMVTaskTest {
// minTimes = 0;
// result = poneId;
mtmvPartitionUtil.isMTMVSync((MTMVRefreshContext) any, (Set<BaseTableInfo>) any, (Set<String>) any);
mtmvPartitionUtil.isMTMVSync((MTMVRefreshContext) any, (Set<BaseTableInfo>) any, (Set<TableName>) any);
minTimes = 0;
result = true;
@ -140,7 +141,7 @@ public class MTMVTaskTest {
public void testCalculateNeedRefreshPartitionsSystemNotSyncComplete() throws AnalysisException {
new Expectations() {
{
mtmvPartitionUtil.isMTMVSync((MTMVRefreshContext) any, (Set<BaseTableInfo>) any, (Set<String>) any);
mtmvPartitionUtil.isMTMVSync((MTMVRefreshContext) any, (Set<BaseTableInfo>) any, (Set<TableName>) any);
minTimes = 0;
result = false;
}
@ -156,7 +157,7 @@ public class MTMVTaskTest {
new Expectations() {
{
mtmvPartitionUtil
.isMTMVSync((MTMVRefreshContext) any, (Set<BaseTableInfo>) any, (Set<String>) any);
.isMTMVSync((MTMVRefreshContext) any, (Set<BaseTableInfo>) any, (Set<TableName>) any);
minTimes = 0;
result = false;

31
fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.mtmv;
import org.apache.doris.analysis.PartitionKeyDesc;
import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.PartitionItem;
@ -27,6 +28,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.RangePartitionItem;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.job.common.IntervalUnit;
import org.apache.doris.job.extensions.mtmv.MTMVTask;
import org.apache.doris.mtmv.MTMVRefreshEnum.BuildMode;
@ -142,4 +144,33 @@ public class MTMVTest {
res.put("mvp1", item1);
return res;
}
@Test
public void testGetExcludedTriggerTables() {
Map<String, String> mvProperties = Maps.newHashMap();
MTMV mtmv = new MTMV();
mtmv.setMvProperties(mvProperties);
mvProperties.put(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES, "t1");
Set<TableName> excludedTriggerTables = mtmv.getExcludedTriggerTables();
Assert.assertEquals(1, excludedTriggerTables.size());
Assert.assertTrue(excludedTriggerTables.contains(new TableName(null, null, "t1")));
mvProperties.put(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES, "db1.t1");
excludedTriggerTables = mtmv.getExcludedTriggerTables();
Assert.assertEquals(1, excludedTriggerTables.size());
Assert.assertTrue(excludedTriggerTables.contains(new TableName(null, "db1", "t1")));
mvProperties.put(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES, "ctl1.db1.t1");
excludedTriggerTables = mtmv.getExcludedTriggerTables();
Assert.assertEquals(1, excludedTriggerTables.size());
Assert.assertTrue(excludedTriggerTables.contains(new TableName("ctl1", "db1", "t1")));
mvProperties.put(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES, "ctl1.db1.t1,db2.t2,t3");
excludedTriggerTables = mtmv.getExcludedTriggerTables();
Assert.assertEquals(3, excludedTriggerTables.size());
Assert.assertTrue(excludedTriggerTables.contains(new TableName("ctl1", "db1", "t1")));
Assert.assertTrue(excludedTriggerTables.contains(new TableName(null, "db2", "t2")));
Assert.assertTrue(excludedTriggerTables.contains(new TableName(null, null, "t3")));
}
}