database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhance](mtmv)Optimize MTMV lock logic (#44967) (#45076)

Browse Source 
pick: https://github.com/apache/doris/pull/44967
This commit is contained in:
zhangdong
2024-12-06 17:17:32 +08:00
committed by GitHub
parent ebb21ef031
commit 4f45fefea2
6 changed files with 52 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -3217,25 +3217,33 @@ public class Env {
hidePassword, false, specificVersion, false, true);
}
public static String getMTMVDdl(MTMV mtmv) {
StringBuilder sb = new StringBuilder("CREATE MATERIALIZED VIEW ");
sb.append(mtmv.getName());
addMTMVCols(mtmv, sb);
sb.append("\n");
sb.append(mtmv.getRefreshInfo());
addMTMVKeyInfo(mtmv, sb);
addTableComment(mtmv, sb);
addMTMVPartitionInfo(mtmv, sb);
DistributionInfo distributionInfo = mtmv.getDefaultDistributionInfo();
sb.append("\n").append(distributionInfo.toSql());
// properties
sb.append("\nPROPERTIES (\n");
addOlapTablePropertyInfo(mtmv, sb, false, false, null);
addMTMVPropertyInfo(mtmv, sb);
sb.append("\n)");
sb.append("\nAS ");
sb.append(mtmv.getQuerySql());
return sb.toString();
public static String getMTMVDdl(MTMV mtmv) throws AnalysisException {
if (!mtmv.tryReadLock(1, TimeUnit.MINUTES)) {
throw new AnalysisException(
"get table read lock timeout, database=" + mtmv.getQualifiedDbName() + ",table=" + mtmv.getName());
}
try {
StringBuilder sb = new StringBuilder("CREATE MATERIALIZED VIEW ");
sb.append(mtmv.getName());
addMTMVCols(mtmv, sb);
sb.append("\n");
sb.append(mtmv.getRefreshInfo());
addMTMVKeyInfo(mtmv, sb);
addTableComment(mtmv, sb);
addMTMVPartitionInfo(mtmv, sb);
DistributionInfo distributionInfo = mtmv.getDefaultDistributionInfo();
sb.append("\n").append(distributionInfo.toSql());
// properties
sb.append("\nPROPERTIES (\n");
addOlapTablePropertyInfo(mtmv, sb, false, false, null);
addMTMVPropertyInfo(mtmv, sb);
sb.append("\n)");
sb.append("\nAS ");
sb.append(mtmv.getQuerySql());
return sb.toString();
} finally {
mtmv.readUnlock();
}
}
private static void addMTMVKeyInfo(MTMV mtmv, StringBuilder sb) {

2
fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
View File

@ -385,7 +385,7 @@ public class MTMV extends OlapTable {
Map<String, String> baseToMv = Maps.newHashMap();
Map<PartitionKeyDesc, Set<String>> relatedPartitionDescs = MTMVPartitionUtil
.generateRelatedPartitionDescs(mvPartitionInfo, mvProperties);
Map<String, PartitionItem> mvPartitionItems = getAndCopyPartitionItems();
Map<String, PartitionItem> mvPartitionItems = getAndCopyPartitionItemsWithoutLock();
for (Entry<String, PartitionItem> entry : mvPartitionItems.entrySet()) {
Set<String> basePartitionNames = relatedPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(),
Sets.newHashSet());

20
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -3029,19 +3029,23 @@ public class OlapTable extends Table implements MTMVRelatedTableIf {
"get table read lock timeout, database=" + getQualifiedDbName() + ",table=" + getName());
}
try {
Map<String, PartitionItem> res = Maps.newHashMap();
for (Entry<Long, PartitionItem> entry : getPartitionInfo().getIdToItem(false).entrySet()) {
Partition partition = idToPartition.get(entry.getKey());
if (partition != null) {
res.put(partition.getName(), entry.getValue());
}
}
return res;
return getAndCopyPartitionItemsWithoutLock();
} finally {
readUnlock();
}
}
public Map<String, PartitionItem> getAndCopyPartitionItemsWithoutLock() throws AnalysisException {
Map<String, PartitionItem> res = Maps.newHashMap();
for (Entry<Long, PartitionItem> entry : getPartitionInfo().getIdToItem(false).entrySet()) {
Partition partition = idToPartition.get(entry.getKey());
if (partition != null) {
res.put(partition.getName(), entry.getValue());
}
}
return res;
}
@Override
public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
return getPartitionColumns();

16
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
View File

@ -172,18 +172,12 @@ public class MTMVPartitionUtil {
}
public static List<Long> getPartitionsIdsByNames(MTMV mtmv, List<String> partitions) throws AnalysisException {
mtmv.readLock();
try {
List<Long> res = Lists.newArrayList();
for (String partitionName : partitions) {
Partition partition = mtmv.getPartitionOrAnalysisException(partitionName);
res.add(partition.getId());
}
return res;
} finally {
mtmv.readUnlock();
List<Long> res = Lists.newArrayList();
for (String partitionName : partitions) {
Partition partition = mtmv.getPartitionOrAnalysisException(partitionName);
res.add(partition.getId());
}
return res;
}
/**

13
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ShowCreateMTMVInfo.java
View File

@ -74,18 +74,13 @@ public class ShowCreateMTMVInfo {
* @throws DdlException DdlException
* @throws IOException IOException
*/
public void run(StmtExecutor executor) throws DdlException, IOException {
public void run(StmtExecutor executor) throws DdlException, IOException, org.apache.doris.common.AnalysisException {
List<List<String>> rows = Lists.newArrayList();
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(mvName.getDb());
MTMV mtmv = (MTMV) db.getTableOrDdlException(mvName.getTbl());
mtmv.readLock();
try {
String mtmvDdl = Env.getMTMVDdl(mtmv);
rows.add(Lists.newArrayList(mtmv.getName(), mtmvDdl));
executor.handleShowCreateMTMVStmt(rows);
} finally {
mtmv.readUnlock();
}
String mtmvDdl = Env.getMTMVDdl(mtmv);
rows.add(Lists.newArrayList(mtmv.getName(), mtmvDdl));
executor.handleShowCreateMTMVStmt(rows);
}
/**

12
fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
View File

@ -1080,15 +1080,9 @@ public class ShowExecutor {
.getDbOrAnalysisException(showStmt.getDb());
MTMV mtmv = (MTMV) db.getTableOrAnalysisException(showStmt.getTable());
List<List<String>> rows = Lists.newArrayList();
mtmv.readLock();
try {
String mtmvDdl = Env.getMTMVDdl(mtmv);
rows.add(Lists.newArrayList(mtmv.getName(), mtmvDdl));
resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
} finally {
mtmv.readUnlock();
}
String mtmvDdl = Env.getMTMVDdl(mtmv);
rows.add(Lists.newArrayList(mtmv.getName(), mtmvDdl));
resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
}
// Describe statement