database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhance](mtmv)Improve the speed of obtaining table snapshots (#40717) (#40995)

Browse Source 
pick: https://github.com/apache/doris/pull/40717
This commit is contained in:
zhangdong
2024-09-19 22:47:50 +08:00
committed by GitHub
parent fd5423145d
commit f81d087bab
1 changed files with 9 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
View File

@ -761,26 +761,26 @@ public class HMSExternalTable extends ExternalTable implements MTMVRelatedTableI
if (getPartitionType() == PartitionType.UNPARTITIONED) {
return new MTMVMaxTimestampSnapshot(getName(), getLastDdlTime());
}
Long maxPartitionId = 0L;
HivePartition maxPartition = null;
long maxVersionTime = 0L;
long visibleVersionTime;
HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
.getMetaStoreCache((HMSExternalCatalog) getCatalog());
HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues(
getDbName(), getName(), getPartitionColumnTypes());
BiMap<Long, String> idToName = hivePartitionValues.getPartitionNameToIdMap().inverse();
if (MapUtils.isEmpty(idToName)) {
throw new AnalysisException("partitions is empty for : " + getName());
List<HivePartition> partitionList = cache.getAllPartitionsWithCache(getDbName(), getName(),
Lists.newArrayList(hivePartitionValues.getPartitionValuesMap().values()));
if (CollectionUtils.isEmpty(partitionList)) {
throw new AnalysisException("partitionList is empty, table name: " + getName());
}
for (Long partitionId : idToName.keySet()) {
visibleVersionTime = getHivePartitionByIdOrAnalysisException(partitionId, hivePartitionValues,
cache).getLastModifiedTime();
for (HivePartition hivePartition : partitionList) {
visibleVersionTime = hivePartition.getLastModifiedTime();
if (visibleVersionTime > maxVersionTime) {
maxVersionTime = visibleVersionTime;
maxPartitionId = partitionId;
maxPartition = hivePartition;
}
}
return new MTMVMaxTimestampSnapshot(idToName.get(maxPartitionId), maxVersionTime);
return new MTMVMaxTimestampSnapshot(maxPartition.getPartitionName(getPartitionColumns()), maxVersionTime);
}
private Long getPartitionIdByNameOrAnalysisException(String partitionName,