database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhance](mtmv) hive cache add partitionId to partitionName Map (#38525) (#39476)

Browse Source 
pick from master #38525
This commit is contained in:
zhangdong
2024-08-16 20:00:22 +08:00
committed by GitHub
parent c9a246b65b
commit c84cb5cf3d
7 changed files with 13 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -2816,16 +2816,6 @@ public class OlapTable extends Table implements MTMVRelatedTableIf {
return new MTMVVersionSnapshot(visibleVersion);
}
@Override
public String getPartitionName(long partitionId) throws AnalysisException {
readLock();
try {
return getPartitionOrAnalysisException(partitionId).getName();
} finally {
readUnlock();
}
}
@Override
public boolean needAutoRefresh() {
return true;

20
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
View File

@ -48,6 +48,7 @@ import org.apache.doris.thrift.THiveTable;
import org.apache.doris.thrift.TTableDescriptor;
import org.apache.doris.thrift.TTableType;
import com.google.common.collect.BiMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@ -733,28 +734,13 @@ public class HMSExternalTable extends ExternalTable implements MTMVRelatedTableI
getDbName(), getName(), getPartitionColumnTypes());
Map<String, PartitionItem> res = Maps.newHashMap();
Map<Long, PartitionItem> idToPartitionItem = hivePartitionValues.getIdToPartitionItem();
BiMap<Long, String> idToName = hivePartitionValues.getPartitionNameToIdMap().inverse();
for (Entry<Long, PartitionItem> entry : idToPartitionItem.entrySet()) {
try {
res.put(getPartitionName(entry.getKey()), entry.getValue());
} catch (AnalysisException e) {
LOG.info("can not get partitionName by: " + entry.getKey());
}
res.put(idToName.get(entry.getKey()), entry.getValue());
}
return res;
}
@Override
public String getPartitionName(long partitionId) throws AnalysisException {
Map<String, Long> partitionNameToIdMap = getHivePartitionValues().getPartitionNameToIdMap();
for (Entry<String, Long> entry : partitionNameToIdMap.entrySet()) {
if (entry.getValue().equals(partitionId)) {
return entry.getKey();
}
}
throw new AnalysisException("can not find partition, partitionId: " + partitionId);
}
private HiveMetaStoreCache.HivePartitionValues getHivePartitionValues() {
HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
.getMetaStoreCache((HMSExternalCatalog) getCatalog());

10
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
View File

@ -54,6 +54,8 @@ import com.github.benmanes.caffeine.cache.LoadingCache;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@ -246,7 +248,7 @@ public class HiveMetaStoreCache {
LOG.debug("load #{} partitions for {} in catalog {}", partitionNames.size(), key, catalog.getName());
}
Map<Long, PartitionItem> idToPartitionItem = Maps.newHashMapWithExpectedSize(partitionNames.size());
Map<String, Long> partitionNameToIdMap = Maps.newHashMapWithExpectedSize(partitionNames.size());
BiMap<String, Long> partitionNameToIdMap = HashBiMap.create(partitionNames.size());
Map<Long, List<UniqueId>> idToUniqueIdsMap = Maps.newHashMapWithExpectedSize(partitionNames.size());
long idx = 0;
for (String partitionName : partitionNames) {
@ -1074,7 +1076,7 @@ public class HiveMetaStoreCache {
@Data
public static class HivePartitionValues {
private long nextPartitionId;
private Map<String, Long> partitionNameToIdMap;
private BiMap<String, Long> partitionNameToIdMap;
private Map<Long, List<UniqueId>> idToUniqueIdsMap;
private Map<Long, PartitionItem> idToPartitionItem;
private Map<Long, List<String>> partitionValuesMap;
@ -1093,7 +1095,7 @@ public class HiveMetaStoreCache {
Map<Range<PartitionKey>, UniqueId> rangeToId,
RangeMap<ColumnBound, UniqueId> singleColumnRangeMap,
long nextPartitionId,
Map<String, Long> partitionNameToIdMap,
BiMap<String, Long> partitionNameToIdMap,
Map<Long, List<UniqueId>> idToUniqueIdsMap,
Map<UniqueId, Range<ColumnBound>> singleUidToColumnRangeMap,
Map<Long, List<String>> partitionValuesMap) {
@ -1111,7 +1113,7 @@ public class HiveMetaStoreCache {
public HivePartitionValues copy() {
HivePartitionValues copy = new HivePartitionValues();
copy.setNextPartitionId(nextPartitionId);
copy.setPartitionNameToIdMap(partitionNameToIdMap == null ? null : Maps.newHashMap(partitionNameToIdMap));
copy.setPartitionNameToIdMap(partitionNameToIdMap == null ? null : HashBiMap.create(partitionNameToIdMap));
copy.setIdToUniqueIdsMap(idToUniqueIdsMap == null ? null : Maps.newHashMap(idToUniqueIdsMap));
copy.setIdToPartitionItem(idToPartitionItem == null ? null : Maps.newHashMap(idToPartitionItem));
copy.setPartitionValuesMap(partitionValuesMap == null ? null : Maps.newHashMap(partitionValuesMap));

9
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
View File

@ -41,7 +41,6 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -166,14 +165,6 @@ public class MTMVPartitionUtil {
return result.getDescs();
}
public static List<String> getPartitionNamesByIds(MTMV mtmv, Collection<Long> ids) throws AnalysisException {
List<String> res = Lists.newArrayList();
for (Long partitionId : ids) {
res.add(mtmv.getPartitionName(partitionId));
}
return res;
}
public static List<Long> getPartitionsIdsByNames(MTMV mtmv, List<String> partitions) throws AnalysisException {
mtmv.readLock();
try {

9
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
View File

@ -79,15 +79,6 @@ public interface MTMVRelatedTableIf extends TableIf {
*/
MTMVSnapshotIf getTableSnapshot() throws AnalysisException;
/**
* getPartitionName
*
* @param partitionId
* @return partitionName
* @throws AnalysisException
*/
String getPartitionName(long partitionId) throws AnalysisException;
/**
* Does the current type of table allow timed triggering
*

5
fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java
View File

@ -63,6 +63,8 @@ import org.apache.doris.utframe.TestWithFeService;
import com.github.benmanes.caffeine.cache.LoadingCache;
import com.google.common.base.Preconditions;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
@ -477,6 +479,7 @@ public class CatalogMgrTest extends TestWithFeService {
partitionValueCacheKey.getTypes());
HivePartitionValues partitionValues = metaStoreCache.getPartitionValues(partitionValueCacheKey);
Assert.assertEquals(partitionValues.getPartitionNameToIdMap().size(), 4);
Assert.assertEquals(partitionValues.getPartitionNameToIdMap().inverse().size(), 4);
}
@Test
@ -520,7 +523,7 @@ public class CatalogMgrTest extends TestWithFeService {
HiveMetaStoreCache metaStoreCache) {
// partition name format: nation=cn/city=beijing
Map<Long, PartitionItem> idToPartitionItem = Maps.newHashMapWithExpectedSize(partitionNames.size());
Map<String, Long> partitionNameToIdMap = Maps.newHashMapWithExpectedSize(partitionNames.size());
BiMap<String, Long> partitionNameToIdMap = HashBiMap.create(partitionNames.size());
Map<Long, List<UniqueId>> idToUniqueIdsMap = Maps.newHashMapWithExpectedSize(partitionNames.size());
long idx = 0;
for (String partitionName : partitionNames) {

8
fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
View File

@ -99,10 +99,6 @@ public class MTMVPartitionUtilTest {
minTimes = 0;
result = baseSnapshotIf;
mtmv.getPartitionName(anyLong);
minTimes = 0;
result = "name1";
mtmv.getRefreshSnapshot();
minTimes = 0;
result = refreshSnapshot;
@ -123,10 +119,6 @@ public class MTMVPartitionUtilTest {
minTimes = 0;
result = baseSnapshotIf;
baseOlapTable.getPartitionName(anyLong);
minTimes = 0;
result = "name1";
refreshSnapshot.equalsWithRelatedPartition(anyString, anyString, (MTMVSnapshotIf) any);
minTimes = 0;
result = true;