database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](metacache) add system table catalog_meta_cache_statistics #40155 (#40210)

Browse Source 
bp #40155
This commit is contained in:
Mingyu Chen
2024-09-02 23:23:35 +08:00
committed by GitHub
parent 1527099e35
commit 92752b90e7
18 changed files with 339 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java
View File

@ -85,7 +85,9 @@ public enum SchemaTableType {
SCH_WORKLOAD_GROUP_RESOURCE_USAGE("WORKLOAD_GROUP_RESOURCE_USAGE",
"WORKLOAD_GROUP_RESOURCE_USAGE", TSchemaTableType.SCH_WORKLOAD_GROUP_RESOURCE_USAGE),
SCH_TABLE_PROPERTIES("TABLE_PROPERTIES", "TABLE_PROPERTIES",
TSchemaTableType.SCH_TABLE_PROPERTIES);
TSchemaTableType.SCH_TABLE_PROPERTIES),
SCH_CATALOG_META_CACHE_STATISTICS("CATALOG_META_CACHE_STATISTICS", "CATALOG_META_CACHE_STATISTICS",
TSchemaTableType.SCH_CATALOG_META_CACHE_STATISTICS);
private static final String dbName = "INFORMATION_SCHEMA";
private static SelectList fullSelectLists;

8
fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
View File

@ -550,6 +550,14 @@ public class SchemaTable extends Table {
.column("PROPERTY_NAME", ScalarType.createStringType())
.column("PROPERTY_VALUE", ScalarType.createStringType())
.build()))
.put("catalog_meta_cache_statistics",
new SchemaTable(SystemIdGenerator.getNextId(), "catalog_meta_cache_statistics", TableType.SCHEMA,
builder().column("CATALOG_NAME", ScalarType.createStringType())
.column("CACHE_NAME", ScalarType.createStringType())
.column("METRIC_NAME", ScalarType.createStringType())
.column("METRIC_VALUE", ScalarType.createStringType())
.build())
)
.build();
private boolean fetchAllFe = false;

12
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
View File

@ -36,6 +36,7 @@ import org.apache.doris.nereids.exceptions.NotSupportedException;
import com.github.benmanes.caffeine.cache.CacheLoader;
import com.github.benmanes.caffeine.cache.RemovalListener;
import com.github.benmanes.caffeine.cache.stats.CacheStats;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -296,4 +297,15 @@ public class ExternalMetaCacheMgr {
maxSize, namesCacheLoader, metaObjCacheLoader, removalListener);
return metaCache;
}
public static Map<String, String> getCacheStats(CacheStats cacheStats, long estimatedSize) {
Map<String, String> stats = Maps.newHashMap();
stats.put("hit_ratio", String.valueOf(cacheStats.hitRate()));
stats.put("hit_count", String.valueOf(cacheStats.hitCount()));
stats.put("read_count", String.valueOf(cacheStats.hitCount() + cacheStats.missCount()));
stats.put("eviction_count", String.valueOf(cacheStats.evictionCount()));
stats.put("average_load_penalty", String.valueOf(cacheStats.averageLoadPenalty()));
stats.put("estimated_size", String.valueOf(estimatedSize));
return stats;
}
}

22
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
View File

@ -36,6 +36,7 @@ import org.apache.doris.common.util.CacheBulkLoader;
import org.apache.doris.common.util.LocationPath;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CacheException;
import org.apache.doris.datasource.ExternalMetaCacheMgr;
import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo;
import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.fs.FileSystemCache;
@ -141,7 +142,7 @@ public class HiveMetaStoreCache {
OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_hive_partition_table_cache_num,
false,
true,
null);
partitionValuesCache = partitionValuesCacheFactory.buildCache(key -> loadPartitionValues(key), null,
refreshExecutor);
@ -150,7 +151,7 @@ public class HiveMetaStoreCache {
OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_hive_partition_cache_num,
false,
true,
null);
partitionCache = partitionCacheFactory.buildCache(new CacheLoader<PartitionCacheKey, HivePartition>() {
@Override
@ -183,7 +184,7 @@ public class HiveMetaStoreCache {
? fileMetaCacheTtlSecond : 28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_external_file_cache_num,
false,
true,
null);
CacheLoader<FileCacheKey, FileCacheValue> loader = new CacheBulkLoader<FileCacheKey, FileCacheValue>() {
@ -1125,4 +1126,19 @@ public class HiveMetaStoreCache {
return copy;
}
}
/**
* get cache stats
* @return <cache name -> <metric name -> metric value>>
*/
public Map<String, Map<String, String>> getStats() {
Map<String, Map<String, String>> res = Maps.newHashMap();
res.put("hive_partition_values_cache", ExternalMetaCacheMgr.getCacheStats(partitionValuesCache.stats(),
partitionCache.estimatedSize()));
res.put("hive_partition_cache",
ExternalMetaCacheMgr.getCacheStats(partitionCache.stats(), partitionCache.estimatedSize()));
res.put("hive_file_cache",
ExternalMetaCacheMgr.getCacheStats(fileCacheRef.get().stats(), fileCacheRef.get().estimatedSize()));
return res;
}
}

12
fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.common.CacheFactory;
import org.apache.doris.common.Config;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CacheException;
import org.apache.doris.datasource.ExternalMetaCacheMgr;
import org.apache.doris.datasource.TablePartitionValues;
import org.apache.doris.datasource.TablePartitionValues.TablePartitionKey;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
@ -28,6 +29,7 @@ import org.apache.doris.datasource.hive.HMSExternalTable;
import com.github.benmanes.caffeine.cache.LoadingCache;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
@ -37,6 +39,7 @@ import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.OptionalLong;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
@ -55,7 +58,7 @@ public class HudiCachedPartitionProcessor extends HudiPartitionProcessor {
OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
Config.max_external_table_cache_num,
false,
true,
null);
this.partitionCache = partitionCacheFactory.buildCache(key -> new TablePartitionValues(), null, executor);
}
@ -167,4 +170,11 @@ public class HudiCachedPartitionProcessor extends HudiPartitionProcessor {
throw new CacheException("Failed to get hudi partitions: " + Util.getRootCauseMessage(e), e);
}
}
public Map<String, Map<String, String>> getCacheStats() {
Map<String, Map<String, String>> res = Maps.newHashMap();
res.put("hudi_partition_cache", ExternalMetaCacheMgr.getCacheStats(partitionCache.stats(),
partitionCache.estimatedSize()));
return res;
}
}

15
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.common.CacheFactory;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.ExternalMetaCacheMgr;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper;
import org.apache.doris.datasource.property.constants.HMSProperties;
@ -31,6 +32,7 @@ import org.apache.doris.thrift.TIcebergMetadataParams;
import com.github.benmanes.caffeine.cache.LoadingCache;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.ManifestFiles;
import org.apache.iceberg.SerializableTable;
@ -58,7 +60,7 @@ public class IcebergMetadataCache {
OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
Config.max_external_table_cache_num,
false,
true,
null);
this.snapshotListCache = snapshotListCacheFactory.buildCache(key -> loadSnapshots(key), null, executor);
@ -66,7 +68,7 @@ public class IcebergMetadataCache {
OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
Config.max_external_table_cache_num,
false,
true,
null);
this.tableCache = tableCacheFactory.buildCache(key -> loadTable(key), null, executor);
}
@ -246,4 +248,13 @@ public class IcebergMetadataCache {
return Objects.hash(catalog.getId(), dbName, tableName);
}
}
public Map<String, Map<String, String>> getCacheStats() {
Map<String, Map<String, String>> res = Maps.newHashMap();
res.put("iceberg_snapshot_cache", ExternalMetaCacheMgr.getCacheStats(snapshotListCache.stats(),
snapshotListCache.estimatedSize()));
res.put("iceberg_table_cache", ExternalMetaCacheMgr.getCacheStats(tableCache.stats(),
tableCache.estimatedSize()));
return res;
}
}

63
fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
View File

@ -38,8 +38,12 @@ import org.apache.doris.common.util.NetUtils;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.ExternalMetaCacheMgr;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.hive.HiveMetaStoreCache;
import org.apache.doris.datasource.hudi.source.HudiCachedPartitionProcessor;
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
import org.apache.doris.datasource.iceberg.IcebergMetadataCache;
import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog;
import org.apache.doris.job.common.JobType;
@ -112,6 +116,8 @@ public class MetadataGenerator {
private static final ImmutableMap<String, Integer> TABLE_PROPERTIES_COLUMN_TO_INDEX;
private static final ImmutableMap<String, Integer> META_CACHE_STATS_COLUMN_TO_INDEX;
static {
ImmutableMap.Builder<String, Integer> activeQueriesbuilder = new ImmutableMap.Builder();
List<Column> activeQueriesColList = SchemaTable.TABLE_MAP.get("active_queries").getFullSchema();
@ -152,6 +158,13 @@ public class MetadataGenerator {
propertiesBuilder.put(propertiesColList.get(i).getName().toLowerCase(), i);
}
TABLE_PROPERTIES_COLUMN_TO_INDEX = propertiesBuilder.build();
ImmutableMap.Builder<String, Integer> metaCacheBuilder = new ImmutableMap.Builder();
List<Column> metaCacheColList = SchemaTable.TABLE_MAP.get("catalog_meta_cache_statistics").getFullSchema();
for (int i = 0; i < metaCacheColList.size(); i++) {
metaCacheBuilder.put(metaCacheColList.get(i).getName().toLowerCase(), i);
}
META_CACHE_STATS_COLUMN_TO_INDEX = metaCacheBuilder.build();
}
public static TFetchSchemaTableDataResult getMetadataTable(TFetchSchemaTableDataRequest request) throws TException {
@ -239,6 +252,10 @@ public class MetadataGenerator {
result = tablePropertiesMetadataResult(schemaTableParams);
columnIndex = TABLE_PROPERTIES_COLUMN_TO_INDEX;
break;
case CATALOG_META_CACHE_STATS:
result = metaCacheStatsMetadataResult(schemaTableParams);
columnIndex = META_CACHE_STATS_COLUMN_TO_INDEX;
break;
default:
return errorResult("invalid schema table name.");
}
@ -1124,4 +1141,50 @@ public class MetadataGenerator {
result.setStatus(new TStatus(TStatusCode.OK));
return result;
}
private static TFetchSchemaTableDataResult metaCacheStatsMetadataResult(TSchemaTableRequestParams params) {
List<TRow> dataBatch = Lists.newArrayList();
TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult();
ExternalMetaCacheMgr mgr = Env.getCurrentEnv().getExtMetaCacheMgr();
for (CatalogIf catalogIf : Env.getCurrentEnv().getCatalogMgr().getCopyOfCatalog()) {
if (catalogIf instanceof HMSExternalCatalog) {
HMSExternalCatalog catalog = (HMSExternalCatalog) catalogIf;
// 1. hive metastore cache
HiveMetaStoreCache cache = mgr.getMetaStoreCache(catalog);
if (cache != null) {
fillBatch(dataBatch, cache.getStats(), catalog.getName());
}
// 2. hudi cache
HudiCachedPartitionProcessor processor
= (HudiCachedPartitionProcessor) mgr.getHudiPartitionProcess(catalog);
fillBatch(dataBatch, processor.getCacheStats(), catalog.getName());
} else if (catalogIf instanceof IcebergExternalCatalog) {
// 3. iceberg cache
IcebergMetadataCache icebergCache = mgr.getIcebergMetadataCache();
fillBatch(dataBatch, icebergCache.getCacheStats(), catalogIf.getName());
}
}
result.setDataBatch(dataBatch);
result.setStatus(new TStatus(TStatusCode.OK));
return result;
}
private static void fillBatch(List<TRow> dataBatch, Map<String, Map<String, String>> stats,
String catalogName) {
for (Map.Entry<String, Map<String, String>> entry : stats.entrySet()) {
String cacheName = entry.getKey();
Map<String, String> cacheStats = entry.getValue();
for (Map.Entry<String, String> cacheStatsEntry : cacheStats.entrySet()) {
String metricName = cacheStatsEntry.getKey();
String metricValue = cacheStatsEntry.getValue();
TRow trow = new TRow();
trow.addToColumnValue(new TCell().setStringVal(catalogName)); // CATALOG_NAME
trow.addToColumnValue(new TCell().setStringVal(cacheName)); // CACHE_NAME
trow.addToColumnValue(new TCell().setStringVal(metricName)); // METRIC_NAME
trow.addToColumnValue(new TCell().setStringVal(metricValue)); // METRIC_VALUE
dataBatch.add(trow);
}
}
}
}