database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](file-cache) support system table file_cache_statistics (#39552)

Browse Source 
1. Add new system table: `file_cache_statistics`

	This table is used for viewing metrics related to file cache on BE side

	```
	mysql> select * from information_schema.file_cache_statistics limit 10;

+-------+---------------+----------------------------+--------------------------------+--------------------+
| BE_ID | BE_IP | CACHE_PATH | METRIC_NAME | METRIC_VALUE |

+-------+---------------+----------------------------+--------------------------------+--------------------+
| 10003 | 172.20.32.136 | /mnt/output/be/file_cache/ |
disposable_queue_curr_elements | 0 |
| 10003 | 172.20.32.136 | /mnt/output/be/file_cache/ |
disposable_queue_curr_size | 0 |
| 10003 | 172.20.32.136 | /mnt/output/be/file_cache/ |
disposable_queue_max_elements | 102400 |
| 10003 | 172.20.32.136 | /mnt/output/be/file_cache/ |
disposable_queue_max_size | 21474836480 |
| 10003 | 172.20.32.136 | /mnt/output/be/file_cache/ | hits_ratio |
0.8539634687001242 |
| 10003 | 172.20.32.136 | /mnt/output/be/file_cache/ | hits_ratio_1h | 0
|
| 10003 | 172.20.32.136 | /mnt/output/be/file_cache/ | hits_ratio_5m | 0
|
| 10003 | 172.20.32.136 | /mnt/output/be/file_cache/ |
index_queue_curr_elements | 0 |
| 10003 | 172.20.32.136 | /mnt/output/be/file_cache/ |
index_queue_curr_size | 0 |
| 10003 | 172.20.32.136 | /mnt/output/be/file_cache/ |
index_queue_max_elements | 102400 |

+-------+---------------+----------------------------+--------------------------------+--------------------+
	```

	It will show metrics of file caches on each BE.

2. Add new metrics `hits_ratio_1h` and `hits_ratio_5m` for file cache

This 2 metrics will show the hit ratio of file cache in recent 1 hour or
5 minutes.
So that we can know recent hit ratio instead of global historical hit
ratio.
This commit is contained in:
Mingyu Chen
2024-08-21 10:03:39 +08:00
committed by GitHub
parent bf26f49505
commit 0bfcee1251
23 changed files with 296 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java
View File

@ -77,7 +77,9 @@ public enum SchemaTableType {
SCH_PROCS_PRIV("procs_priv", "procs_priv", TSchemaTableType.SCH_PROCS_PRIV),
SCH_WORKLOAD_POLICY("WORKLOAD_POLICY", "WORKLOAD_POLICY",
TSchemaTableType.SCH_WORKLOAD_POLICY);
TSchemaTableType.SCH_WORKLOAD_POLICY),
SCH_FILE_CACHE_STATISTICS("FILE_CACHE_STATISTICS", "FILE_CACHE_STATISTICS",
TSchemaTableType.SCH_FILE_CACHE_STATISTICS);
private static final String dbName = "INFORMATION_SCHEMA";
private static SelectList fullSelectLists;
@ -114,3 +116,5 @@ public enum SchemaTableType {
return tableType;
}
}

8
fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
View File

@ -516,6 +516,14 @@ public class SchemaTable extends Table {
.column("VERSION", ScalarType.createType(PrimitiveType.INT))
.column("WORKLOAD_GROUP", ScalarType.createStringType())
.build()))
.put("file_cache_statistics",
new SchemaTable(SystemIdGenerator.getNextId(), "file_cache_statistics", TableType.SCHEMA,
builder().column("BE_ID", ScalarType.createType(PrimitiveType.BIGINT))
.column("BE_IP", ScalarType.createVarchar(256))
.column("CACHE_PATH", ScalarType.createVarchar(256))
.column("METRIC_NAME", ScalarType.createVarchar(256))
.column("METRIC_VALUE", ScalarType.createType(PrimitiveType.DOUBLE))
.build()))
.build();
private boolean fetchAllFe = false;

3
fe/fe-core/src/main/java/org/apache/doris/planner/BackendPartitionedSchemaScanNode.java
View File

@ -66,6 +66,9 @@ public class BackendPartitionedSchemaScanNode extends SchemaScanNode {
BACKEND_TABLE.add("backend_active_tasks");
BEACKEND_ID_COLUMN_SET.add("be_id");
BACKEND_TABLE.add("file_cache_statistics");
BEACKEND_ID_COLUMN_SET.add("be_id");
}
public static boolean isBackendPartitionedSchemaTable(String tableName) {