diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md index 6bb154f572..2d35568f49 100644 --- a/docs/en/docs/admin-manual/config/fe-config.md +++ b/docs/en/docs/admin-manual/config/fe-config.md @@ -2701,6 +2701,19 @@ If false, when select from tables in information_schema database, the result will not contain the information of the table in external catalog. This is to avoid query time when external catalog is not reachable. + +#### `enable_query_hit_stats` + + + +Default: false + +IsMutable: true + +MasterOnly: false + +Controls whether to enable query hit statistics. The default is false. + #### `max_instance_num` @@ -2708,4 +2721,4 @@ This is to avoid query time when external catalog is not reachable. Default: 128 This is used to limit the setting of "parallel_fragment_exec_instance_num". -"parallel_fragment_exec_instance_num" cannot be set higher than "max_instance_num". \ No newline at end of file +"parallel_fragment_exec_instance_num" cannot be set higher than "max_instance_num". diff --git a/docs/en/docs/admin-manual/http-actions/fe/query-stats-action.md b/docs/en/docs/admin-manual/http-actions/fe/query-stats-action.md new file mode 100644 index 0000000000..9ef5035ea1 --- /dev/null +++ b/docs/en/docs/admin-manual/http-actions/fe/query-stats-action.md @@ -0,0 +1,99 @@ +--- +{ +"title": "Query Stats Action", +"language": "en" +} +--- + + + +# Query Stats Action + + + +## Request + +``` +查看 +get api/query_stats/ +get api/query_stats// +get api/query_stats/// + +清空 +delete api/query_stats// +delete api/query_stats/// +``` + +## Description + +Get or delete the statistics information of the specified catalog database or table, if it is a doris catalog, you can use default_cluster + +## Path parameters + +* `` + specified catalog name +* +* `` + specified database name + +* `` + specified table name + +## Query parameters +* `summary` + if true, only return summary information, otherwise return all the detailed statistics information of the table, only used in get + +## Request body + +``` +GET /api/query_stats/default_cluster/test_query_db/baseall?summary=false +{ + "msg": "success", + "code": 0, + "data": { + "summary": { + "query": 2 + }, + "detail": { + "baseall": { + "summary": { + "query": 2 + } + } + } + }, + "count": 0 +} + +``` + +## Response + +* return statistics information + + +## Example + + +2. use curl + + ``` + curl --location -u root: 'http://127.0.0.1:8030/api/query_stats/default_cluster/test_query_db/baseall?summary=false' + ``` diff --git a/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-QUERY-STATS.md b/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-QUERY-STATS.md new file mode 100644 index 0000000000..1214272c97 --- /dev/null +++ b/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-QUERY-STATS.md @@ -0,0 +1,196 @@ +--- +{ + "title": "SHOW-QUERY-STATS", + "language": "en" +} +--- + + + +## SHOW-QUERY-STATS + +### Name + + +SHOW QUERY STATS + + +### Description + +This statement is used to show the query hit statistics of the database and table + +grammar: + +```sql +SHOW QUERY STATS [[FOR db_name]|[FROM table_name]] [ALL] [VERBOSE]]; +``` + +Remarks： + +1. Support query database and table history query hit statistics, restart fe after data will be reset, each fe separately statistics +2. Use FOR DATABASE and FROM TABLE to specify the query database or table hit statistics, respectively followed by the database name or table name +3. ALL can specify whether to display all index query hit statistics, VERBOSE can display more detailed hit statistics, these two parameters can be used separately, but must be placed at the end and can only be used on table queries +4. If no database is used, execute `SHOW QUERY STATS` directly to display the hit statistics of all databases +5. The result may have two columns: + QueryCount: The number of times the column was queried + FilterCount: The number of times the column was queried as a where condition +### Example + +1. Show the query hit statistics for `baseall` + + ```sql + MySQL [test_query_db]> show query stats from baseall; + +-------+------------+-------------+ + | Field | QueryCount | FilterCount | + +-------+------------+-------------+ + | k0 | 0 | 0 | + | k1 | 0 | 0 | + | k2 | 0 | 0 | + | k3 | 0 | 0 | + | k4 | 0 | 0 | + | k5 | 0 | 0 | + | k6 | 0 | 0 | + | k10 | 0 | 0 | + | k11 | 0 | 0 | + | k7 | 0 | 0 | + | k8 | 0 | 0 | + | k9 | 0 | 0 | + | k12 | 0 | 0 | + | k13 | 0 | 0 | + +-------+------------+-------------+ + 14 rows in set (0.002 sec) + + MySQL [test_query_db]> select k0, k1,k2, sum(k3) from baseall where k9 > 1 group by k0,k1,k2; + +------+------+--------+-------------+ + | k0 | k1 | k2 | sum(`k3`) | + +------+------+--------+-------------+ + | 0 | 6 | 32767 | 3021 | + | 1 | 12 | 32767 | -2147483647 | + | 0 | 3 | 1989 | 1002 | + | 0 | 7 | -32767 | 1002 | + | 1 | 8 | 255 | 2147483647 | + | 1 | 9 | 1991 | -2147483647 | + | 1 | 11 | 1989 | 25699 | + | 1 | 13 | -32767 | 2147483647 | + | 1 | 14 | 255 | 103 | + | 0 | 1 | 1989 | 1001 | + | 0 | 2 | 1986 | 1001 | + | 1 | 15 | 1992 | 3021 | + +------+------+--------+-------------+ + 12 rows in set (0.050 sec) + + MySQL [test_query_db]> show query stats from baseall; + +-------+------------+-------------+ + | Field | QueryCount | FilterCount | + +-------+------------+-------------+ + | k0 | 1 | 0 | + | k1 | 1 | 0 | + | k2 | 1 | 0 | + | k3 | 1 | 0 | + | k4 | 0 | 0 | + | k5 | 0 | 0 | + | k6 | 0 | 0 | + | k10 | 0 | 0 | + | k11 | 0 | 0 | + | k7 | 0 | 0 | + | k8 | 0 | 0 | + | k9 | 1 | 1 | + | k12 | 0 | 0 | + | k13 | 0 | 0 | + +-------+------------+-------------+ + 14 rows in set (0.001 sec) + ``` + +2. Show the query hit statistics summary for all the mv in a table + + ```sql + MySQL [test_query_db]> show query stats from baseall all; + +-----------+------------+ + | IndexName | QueryCount | + +-----------+------------+ + | baseall | 1 | + +-----------+------------+ + 1 row in set (0.005 sec) + ``` + +3. Show the query hit statistics detail info for all the mv in a table + + ```sql + MySQL [test_query_db]> show query stats from baseall all verbose; + +-----------+-------+------------+-------------+ + | IndexName | Field | QueryCount | FilterCount | + +-----------+-------+------------+-------------+ + | baseall | k0 | 1 | 0 | + | | k1 | 1 | 0 | + | | k2 | 1 | 0 | + | | k3 | 1 | 0 | + | | k4 | 0 | 0 | + | | k5 | 0 | 0 | + | | k6 | 0 | 0 | + | | k10 | 0 | 0 | + | | k11 | 0 | 0 | + | | k7 | 0 | 0 | + | | k8 | 0 | 0 | + | | k9 | 1 | 1 | + | | k12 | 0 | 0 | + | | k13 | 0 | 0 | + +-----------+-------+------------+-------------+ + 14 rows in set (0.017 sec) + ``` + +4. Show the query hit for a database + + ```sql + MySQL [test_query_db]> show query stats for test_query_db; + +----------------------------+------------+ + | TableName | QueryCount | + +----------------------------+------------+ + | compaction_tbl | 0 | + | bigtable | 0 | + | empty | 0 | + | tempbaseall | 0 | + | test | 0 | + | test_data_type | 0 | + | test_string_function_field | 0 | + | baseall | 1 | + | nullable | 0 | + +----------------------------+------------+ + 9 rows in set (0.005 sec) + ``` + +5. Show query hit statistics for all the databases + + ```sql + MySQL [(none)]> show query stats; + +-----------------+------------+ + | Database | QueryCount | + +-----------------+------------+ + | test_query_db | 1 | + +-----------------+------------+ + 1 rows in set (0.005 sec) + ``` + SHOW QUERY STATS; + ``` + +### Keywords + + SHOW， QUERY, STATS; + +### Best Practice diff --git a/docs/en/docs/sql-manual/sql-reference/Utility-Statements/CLEAN-QUERY-STATS.md b/docs/en/docs/sql-manual/sql-reference/Utility-Statements/CLEAN-QUERY-STATS.md new file mode 100644 index 0000000000..d7e0776de0 --- /dev/null +++ b/docs/en/docs/sql-manual/sql-reference/Utility-Statements/CLEAN-QUERY-STATS.md @@ -0,0 +1,75 @@ +--- +{ + "title": "CLEAN-QUERY-STATS", + "language": "en" +} +--- + + + +## CLEAN-QUERY-STATS + +### Name + + +CLEAN QUERY STATS + + +### Description + +This statement is used to clear query statistics + +grammar： + +```sql +CLEAN [ALL| DATABASE | TABLE] QUERY STATS [[FOR db_name]|[FROM|IN] table_name]]; +``` + +Remarks： + +1. If ALL is specified, all query statistics are cleared, including database and table, admin privilege is needed +2. If DATABASE is specified, the query statistics of the specified database are cleared, alter privilege for this database is needed +3. If TABLE is specified, the query statistics of the specified table are cleared, alter privilege for this table is needed + +### Example + +1. Clear all statistics +2. + ```sql + clean all query stats; + ``` + +2. Clear the specified database statistics + + ```sql + clean database query stats for test_query_db; + ``` +3. Clear the specified table statistics + + ```sql + clean table query stats from test_query_db.baseall; + ``` + +### Keywords + + CLEAN, QUERY, STATS + +### Best Practice + diff --git a/docs/sidebars.json b/docs/sidebars.json index f27813a884..7c7e554166 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -984,7 +984,8 @@ "sql-manual/sql-reference/Show-Statements/SHOW-TABLE-ID", "sql-manual/sql-reference/Show-Statements/SHOW-SMALL-FILES", "sql-manual/sql-reference/Show-Statements/SHOW-POLICY", - "sql-manual/sql-reference/Show-Statements/SHOW-CATALOG-RECYCLE-BIN" + "sql-manual/sql-reference/Show-Statements/SHOW-CATALOG-RECYCLE-BIN", + "sql-manual/sql-reference/Show-Statements/SHOW-QUERY-STATS" ] }, { @@ -1031,7 +1032,8 @@ "sql-manual/sql-reference/Utility-Statements/DESCRIBE", "sql-manual/sql-reference/Utility-Statements/SWITCH", "sql-manual/sql-reference/Utility-Statements/REFRESH", - "sql-manual/sql-reference/Utility-Statements/SYNC" + "sql-manual/sql-reference/Utility-Statements/SYNC", + "sql-manual/sql-reference/Utility-Statements/CLEAN-QUERY-STATS" ] } ] @@ -1167,6 +1169,7 @@ "admin-manual/http-actions/fe/profile-action", "admin-manual/http-actions/fe/query-detail-action", "admin-manual/http-actions/fe/query-schema-action", + "admin-manual/http-actions/fe/query-stats-action", "admin-manual/http-actions/fe/row-count-action", "admin-manual/http-actions/fe/set-config-action", "admin-manual/http-actions/fe/show-data-action", diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md index 57612c7d26..fb440bfa1d 100644 --- a/docs/zh-CN/docs/admin-manual/config/fe-config.md +++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md @@ -2710,4 +2710,16 @@ show data （其他用法：HELP SHOW DATA） 默认值：128 -用于限制parallel_fragment_exec_instance_num的设置，set parallel_fragment_exec_instance_num不能超过max_instance_num \ No newline at end of file +用于限制parallel_fragment_exec_instance_num的设置，set parallel_fragment_exec_instance_num不能超过max_instance_num + +#### `enable_query_hit_stats` + + + +默认值：false + +是否可以动态配置：true + +是否为 Master FE 节点独有的配置项：false + +控制是否启用查询命中率统计。默认为 false。 \ No newline at end of file diff --git a/docs/zh-CN/docs/admin-manual/http-actions/fe/query-stats-action.md b/docs/zh-CN/docs/admin-manual/http-actions/fe/query-stats-action.md new file mode 100644 index 0000000000..29a0297686 --- /dev/null +++ b/docs/zh-CN/docs/admin-manual/http-actions/fe/query-stats-action.md @@ -0,0 +1,100 @@ +--- +{ + "title": "Query Stats Action", + "language": "zh-CN" +} +--- + + + +# Query Stats Action + + + +## Request + +``` +查看 +get api/query_stats/ +get api/query_stats// +get api/query_stats/// + +清空 +delete api/query_stats// +delete api/query_stats/// +``` + +## Description + +获取或者删除指定的catalog 数据库或者表中的统计信息， 如果是doris catalog 可以使用default_cluster + +## Path parameters + +* `` + + 指定的catalog 名称 +* `` + + 指定的数据库名称 +* `` + + 指定的表名称 + +## Query parameters +* `summary` +如果为true 则只返回summary信息， 否则返回所有的表的详细统计信息，只在get 时使用 + +## Request body + +``` +GET /api/query_stats/default_cluster/test_query_db/baseall?summary=false +{ + "msg": "success", + "code": 0, + "data": { + "summary": { + "query": 2 + }, + "detail": { + "baseall": { + "summary": { + "query": 2 + } + } + } + }, + "count": 0 +} + +``` + +## Response + +* 返回结果集 + + +## Example + + +2. 使用 curl 命令获取统计信息 + + ``` + curl --location -u root: 'http://127.0.0.1:8030/api/query_stats/default_cluster/test_query_db/baseall?summary=false' + ``` diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-QUERY-STATS.md b/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-QUERY-STATS.md new file mode 100644 index 0000000000..af1b868323 --- /dev/null +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-QUERY-STATS.md @@ -0,0 +1,195 @@ +--- +{ + "title": "SHOW-QUERY-STATS", + "language": "zh-CN" +} +--- + + + +## SHOW-QUERY-STATS + + +### Name + +SHOW QUERY STATS + + +### Description + +该语句用于展示数据库中历史查询命中的库表列的情况 + +```sql +SHOW QUERY STATS [[FOR db_name]|[FROM table_name]] [ALL] [VERBOSE]]; +``` + +说明： + +1. 支持查询数据库和表的历史查询命中情况，重启fe后数据会重置，每个fe单独统计 +2. 通过 FOR DATABASE 和FROM TABLE 可以指定查询数据库或者表的命中情况，后面分别接数据库名或者表名 +3. ALL 可以指定是否展示所有index的查询命中情况，VERBOSE 可以展示更详细的命中情况， 这两个参数可以单独使用， + 也可以一起使用，但是必须放在最后 而且只能用在表的查询上 +4. 如果没有 use 任何数据库那么直接执行`SHOW QUERY STATS` 将展示所有数据库的命中情况 +5. 命中结果中可能有两列： + QueryCount：该列被查询次数 + FilterCount: 该列作为where 条件被查询的次数 +### Example + +1. 展示表`baseall` 的查询命中情况 + + ```sql + MySQL [test_query_db]> show query stats from baseall; + +-------+------------+-------------+ + | Field | QueryCount | FilterCount | + +-------+------------+-------------+ + | k0 | 0 | 0 | + | k1 | 0 | 0 | + | k2 | 0 | 0 | + | k3 | 0 | 0 | + | k4 | 0 | 0 | + | k5 | 0 | 0 | + | k6 | 0 | 0 | + | k10 | 0 | 0 | + | k11 | 0 | 0 | + | k7 | 0 | 0 | + | k8 | 0 | 0 | + | k9 | 0 | 0 | + | k12 | 0 | 0 | + | k13 | 0 | 0 | + +-------+------------+-------------+ + 14 rows in set (0.002 sec) + + MySQL [test_query_db]> select k0, k1,k2, sum(k3) from baseall where k9 > 1 group by k0,k1,k2; + +------+------+--------+-------------+ + | k0 | k1 | k2 | sum(`k3`) | + +------+------+--------+-------------+ + | 0 | 6 | 32767 | 3021 | + | 1 | 12 | 32767 | -2147483647 | + | 0 | 3 | 1989 | 1002 | + | 0 | 7 | -32767 | 1002 | + | 1 | 8 | 255 | 2147483647 | + | 1 | 9 | 1991 | -2147483647 | + | 1 | 11 | 1989 | 25699 | + | 1 | 13 | -32767 | 2147483647 | + | 1 | 14 | 255 | 103 | + | 0 | 1 | 1989 | 1001 | + | 0 | 2 | 1986 | 1001 | + | 1 | 15 | 1992 | 3021 | + +------+------+--------+-------------+ + 12 rows in set (0.050 sec) + + MySQL [test_query_db]> show query stats from baseall; + +-------+------------+-------------+ + | Field | QueryCount | FilterCount | + +-------+------------+-------------+ + | k0 | 1 | 0 | + | k1 | 1 | 0 | + | k2 | 1 | 0 | + | k3 | 1 | 0 | + | k4 | 0 | 0 | + | k5 | 0 | 0 | + | k6 | 0 | 0 | + | k10 | 0 | 0 | + | k11 | 0 | 0 | + | k7 | 0 | 0 | + | k8 | 0 | 0 | + | k9 | 1 | 1 | + | k12 | 0 | 0 | + | k13 | 0 | 0 | + +-------+------------+-------------+ + 14 rows in set (0.001 sec) + ``` + +2. 展示表的所物化视图的的命中的汇总情况 + + ```sql + MySQL [test_query_db]> show query stats from baseall all; + +-----------+------------+ + | IndexName | QueryCount | + +-----------+------------+ + | baseall | 1 | + +-----------+------------+ + 1 row in set (0.005 sec) + ``` + +3. 展示表的所物化视图的的命中的详细情况 + + ```sql + MySQL [test_query_db]> show query stats from baseall all verbose; + +-----------+-------+------------+-------------+ + | IndexName | Field | QueryCount | FilterCount | + +-----------+-------+------------+-------------+ + | baseall | k0 | 1 | 0 | + | | k1 | 1 | 0 | + | | k2 | 1 | 0 | + | | k3 | 1 | 0 | + | | k4 | 0 | 0 | + | | k5 | 0 | 0 | + | | k6 | 0 | 0 | + | | k10 | 0 | 0 | + | | k11 | 0 | 0 | + | | k7 | 0 | 0 | + | | k8 | 0 | 0 | + | | k9 | 1 | 1 | + | | k12 | 0 | 0 | + | | k13 | 0 | 0 | + +-----------+-------+------------+-------------+ + 14 rows in set (0.017 sec) + ``` + +4. 展示数据库的命中情况 + + ```sql + MySQL [test_query_db]> show query stats for test_query_db; + +----------------------------+------------+ + | TableName | QueryCount | + +----------------------------+------------+ + | compaction_tbl | 0 | + | bigtable | 0 | + | empty | 0 | + | tempbaseall | 0 | + | test | 0 | + | test_data_type | 0 | + | test_string_function_field | 0 | + | baseall | 1 | + | nullable | 0 | + +----------------------------+------------+ + 9 rows in set (0.005 sec) + ``` + +5. 展示所有数据库的命中情况，这时不能use 任何数据库 + + ```sql + MySQL [(none)]> show query stats; + +-----------------+------------+ + | Database | QueryCount | + +-----------------+------------+ + | test_query_db | 1 | + +-----------------+------------+ + 1 rows in set (0.005 sec) + ``` + SHOW QUERY STATS; + ``` + +### Keywords + + SHOW， QUERY, STATS; + +### Best Practice diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/CLEAN-QUERY-STATS.md b/docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/CLEAN-QUERY-STATS.md new file mode 100644 index 0000000000..0996c56328 --- /dev/null +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/CLEAN-QUERY-STATS.md @@ -0,0 +1,75 @@ +--- +{ + "title": "CLEAN-QUERY-STATS", + "language": "zh-CN" +} +--- + + + +## CLEAN-QUERY-STATS + +### Name + + +CLEAN QUERY STATS + + +### Description + +该语句用请空查询统计信息 + +语法： + +```sql +CLEAN [ALL| DATABASE | TABLE] QUERY STATS [[FOR db_name]|[FROM|IN] table_name]]; +``` + +说明： + +1. 如果指定 ALL，则清空所有查询统计信息，包括数据库和表的统计信息，需要admin 权限 +2. 如果指定 DATABASE，则清空指定数据库的查询统计信息，需要对应database 的alter 权限 +3. 如果指定 TABLE，则清空指定表的查询统计信息，需要对应表的alter 权限 + +### Example + +1. 清空所有统计信息 + + ```sql + clean all query stats; + ``` + +2. 清空指定数据库的统计信息 + + ```sql + clean database query stats for test_query_db; + ``` +3. 清空指定表的统计信息 + + ```sql + clean table query stats from test_query_db.baseall; + ``` + +### Keywords + + CLEAN, QUERY, STATS + +### Best Practice + diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index b52e4a0822..18d773b7e8 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1902,7 +1902,7 @@ public class Config extends ConfigBase { * This is to solve the case that user forgot the password. */ @ConfField(mutable = false) - public static boolean skip_localhost_auth_check = false; + public static boolean skip_localhost_auth_check = true; @ConfField(mutable = true) public static boolean enable_round_robin_create_tablet = false; @@ -1940,12 +1940,12 @@ public class Config extends ConfigBase { @ConfField(mutable = true) public static int max_instance_num = 128; - /** * This config used for export/outfile. * Whether delete all files in the directory specified by export/outfile. * It is a very dangerous operation, should only be used in test env. */ + @ConfField(mutable = false) public static boolean enable_delete_existing_files = false; /* @@ -1961,4 +1961,13 @@ public class Config extends ConfigBase { */ @ConfField public static long stats_cache_size = 10_0000; + + /** + * This configuration is used to enable the statistics of query information, which will record + * the access status of databases, tables, and columns, and can be used to guide the + * optimization of table structures + * + */ + @ConfField(mutable = true, masterOnly = false) + public static boolean enable_query_hit_stats = false; } diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 28319e127b..a9f67cf270 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -1209,6 +1209,18 @@ clean_stmt ::= {: RESULT = new CleanProfileStmt(); :} + | KW_CLEAN KW_QUERY KW_STATS KW_FOR ident:db + {: + RESULT = new CleanQueryStatsStmt(db, CleanQueryStatsStmt.Scope.DB); + :} + | KW_CLEAN KW_ALL KW_QUERY KW_STATS + {: + RESULT = new CleanQueryStatsStmt(); + :} + | KW_CLEAN KW_QUERY KW_STATS from_or_in table_name:tbl + {: + RESULT = new CleanQueryStatsStmt(tbl, CleanQueryStatsStmt.Scope.TABLE); + :} ; // plugin statement @@ -3977,6 +3989,26 @@ show_param ::= | KW_CATALOG KW_RECYCLE KW_BIN opt_wild_where {: RESULT = new ShowCatalogRecycleBinStmt(parser.where); + :} + | KW_QUERY KW_STATS + {: + RESULT = new ShowQueryStatsStmt(); + :} + | KW_QUERY KW_STATS KW_FOR ident:dbName + {: + RESULT = new ShowQueryStatsStmt(dbName); + :} + | KW_QUERY KW_STATS KW_FROM table_name:dbTblName + {: + RESULT = new ShowQueryStatsStmt(dbTblName, false, false); + :} + | KW_QUERY KW_STATS KW_FROM table_name:dbTblName KW_ALL + {: + RESULT = new ShowQueryStatsStmt(dbTblName, true, false); + :} + | KW_QUERY KW_STATS KW_FROM table_name:dbTblName KW_ALL KW_VERBOSE + {: + RESULT = new ShowQueryStatsStmt(dbTblName, true, true); :} ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java index 38d195c6d8..358b6d4ccf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java @@ -970,6 +970,13 @@ public class MaterializedViewHandler extends AlterHandler { } } olapTable.deleteIndexInfo(mvName); + try { + Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentInternalCatalog().getId(), + Env.getCurrentInternalCatalog().getDbOrDdlException(olapTable.getQualifiedDbName()).getId(), + olapTable.getId(), mvIndexId); + } catch (DdlException e) { + LOG.info("failed to clean stats for mv {} from {}", mvName, olapTable.getName(), e); + } return mvIndexId; } @@ -993,9 +1000,11 @@ public class MaterializedViewHandler extends AlterHandler { } } } - String rollupIndexName = olapTable.getIndexNameById(rollupIndexId); olapTable.deleteIndexInfo(rollupIndexName); + + env.getQueryStats().clear(env.getCurrentCatalog().getId(), db.getId(), + olapTable.getId(), rollupIndexId); } finally { olapTable.writeUnlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CleanQueryStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CleanQueryStatsStmt.java new file mode 100644 index 0000000000..da53daa4be --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CleanQueryStatsStmt.java @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; +import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.UserException; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.SystemInfoService; + +import org.apache.commons.lang3.StringUtils; + +/** + * CLEAN ALL QUERY STATS; + * CLEAN DATABASE QUERY STATS FROM db; + * CLEAN TABLE QUERY STATS FROM db.table; + */ +public class CleanQueryStatsStmt extends DdlStmt { + private String dbName; + private TableName tableName; + private Scope scope; + + /** + * CLEAN DATABASE QUERY STATS FROM db; + */ + public CleanQueryStatsStmt(String dbName, Scope scope) { + this.dbName = dbName; + this.tableName = null; + this.scope = scope; + } + + /** + * CLEAN TABLE QUERY STATS FROM db.table; + */ + public CleanQueryStatsStmt(TableName tableName, Scope scope) { + this.dbName = null; + this.tableName = tableName; + this.scope = scope; + } + + public CleanQueryStatsStmt() { + this.scope = Scope.ALL; + this.tableName = null; + this.dbName = null; + } + + public String getDbName() { + return dbName; + } + + public TableName getTableName() { + return tableName; + } + + public Scope getScope() { + return scope; + } + + @Override + public void analyze(Analyzer analyzer) throws UserException { + super.analyze(analyzer); + switch (scope) { + case ALL: + if (!Env.getCurrentEnv().getAccessManager() + .checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, + "CLEAN ALL QUERY STATS"); + } + break; + case DB: + if (StringUtils.isEmpty(dbName)) { + dbName = analyzer.getDefaultDb(); + } + if (StringUtils.isEmpty(dbName)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); + } + dbName = ClusterNamespace.getFullName(SystemInfoService.DEFAULT_CLUSTER, dbName); + + Env.getCurrentEnv().getCurrentCatalog().getDbOrAnalysisException(dbName); + if (!Env.getCurrentEnv().getAccessManager() + .checkDbPriv(ConnectContext.get(), dbName, PrivPredicate.ALTER)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, + "CLEAN DATABASE QUERY STATS FOR " + ClusterNamespace.getNameFromFullName(dbName)); + } + break; + case TABLE: + tableName.analyze(analyzer); + dbName = tableName.getDb(); + if (StringUtils.isEmpty(dbName)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); + } + DatabaseIf db = Env.getCurrentEnv().getCurrentCatalog().getDbOrAnalysisException(dbName); + db.getTableOrAnalysisException(tableName.getTbl()); + if (!Env.getCurrentEnv().getAccessManager() + .checkTblPriv(ConnectContext.get(), dbName, tableName.getTbl(), PrivPredicate.ALTER)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, + "CLEAN TABLE QUERY STATS FROM " + tableName); + } + break; + default: + throw new IllegalStateException("Unexpected value: " + scope); + } + } + + @Override + public String toSql() { + switch (scope) { + case ALL: + return "CLEAN ALL QUERY STATS"; + case DB: + return "CLEAN DATABASE QUERY STATS FOR " + dbName; + case TABLE: + return "CLEAN TABLE QUERY STATS FROM " + tableName; + default: + throw new IllegalStateException("Unexpected value: " + scope); + } + } + + /** + * Scope of clean query stats + */ + public enum Scope { + ALL, DB, TABLE + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryStatsStmt.java new file mode 100644 index 0000000000..9454160c7e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryStatsStmt.java @@ -0,0 +1,234 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.UserException; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.doris.statistics.query.QueryStatsUtil; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public class ShowQueryStatsStmt extends ShowStmt { + + private static final ShowResultSetMetaData SHOW_QUERY_STATS_CATALOG_META_DATA = ShowResultSetMetaData.builder() + .addColumn(new Column("Database", ScalarType.createVarchar(20))) + .addColumn(new Column("QueryCount", ScalarType.createVarchar(30))).build(); + private static final ShowResultSetMetaData SHOW_QUERY_STATS_DATABASE_META_DATA = ShowResultSetMetaData.builder() + .addColumn(new Column("TableName", ScalarType.createVarchar(20))) + .addColumn(new Column("QueryCount", ScalarType.createVarchar(30))).build(); + private static final ShowResultSetMetaData SHOW_QUERY_STATS_TABLE_META_DATA = ShowResultSetMetaData.builder() + .addColumn(new Column("Field", ScalarType.createVarchar(20))) + .addColumn(new Column("QueryCount", ScalarType.createVarchar(30))) + .addColumn(new Column("FilterCount", ScalarType.createVarchar(30))).build(); + private static final ShowResultSetMetaData SHOW_QUERY_STATS_TABLE_ALL_META_DATA = ShowResultSetMetaData.builder() + .addColumn(new Column("IndexName", ScalarType.createVarchar(20))) + .addColumn(new Column("QueryCount", ScalarType.createVarchar(30))).build(); + private static final ShowResultSetMetaData SHOW_QUERY_STATS_TABLE_ALL_VERBOSE_META_DATA + = ShowResultSetMetaData.builder().addColumn(new Column("IndexName", ScalarType.createVarchar(20))) + .addColumn(new Column("Field", ScalarType.createVarchar(20))) + .addColumn(new Column("QueryCount", ScalarType.createVarchar(30))) + .addColumn(new Column("FilterCount", ScalarType.createVarchar(30))).build(); + + TableName tableName; + String dbName; + boolean all; + boolean verbose; + List> totalRows; + ShowQueryStatsType type; + + /** + * for SHOW QUERY STATS FROM TABLE + */ + public ShowQueryStatsStmt(TableName tableName, boolean all, boolean verbose) { + this.tableName = tableName; + this.all = all; + this.verbose = verbose; + this.totalRows = Lists.newArrayList(); + } + + /** + * for SHOW QUERY STATS FROM DATABASE + */ + public ShowQueryStatsStmt(String dbName) { + this.tableName = null; + this.all = false; + this.verbose = false; + this.dbName = dbName; + this.totalRows = Lists.newArrayList(); + this.type = ShowQueryStatsType.DATABASE; + } + + /** + * for SHOW QUERY STATS + */ + public ShowQueryStatsStmt() { + this.tableName = null; + this.all = false; + this.verbose = false; + this.dbName = null; + this.totalRows = Lists.newArrayList(); + this.type = ShowQueryStatsType.CATALOG; + } + + @Override + public void analyze(Analyzer analyzer) throws UserException { + super.analyze(analyzer); + if (StringUtils.isEmpty(dbName)) { + dbName = analyzer.getDefaultDb(); + type = ShowQueryStatsType.DATABASE; + } + if (Strings.isNullOrEmpty(dbName)) { + dbName = analyzer.getDefaultDb(); + } else { + dbName = ClusterNamespace.getFullName(getClusterName(), dbName); + } + String catalog = Env.getCurrentEnv().getCurrentCatalog().getName(); + if (tableName == null && StringUtils.isEmpty(dbName)) { + if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "SHOW QUERY STATS"); + } + Map result = QueryStatsUtil.getMergedCatalogStats(catalog); + result.forEach((dbName, queryHit) -> { + totalRows.add(Arrays.asList(dbName, String.valueOf(queryHit))); + }); + type = ShowQueryStatsType.CATALOG; + return; + } + if (tableName != null) { + tableName.analyze(analyzer); + dbName = tableName.getDb(); + } + Database db = (Database) Env.getCurrentEnv().getCurrentCatalog().getDbOrDdlException(dbName); + if (tableName != null) { + db.getTableOrDdlException(tableName.getTbl()); + } + if (tableName == null) { + Map stats = QueryStatsUtil.getMergedDatabaseStats(catalog, dbName); + stats.forEach((tableName, queryHit) -> { + if (Env.getCurrentEnv().getAccessManager() + .checkTblPriv(ConnectContext.get(), dbName, tableName, PrivPredicate.SHOW)) { + totalRows.add(Arrays.asList(tableName, String.valueOf(queryHit))); + } + }); + } else { + if (!Env.getCurrentEnv().getAccessManager() + .checkTblPriv(ConnectContext.get(), tableName, PrivPredicate.SHOW)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SHOW QUERY STATS", + ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(), + dbName + ": " + tableName); + } + if (all && verbose) { + type = ShowQueryStatsType.TABLE_ALL_VERBOSE; + QueryStatsUtil.getMergedTableAllVerboseStats(catalog, dbName, tableName.getTbl()) + .forEach((indexName, stat) -> { + final boolean[] firstRow = new boolean[] {true}; + stat.forEach((col, statCount) -> { + totalRows.add(Arrays.asList(firstRow[0] ? indexName : "", col, + String.valueOf(statCount.first), String.valueOf(statCount.second))); + firstRow[0] = false; + }); + }); + } else if (all) { + type = ShowQueryStatsType.TABLE_ALL; + + Map stats = QueryStatsUtil.getMergedTableAllStats(catalog, dbName, tableName.getTbl()); + stats.forEach((indexName, queryHit) -> { + totalRows.add(Arrays.asList(indexName, String.valueOf(queryHit))); + }); + } else if (verbose) { + Preconditions.checkState(false, "verbose is not supported if all is not set"); + } else { + type = ShowQueryStatsType.TABLE; + QueryStatsUtil.getMergedTableStats(catalog, dbName, tableName.getTbl()).forEach((col, statCount) -> { + totalRows.add( + Arrays.asList(col, String.valueOf(statCount.first), String.valueOf(statCount.second))); + }); + } + } + } + + @Override + public String toSql() { + StringBuilder builder = new StringBuilder(); + builder.append("SHOW QUERY STATS"); + if (tableName != null) { + builder.append(" FROM ").append(tableName.toSql()); + } else if (StringUtils.isNotEmpty(dbName)) { + builder.append(" FROM ").append("`").append(dbName).append("`"); + } + if (all) { + builder.append(" ALL"); + } + if (verbose) { + builder.append(" VERBOSE"); + } + return builder.toString(); + } + + public List> getResultRows() throws AnalysisException { + return totalRows; + } + + @Override + public ShowResultSetMetaData getMetaData() { + switch (type) { + case CATALOG: + return SHOW_QUERY_STATS_CATALOG_META_DATA; + case DATABASE: + return SHOW_QUERY_STATS_DATABASE_META_DATA; + case TABLE: + return SHOW_QUERY_STATS_TABLE_META_DATA; + case TABLE_ALL: + return SHOW_QUERY_STATS_TABLE_ALL_META_DATA; + case TABLE_ALL_VERBOSE: + return SHOW_QUERY_STATS_TABLE_ALL_VERBOSE_META_DATA; + default: + Preconditions.checkState(false); + return null; + } + } + + @Override + public String toString() { + return toSql(); + } + + /** + * Show query statistics type + */ + public enum ShowQueryStatsType { + CATALOG, DATABASE, TABLE, TABLE_ALL, TABLE_ALL_VERBOSE + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTabletStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTabletStmt.java index 2666dafedc..6ea93fa002 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTabletStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTabletStmt.java @@ -315,6 +315,7 @@ public class ShowTabletStmt extends ShowStmt { builder.addColumn(new Column("IndexId", ScalarType.createVarchar(30))); builder.addColumn(new Column("IsSync", ScalarType.createVarchar(30))); builder.addColumn(new Column("Order", ScalarType.createVarchar(30))); + builder.addColumn(new Column("QueryHits", ScalarType.createVarchar(30))); builder.addColumn(new Column("DetailCmd", ScalarType.createVarchar(30))); } else { for (String title : TabletsProcDir.TITLE_NAMES) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index 3be065af44..2300cd1e01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -776,8 +776,7 @@ public class BackupJob extends AbstractJob { Collections.sort(replicaIds); for (Long replicaId : replicaIds) { Replica replica = tablet.getReplicaById(replicaId); - if (replica.getLastFailedVersion() < 0 && (replica.getVersion() > visibleVersion - || (replica.getVersion() == visibleVersion))) { + if (replica.getLastFailedVersion() < 0 && replica.getVersion() >= visibleVersion) { return replica; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 0edd2536fd..e853abf798 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -172,6 +172,7 @@ import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.persist.AlterMultiMaterializedView; import org.apache.doris.persist.BackendReplicasInfo; import org.apache.doris.persist.BackendTabletsInfo; +import org.apache.doris.persist.CleanQueryStatsInfo; import org.apache.doris.persist.DropPartitionInfo; import org.apache.doris.persist.EditLog; import org.apache.doris.persist.GlobalVarPersistInfo; @@ -211,6 +212,7 @@ import org.apache.doris.statistics.AnalysisTaskScheduler; import org.apache.doris.statistics.StatisticsAutoAnalyzer; import org.apache.doris.statistics.StatisticsCache; import org.apache.doris.statistics.StatisticsCleaner; +import org.apache.doris.statistics.query.QueryStats; import org.apache.doris.system.Backend; import org.apache.doris.system.Frontend; import org.apache.doris.system.HeartbeatMgr; @@ -438,6 +440,8 @@ public class Env { private ResourceGroupMgr resourceGroupMgr; + private QueryStats queryStats; + private StatisticsCleaner statisticsCleaner; /** @@ -652,6 +656,7 @@ public class Env { } this.globalFunctionMgr = new GlobalFunctionMgr(); this.resourceGroupMgr = new ResourceGroupMgr(); + this.queryStats = new QueryStats(); this.loadManagerAdapter = new LoadManagerAdapter(); this.hiveTransactionMgr = new HiveTransactionMgr(); } @@ -4183,6 +4188,9 @@ public class Env { if (column != null) { column.setName(newColName); hasColumn = true; + Env.getCurrentEnv().getQueryStats() + .rename(Env.getCurrentEnv().getCurrentCatalog().getId(), db.getId(), + table.getId(), entry.getKey(), colName, newColName); } } if (!hasColumn) { @@ -5235,7 +5243,6 @@ public class Env { return analysisManager; } - public GlobalFunctionMgr getGlobalFunctionMgr() { return globalFunctionMgr; } @@ -5251,4 +5258,13 @@ public class Env { public StatisticsAutoAnalyzer getStatisticsAutoAnalyzer() { return statisticsAutoAnalyzer; } + + public QueryStats getQueryStats() { + return queryStats; + } + + public void cleanQueryStats(CleanQueryStatsInfo info) throws DdlException { + queryStats.clear(info); + editLog.logCleanQueryStats(info); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/ReplicasProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/ReplicasProcNode.java index 82bbb2618c..383ba14ab0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/ReplicasProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/ReplicasProcNode.java @@ -22,8 +22,10 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletMeta; +import org.apache.doris.common.Config; import org.apache.doris.common.util.NetUtils; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.statistics.query.QueryStatsUtil; import org.apache.doris.system.Backend; import com.google.common.collect.ImmutableList; @@ -41,7 +43,7 @@ public class ReplicasProcNode implements ProcNodeInterface { .add("BackendId").add("Version").add("LstSuccessVersion").add("LstFailedVersion").add("LstFailedTime") .add("SchemaHash").add("LocalDataSize").add("RemoteDataSize").add("RowCount").add("State").add("IsBad") .add("VersionCount").add("PathHash").add("MetaUrl").add("CompactionStatus").add("CooldownReplicaId") - .add("CooldownMetaId").build(); + .add("CooldownMetaId").add("QueryHits").build(); private long tabletId; private List replicas; @@ -85,6 +87,10 @@ public class ReplicasProcNode implements ProcNodeInterface { if (replica.getCooldownMetaId() != null) { cooldownMetaId = replica.getCooldownMetaId().toString(); } + long queryHits = 0L; + if (Config.enable_query_hit_stats) { + queryHits = QueryStatsUtil.getMergedReplicaStats(replica.getId()); + } result.addRow(Arrays.asList(String.valueOf(replica.getId()), String.valueOf(replica.getBackendId()), String.valueOf(replica.getVersion()), @@ -102,7 +108,8 @@ public class ReplicasProcNode implements ProcNodeInterface { metaUrl, compactionUrl, String.valueOf(tablet.getCooldownConf().first), - cooldownMetaId)); + cooldownMetaId, + String.valueOf(queryHits))); } return result; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java index 47a1790cfa..e13d1846a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java @@ -24,10 +24,12 @@ import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletInvertedIndex; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.NetUtils; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.statistics.query.QueryStatsUtil; import org.apache.doris.system.Backend; import com.google.common.base.Preconditions; @@ -36,7 +38,9 @@ import com.google.common.collect.ImmutableMap; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; /* * SHOW PROC /dbs/dbId/tableId/partitions/partitionId/indexId @@ -48,8 +52,8 @@ public class TabletsProcDir implements ProcDirInterface { .add("LstSuccessVersion").add("LstFailedVersion").add("LstFailedTime") .add("LocalDataSize").add("RemoteDataSize").add("RowCount").add("State") .add("LstConsistencyCheckTime").add("CheckVersion") - .add("VersionCount").add("PathHash").add("MetaUrl").add("CompactionStatus").add("CooldownReplicaId") - .add("CooldownMetaId").build(); + .add("VersionCount").add("QueryHits").add("PathHash").add("MetaUrl").add("CompactionStatus") + .add("CooldownReplicaId").add("CooldownMetaId").build(); private Table table; private MaterializedIndex index; @@ -67,6 +71,16 @@ public class TabletsProcDir implements ProcDirInterface { List> tabletInfos = new ArrayList>(); table.readLock(); try { + Map replicaIdToQueryHits = new HashMap<>(); + if (Config.enable_query_hit_stats) { + List replicaIds = new ArrayList(); + for (Tablet tablet : index.getTablets()) { + for (Replica replica : tablet.getReplicas()) { + replicaIds.add(replica.getId()); + } + } + replicaIdToQueryHits = QueryStatsUtil.getMergedReplicasStats(replicaIds); + } // get infos for (Tablet tablet : index.getTablets()) { long tabletId = tablet.getId(); @@ -92,6 +106,7 @@ public class TabletsProcDir implements ProcDirInterface { tabletInfo.add(-1); // check version tabletInfo.add(-1); // check version hash tabletInfo.add(-1); // version count + tabletInfo.add(0L); // query hits tabletInfo.add(-1); // path hash tabletInfo.add(FeConstants.null_string); // meta url tabletInfo.add(FeConstants.null_string); // compaction status @@ -124,6 +139,7 @@ public class TabletsProcDir implements ProcDirInterface { tabletInfo.add(TimeUtils.longToTimeString(tablet.getLastCheckTime())); tabletInfo.add(tablet.getCheckedVersion()); tabletInfo.add(replica.getVersionCount()); + tabletInfo.add(replicaIdToQueryHits.getOrDefault(replica.getId(), 0L)); tabletInfo.add(replica.getPathHash()); Backend be = backendMap.get(replica.getBackendId()); String host = (be == null ? Backend.DUMMY_IP : be.getHost()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java index 6b073b05e2..0fbc19596d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java @@ -49,6 +49,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.function.LongUnaryOperator; import java.util.function.Predicate; public class Util { @@ -92,6 +93,21 @@ public class Util { TYPE_STRING_MAP.put(PrimitiveType.NULL_TYPE, "null"); } + public static LongUnaryOperator overflowSafeIncrement() { + return original -> { + if (original == Long.MAX_VALUE) { + return Long.MAX_VALUE; + } + long r = original + 1; + if (r == Long.MAX_VALUE || ((original ^ r) & (1 ^ r)) < 0) { + // unbounded reached + return Long.MAX_VALUE; + } else { + return r; + } + }; + } + private static class CmdWorker extends Thread { private final Process process; private Integer exitValue; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 566b1d455d..2c400786e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -132,6 +132,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable { catalogResource.removeReference(catalog.getName(), ReferenceType.CATALOG); } } + Env.getCurrentEnv().getQueryStats().clear(catalog.getId()); } return catalog; } @@ -158,6 +159,20 @@ public class CatalogMgr implements Writable, GsonPostProcessable { return idToCatalog.get(id); } + public CatalogIf getCatalogOrAnalysisException(long id) throws AnalysisException { + return getCatalogOrException(id, + catalog -> new AnalysisException(ErrorCode.ERR_UNKNOWN_CATALOG.formatErrorMsg(catalog), + ErrorCode.ERR_UNKNOWN_CATALOG)); + } + + public CatalogIf getCatalogOrException(long id, Function e) throws E { + CatalogIf catalog = idToCatalog.get(id); + if (catalog == null) { + throw e.apply(id); + } + return catalog; + } + public CatalogIf getCatalogOrException(String name, Function e) throws E { CatalogIf catalog = nameToCatalog.get(name); if (catalog == null) { @@ -277,6 +292,8 @@ public class CatalogMgr implements Writable, GsonPostProcessable { Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_DROP_CATALOG, log); lastDBOfCatalog.remove(stmt.getCatalogName()); + Env.getCurrentEnv().getQueryStats().clear(catalog.getId()); + } finally { writeUnlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index afade5022e..9713341a23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -533,6 +533,7 @@ public class InternalCatalog implements CatalogIf { fullNameToDb.remove(db.getFullName()); DropDbInfo info = new DropDbInfo(dbName, stmt.isForceDrop(), recycleTime); Env.getCurrentEnv().getEditLog().logDropDb(info); + Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentEnv().getCurrentCatalog().getId(), db.getId()); } finally { unlock(); } @@ -574,6 +575,7 @@ public class InternalCatalog implements CatalogIf { } else { Env.getCurrentEnv().eraseDatabase(db.getId(), false); } + Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentEnv().getInternalCatalog().getId(), db.getId()); } finally { db.writeUnlock(); } @@ -865,6 +867,8 @@ public class InternalCatalog implements CatalogIf { } DropInfo info = new DropInfo(db.getId(), table.getId(), -1L, stmt.isForceDrop(), recycleTime); Env.getCurrentEnv().getEditLog().logDropTable(info); + Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentEnv().getCurrentCatalog().getId(), + db.getId(), table.getId()); } finally { db.writeUnlock(); } @@ -910,6 +914,8 @@ public class InternalCatalog implements CatalogIf { table.writeLock(); try { unprotectDropTable(db, table, isForceDrop, true, recycleTime); + Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentInternalCatalog().getId(), db.getId(), + tableId); } finally { table.writeUnlock(); db.writeUnlock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/QueryStatsAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/QueryStatsAction.java new file mode 100644 index 0000000000..7656fbb0a5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/QueryStatsAction.java @@ -0,0 +1,292 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.httpv2.rest; + +import org.apache.doris.catalog.Env; +import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.SystemInfoService; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiFunction; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * This class is used to get query stats or clear query stats. + */ +@RestController +public class QueryStatsAction extends RestBaseController { + private static final Logger LOG = LogManager.getLogger(QueryStatsAction.class); + + @RequestMapping(path = "/api/query_stats/{catalog}", method = RequestMethod.GET) + protected Object getQueryStatsFromCatalog(@PathVariable("catalog") String catalog, + @RequestParam(name = "summary", required = false, defaultValue = "true") boolean summary, + @RequestParam(name = "pretty", required = false, defaultValue = "false") boolean pretty, + HttpServletRequest request, HttpServletResponse response) { + if (pretty && summary) { + return ResponseEntityBuilder.badRequest("pretty and summary can not be true at the same time"); + } + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + // use NS_KEY as catalog, but NS_KEY's default value is 'default_cluster'. + if (catalog.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + catalog = InternalCatalog.INTERNAL_CATALOG_NAME; + } + try { + Map result = Env.getCurrentEnv().getQueryStats().getStats(catalog, summary); + if (pretty) { + return ResponseEntityBuilder.ok(getPrettyJson(result.get("detail"), QueryStatsType.DATABASE)); + } + return ResponseEntityBuilder.ok(result); + } catch (Exception e) { + LOG.warn("get query stats from catalog {} failed", catalog, e); + return ResponseEntityBuilder.internalError(e.getMessage()); + } + } + + @RequestMapping(path = "/api/query_stats/{catalog}/{database}", method = RequestMethod.GET) + protected Object getQueryStatsFromDatabase(@PathVariable("catalog") String catalog, + @PathVariable("database") String database, + @RequestParam(name = "summary", required = false, defaultValue = "true") boolean summary, + @RequestParam(name = "pretty", required = false, defaultValue = "false") boolean pretty, + HttpServletRequest request, HttpServletResponse response) { + if (pretty && summary) { + return ResponseEntityBuilder.badRequest("pretty and summary can not be true at the same time"); + } + executeCheckPassword(request, response); + checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), database, PrivPredicate.SHOW); + // use NS_KEY as catalog, but NS_KEY's default value is 'default_cluster'. + String clasterName = catalog; + if (catalog.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + catalog = InternalCatalog.INTERNAL_CATALOG_NAME; + } + database = ClusterNamespace.getFullName(clasterName, database); + try { + Map result = Env.getCurrentEnv().getQueryStats().getStats(catalog, database, summary); + if (pretty) { + return ResponseEntityBuilder.ok(getPrettyJson(result.get("detail"), QueryStatsType.TABLE)); + } + return ResponseEntityBuilder.ok(result); + } catch (Exception e) { + LOG.warn("get query stats from catalog {} failed", catalog, e); + return ResponseEntityBuilder.internalError(e.getMessage()); + } + } + + @RequestMapping(path = "/api/query_stats/{catalog}/{database}/{table}", method = RequestMethod.GET) + protected Object getQueryStatsFromTable(@PathVariable("catalog") String catalog, + @PathVariable("database") String database, @PathVariable("table") String table, + @RequestParam(name = "summary", required = false, defaultValue = "true") boolean summary, + @RequestParam(name = "pretty", required = false, defaultValue = "false") boolean pretty, + HttpServletRequest request, HttpServletResponse response) { + if (pretty && summary) { + return ResponseEntityBuilder.badRequest("pretty and summary can not be true at the same time"); + } + executeCheckPassword(request, response); + checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), database, table, PrivPredicate.SHOW); + // use NS_KEY as catalog, but NS_KEY's default value is 'default_cluster'. + String clasterName = catalog; + if (catalog.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + catalog = InternalCatalog.INTERNAL_CATALOG_NAME; + } + database = ClusterNamespace.getFullName(clasterName, database); + try { + Map result = Env.getCurrentEnv().getQueryStats().getStats(catalog, database, table, summary); + if (pretty) { + return ResponseEntityBuilder.ok(getPrettyJson(result.get("detail"), QueryStatsType.INDEX)); + } + return ResponseEntityBuilder.ok(result); + } catch (Exception e) { + LOG.warn("get query stats from catalog {} failed", catalog, e); + return ResponseEntityBuilder.internalError(e.getMessage()); + } + } + + @RequestMapping(path = "/api/query_stats/{catalog}/{database}/{table}/{index}", method = RequestMethod.GET) + protected Object getQueryStatsFromIndex(@PathVariable("catalog") String catalog, + @PathVariable("database") String database, @PathVariable("table") String table, + @PathVariable("index") String index, + @RequestParam(name = "summary", required = false, defaultValue = "true") boolean summary, + @RequestParam(name = "pretty", required = false, defaultValue = "false") boolean pretty, + HttpServletRequest request, HttpServletResponse response) { + if (pretty && summary) { + return ResponseEntityBuilder.badRequest("pretty and summary can not be true at the same time"); + } + executeCheckPassword(request, response); + checkTblAuth(ConnectContext.get().getCurrentUserIdentity(), database, table, PrivPredicate.SHOW); + // use NS_KEY as catalog, but NS_KEY's default value is 'default_cluster'. + String clasterName = catalog; + if (catalog.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + catalog = InternalCatalog.INTERNAL_CATALOG_NAME; + } + database = ClusterNamespace.getFullName(clasterName, database); + try { + Map result = Env.getCurrentEnv().getQueryStats() + .getStats(catalog, database, table, index, summary); + if (pretty) { + return ResponseEntityBuilder.ok(getPrettyJson(result.get("detail"), QueryStatsType.COLUMN)); + } + return ResponseEntityBuilder.ok(result); + } catch (Exception e) { + LOG.warn("get query stats from catalog {} failed", catalog, e); + return ResponseEntityBuilder.internalError(e.getMessage()); + } + } + + @RequestMapping(path = "/api/query_stats/{catalog}/{database}", method = RequestMethod.DELETE) + protected Object clearQueryStatsFromDatabase(@PathVariable("catalog") String catalog, + @PathVariable("database") String database, HttpServletRequest request, HttpServletResponse response) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + // use NS_KEY as catalog, but NS_KEY's default value is 'default_cluster'. + String clasterName = catalog; + if (catalog.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + catalog = InternalCatalog.INTERNAL_CATALOG_NAME; + } + database = ClusterNamespace.getFullName(clasterName, database); + Env.getCurrentEnv().getQueryStats().clear(catalog, database); + return ResponseEntityBuilder.ok(); + } + + @RequestMapping(path = "/api/query_stats/{catalog}/{database}/{table}", method = RequestMethod.DELETE) + protected Object clearQueryStatsFromTable(@PathVariable("catalog") String catalog, + @PathVariable("database") String database, @PathVariable("table") String table, HttpServletRequest request, + HttpServletResponse response) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + // use NS_KEY as catalog, but NS_KEY's default value is 'default_cluster'. + String clasterName = catalog; + if (catalog.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + catalog = InternalCatalog.INTERNAL_CATALOG_NAME; + } + database = ClusterNamespace.getFullName(clasterName, database); + Env.getCurrentEnv().getQueryStats().clear(catalog, database, table); + return ResponseEntityBuilder.ok(); + } + + private JSONArray getPrettyJson(Map stats, QueryStatsType type) { + JSONArray result = new JSONArray(); + QueryStatsType nextType = QueryStatsType.INVALID; + switch (type) { + case CATALOG: + nextType = QueryStatsType.DATABASE; + break; + case DATABASE: + nextType = QueryStatsType.TABLE; + break; + case TABLE: + nextType = QueryStatsType.INDEX; + break; + case INDEX: + nextType = QueryStatsType.COLUMN; + break; + case COLUMN: + nextType = QueryStatsType.DETAIL; + break; + default: + break; + } + for (Map.Entry entry : stats.entrySet()) { + JSONObject obj = new JSONObject(); + if (type == QueryStatsType.COLUMN) { + obj.put("name", entry.getKey()); + obj.put("type", type.toString()); + Map columnStats = (Map) entry.getValue(); + obj.put("value", + Math.max(columnStats.getOrDefault("query", 0L), columnStats.getOrDefault("filter", 0L))); + JSONArray children = new JSONArray(); + BiFunction, Boolean, JSONObject> genDetail + = (Map detail, Boolean query) -> { + JSONObject detailObj = new JSONObject(); + detailObj.put("type", "DETAIL"); + if (query) { + detailObj.put("name", "query"); + detailObj.put("value", detail.getOrDefault("query", 0L)); + } else { + detailObj.put("name", "filter"); + detailObj.put("value", detail.getOrDefault("filter", 0L)); + } + return detailObj; + }; + children.add(genDetail.apply(columnStats, true)); + children.add(genDetail.apply(columnStats, false)); + obj.put("children", children); + } else { + if (entry.getValue() == null || entry.getValue().isEmpty()) { + continue; + } + if (type == QueryStatsType.DATABASE && entry.getKey().contains(":")) { + obj.put("name", entry.getKey().split(":")[1]); + } else { + obj.put("name", entry.getKey()); + } + obj.put("type", type.toString()); + Map summary = (Map) entry.getValue().get("summary"); + obj.put("value", summary.getOrDefault("query", 0L)); + if (entry.getValue().containsKey("detail") && nextType != QueryStatsType.INVALID) { + obj.put("children", getPrettyJson((Map) entry.getValue().get("detail"), nextType)); + } + } + result.add(obj); + } + return result; + } + + enum QueryStatsType { + CATALOG(1), DATABASE(2), TABLE(3), INDEX(4), COLUMN(5), DETAIL(6), INVALID(99); + private static Map map = new HashMap<>(); + private int value; + + static { + for (QueryStatsType queryStatsType : QueryStatsType.values()) { + map.put(queryStatsType.value, queryStatsType); + } + } + + QueryStatsType(int i) { + this.value = value; + } + + public static QueryStatsType valueOf(int value) { + QueryStatsType queryStatsType = (QueryStatsType) map.get(value); + if (queryStatsType == null) { + return QueryStatsType.INVALID; + } + return queryStatsType; + } + + public int getValue() { + return value; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index 1d8908e6c9..c2e0e8551e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -71,6 +71,7 @@ import org.apache.doris.persist.BatchModifyPartitionsInfo; import org.apache.doris.persist.BatchRemoveTransactionsOperation; import org.apache.doris.persist.BatchRemoveTransactionsOperationV2; import org.apache.doris.persist.CleanLabelOperationLog; +import org.apache.doris.persist.CleanQueryStatsInfo; import org.apache.doris.persist.ColocatePersistInfo; import org.apache.doris.persist.ConsistencyCheckInfo; import org.apache.doris.persist.CreateTableInfo; @@ -777,6 +778,11 @@ public class JournalEntity implements Writable { isRead = true; break; } + case OperationType.OP_CLEAN_QUERY_STATS: { + data = CleanQueryStatsInfo.read(in); + isRead = true; + break; + } default: { IOException e = new IOException(); LOG.error("UNKNOWN Operation Type {}", opCode, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/CleanQueryStatsInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/CleanQueryStatsInfo.java new file mode 100644 index 0000000000..d487b05b94 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/CleanQueryStatsInfo.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.persist; + +import org.apache.doris.analysis.CleanQueryStatsStmt.Scope; +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.gson.annotations.SerializedName; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +public class CleanQueryStatsInfo implements Writable { + @SerializedName(value = "catalog") + private String catalog; + @SerializedName(value = "dbName") + private String dbName; + @SerializedName(value = "tableName") + private String tableName; + @SerializedName(value = "scope") + private Scope scope; + + public CleanQueryStatsInfo(Scope scope, String catalog, String dbName, String tableName) { + this.catalog = catalog; + this.dbName = dbName; + this.tableName = tableName; + this.scope = scope; + } + + public static CleanQueryStatsInfo read(DataInput in) throws IOException { + return GsonUtils.GSON.fromJson(Text.readString(in), CleanQueryStatsInfo.class); + } + + public String getCatalog() { + return catalog; + } + + public String getDbName() { + return dbName; + } + + public String getTableName() { + return tableName; + } + + public Scope getScope() { + return scope; + } + + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index f491253ea8..1e2276edc6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -858,6 +858,11 @@ public class EditLog { env.getSchemaChangeHandler().replayAlterLightSchChange(info); break; } + case OperationType.OP_CLEAN_QUERY_STATS: { + final CleanQueryStatsInfo info = (CleanQueryStatsInfo) journal.getData(); + env.getQueryStats().clear(info); + break; + } case OperationType.OP_MODIFY_TABLE_ADD_OR_DROP_INVERTED_INDICES: { final TableAddOrDropInvertedIndicesInfo info = (TableAddOrDropInvertedIndicesInfo) journal.getData(); @@ -1713,4 +1718,8 @@ public class EditLog { public void logAlterMTMV(AlterMultiMaterializedView log) { logEdit(OperationType.OP_ALTER_MTMV_STMT, log); } + + public void logCleanQueryStats(CleanQueryStatsInfo log) { + logEdit(OperationType.OP_CLEAN_QUERY_STATS, log); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java index fcdfdfe628..26a0badd3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java @@ -286,6 +286,9 @@ public class OperationType { public static final short OP_DROP_RESOURCE_GROUP = 411; public static final short OP_ALTER_RESOURCE_GROUP = 412; + // query stats 440 ~ 424 + public static final short OP_CLEAN_QUERY_STATS = 420; + /** * Get opcode name by op code. **/ diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java index 050b810dca..e2ddb34509 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.EsResource; import org.apache.doris.catalog.EsTable; import org.apache.doris.catalog.PartitionInfo; @@ -38,6 +39,7 @@ import org.apache.doris.external.elasticsearch.QueryBuilders.BuilderOptions; import org.apache.doris.external.elasticsearch.QueryBuilders.QueryBuilder; import org.apache.doris.planner.external.FederationBackendPolicy; import org.apache.doris.statistics.StatisticalType; +import org.apache.doris.statistics.query.StatsDelta; import org.apache.doris.system.Backend; import org.apache.doris.thrift.TEsScanNode; import org.apache.doris.thrift.TEsScanRange; @@ -363,4 +365,11 @@ public class EsScanNode extends ScanNode { conjuncts.removeIf(expr -> !notPushDownList.contains(expr)); } } + + @Override + public StatsDelta genStatsDelta() throws AnalysisException { + return new StatsDelta(Env.getCurrentEnv().getCurrentCatalog().getId(), + Env.getCurrentEnv().getCurrentCatalog().getDbOrAnalysisException(table.getQualifiedDbName()).getId(), + table.getId(), -1L); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java index 4685046079..e718bad667 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java @@ -25,12 +25,15 @@ import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.JdbcTable; import org.apache.doris.catalog.OdbcTable; import org.apache.doris.catalog.external.JdbcExternalTable; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.statistics.StatisticalType; import org.apache.doris.statistics.StatsRecursiveDerive; +import org.apache.doris.statistics.query.StatsDelta; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TJdbcScanNode; import org.apache.doris.thrift.TOdbcTableType; @@ -56,9 +59,10 @@ public class JdbcScanNode extends ScanNode { private TOdbcTableType jdbcType; private String graphQueryString = ""; + private JdbcTable tbl; + public JdbcScanNode(PlanNodeId id, TupleDescriptor desc, boolean isJdbcExternalTable) { super(id, desc, "JdbcScanNode", StatisticalType.JDBC_SCAN_NODE); - JdbcTable tbl = null; if (isJdbcExternalTable) { JdbcExternalTable jdbcExternalTable = (JdbcExternalTable) (desc.getTable()); tbl = jdbcExternalTable.getJdbcTable(); @@ -244,4 +248,11 @@ public class JdbcScanNode extends ScanNode { public int getNumInstances() { return 1; } + + @Override + public StatsDelta genStatsDelta() throws AnalysisException { + return new StatsDelta(Env.getCurrentEnv().getCurrentCatalog().getId(), + Env.getCurrentEnv().getCurrentCatalog().getDbOrAnalysisException(tbl.getQualifiedDbName()).getId(), + tbl.getId(), -1L); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java index f47be29d38..32f8fe2f65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java @@ -27,12 +27,15 @@ import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OdbcTable; import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.UserException; import org.apache.doris.statistics.StatisticalType; import org.apache.doris.statistics.StatsRecursiveDerive; +import org.apache.doris.statistics.query.StatsDelta; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TOdbcScanNode; import org.apache.doris.thrift.TOdbcTableType; @@ -91,6 +94,8 @@ public class OdbcScanNode extends ScanNode { private String connectString; private TOdbcTableType odbcType; + private OdbcTable tbl; + /** * Constructs node to scan given data files of table 'tbl'. */ @@ -99,6 +104,7 @@ public class OdbcScanNode extends ScanNode { connectString = tbl.getConnectString(); odbcType = tbl.getOdbcTableType(); tblName = OdbcTable.databaseProperName(odbcType, tbl.getOdbcTableName()); + this.tbl = tbl; } @Override @@ -246,4 +252,11 @@ public class OdbcScanNode extends ScanNode { StatsRecursiveDerive.getStatsRecursiveDerive().statsRecursiveDerive(this); cardinality = (long) statsDeriveResult.getRowCount(); } + + @Override + public StatsDelta genStatsDelta() throws AnalysisException { + return new StatsDelta(Env.getCurrentEnv().getCurrentCatalog().getId(), + Env.getCurrentEnv().getCurrentCatalog().getDbOrAnalysisException(tbl.getQualifiedDbName()).getId(), + tbl.getId(), -1L); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 549718a8f0..8ca8386bae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -64,6 +64,7 @@ import org.apache.doris.resource.Tag; import org.apache.doris.statistics.StatisticalType; import org.apache.doris.statistics.StatsDeriveResult; import org.apache.doris.statistics.StatsRecursiveDerive; +import org.apache.doris.statistics.query.StatsDelta; import org.apache.doris.system.Backend; import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TExplainLevel; @@ -172,6 +173,8 @@ public class OlapScanNode extends ScanNode { // List of tablets will be scanned by current olap_scan_node private ArrayList scanTabletIds = Lists.newArrayList(); + private ArrayList scanReplicaIds = Lists.newArrayList(); + private Set sampleTabletIds = Sets.newHashSet(); private TableSample tableSample; @@ -782,6 +785,7 @@ public class OlapScanNode extends ScanNode { errs.add(err); continue; } + scanReplicaIds.add(replica.getId()); String ip = backend.getHost(); int port = backend.getBePort(); TScanRangeLocation scanRangeLocation = new TScanRangeLocation(new TNetworkAddress(ip, port)); @@ -1512,4 +1516,13 @@ public class OlapScanNode extends ScanNode { } } } + + @Override + public StatsDelta genStatsDelta() throws AnalysisException { + return new StatsDelta(Env.getCurrentEnv().getCurrentCatalog().getId(), + Env.getCurrentEnv().getCurrentCatalog().getDbOrAnalysisException( + olapTable.getQualifiedDbName()).getId(), + olapTable.getId(), selectedIndexId == -1 ? olapTable.getBaseIndexId() : selectedIndexId, + scanReplicaIds); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java index c53670544a..49058a021c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java @@ -43,12 +43,14 @@ import org.apache.doris.catalog.Type; import org.apache.doris.catalog.external.ExternalTable; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.UserException; import org.apache.doris.common.util.VectorizedUtil; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import org.apache.doris.rewrite.mvrewrite.MVSelectFailedException; +import org.apache.doris.statistics.query.StatsDelta; import org.apache.doris.thrift.TQueryOptions; import org.apache.doris.thrift.TRuntimeFilterMode; @@ -205,7 +207,10 @@ public class OriginalPlanner extends Planner { */ analyzer.getDescTbl().computeMemLayout(); singleNodePlan.finalize(analyzer); - + if (Config.enable_query_hit_stats && plannerContext.getStatement() != null + && plannerContext.getStatement().getExplainOptions() == null) { + collectQueryStat(singleNodePlan); + } // check and set flag for topn detail query opt if (VectorizedUtil.isVectorized()) { checkAndSetTopnOpt(singleNodePlan); @@ -442,7 +447,6 @@ public class OriginalPlanner extends Planner { topPlanFragment.getPlanRoot().resetTupleIds(Lists.newArrayList(fileStatusDesc.getId())); } - private SlotDescriptor injectRowIdColumnSlot(Analyzer analyzer, TupleDescriptor tupleDesc) { SlotDescriptor slotDesc = analyzer.getDescTbl().addSlotDescriptor(tupleDesc); LOG.debug("inject slot {}", slotDesc); @@ -537,7 +541,7 @@ public class OriginalPlanner extends Planner { * */ private void pushOutColumnUniqueIdsToOlapScan(PlanFragment rootFragment, Analyzer analyzer) { - Set outputColumnUniqueIds = new HashSet<>(); + Set outputColumnUniqueIds = new HashSet<>(); ArrayList outputExprs = rootFragment.getOutputExprs(); for (Expr expr : outputExprs) { if (expr instanceof SlotRef) { @@ -705,4 +709,23 @@ public class OriginalPlanner extends Planner { public DescriptorTable getDescTable() { return analyzer.getDescTbl(); } + + private void collectQueryStat(PlanNode root) { + try { + if (root instanceof ScanNode) { + StatsDelta delta = ((ScanNode) root).genQueryStats(); + if (delta != null && !delta.empty()) { + Env.getCurrentEnv().getQueryStats().addStats(delta); + if (!delta.getTabletStats().isEmpty()) { + Env.getCurrentEnv().getQueryStats().addStats(delta.getTabletStats()); + } + } + } + for (PlanNode child : root.getChildren()) { + collectQueryStat(child); + } + } catch (UserException e) { + LOG.info("failed to collect query stat: {}", e.getMessage()); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlannerContext.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlannerContext.java index 3ff4d5e7b7..a8ce9323ca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlannerContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlannerContext.java @@ -63,6 +63,10 @@ public class PlannerContext { return queryStmt; } + public StatementBase getStatement() { + return statement; + } + public TQueryOptions getQueryOptions() { return queryOptions; } // getRootAnalyzer().getQueryOptions(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index 6c4997ea77..e63662b053 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -37,11 +37,13 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.UserException; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.spi.Split; import org.apache.doris.statistics.StatisticalType; +import org.apache.doris.statistics.query.StatsDelta; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TScanRangeLocations; @@ -563,4 +565,32 @@ public abstract class ScanNode extends PlanNode { } return tupleIds; } + + public StatsDelta genStatsDelta() throws AnalysisException { + return null; + } + + public StatsDelta genQueryStats() throws UserException { + StatsDelta delta = genStatsDelta(); + if (delta == null) { + return null; + } + for (SlotDescriptor slot : desc.getMaterializedSlots()) { + if (slot.isScanSlot() && slot.getColumn() != null) { + delta.addQueryStats(slot.getColumn().getName()); + } + } + + for (Expr expr : conjuncts) { + List slotIds = Lists.newArrayList(); + expr.getIds(null, slotIds); + for (SlotId slotId : slotIds) { + SlotDescriptor slot = desc.getSlot(slotId.asInt()); + if (slot.getColumn() != null) { + delta.addFilterStats(slot.getColumn().getName()); + } + } + } + return delta; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java index 8d5f063cc2..356939ea33 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java @@ -52,6 +52,7 @@ import org.apache.doris.analysis.CancelExportStmt; import org.apache.doris.analysis.CancelLoadStmt; import org.apache.doris.analysis.CleanLabelStmt; import org.apache.doris.analysis.CleanProfileStmt; +import org.apache.doris.analysis.CleanQueryStatsStmt; import org.apache.doris.analysis.CreateCatalogStmt; import org.apache.doris.analysis.CreateDataSyncJobStmt; import org.apache.doris.analysis.CreateDbStmt; @@ -116,6 +117,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.DdlException; import org.apache.doris.common.util.ProfileManager; import org.apache.doris.load.sync.SyncJobManager; +import org.apache.doris.persist.CleanQueryStatsInfo; import org.apache.doris.statistics.StatisticsRepository; /** @@ -260,8 +262,8 @@ public class DdlExecutor { if (!syncJobMgr.isJobNameExist(createSyncJobStmt.getDbName(), createSyncJobStmt.getJobName())) { syncJobMgr.addDataSyncJob((CreateDataSyncJobStmt) ddlStmt); } else { - throw new DdlException("The syncJob with jobName '" + createSyncJobStmt.getJobName() - + "' in database [" + createSyncJobStmt.getDbName() + "] is already exists."); + throw new DdlException("The syncJob with jobName '" + createSyncJobStmt.getJobName() + "' in database [" + + createSyncJobStmt.getDbName() + "] is already exists."); } } else if (ddlStmt instanceof ResumeSyncJobStmt) { env.getSyncJobManager().resumeSyncJob((ResumeSyncJobStmt) ddlStmt); @@ -325,6 +327,26 @@ public class DdlExecutor { env.getAnalysisManager().dropStats((DropStatsStmt) ddlStmt); } else if (ddlStmt instanceof KillAnalysisJobStmt) { env.getAnalysisManager().handleKillAnalyzeStmt((KillAnalysisJobStmt) ddlStmt); + } else if (ddlStmt instanceof CleanQueryStatsStmt) { + CleanQueryStatsStmt stmt = (CleanQueryStatsStmt) ddlStmt; + CleanQueryStatsInfo cleanQueryStatsInfo = null; + switch (stmt.getScope()) { + case ALL: + cleanQueryStatsInfo = new CleanQueryStatsInfo( + CleanQueryStatsStmt.Scope.ALL, env.getCurrentCatalog().getName(), null, null); + break; + case DB: + cleanQueryStatsInfo = new CleanQueryStatsInfo(CleanQueryStatsStmt.Scope.DB, + env.getCurrentCatalog().getName(), stmt.getDbName(), null); + break; + case TABLE: + cleanQueryStatsInfo = new CleanQueryStatsInfo(CleanQueryStatsStmt.Scope.TABLE, + env.getCurrentCatalog().getName(), stmt.getDbName(), stmt.getTableName().getTbl()); + break; + default: + throw new DdlException("Unknown scope: " + stmt.getScope()); + } + env.cleanQueryStats(cleanQueryStatsInfo); } else { throw new DdlException("Unknown statement."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index f1f52357fc..52b6643cda 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -73,6 +73,7 @@ import org.apache.doris.analysis.ShowPolicyStmt; import org.apache.doris.analysis.ShowProcStmt; import org.apache.doris.analysis.ShowProcesslistStmt; import org.apache.doris.analysis.ShowQueryProfileStmt; +import org.apache.doris.analysis.ShowQueryStatsStmt; import org.apache.doris.analysis.ShowRepositoriesStmt; import org.apache.doris.analysis.ShowResourceGroupsStmt; import org.apache.doris.analysis.ShowResourcesStmt; @@ -139,6 +140,7 @@ import org.apache.doris.clone.DynamicPartitionScheduler; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.CaseSensibility; +import org.apache.doris.common.Config; import org.apache.doris.common.ConfigBase; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; @@ -188,6 +190,7 @@ import org.apache.doris.statistics.ColumnStatistic; import org.apache.doris.statistics.Histogram; import org.apache.doris.statistics.StatisticsRepository; import org.apache.doris.statistics.TableStatistic; +import org.apache.doris.statistics.query.QueryStatsUtil; import org.apache.doris.system.Backend; import org.apache.doris.system.Diagnoser; import org.apache.doris.system.SystemInfoService; @@ -310,6 +313,8 @@ public class ShowExecutor { handleShowUserProperty(); } else if (stmt instanceof ShowDataStmt) { handleShowData(); + } else if (stmt instanceof ShowQueryStatsStmt) { + handleShowQueryStats(); } else if (stmt instanceof ShowCollationStmt) { handleShowCollation(); } else if (stmt instanceof ShowPartitionsStmt) { @@ -1524,6 +1529,11 @@ public class ShowExecutor { resultSet = new ShowResultSet(showStmt.getMetaData(), showStmt.getResultRows()); } + private void handleShowQueryStats() throws AnalysisException { + ShowQueryStatsStmt showStmt = (ShowQueryStatsStmt) stmt; + resultSet = new ShowResultSet(showStmt.getMetaData(), showStmt.getResultRows()); + } + private void handleShowPartitions() throws AnalysisException { ShowPartitionsStmt showStmt = (ShowPartitionsStmt) stmt; if (showStmt.getCatalog().isInternalCatalog()) { @@ -1575,6 +1585,7 @@ public class ShowExecutor { Long indexId = tabletMeta != null ? tabletMeta.getIndexId() : TabletInvertedIndex.NOT_EXIST_VALUE; String indexName = FeConstants.null_string; Boolean isSync = true; + long queryHits = 0L; int tabletIdx = -1; // check real meta @@ -1590,6 +1601,15 @@ public class ShowExecutor { isSync = false; break; } + if (Config.enable_query_hit_stats) { + MaterializedIndex mi = ((OlapTable) table).getPartition(partitionId).getIndex(indexId); + if (mi != null) { + Tablet t = mi.getTablet(tabletId); + for (Replica r : t.getReplicas()) { + queryHits += QueryStatsUtil.getMergedReplicaStats(r.getId()); + } + } + } table.readLock(); try { @@ -1641,7 +1661,7 @@ public class ShowExecutor { rows.add(Lists.newArrayList(dbName, tableName, partitionName, indexName, dbId.toString(), tableId.toString(), partitionId.toString(), indexId.toString(), - isSync.toString(), String.valueOf(tabletIdx), detailCmd)); + isSync.toString(), String.valueOf(tabletIdx), String.valueOf(queryHits), detailCmd)); } else { Database db = env.getInternalCatalog().getDbOrAnalysisException(showStmt.getDbName()); OlapTable olapTable = db.getOlapTableOrAnalysisException(showStmt.getTableName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 76c8a1ebfb..d3aaa2b142 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -66,6 +66,7 @@ import org.apache.doris.qe.ConnectProcessor; import org.apache.doris.qe.QeProcessorImpl; import org.apache.doris.qe.QueryState; import org.apache.doris.qe.VariableMgr; +import org.apache.doris.statistics.query.QueryStats; import org.apache.doris.system.Frontend; import org.apache.doris.system.SystemInfoService; import org.apache.doris.tablefunction.MetadataGenerator; @@ -96,6 +97,7 @@ import org.apache.doris.thrift.TFrontendPingFrontendResult; import org.apache.doris.thrift.TFrontendPingFrontendStatusCode; import org.apache.doris.thrift.TGetDbsParams; import org.apache.doris.thrift.TGetDbsResult; +import org.apache.doris.thrift.TGetQueryStatsRequest; import org.apache.doris.thrift.TGetTablesParams; import org.apache.doris.thrift.TGetTablesResult; import org.apache.doris.thrift.TInitExternalCtlMetaRequest; @@ -119,6 +121,7 @@ import org.apache.doris.thrift.TPrivilegeCtrl; import org.apache.doris.thrift.TPrivilegeHier; import org.apache.doris.thrift.TPrivilegeStatus; import org.apache.doris.thrift.TPrivilegeType; +import org.apache.doris.thrift.TQueryStatsResult; import org.apache.doris.thrift.TReportExecStatusParams; import org.apache.doris.thrift.TReportExecStatusResult; import org.apache.doris.thrift.TReportRequest; @@ -129,6 +132,8 @@ import org.apache.doris.thrift.TStatus; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TStreamLoadPutRequest; import org.apache.doris.thrift.TStreamLoadPutResult; +import org.apache.doris.thrift.TTableIndexQueryStats; +import org.apache.doris.thrift.TTableQueryStats; import org.apache.doris.thrift.TTableStatus; import org.apache.doris.thrift.TUpdateExportTaskStatusRequest; import org.apache.doris.thrift.TWaitingTxnStatusRequest; @@ -1550,5 +1555,134 @@ public class FrontendServiceImpl implements FrontendService.Iface { return null; } } + + @Override + public TQueryStatsResult getQueryStats(TGetQueryStatsRequest request) throws TException { + TQueryStatsResult result = new TQueryStatsResult(); + result.setStatus(new TStatus(TStatusCode.OK)); + if (!request.isSetType()) { + TStatus status = new TStatus(TStatusCode.ANALYSIS_ERROR); + status.addToErrorMsgs("type is not set"); + result.setStatus(status); + return result; + } + try { + switch (request.getType()) { + case CATALOG: { + if (!request.isSetCatalog()) { + TStatus status = new TStatus(TStatusCode.ANALYSIS_ERROR); + status.addToErrorMsgs("catalog is not set"); + result.setStatus(status); + } else { + result.setSimpleResult(Env.getCurrentEnv().getQueryStats().getCatalogStats(request.catalog)); + } + break; + } + case DATABASE: { + if (!request.isSetCatalog() || !request.isSetDb()) { + TStatus status = new TStatus(TStatusCode.ANALYSIS_ERROR); + status.addToErrorMsgs("catalog or db is not set"); + result.setStatus(status); + return result; + } else { + result.setSimpleResult( + Env.getCurrentEnv().getQueryStats().getDbStats(request.catalog, request.db)); + } + break; + } + case TABLE: { + if (!request.isSetCatalog() || !request.isSetDb() || !request.isSetTbl()) { + TStatus status = new TStatus(TStatusCode.ANALYSIS_ERROR); + status.addToErrorMsgs("catalog or db or table is not set"); + result.setStatus(status); + return result; + } else { + Env.getCurrentEnv().getQueryStats().getTblStats(request.catalog, request.db, request.tbl) + .forEach((col, stat) -> { + TTableQueryStats colunmStat = new TTableQueryStats(); + colunmStat.setField(col); + colunmStat.setQueryStats(stat.first); + colunmStat.setFilterStats(stat.second); + result.addToTableStats(colunmStat); + }); + } + break; + } + case TABLE_ALL: { + if (!request.isSetCatalog() || !request.isSetDb() || !request.isSetTbl()) { + TStatus status = new TStatus(TStatusCode.ANALYSIS_ERROR); + status.addToErrorMsgs("catalog or db or table is not set"); + result.setStatus(status); + } else { + result.setSimpleResult(Env.getCurrentEnv().getQueryStats() + .getTblAllStats(request.catalog, request.db, request.tbl)); + } + break; + } + case TABLE_ALL_VERBOSE: { + if (!request.isSetCatalog() || !request.isSetDb() || !request.isSetTbl()) { + TStatus status = new TStatus(TStatusCode.ANALYSIS_ERROR); + status.addToErrorMsgs("catalog or db or table is not set"); + result.setStatus(status); + } else { + Env.getCurrentEnv().getQueryStats() + .getTblAllVerboseStats(request.catalog, request.db, request.tbl) + .forEach((indexName, indexStats) -> { + TTableIndexQueryStats indexStat = new TTableIndexQueryStats(); + indexStat.setIndexName(indexName); + indexStats.forEach((col, stat) -> { + TTableQueryStats colunmStat = new TTableQueryStats(); + colunmStat.setField(col); + colunmStat.setQueryStats(stat.first); + colunmStat.setFilterStats(stat.second); + indexStat.addToTableStats(colunmStat); + }); + result.addToTableVerbosStats(indexStat); + }); + } + break; + } + case TABLET: { + if (!request.isSetReplicaId()) { + TStatus status = new TStatus(TStatusCode.ANALYSIS_ERROR); + status.addToErrorMsgs("Replica Id is not set"); + result.setStatus(status); + } else { + Map tabletStats = new HashMap<>(); + tabletStats.put(request.getReplicaId(), + Env.getCurrentEnv().getQueryStats().getStats(request.getReplicaId())); + result.setTabletStats(tabletStats); + } + break; + } + case TABLETS: { + if (!request.isSetReplicaIds()) { + TStatus status = new TStatus(TStatusCode.ANALYSIS_ERROR); + status.addToErrorMsgs("Replica Ids is not set"); + result.setStatus(status); + } else { + Map tabletStats = new HashMap<>(); + QueryStats qs = Env.getCurrentEnv().getQueryStats(); + for (long replicaId : request.getReplicaIds()) { + tabletStats.put(replicaId, qs.getStats(replicaId)); + } + result.setTabletStats(tabletStats); + } + break; + } + default: { + TStatus status = new TStatus(TStatusCode.ANALYSIS_ERROR); + status.addToErrorMsgs("unknown type: " + request.getType()); + result.setStatus(status); + break; + } + } + } catch (UserException e) { + TStatus status = new TStatus(TStatusCode.ANALYSIS_ERROR); + status.addToErrorMsgs(e.getMessage()); + result.setStatus(status); + } + return result; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/CatalogStats.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/CatalogStats.java new file mode 100644 index 0000000000..632a17fdda --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/CatalogStats.java @@ -0,0 +1,237 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics.query; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.datasource.CatalogIf; + +import com.google.common.collect.ImmutableMap; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * CatalogStats is used to store the query statistics of the catalog. + * The query statistics of the catalog is the sum of the query statistics of all databases. + */ +public class CatalogStats { + private ConcurrentHashMap dataBaseStats; + private CatalogIf catalog; + + public CatalogStats(long catalogId) throws AnalysisException { + this.dataBaseStats = new ConcurrentHashMap<>(); + this.catalog = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalogId); + } + + public CatalogIf getCatalog() { + return catalog; + } + + /** + * Add the query statistics of the database to the catalog. + * If it does not exist, it will be created. + */ + public void addStats(StatsDelta statsDelta) throws AnalysisException { + long databaseId = statsDelta.getDatabase(); + DataBaseStats d = dataBaseStats.get(databaseId); + if (d == null) { + d = new DataBaseStats(catalog, databaseId); + DataBaseStats old = dataBaseStats.putIfAbsent(databaseId, d); + if (old == null) { + d.addStats(statsDelta); + } else { + old.addStats(statsDelta); + } + } else { + d.addStats(statsDelta); + } + } + + public ConcurrentHashMap getDataBaseStats() { + return dataBaseStats; + } + + /** + * Get the query statistics of the table. + * If it does not exist, return 0. + */ + public long getQueryStats(long database, long table) { + DataBaseStats d = dataBaseStats.get(database); + if (d == null) { + return 0; + } else { + return d.getQueryStats(table); + } + } + + /** + * Get the query statistics of the index. + * If it does not exist, return 0. + */ + public long getQueryStats(long database, long table, long index) { + DataBaseStats d = dataBaseStats.get(database); + if (d == null) { + return 0; + } else { + return d.getQueryStats(table, index); + } + } + + /** + * Get the query statistics of the column. + * If it does not exist, return 0. + */ + public long getQueryStats(long database, long table, long index, String column) { + DataBaseStats d = dataBaseStats.get(database); + if (d == null) { + return 0; + } else { + return d.getQueryStats(table, index, column); + } + } + + /** + * Get the query statistics of the database. + * If it does not exist, return 0. + */ + public long getQueryStats(long database) { + DataBaseStats d = dataBaseStats.get(database); + if (d == null) { + return 0; + } else { + return d.getQueryStats(); + } + } + + /** + * Get the all query statistics of the catalog. + * If it does not exist, return 0. + */ + public long getQueryStats() { + long total = 0; + for (DataBaseStats d : dataBaseStats.values()) { + total = total + d.getQueryStats(); + } + return total; + } + + /** + * Get the filter statistics of the column. + * If it does not exist, return 0. + */ + public long getFilterStats(long database, long table, long index, String column) { + DataBaseStats d = dataBaseStats.get(database); + if (d == null) { + return 0; + } else { + return d.getFilterStats(table, index, column); + } + } + + /** + * Get the all statistics of all the catalog. + * If it does not exist, return 0. + */ + public Map getStats(boolean summary) throws AnalysisException { + Map stat = new HashMap<>(); + Map dstat = new HashMap<>(); + + stat.put("summary", ImmutableMap.of("query", getQueryStats())); + List dbIds = catalog.getDbIds(); + for (long dbId : dbIds) { + String dbName = catalog.getDbOrAnalysisException(dbId).getFullName(); + if (dataBaseStats.containsKey(dbId)) { + dstat.put(dbName, dataBaseStats.get(dbId).getStats(summary)); + } else { + dstat.put(dbName, new HashMap<>()); + } + } + stat.put("detail", dstat); + return stat; + } + + /** + * Get the all statistics of the database. + * If it does not exist, return empty. + */ + public Map getStats(long database, boolean summary) throws AnalysisException { + if (dataBaseStats.containsKey(database)) { + return dataBaseStats.get(database).getStats(summary); + } else { + return new HashMap<>(); + } + } + + /** + * Get the all statistics of the table. + * If it does not exist, return empty. + */ + public Map getStats(long database, long table, boolean summary) throws AnalysisException { + if (dataBaseStats.containsKey(database)) { + return dataBaseStats.get(database).getStats(table, summary); + } else { + return new HashMap<>(); + } + } + + /** + * Get the all statistics of the index. + * If it does not exist, return empty. + */ + public Map getStats(long database, long table, long index, boolean summary) + throws AnalysisException { + if (dataBaseStats.containsKey(database)) { + return dataBaseStats.get(database).getStats(table, index, summary); + } else { + return new HashMap<>(); + } + } + + public void clear() { + dataBaseStats.clear(); + } + + // remove database if exist + public void clear(long database) { + dataBaseStats.remove(database); + } + + public void clear(long database, long table) { + dataBaseStats.computeIfPresent(database, (k, v) -> { + v.clear(table); + return v; + }); + } + + public void clear(long database, long table, long index) { + dataBaseStats.computeIfPresent(database, (k, v) -> { + v.clear(table, index); + return v; + }); + } + + public void rename(long database, long table, long index, String column, String newName) { + dataBaseStats.computeIfPresent(database, (k, v) -> { + v.rename(table, index, column, newName); + return v; + }); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/ColumnStatsDelta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/ColumnStatsDelta.java new file mode 100644 index 0000000000..dc555d30a6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/ColumnStatsDelta.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics.query; + +/** + * ColumnStatsDelta is used to record the stats delta of a column + */ +public class ColumnStatsDelta { + public boolean queryHit; + public boolean filterHit; + + public ColumnStatsDelta(boolean queryHit, boolean filterHit) { + this.queryHit = queryHit; + this.filterHit = filterHit; + } + + public ColumnStatsDelta() { + this.queryHit = false; + this.filterHit = false; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/DataBaseStats.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/DataBaseStats.java new file mode 100644 index 0000000000..ac8680d32a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/DataBaseStats.java @@ -0,0 +1,187 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics.query; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.datasource.CatalogIf; + +import com.google.common.collect.ImmutableMap; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * This class is used to store the query statistics of all tables in a database. + */ +public class DataBaseStats { + private ConcurrentHashMap tableStats; + private CatalogIf catalog; + private DatabaseIf db; + + public DatabaseIf getDb() { + return db; + } + + public DataBaseStats(CatalogIf catalog, long databaseId) throws AnalysisException { + tableStats = new ConcurrentHashMap<>(); + this.catalog = catalog; + this.db = catalog.getDbOrAnalysisException(databaseId); + } + + /** + * Add the query statistics of the table to the database. + */ + public void addStats(StatsDelta statsDelta) throws AnalysisException { + long tableId = statsDelta.getTable(); + TableStats t = tableStats.get(tableId); + if (t == null) { + t = new TableStats(db, tableId); + TableStats old = tableStats.putIfAbsent(tableId, t); + if (old == null) { + t.addStats(statsDelta); + } else { + old.addStats(statsDelta); + } + } else { + t.addStats(statsDelta); + } + } + + public ConcurrentHashMap getTableStats() { + return tableStats; + } + + /** + * Get the query statistics of the table. + */ + public long getQueryStats(long table) { + TableStats t = tableStats.get(table); + if (t == null) { + return 0; + } else { + return t.getQueryStats(); + } + } + + /** + * Get the query statistics of the index. + */ + public long getQueryStats(long table, long index) { + TableStats t = tableStats.get(table); + if (t == null) { + return 0; + } else { + return t.getQueryStats(index); + } + } + + /** + * Get the query statistics of the column. + */ + public long getQueryStats(long table, long index, String column) { + TableStats t = tableStats.get(table); + if (t == null) { + return 0; + } else { + return t.getQueryStats(index, column); + } + } + + public long getQueryStats() { + long total = 0; + for (TableStats t : tableStats.values()) { + total = total + t.getQueryStats(); + } + return total; + } + + /** + * Get the filter statistics of the column. + */ + public long getFilterStats(long table, long index, String column) { + TableStats t = tableStats.get(table); + if (t == null) { + return 0; + } else { + return t.getFilterStats(index, column); + } + } + + /** + * Get the filter statistics of the database. + */ + public Map getStats(boolean summary) { + Map stat = new HashMap<>(); + Map dstat = new HashMap<>(); + List tables = db.getTablesOrEmpty(); + stat.put("summary", ImmutableMap.of("query", getQueryStats())); + + for (TableIf table : tables) { + if (tableStats.containsKey(table.getId())) { + dstat.put(table.getName(), tableStats.get(table.getId()).getStats(summary)); + } else { + dstat.put(table.getName(), new HashMap<>()); + } + } + stat.put("detail", dstat); + return stat; + } + + public Map getStats(long table, boolean summary) { + if (tableStats.containsKey(table)) { + return tableStats.get(table).getStats(summary); + } else { + return new HashMap<>(); + } + } + + public Map getStats(long table, long index, boolean summary) { + if (tableStats.containsKey(table)) { + return tableStats.get(table).getStats(index, summary); + } else { + return new HashMap<>(); + } + } + + public void clear() { + tableStats.clear(); + } + + public void clear(long table) { + tableStats.remove(table); + } + + public void clear(long table, long index) { + tableStats.computeIfPresent(table, (k, v) -> { + v.clear(index); + return v; + }); + } + + public void rename(long table, long index, String column, String newName) { + tableStats.computeIfPresent(table, (k, v) -> { + v.rename(index, column, newName); + return v; + }); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/IndexStats.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/IndexStats.java new file mode 100644 index 0000000000..df1265b735 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/IndexStats.java @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics.query; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.common.util.Util; + +import com.google.common.collect.ImmutableMap; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +/** + * This class is used to store the query statistics of materialized views. + */ +public class IndexStats { + private ConcurrentHashMap columnQueryStats; + private ConcurrentHashMap columnFilterStats; + private AtomicLong queryHit; + private TableIf table; + private long indexId; + + /** + * The statistics of the column in the materialized view. + */ + public IndexStats(TableIf table, long indexId) { + this.indexId = indexId; + columnQueryStats = new ConcurrentHashMap<>(); + columnFilterStats = new ConcurrentHashMap<>(); + queryHit = new AtomicLong(0L); + this.table = table; + } + + public TableIf getTable() { + return table; + } + + public ConcurrentHashMap getColumnQueryStats() { + return columnQueryStats; + } + + public ConcurrentHashMap getColumnFilterStats() { + return columnFilterStats; + } + + private void addStats(String column, ConcurrentHashMap stats) { + AtomicLong l = stats.get(column); + if (l == null) { + l = new AtomicLong(0); + AtomicLong old = stats.putIfAbsent(column, l); + if (old == null) { + l.updateAndGet(Util.overflowSafeIncrement()); + } else { + old.updateAndGet(Util.overflowSafeIncrement()); + } + } else { + l.updateAndGet(Util.overflowSafeIncrement()); + } + } + + /** + * Add the query statistics of the column to the materialized view. + */ + public void addStats(Map columnStats) { + queryHit.updateAndGet(Util.overflowSafeIncrement()); + for (Map.Entry columnStat : columnStats.entrySet()) { + if (columnStat.getValue().filterHit) { + addFilterStats(columnStat.getKey()); + } + if (columnStat.getValue().queryHit) { + addQueryStats(columnStat.getKey()); + } + } + } + + public void addQueryStats(String column) { + addStats(column, columnQueryStats); + } + + public void addFilterStats(String column) { + addStats(column, columnFilterStats); + } + + /** + * Get the query statistics of the column in the materialized view. + */ + public long getQueryStats(String column) { + AtomicLong l = columnQueryStats.get(column); + if (l == null) { + return 0; + } else { + return l.get(); + } + } + + /** + * Get the query statistics of the materialized view. + */ + public Long getQueryStats() { + return queryHit.get(); + } + + /** + * Get the filter statistics of all columns in the materialized view. + */ + public long getFilterStats(String column) { + AtomicLong l = columnFilterStats.get(column); + if (l == null) { + return 0; + } else { + return l.get(); + } + } + + /** + * Get the maximum filter statistics of all columns in the materialized view. + */ + public Map getStats(boolean summary) { + List indexColumns; + if (table.getType() == TableType.OLAP) { + OlapTable olapTable = (OlapTable) table; + indexColumns = olapTable.getSchemaByIndexId(indexId); + } else { + indexColumns = table.getBaseSchema(); + } + Map stat = new HashMap<>(); + stat.put("summary", ImmutableMap.of("query", getQueryStats())); + if (!summary) { + Map columnStats = new HashMap<>(); + + for (Column column : indexColumns) { + Map dstat = new HashMap<>(); + dstat.put("query", getQueryStats(column.getName())); + dstat.put("filter", getFilterStats(column.getName())); + columnStats.put(column.getName(), dstat); + } + stat.put("detail", columnStats); + } + return stat; + } + + public void rename(String column, String newName) { + AtomicLong queryStata = columnQueryStats.get(column); + if (queryStata != null) { + columnQueryStats.put(newName, queryStata); + columnQueryStats.remove(column); + } + AtomicLong filterStats = columnFilterStats.get(column); + if (filterStats != null) { + columnFilterStats.put(newName, filterStats); + columnFilterStats.remove(column); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStats.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStats.java new file mode 100644 index 0000000000..3cb9502dbb --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStats.java @@ -0,0 +1,471 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics.query; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.Pair; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.persist.CleanQueryStatsInfo; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +/** + * QueryStats is used to record the query statistics of each table. + * The statistics include the number of column/index/table hits in queries, the number of column hit in where clause, + * The statistics are used to find hot table and columns. + */ +public class QueryStats { + private static final Logger LOG = LogManager.getLogger(QueryStats.class); + + ConcurrentHashMap catalogStats; + TabletStats tabletStats; + + public QueryStats() { + catalogStats = new ConcurrentHashMap<>(); + tabletStats = new TabletStats(); + } + + /** + * Add query statistics columns + */ + public void addStats(StatsDelta statsDelta) throws AnalysisException { + long catalogId = statsDelta.getCatalog(); + CatalogStats c = catalogStats.get(catalogId); + if (c == null) { + c = new CatalogStats(catalogId); + CatalogStats old = catalogStats.putIfAbsent(catalogId, c); + if (old == null) { + c.addStats(statsDelta); + } else { + old.addStats(statsDelta); + } + } else { + c.addStats(statsDelta); + } + } + + /** + * Add tablet statistics + */ + public void addStats(List replicaIds) { + tabletStats.addStats(replicaIds); + } + + public long getQueryStats(long catalog, long database, long table) { + CatalogStats c = catalogStats.get(catalog); + if (c == null) { + return 0; + } else { + return c.getQueryStats(database, table); + } + } + + public long getQueryStats(long catalog, long database, long table, long index) { + CatalogStats c = catalogStats.get(catalog); + if (c == null) { + return 0; + } else { + return c.getQueryStats(database, table, index); + } + } + + public long getQueryStats(long catalog, long database, long table, long index, String column) { + CatalogStats c = catalogStats.get(catalog); + if (c == null) { + return 0; + } else { + return c.getQueryStats(database, table, index, column); + } + } + + public long getFilterStats(long catalog, long database, long table, long index, String column) { + CatalogStats c = catalogStats.get(catalog); + if (c == null) { + return 0; + } else { + return c.getFilterStats(database, table, index, column); + } + } + + public long getStats(long replicaId) { + return tabletStats.getTabletQueryStats(replicaId); + } + + public Map getStats(String catalog, boolean summary) throws AnalysisException { + try { + CatalogIf c = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalog); + return getStats(c.getId(), summary); + } catch (AnalysisException e) { + LOG.info("get stats failed. catalog: {}", catalog, e); + return new HashMap<>(); + } + } + + public Map getStats(long catalog, boolean summary) throws AnalysisException { + if (catalogStats.containsKey(catalog)) { + return catalogStats.get(catalog).getStats(summary); + } else { + return new HashMap<>(); + } + } + + public Map getStats(String catalog, String database, boolean summary) throws AnalysisException { + try { + CatalogIf c = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalog); + DatabaseIf d = c.getDbOrAnalysisException(database); + return getStats(c.getId(), d.getId(), summary); + } catch (AnalysisException e) { + LOG.info("get stats failed. catalog: {}, database: {}", catalog, database, e); + return new HashMap<>(); + } + } + + public Map getStats(long catalog, long database, boolean summary) throws AnalysisException { + if (catalogStats.containsKey(catalog)) { + return catalogStats.get(catalog).getStats(database, summary); + } else { + return new HashMap<>(); + } + } + + public Map getStats(String catalog, String database, String table, boolean summary) + throws AnalysisException { + try { + CatalogIf c = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalog); + DatabaseIf d = c.getDbOrAnalysisException(database); + TableIf t = d.getTableOrAnalysisException(table); + return getStats(c.getId(), d.getId(), t.getId(), summary); + } catch (AnalysisException e) { + LOG.info("get stats failed. catalog: {}, database: {}, table: {}", catalog, database, table, e); + return new HashMap<>(); + } + } + + public Map getStats(long catalog, long database, long table, boolean summary) + throws AnalysisException { + if (catalogStats.containsKey(catalog)) { + return catalogStats.get(catalog).getStats(database, table, summary); + } else { + return new HashMap<>(); + } + } + + public Map getStats(String catalog, String database, String table, String index, boolean summary) + throws AnalysisException { + try { + CatalogIf c = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalog); + DatabaseIf d = c.getDbOrAnalysisException(database); + TableIf t = d.getTableOrAnalysisException(table); + long indexId = TableStats.DEFAULT_INDEX_ID; + if (t.getType() == TableType.OLAP) { + indexId = ((OlapTable) t).getIndexIdByName(index); + } + return getStats(c.getId(), d.getId(), t.getId(), indexId, summary); + } catch (AnalysisException e) { + LOG.info("get stats failed. catalog: {}, database: {}, table: {}, index: {}", catalog, database, table, + index, e); + return new HashMap<>(); + } + } + + public Map getStats(long catalog, long database, long table, long index, boolean summary) + throws AnalysisException { + if (catalogStats.containsKey(catalog)) { + return catalogStats.get(catalog).getStats(database, table, index, summary); + } else { + return new HashMap<>(); + } + } + + public Map getCatalogStats(String catalog) throws AnalysisException { + Map result = new LinkedHashMap<>(); + CatalogIf c = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalog); + c.getDbNamesOrEmpty().forEach(dbName -> result.put(dbName.toString(), 0L)); + if (!catalogStats.containsKey(c.getId())) { + return result; + } + for (Map.Entry entry : catalogStats.get(c.getId()).getDataBaseStats().entrySet()) { + if (result.containsKey(entry.getKey())) { + result.put(c.getDbOrAnalysisException(entry.getKey()).getFullName(), entry.getValue().getQueryStats()); + } + } + return result; + } + + + public Map getDbStats(String catalog, String db) throws AnalysisException { + Map result = new LinkedHashMap<>(); + CatalogIf c = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalog); + DatabaseIf d = c.getDbOrAnalysisException(db); + d.getTableNamesOrEmptyWithLock().forEach(tblName -> result.put(tblName.toString(), 0L)); + if (!catalogStats.containsKey(c.getId())) { + return result; + } + if (!catalogStats.get(c.getId()).getDataBaseStats().containsKey(d.getId())) { + return result; + } + DataBaseStats dbStats = catalogStats.get(c.getId()).getDataBaseStats().get(d.getId()); + for (Map.Entry entry : dbStats.getTableStats().entrySet()) { + TableIf tableIf = d.getTableNullable(entry.getKey()); + if (tableIf == null) { + continue; + } + if (result.containsKey(tableIf.getName())) { + result.put(tableIf.getName(), entry.getValue().getQueryStats()); + } + } + return result; + } + + public Map> getTblStats(String catalog, String db, String tbl) throws AnalysisException { + Map> result = new LinkedHashMap<>(); + CatalogIf c = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalog); + DatabaseIf d = c.getDbOrAnalysisException(db); + TableIf t = d.getTableOrAnalysisException(tbl); + d.getTableOrAnalysisException(tbl).getBaseSchema().forEach(col -> result.put(col.getName(), Pair.of(0L, 0L))); + if (!catalogStats.containsKey(c.getId())) { + return result; + } + if (!catalogStats.get(c.getId()).getDataBaseStats().containsKey(d.getId())) { + return result; + } + if (!catalogStats.get(c.getId()).getDataBaseStats().get(d.getId()).getTableStats().containsKey(t.getId())) { + return result; + } + + ConcurrentHashMap indexStats = catalogStats.get(c.getId()).getDataBaseStats().get(d.getId()) + .getTableStats().get(t.getId()).getIndexStats(); + + if (t.getType() == TableType.OLAP) { + for (Map.Entry entry : indexStats.entrySet()) { + for (Map.Entry indexEntry : entry.getValue().getColumnQueryStats().entrySet()) { + if (result.containsKey(indexEntry.getKey())) { + result.get(indexEntry.getKey()).first += indexEntry.getValue().get(); + } + } + for (Map.Entry indexEntry : entry.getValue().getColumnFilterStats().entrySet()) { + if (result.containsKey(indexEntry.getKey())) { + result.get(indexEntry.getKey()).second += indexEntry.getValue().get(); + } + } + } + } else { + IndexStats stats = indexStats.get(TableStats.DEFAULT_INDEX_ID); + for (Map.Entry entry : stats.getColumnQueryStats().entrySet()) { + if (result.containsKey(entry.getKey())) { + result.get(entry.getKey()).first = entry.getValue().get(); + } + } + for (Map.Entry entry : stats.getColumnFilterStats().entrySet()) { + if (result.containsKey(entry.getKey())) { + result.get(entry.getKey()).second = entry.getValue().get(); + } + } + } + return result; + } + + public Map getTblAllStats(String catalog, String db, String tbl) throws AnalysisException { + Map result = new LinkedHashMap<>(); + CatalogIf c = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalog); + DatabaseIf d = c.getDbOrAnalysisException(db); + TableIf t = d.getTableOrAnalysisException(tbl); + if (t.getType() == TableType.OLAP) { + ((OlapTable) t).getIndexNameToId().keySet().forEach(indexName -> result.put(indexName, 0L)); + } else { + result.put(tbl, 0L); + } + if (!catalogStats.containsKey(c.getId())) { + return result; + } + if (!catalogStats.get(c.getId()).getDataBaseStats().containsKey(d.getId())) { + return result; + } + if (!catalogStats.get(c.getId()).getDataBaseStats().get(d.getId()).getTableStats().containsKey(t.getId())) { + return result; + } + ConcurrentHashMap indexStats = catalogStats.get(c.getId()).getDataBaseStats().get(d.getId()) + .getTableStats().get(t.getId()).getIndexStats(); + if (t.getType() == TableType.OLAP) { + for (Map.Entry entry : indexStats.entrySet()) { + result.put(((OlapTable) t).getIndexNameById(entry.getKey()), entry.getValue().getQueryStats()); + } + } else { + for (Map.Entry entry : indexStats.entrySet()) { + result.put(tbl, entry.getValue().getQueryStats()); + } + } + return result; + } + + public Map>> getTblAllVerboseStats(String catalog, String db, String tbl) + throws AnalysisException { + Map>> result = new LinkedHashMap<>(); + CatalogIf c = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalog); + DatabaseIf d = c.getDbOrAnalysisException(db); + TableIf t = d.getTableOrAnalysisException(tbl); + if (t.getType() == TableType.OLAP) { + ((OlapTable) t).getIndexNameToId().forEach((indexName, indexId) -> { + Map> indexResult = new LinkedHashMap<>(); + ((OlapTable) t).getSchemaByIndexId(indexId) + .forEach(col -> indexResult.put(col.getName(), Pair.of(0L, 0L))); + result.put(indexName, indexResult); + }); + } else { + Map> indexResult = new LinkedHashMap<>(); + t.getBaseSchema().forEach(col -> indexResult.put(col.getName(), Pair.of(0L, 0L))); + result.put(tbl, indexResult); + } + if (!catalogStats.containsKey(c.getId())) { + return result; + } + if (!catalogStats.get(c.getId()).getDataBaseStats().containsKey(d.getId())) { + return result; + } + if (!catalogStats.get(c.getId()).getDataBaseStats().get(d.getId()).getTableStats().containsKey(t.getId())) { + return result; + } + ConcurrentHashMap indexStats = catalogStats.get(c.getId()).getDataBaseStats().get(d.getId()) + .getTableStats().get(t.getId()).getIndexStats(); + for (Map.Entry entry : indexStats.entrySet()) { + String indexName = t.getType() == TableType.OLAP ? ((OlapTable) t).getIndexNameById(entry.getKey()) : tbl; + if (!result.containsKey(indexName)) { + continue; + } + Map> indexResult = result.get(indexName); + for (Map.Entry indexEntry : entry.getValue().getColumnQueryStats().entrySet()) { + if (indexResult.containsKey(indexEntry.getKey())) { + indexResult.get(indexEntry.getKey()).first = indexEntry.getValue().get(); + } + } + for (Map.Entry indexEntry : entry.getValue().getColumnFilterStats().entrySet()) { + if (indexResult.containsKey(indexEntry.getKey())) { + indexResult.get(indexEntry.getKey()).second = indexEntry.getValue().get(); + } + } + if (t.getType() == TableType.OLAP) { + result.get(((OlapTable) t).getIndexNameById(entry.getKey())).putAll(indexResult); + } else { + result.put(tbl, indexResult); + } + } + return result; + } + + public void clear() { + catalogStats.clear(); + tabletStats.clear(); + } + + public void clear(CleanQueryStatsInfo cleanQueryStatsInfo) throws DdlException { + switch (cleanQueryStatsInfo.getScope()) { + case ALL: + clear(cleanQueryStatsInfo.getCatalog()); + break; + case DB: + clear(cleanQueryStatsInfo.getCatalog(), cleanQueryStatsInfo.getDbName()); + break; + case TABLE: + clear(cleanQueryStatsInfo.getCatalog(), cleanQueryStatsInfo.getDbName(), + cleanQueryStatsInfo.getTableName()); + break; + default: + throw new DdlException("Unknown scope: " + cleanQueryStatsInfo.getScope()); + } + } + + public void clear(String catalog) { + CatalogIf c = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalog); + if (c != null) { + clear(c.getId()); + } + } + + public void clear(long catalog) { + catalogStats.remove(catalog); + if (catalog == InternalCatalog.INTERNAL_CATALOG_ID) { + tabletStats.clear(); + } + } + + public void clear(String catalog, String database) { + try { + CatalogIf c = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalog); + DatabaseIf d = c.getDbOrAnalysisException(database); + clear(c.getId(), d.getId()); + } catch (AnalysisException e) { + LOG.warn("Failed to clear query stats", e); + } + } + + public void clear(long catalog, long database) { + catalogStats.computeIfPresent(catalog, (k, v) -> { + v.clear(database); + return v; + }); + } + + public void clear(String catalog, String database, String table) { + try { + CatalogIf c = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(catalog); + DatabaseIf d = c.getDbOrAnalysisException(database); + TableIf t = d.getTableOrAnalysisException(table); + clear(c.getId(), d.getId(), t.getId()); + } catch (AnalysisException e) { + LOG.warn("Failed to clear query stats", e); + } + } + + public void clear(long catalog, long database, long table) { + catalogStats.computeIfPresent(catalog, (k, v) -> { + v.clear(database, table); + return v; + }); + } + + + public void clear(long catalog, long database, long table, long index) { + catalogStats.computeIfPresent(catalog, (k, v) -> { + v.clear(database, table, index); + return v; + }); + } + + public void rename(long catalog, long database, long table, long index, String column, String newName) { + catalogStats.computeIfPresent(catalog, (k, v) -> { + v.rename(database, table, index, column, newName); + return v; + }); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsUtil.java new file mode 100644 index 0000000000..fd7b257715 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsUtil.java @@ -0,0 +1,244 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics.query; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.ClientPool; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.Frontend; +import org.apache.doris.thrift.FrontendService; +import org.apache.doris.thrift.TGetQueryStatsRequest; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TQueryStatsResult; +import org.apache.doris.thrift.TQueryStatsType; +import org.apache.doris.thrift.TStatusCode; +import org.apache.doris.thrift.TTableIndexQueryStats; +import org.apache.doris.thrift.TTableQueryStats; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class QueryStatsUtil { + private static final Logger LOG = LogManager.getLogger(QueryStatsUtil.class); + + public static Map getMergedCatalogStats(String catalog) throws UserException { + Map result = Env.getCurrentEnv().getQueryStats().getCatalogStats(catalog); + TGetQueryStatsRequest request = new TGetQueryStatsRequest(); + request.setType(TQueryStatsType.CATALOG); + request.setCatalog(catalog); + for (TQueryStatsResult other : getStats(request)) { + other.getSimpleResult().forEach((k, v) -> { + result.merge(k, v, Long::sum); + }); + } + return result; + } + + public static Map getMergedDatabaseStats(String catalog, String db) throws UserException { + Map result = Env.getCurrentEnv().getQueryStats().getDbStats(catalog, db); + TGetQueryStatsRequest request = new TGetQueryStatsRequest(); + request.setType(TQueryStatsType.DATABASE); + request.setCatalog(catalog); + request.setDb(db); + for (TQueryStatsResult other : getStats(request)) { + other.getSimpleResult().forEach((k, v) -> { + result.merge(k, v, Long::sum); + }); + } + return result; + } + + public static Map> getMergedTableStats(String catalog, String db, String table) + throws UserException { + Map> result = Env.getCurrentEnv().getQueryStats().getTblStats(catalog, db, table); + TGetQueryStatsRequest request = new TGetQueryStatsRequest(); + request.setType(TQueryStatsType.TABLE); + request.setCatalog(catalog); + request.setDb(db); + request.setTbl(table); + for (TQueryStatsResult other : getStats(request)) { + for (TTableQueryStats ts : other.getTableStats()) { + if (result.containsKey(ts.getField())) { + result.get(ts.getField()).first += ts.getQueryStats(); + result.get(ts.getField()).second += ts.getFilterStats(); + } else { + result.put(ts.getField(), Pair.of(ts.getQueryStats(), ts.getFilterStats())); + } + } + } + return result; + } + + public static Map getMergedTableAllStats(String catalog, String db, String table) + throws UserException { + Map result = Env.getCurrentEnv().getQueryStats().getTblAllStats(catalog, db, table); + TGetQueryStatsRequest request = new TGetQueryStatsRequest(); + request.setType(TQueryStatsType.TABLE_ALL); + request.setCatalog(catalog); + request.setDb(db); + request.setTbl(table); + for (TQueryStatsResult other : getStats(request)) { + other.getSimpleResult().forEach((k, v) -> { + result.merge(k, v, Long::sum); + }); + } + return result; + } + + public static Map>> getMergedTableAllVerboseStats(String catalog, String db, + String table) throws UserException { + Map>> result = Env.getCurrentEnv().getQueryStats() + .getTblAllVerboseStats(catalog, db, table); + TGetQueryStatsRequest request = new TGetQueryStatsRequest(); + request.setType(TQueryStatsType.TABLE_ALL_VERBOSE); + request.setCatalog(catalog); + request.setDb(db); + request.setTbl(table); + for (TQueryStatsResult other : getStats(request)) { + for (TTableIndexQueryStats tis : other.getTableVerbosStats()) { + if (result.containsKey(tis.getIndexName())) { + for (TTableQueryStats ts : tis.getTableStats()) { + if (result.get(tis.getIndexName()).containsKey(ts.getField())) { + result.get(tis.getIndexName()).get(ts.getField()).first += ts.getQueryStats(); + result.get(tis.getIndexName()).get(ts.getField()).second += ts.getFilterStats(); + } else { + result.get(tis.getIndexName()) + .put(ts.getField(), Pair.of(ts.getQueryStats(), ts.getFilterStats())); + } + } + } else { + Map> indexMap = new HashMap<>(); + for (TTableQueryStats ts : tis.getTableStats()) { + indexMap.put(ts.getField(), Pair.of(ts.getQueryStats(), ts.getFilterStats())); + } + result.put(tis.getIndexName(), indexMap); + } + } + } + return result; + } + + public static long getMergedReplicaStats(long replicaId) { + long queryHits = Env.getCurrentEnv().getQueryStats().getStats(replicaId); + TGetQueryStatsRequest request = new TGetQueryStatsRequest(); + request.setType(TQueryStatsType.TABLET); + request.setReplicaId(replicaId); + for (TQueryStatsResult other : getStats(request)) { + queryHits += other.getTabletStats().get(replicaId); + } + return queryHits; + } + + public static Map getMergedReplicasStats(List replicaIds) { + Map result = new HashMap<>(); + QueryStats qs = Env.getCurrentEnv().getQueryStats(); + for (long replicaId : replicaIds) { + result.put(replicaId, qs.getStats(replicaId)); + } + TGetQueryStatsRequest request = new TGetQueryStatsRequest(); + request.setType(TQueryStatsType.TABLETS); + request.setReplicaIds(replicaIds); + for (TQueryStatsResult other : getStats(request)) { + other.getTabletStats().forEach((k, v) -> { + result.merge(k, v, Long::sum); + }); + } + return result; + } + + private static List getStats(TGetQueryStatsRequest request) { + List results = new ArrayList<>(); + for (Frontend fe : Env.getCurrentEnv().getFrontends(null /* all */)) { + if (!fe.isAlive() || fe.getHost().equals(Env.getCurrentEnv().getSelfNode().getHost())) { + continue; + } + FrontendService.Client client = null; + try { + int waitTimeOut = ConnectContext.get() == null ? 300 : ConnectContext.get().getExecTimeout(); + client = ClientPool.frontendPool.borrowObject(new TNetworkAddress(fe.getHost(), fe.getRpcPort()), + waitTimeOut * 1000); + TQueryStatsResult other = client.getQueryStats(request); + if (!other.isSetStatus() || other.getStatus().getStatusCode() != TStatusCode.OK) { + LOG.info("Failed to collect stats from " + fe.getHost()); + continue; + } + switch (request.getType()) { + case TABLET: + case TABLETS: { + if (!other.isSetTabletStats()) { + throw new DdlException("Failed to collect stats from " + fe.getHost()); + } + if (other.getTabletStats().isEmpty()) { + LOG.info("get empty stats from " + fe.getHost()); + continue; + } + break; + } + case TABLE: { + if (!other.isSetTableStats()) { + throw new DdlException("Failed to collect stats from " + fe.getHost()); + } + if (other.getTableStats().isEmpty()) { + LOG.info("get empty stats from " + fe.getHost()); + continue; + } + break; + } + case CATALOG: + case DATABASE: + case TABLE_ALL: { + if (!other.isSetSimpleResult()) { + throw new DdlException("Failed to collect stats from " + fe.getHost()); + } + if (other.getSimpleResult().isEmpty()) { + LOG.info("get empty stats from " + fe.getHost()); + continue; + } + break; + } + + case TABLE_ALL_VERBOSE: { + if (!other.isSetTableVerbosStats()) { + throw new DdlException("Failed to collect stats from " + fe.getHost()); + } + if (other.getTableVerbosStats().isEmpty()) { + continue; + } + break; + } + default: { + throw new DdlException("Unknown stats type: " + request.getType()); + } + } + results.add(other); + } catch (Exception e) { + LOG.info("Failed to get fe client.", e); + } + } + return results; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/StatsDelta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/StatsDelta.java new file mode 100644 index 0000000000..09362d182a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/StatsDelta.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics.query; + +import com.google.gson.Gson; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +/** + * StatsDelta is used to record the stats delta of a table or index + */ +public class StatsDelta { + private long catalogId; + private long databaseId; + private long tableId; + private long indexId; + private HashMap columnStats; // column name -> column stats + private List tabletStats; + + public StatsDelta(long catalogId, long databaseId, long tableId, long indexId) { + this.catalogId = catalogId; + this.databaseId = databaseId; + this.tableId = tableId; + this.indexId = indexId; + this.columnStats = new HashMap<>(); + this.tabletStats = new ArrayList<>(); + } + + public StatsDelta(long catalogId, long databaseId, long tableId, long indexId, List tabletStats) { + this.catalogId = catalogId; + this.databaseId = databaseId; + this.tableId = tableId; + this.indexId = indexId; + this.columnStats = new HashMap<>(); + this.tabletStats = tabletStats; + } + + public long getCatalog() { + return catalogId; + } + + public long getDatabase() { + return databaseId; + } + + public long getTable() { + return tableId; + } + + public long getIndex() { + return indexId; + } + + public HashMap getColumnStats() { + return columnStats; + } + + /** + * add column stats to this delta + */ + public void addColumnStats(String column, ColumnStatsDelta delta) { + if (this.columnStats.containsKey(column)) { + this.columnStats.get(column).queryHit |= delta.queryHit; + this.columnStats.get(column).filterHit |= delta.filterHit; + } else { + this.columnStats.put(column, delta); + } + } + + public void addQueryStats(String column) { + if (this.columnStats.containsKey(column)) { + this.columnStats.get(column).queryHit = true; + } else { + this.columnStats.put(column, new ColumnStatsDelta(true, false)); + } + } + + public void addFilterStats(String column) { + if (this.columnStats.containsKey(column)) { + this.columnStats.get(column).filterHit = true; + } else { + this.columnStats.put(column, new ColumnStatsDelta(false, true)); + } + } + + /** + * add column stats to this delta + */ + public void addStats(String column, boolean queryHit, boolean filterHit) { + if (this.columnStats.containsKey(column)) { + this.columnStats.get(column).queryHit |= queryHit; + this.columnStats.get(column).filterHit |= filterHit; + } else { + this.columnStats.put(column, new ColumnStatsDelta(queryHit, filterHit)); + } + } + + public boolean empty() { + return columnStats.isEmpty(); + } + + public List getTabletStats() { + return tabletStats; + } + + public void addTabletStats(List tabletIds) { + this.tabletStats.addAll(tabletIds); + } + + @Override + public String toString() { + Gson gson = new Gson(); + return gson.toJson(this); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/TableStats.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/TableStats.java new file mode 100644 index 0000000000..e6c314d39a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/TableStats.java @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics.query; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.common.AnalysisException; + +import com.google.common.collect.ImmutableMap; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * TableStats is used to store the query stats of a table. + * It contains a map of index stats. + */ +public class TableStats { + public static final long DEFAULT_INDEX_ID = -1L; + private ConcurrentHashMap indexStats; + private long tableId; + private TableIf table; + + public TableStats(DatabaseIf db, long tableId) throws AnalysisException { + this.tableId = tableId; + indexStats = new ConcurrentHashMap<>(); + table = db.getTableOrAnalysisException(tableId); + } + + public TableIf getTable() { + return table; + } + + public ConcurrentHashMap getIndexStats() { + return indexStats; + } + + /** + * Add the query statistics of the index to the table. + */ + public void addStats(StatsDelta statsDelta) { + long indexId = statsDelta.getIndex(); + IndexStats i = indexStats.get(indexId); + if (i == null) { + i = new IndexStats(table, indexId); + IndexStats old = indexStats.putIfAbsent(indexId, i); + if (old == null) { + i.addStats(statsDelta.getColumnStats()); + } else { + old.addStats(statsDelta.getColumnStats()); + } + } else { + i.addStats(statsDelta.getColumnStats()); + } + } + + /** + * Get the query statistics of the table. + */ + public long getQueryStats() { + long total = 0; + for (IndexStats i : indexStats.values()) { + total = total + i.getQueryStats(); + } + return total; + } + + /** + * Get the query statistics of the index. + */ + public long getQueryStats(long index) { + IndexStats i = indexStats.get(index); + if (i == null) { + return 0; + } else { + return i.getQueryStats(); + } + } + + /** + * Get the query statistics of the column. + */ + public long getQueryStats(long index, String column) { + IndexStats i = indexStats.get(index); + if (i == null) { + return 0; + } else { + return i.getQueryStats(column); + } + } + + /** + * Get the filter statistics of the column. + */ + public long getFilterStats(long index, String column) { + IndexStats i = indexStats.get(index); + if (i == null) { + return 0; + } else { + return i.getFilterStats(column); + } + } + + /** + * Get the query statistics of the column. + */ + public Map getStats(boolean summary) { + Map stat = new HashMap<>(); + stat.put("summary", ImmutableMap.of("query", getQueryStats())); + Map dstat = new HashMap<>(); + if (table.getType() == TableType.OLAP) { + OlapTable olapTable = (OlapTable) table; + for (Map.Entry entry : olapTable.getIndexNameToId().entrySet()) { + if (indexStats.containsKey(entry.getValue())) { + dstat.put(entry.getKey(), indexStats.get(entry.getValue()).getStats(summary)); + } else { + dstat.put(entry.getKey(), new HashMap()); + } + } + } else { + if (indexStats.containsKey(DEFAULT_INDEX_ID)) { + dstat.put(table.getName(), indexStats.get(DEFAULT_INDEX_ID).getStats(summary)); + } else { + dstat.put(table.getName(), new HashMap()); + } + } + stat.put("detail", dstat); + return stat; + } + + public Map getStats(long index, boolean summary) { + if (indexStats.containsKey(index)) { + return indexStats.get(index).getStats(summary); + } else { + return new HashMap<>(); + } + } + + public void clear() { + indexStats.clear(); + } + + public void clear(long index) { + indexStats.remove(index); + } + + public void rename(long index, String column, String newName) { + + indexStats.computeIfPresent(index, (k, v) -> { + v.rename(column, newName); + return v; + }); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/TabletStats.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/TabletStats.java new file mode 100644 index 0000000000..2e1f52e38b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/TabletStats.java @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics.query; + +import org.apache.doris.common.util.Util; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +/** + * TabletStats is used to store the query stats of a tablet. + */ +public class TabletStats { + private ConcurrentHashMap tabletQueryStats; + + public TabletStats() { + tabletQueryStats = new ConcurrentHashMap<>(); + } + + /** + * Add the query statistics of the tablet. + * + * @param replicaIds The tablet replica ids + */ + public void addStats(List replicaIds) { + for (Long replicaId : replicaIds) { + AtomicLong l = tabletQueryStats.get(replicaId); + if (l == null) { + l = new AtomicLong(0); + AtomicLong old = tabletQueryStats.putIfAbsent(replicaId, l); + if (old == null) { + l.updateAndGet(Util.overflowSafeIncrement()); + } else { + old.updateAndGet(Util.overflowSafeIncrement()); + } + } else { + l.updateAndGet(Util.overflowSafeIncrement()); + } + } + } + + /** + * Get the query statistics of the tablet. + * + * @param replicaIds The tablet replica ids + * @return The query statistics of the tablets + */ + public Map getTabletQueryStats(List replicaIds) { + Map stats = new HashMap<>(); + for (Long replicaId : replicaIds) { + AtomicLong l = this.tabletQueryStats.get(replicaId); + if (l != null) { + stats.put(replicaId, l.get()); + } else { + stats.put(replicaId, 0L); + } + } + return stats; + } + + /** + * Get the query statistics of the tablet. + * @param replicaId The tablet replica id + * @return The query statistics of the tablet + */ + public long getTabletQueryStats(long replicaId) { + AtomicLong l = this.tabletQueryStats.get(replicaId); + if (l != null) { + return l.get(); + } else { + return 0L; + } + } + + public Map getTabletQueryStats() { + Map stats = new HashMap<>(); + for (Map.Entry entry : this.tabletQueryStats.entrySet()) { + stats.put(entry.getKey(), entry.getValue().get()); + } + return stats; + } + + public void clear() { + this.tabletQueryStats.clear(); + } + + public void clear(long replicaId) { + this.tabletQueryStats.remove(replicaId); + } + + public void clear(List replicaIds) { + for (Long replicaId : replicaIds) { + this.tabletQueryStats.remove(replicaId); + } + } + + public boolean isEmpty() { + return tabletQueryStats.isEmpty(); + } +} diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 82b52a1cf2..b4c13df7f1 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -832,6 +832,44 @@ struct TCheckAuthResult { 1: required Status.TStatus status } +enum TQueryStatsType { + CATALOG = 0, + DATABASE = 1, + TABLE = 2, + TABLE_ALL = 3, + TABLE_ALL_VERBOSE = 4, + TABLET = 5, + TABLETS = 6 +} + +struct TGetQueryStatsRequest { + 1: optional TQueryStatsType type + 2: optional string catalog + 3: optional string db + 4: optional string tbl + 5: optional i64 replica_id + 6: optional list replica_ids +} + +struct TTableQueryStats { + 1: optional string field + 2: optional i64 query_stats + 3: optional i64 filter_stats +} + +struct TTableIndexQueryStats { + 1: optional string index_name + 2: optional list table_stats +} + +struct TQueryStatsResult { + 1: optional Status.TStatus status + 2: optional map simple_result + 3: optional list table_stats + 4: optional list table_verbos_stats + 5: optional map tablet_stats +} + service FrontendService { TGetDbsResult getDbNames(1: TGetDbsParams params) TGetTablesResult getTableNames(1: TGetTablesParams params) @@ -879,4 +917,6 @@ service FrontendService { TConfirmUnusedRemoteFilesResult confirmUnusedRemoteFiles(1: TConfirmUnusedRemoteFilesRequest request) TCheckAuthResult checkAuth(1: TCheckAuthRequest request) + + TQueryStatsResult getQueryStats(1: TGetQueryStatsRequest request) } diff --git a/regression-test/data/query_p0/stats/query_stats_test.out b/regression-test/data/query_p0/stats/query_stats_test.out new file mode 100644 index 0000000000..261439870f --- /dev/null +++ b/regression-test/data/query_p0/stats/query_stats_test.out @@ -0,0 +1,51 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +k0 0 0 +k1 0 0 +k2 0 0 +k3 0 0 +k4 0 0 +k5 0 0 +k6 0 0 +k10 0 0 +k11 0 0 +k7 0 0 +k8 0 0 +k9 0 0 +k12 0 0 +k13 0 0 + +-- !sql -- +k0 1 1 +k1 1 0 +k2 1 1 +k3 0 0 +k4 1 0 +k5 0 0 +k6 0 0 +k10 0 0 +k11 0 0 +k7 0 0 +k8 0 0 +k9 0 0 +k12 0 0 +k13 0 0 + +-- !sql -- +baseall 2 + +-- !sql -- +baseall k0 1 1 + k1 1 0 + k2 1 1 + k3 0 0 + k4 1 0 + k5 0 0 + k6 0 0 + k10 0 0 + k11 0 0 + k7 0 0 + k8 0 0 + k9 0 0 + k12 0 0 + k13 0 0 diff --git a/regression-test/suites/cold_heat_separation_p2/modify_replica_use_partition.groovy b/regression-test/suites/cold_heat_separation_p2/modify_replica_use_partition.groovy index 4ef34f25a5..1373ef5685 100644 --- a/regression-test/suites/cold_heat_separation_p2/modify_replica_use_partition.groovy +++ b/regression-test/suites/cold_heat_separation_p2/modify_replica_use_partition.groovy @@ -265,12 +265,12 @@ suite("modify_replica_use_partition") { def iterate_num = tablets.size() / 3; for (int i = 0; i < iterate_num; i++) { int idx = i * 3; - def dst = tablets[idx][17] + def dst = tablets[idx][18] def text = get_meta(dst) def obj = new JsonSlurper().parseText(text) def rowsets = obj.rowsets for (x in [1,2]) { - dst = tablets[idx + x][17] + dst = tablets[idx + x][18] text = get_meta(dst) obj = new JsonSlurper().parseText(text) log.info( "test rowset meta is the same") @@ -485,12 +485,12 @@ suite("modify_replica_use_partition") { iterate_num = tablets.size() / 3; for (int i = 0; i < iterate_num; i++) { int idx = i * 3; - def dst = tablets[idx][17] + def dst = tablets[idx][18] def text = get_meta(dst) def obj = new JsonSlurper().parseText(text) def rowsets = obj.rowsets for (x in [1,2]) { - dst = tablets[idx + x][17] + dst = tablets[idx + x][18] text = get_meta(dst) obj = new JsonSlurper().parseText(text) log.info( "test rowset meta is the same") diff --git a/regression-test/suites/compaction/test_compacation_with_delete.groovy b/regression-test/suites/compaction/test_compacation_with_delete.groovy index 96943e1c0d..8f5d611b18 100644 --- a/regression-test/suites/compaction/test_compacation_with_delete.groovy +++ b/regression-test/suites/compaction/test_compacation_with_delete.groovy @@ -213,7 +213,7 @@ suite("test_compaction_with_delete") { for (String[] tablet in tablets) { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); - def compactionStatusUrlIndex = 17 + def compactionStatusUrlIndex = 18 sb.append("curl -X GET ") sb.append(tablet[compactionStatusUrlIndex]) String command = sb.toString() diff --git a/regression-test/suites/compaction/test_compaction_agg_keys.groovy b/regression-test/suites/compaction/test_compaction_agg_keys.groovy index c29675f7fa..f63b13cb42 100644 --- a/regression-test/suites/compaction/test_compaction_agg_keys.groovy +++ b/regression-test/suites/compaction/test_compaction_agg_keys.groovy @@ -116,7 +116,7 @@ suite("test_compaction_agg_keys") { qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """ - //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus + //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,QueryHits,VersionCount,PathHash,MetaUrl,CompactionStatus String[][] tablets = sql """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} @@ -182,7 +182,7 @@ suite("test_compaction_agg_keys") { for (String[] tablet in tablets) { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); - def compactionStatusUrlIndex = 17 + def compactionStatusUrlIndex = 18 sb.append("curl -X GET ") sb.append(tablet[compactionStatusUrlIndex]) String command = sb.toString() diff --git a/regression-test/suites/compaction/test_compaction_agg_keys_with_delete.groovy b/regression-test/suites/compaction/test_compaction_agg_keys_with_delete.groovy index c641fa4cbe..112a66d519 100644 --- a/regression-test/suites/compaction/test_compaction_agg_keys_with_delete.groovy +++ b/regression-test/suites/compaction/test_compaction_agg_keys_with_delete.groovy @@ -128,7 +128,7 @@ suite("test_compaction_agg_keys_with_delete") { qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """ - //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus + //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,QueryHits,PathHash,MetaUrl,CompactionStatus String[][] tablets = sql """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} @@ -194,7 +194,7 @@ suite("test_compaction_agg_keys_with_delete") { for (String[] tablet in tablets) { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); - def compactionStatusUrlIndex = 17 + def compactionStatusUrlIndex = 18 sb.append("curl -X GET ") sb.append(tablet[compactionStatusUrlIndex]) String command = sb.toString() diff --git a/regression-test/suites/compaction/test_compaction_dup_keys.groovy b/regression-test/suites/compaction/test_compaction_dup_keys.groovy index 374153c224..b5e393a667 100644 --- a/regression-test/suites/compaction/test_compaction_dup_keys.groovy +++ b/regression-test/suites/compaction/test_compaction_dup_keys.groovy @@ -114,7 +114,7 @@ suite("test_compaction_dup_keys") { qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time; """ - //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus + //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,QueryHits,PathHash,MetaUrl,CompactionStatus String[][] tablets = sql """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} @@ -180,7 +180,7 @@ suite("test_compaction_dup_keys") { for (String[] tablet in tablets) { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); - def compactionStatusUrlIndex = 17 + def compactionStatusUrlIndex = 18 sb.append("curl -X GET ") sb.append(tablet[compactionStatusUrlIndex]) String command = sb.toString() diff --git a/regression-test/suites/compaction/test_compaction_dup_keys_with_delete.groovy b/regression-test/suites/compaction/test_compaction_dup_keys_with_delete.groovy index f57229072d..dbbf899c15 100644 --- a/regression-test/suites/compaction/test_compaction_dup_keys_with_delete.groovy +++ b/regression-test/suites/compaction/test_compaction_dup_keys_with_delete.groovy @@ -126,7 +126,7 @@ suite("test_compaction_dup_keys_with_delete") { qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time; """ - //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus + //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,QueryHits,PathHash,MetaUrl,CompactionStatus String[][] tablets = sql """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} @@ -192,7 +192,7 @@ suite("test_compaction_dup_keys_with_delete") { for (String[] tablet in tablets) { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); - def compactionStatusUrlIndex = 17 + def compactionStatusUrlIndex = 18 sb.append("curl -X GET ") sb.append(tablet[compactionStatusUrlIndex]) String command = sb.toString() diff --git a/regression-test/suites/compaction/test_compaction_uniq_keys.groovy b/regression-test/suites/compaction/test_compaction_uniq_keys.groovy index 2ff2a7137d..970b162a51 100644 --- a/regression-test/suites/compaction/test_compaction_uniq_keys.groovy +++ b/regression-test/suites/compaction/test_compaction_uniq_keys.groovy @@ -114,7 +114,7 @@ suite("test_compaction_uniq_keys") { qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """ - //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus + //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,QueryHits,PathHash,MetaUrl,CompactionStatus String[][] tablets = sql """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} @@ -180,7 +180,7 @@ suite("test_compaction_uniq_keys") { for (String[] tablet in tablets) { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); - def compactionStatusUrlIndex = 17 + def compactionStatusUrlIndex = 18 sb.append("curl -X GET ") sb.append(tablet[compactionStatusUrlIndex]) String command = sb.toString() diff --git a/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy b/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy index 0b54e16a35..3350449ea1 100644 --- a/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy +++ b/regression-test/suites/compaction/test_compaction_uniq_keys_row_store.groovy @@ -168,7 +168,7 @@ suite("test_compaction_uniq_keys_row_store") { sql """ INSERT INTO ${tableName} VALUES (4, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.028', '2017-10-01 11:11:11.018', 'Beijing', 10, 1, NULL, NULL, NULL, NULL, '2020-01-05', 1, 34, 20) """ - //TabletId,ReplicaIdBackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus + //TabletId,ReplicaIdBackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,QueryHits,PathHash,MetaUrl,CompactionStatus tablets = sql """ show tablets from ${tableName}; """ checkValue() @@ -236,7 +236,7 @@ suite("test_compaction_uniq_keys_row_store") { for (String[] tablet in tablets) { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); - def compactionStatusUrlIndex = 17 + def compactionStatusUrlIndex = 18 sb.append("curl -X GET ") sb.append(tablet[compactionStatusUrlIndex]) String command = sb.toString() diff --git a/regression-test/suites/compaction/test_compaction_uniq_keys_with_delete.groovy b/regression-test/suites/compaction/test_compaction_uniq_keys_with_delete.groovy index caffb945d5..772f9b33e8 100644 --- a/regression-test/suites/compaction/test_compaction_uniq_keys_with_delete.groovy +++ b/regression-test/suites/compaction/test_compaction_uniq_keys_with_delete.groovy @@ -196,7 +196,7 @@ suite("test_compaction_uniq_keys_with_delete") { for (String[] tablet in tablets) { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); - def compactionStatusUrlIndex = 17 + def compactionStatusUrlIndex = 18 sb.append("curl -X GET ") sb.append(tablet[compactionStatusUrlIndex]) String command = sb.toString() diff --git a/regression-test/suites/compaction/test_vertical_compaction_agg_keys.groovy b/regression-test/suites/compaction/test_vertical_compaction_agg_keys.groovy index 981c2448af..9074755a09 100644 --- a/regression-test/suites/compaction/test_vertical_compaction_agg_keys.groovy +++ b/regression-test/suites/compaction/test_vertical_compaction_agg_keys.groovy @@ -164,7 +164,7 @@ suite("test_vertical_compaction_agg_keys") { qt_select_default2 """ SELECT * FROM ${tableName} t ORDER BY user_id; """ - //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus + //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,QueryHits,PathHash,MetaUrl,CompactionStatus String[][] tablets = sql """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} @@ -230,7 +230,7 @@ suite("test_vertical_compaction_agg_keys") { for (String[] tablet in tablets) { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); - def compactionStatusUrlIndex = 17 + def compactionStatusUrlIndex = 18 sb.append("curl -X GET ") sb.append(tablet[compactionStatusUrlIndex]) String command = sb.toString() diff --git a/regression-test/suites/compaction/test_vertical_compaction_dup_keys.groovy b/regression-test/suites/compaction/test_vertical_compaction_dup_keys.groovy index ac91094ed9..373ff8653b 100644 --- a/regression-test/suites/compaction/test_vertical_compaction_dup_keys.groovy +++ b/regression-test/suites/compaction/test_vertical_compaction_dup_keys.groovy @@ -163,7 +163,7 @@ suite("test_vertical_compaction_dup_keys") { qt_select_default2 """ SELECT * FROM ${tableName} t ORDER BY user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time; """ - //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus + //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,QueryHits,PathHash,MetaUrl,CompactionStatus String[][] tablets = sql """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} @@ -229,7 +229,7 @@ suite("test_vertical_compaction_dup_keys") { for (String[] tablet in tablets) { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); - def compactionStatusUrlIndex = 17 + def compactionStatusUrlIndex = 18 sb.append("curl -X GET ") sb.append(tablet[compactionStatusUrlIndex]) String command = sb.toString() diff --git a/regression-test/suites/compaction/test_vertical_compaction_uniq_keys.groovy b/regression-test/suites/compaction/test_vertical_compaction_uniq_keys.groovy index 02ec99aefd..dc3f9508ec 100644 --- a/regression-test/suites/compaction/test_vertical_compaction_uniq_keys.groovy +++ b/regression-test/suites/compaction/test_vertical_compaction_uniq_keys.groovy @@ -161,7 +161,7 @@ suite("test_vertical_compaction_uniq_keys") { qt_select_default2 """ SELECT * FROM ${tableName} t ORDER BY user_id; """ - //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus + //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,QueryHits,PathHash,MetaUrl,CompactionStatus String[][] tablets = sql """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} @@ -227,7 +227,7 @@ suite("test_vertical_compaction_uniq_keys") { for (String[] tablet in tablets) { String tablet_id = tablet[0] StringBuilder sb = new StringBuilder(); - def compactionStatusUrlIndex = 17 + def compactionStatusUrlIndex = 18 sb.append("curl -X GET ") sb.append(tablet[compactionStatusUrlIndex]) String command = sb.toString() diff --git a/regression-test/suites/load_p0/stream_load/test_map_load_and_compaction.groovy b/regression-test/suites/load_p0/stream_load/test_map_load_and_compaction.groovy index c8b0a98570..39999b8514 100644 --- a/regression-test/suites/load_p0/stream_load/test_map_load_and_compaction.groovy +++ b/regression-test/suites/load_p0/stream_load/test_map_load_and_compaction.groovy @@ -104,7 +104,7 @@ suite("test_map_load_and_compaction", "p0") { String[][] tablets = sql """ show tablets from ${testTable}; """ String[] tablet = tablets[0] // check rowsets number - String compactionStatus = tablet[17] + String compactionStatus = tablet[18] checkCompactionStatus.call(compactionStatus, 6) // trigger compaction diff --git a/regression-test/suites/query_p0/stats/query_stats_test.groovy b/regression-test/suites/query_p0/stats/query_stats_test.groovy new file mode 100644 index 0000000000..c081869087 --- /dev/null +++ b/regression-test/suites/query_p0/stats/query_stats_test.groovy @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("query_stats_test") { + sql "use test_query_db" + sql "admin set frontend config (\"enable_query_hit_stats\"=\"true\");" + sql "clean all query stats" + + explain { + sql("select k1 from baseall where k1 = 1") + } + + qt_sql "show query stats from baseall" + + sql "select k1 from baseall where k0 = 1" + sql "select k4 from baseall where k2 = 1991" + + qt_sql "show query stats from baseall" + qt_sql "show query stats from baseall all" + qt_sql "show query stats from baseall all verbose" + sql "admin set frontend config (\"enable_query_hit_stats\"=\"false\");" +}