From a87d34b19baf87e99e8c699e41359d041be80dfb Mon Sep 17 00:00:00 2001 From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com> Date: Thu, 27 Jul 2023 20:01:14 +0800 Subject: [PATCH] [Fix](multi catalog statistics)Improve external table statistics collection (#22224) Improve external table statistics collection, including log, observability and fix some bugs. 1. Add Running state for statistics job. 2. Add progress for show analyze job. (n/m tasks finished, n/m task failed and so on) 3. Add analyze time cost for show analyze task. 4. Make task failure message more clear. 5. Synchronize the job status updating code in updateTaskStatus. 6. Fix NPE in HMSAnalyzeTask. (Avoid refreshing statistics cache if the collection sql failed) 7. Return error message for with sync collection while timeout. 8. Log level improvement 9. Fix misuse of logCreateAnalysisJob for tasks. --- docs/en/docs/lakehouse/external_statistics.md | 122 +++++----- .../docs/lakehouse/external_statistics.md | 122 +++++----- .../apache/doris/analysis/AnalyzeTblStmt.java | 8 + .../doris/analysis/ShowAnalyzeStmt.java | 1 + .../doris/analysis/ShowAnalyzeTaskStatus.java | 3 +- .../org/apache/doris/qe/ShowExecutor.java | 2 + .../org/apache/doris/qe/StmtExecutor.java | 11 +- .../apache/doris/statistics/AnalysisInfo.java | 15 +- .../doris/statistics/AnalysisInfoBuilder.java | 10 +- .../doris/statistics/AnalysisManager.java | 143 ++++++++---- .../statistics/AnalysisTaskExecutor.java | 4 - .../doris/statistics/AnalysisTaskWrapper.java | 8 +- .../doris/statistics/BaseAnalysisTask.java | 17 +- .../doris/statistics/HMSAnalysisTask.java | 33 ++- .../doris/statistics/HistogramTask.java | 1 + .../doris/statistics/MVAnalysisTask.java | 1 + .../doris/statistics/OlapAnalysisTask.java | 1 + .../doris/statistics/StatisticsCache.java | 6 +- .../doris/statistics/util/StatisticsUtil.java | 2 + .../doris/statistics/AnalysisManagerTest.java | 73 ++++++ .../hive/test_hive_analyze_db.out | 20 -- .../hive/test_hive_statistic.out | 57 ----- .../hive/test_hive_statistic_cache.out | 58 ----- .../hive/test_hive_analyze_db.groovy | 157 ++++++++++++- .../hive/test_hive_statistic.groovy | 213 ++++++++++++++++-- .../hive/test_hive_statistic_cache.groovy | 191 ++++++++++++++-- 26 files changed, 914 insertions(+), 365 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java delete mode 100644 regression-test/data/external_table_emr_p2/hive/test_hive_analyze_db.out delete mode 100644 regression-test/data/external_table_emr_p2/hive/test_hive_statistic.out delete mode 100644 regression-test/data/external_table_emr_p2/hive/test_hive_statistic_cache.out diff --git a/docs/en/docs/lakehouse/external_statistics.md b/docs/en/docs/lakehouse/external_statistics.md index 7329310f0b..6c469961a8 100644 --- a/docs/en/docs/lakehouse/external_statistics.md +++ b/docs/en/docs/lakehouse/external_statistics.md @@ -53,45 +53,45 @@ mysql> ANALYZE TABLE hive.tpch100.lineitem; +--------------+-------------------------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ | Catalog_Name | DB_Name | Table_Name | Columns | Job_Id | +--------------+-------------------------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ -| hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | 126039 | +| hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | 16990 | +--------------+-------------------------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ -1 row in set (0.03 sec) +1 row in set (0.06 sec) ``` This operation is performed asynchronously, a collection job will be created in the background, and the progress of the job can be viewed using job_id ``` -mysql> SHOW ANALYZE 126039; -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------+ -| job_id | catalog_name | db_name | tbl_name | col_name | job_type | analysis_type | message | last_exec_time_in_ms | state | schedule_type | -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------+ -| 126039 | hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | MANUAL | FUNDAMENTALS | | 2023-07-13 10:33:44 | PENDING | ONCE | -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------+ +mysql> SHOW ANALYZE 16990; ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------------------------------------+---------------+ +| job_id | catalog_name | db_name | tbl_name | col_name | job_type | analysis_type | message | last_exec_time_in_ms | state | progress | schedule_type | ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------------------------------------+---------------+ +| 16990 | hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | MANUAL | FUNDAMENTALS | | 2023-07-27 16:01:52 | RUNNING | 2 Finished/0 Failed/15 In Progress/17 Total | ONCE | ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------------------------------------+---------------+ 1 row in set (0.00 sec) ``` And view the task status of each column. ``` -mysql> SHOW ANALYZE TASK STATUS 126039; -+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+----------------------+----------+ -| task_id | col_name | message | last_exec_time_in_ms | state | -+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+----------------------+----------+ -| 126040 | l_receiptdate | | 2023-07-13 10:33:44 | RUNNING | -| 126041 | l_returnflag | | 2023-07-13 10:33:44 | RUNNING | -| 126042 | l_tax | | 2023-07-13 10:33:44 | RUNNING | -| 126043 | l_shipmode | | 2023-07-13 10:33:44 | RUNNING | -| 126044 | l_suppkey | | 2023-07-13 10:33:44 | RUNNING | -| 126045 | l_shipdate | | 2023-07-13 10:33:44 | RUNNING | -| 126046 | l_commitdate | | 2023-07-13 10:33:44 | RUNNING | -| 126047 | l_partkey | | 2023-07-13 10:33:44 | RUNNING | -| 126048 | l_quantity | | 2023-07-13 10:33:44 | RUNNING | -| 126049 | l_orderkey | | 2023-07-13 10:33:44 | RUNNING | -| 126050 | l_comment | | 2023-07-13 10:33:44 | RUNNING | -| 126051 | l_linestatus | | 2023-07-13 10:33:44 | RUNNING | -| 126052 | l_extendedprice | | 2023-07-13 10:33:44 | RUNNING | -| 126053 | l_linenumber | | 2023-07-13 10:33:44 | RUNNING | -| 126054 | l_shipinstruct | | 2023-07-13 10:33:44 | RUNNING | -| 126055 | l_discount | | 2023-07-13 10:33:44 | RUNNING | -| 126056 | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | | 2023-07-13 10:33:56 | FINISHED | -+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+----------------------+----------+ -17 rows in set (0.01 sec) +mysql> SHOW ANALYZE TASK STATUS 16990; ++---------+-----------------+---------+------------------------+-----------------+----------+ +| task_id | col_name | message | last_state_change_time | time_cost_in_ms | state | ++---------+-----------------+---------+------------------------+-----------------+----------+ +| 16991 | l_receiptdate | | 2023-07-27 16:01:29 | 0 | PENDING | +| 16992 | l_returnflag | | 2023-07-27 16:01:44 | 14394 | FINISHED | +| 16993 | l_tax | | 2023-07-27 16:01:52 | 7975 | FINISHED | +| 16994 | l_shipmode | | 2023-07-27 16:02:11 | 18961 | FINISHED | +| 16995 | l_suppkey | | 2023-07-27 16:02:17 | 6684 | FINISHED | +| 16996 | l_shipdate | | 2023-07-27 16:02:26 | 8518 | FINISHED | +| 16997 | l_commitdate | | 2023-07-27 16:02:26 | 0 | RUNNING | +| 16998 | l_partkey | | 2023-07-27 16:01:29 | 0 | PENDING | +| 16999 | l_quantity | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17000 | l_orderkey | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17001 | l_comment | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17002 | l_linestatus | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17003 | l_extendedprice | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17004 | l_linenumber | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17005 | l_shipinstruct | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17006 | l_discount | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17007 | TableRowCount | | 2023-07-27 16:01:29 | 0 | PENDING | ++---------+-----------------+---------+------------------------+-----------------+----------+ +17 rows in set (0.00 sec) ``` - Collect statistics about all tables in the tpch100 database @@ -101,14 +101,14 @@ mysql> ANALYZE DATABASE hive.tpch100; +--------------+---------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ | Catalog_Name | DB_Name | Table_Name | Columns | Job_Id | +--------------+---------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ -| hive | tpch100 | partsupp | [ps_suppkey,ps_availqty,ps_comment,ps_partkey,ps_supplycost] | 124192 | -| hive | tpch100 | orders | [o_orderstatus,o_clerk,o_orderdate,o_shippriority,o_custkey,o_totalprice,o_orderkey,o_comment,o_orderpriority] | 124199 | -| hive | tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | 124210 | -| hive | tpch100 | part | [p_partkey,p_container,p_name,p_comment,p_brand,p_type,p_retailprice,p_mfgr,p_size] | 124228 | -| hive | tpch100 | customer | [c_custkey,c_phone,c_acctbal,c_mktsegment,c_address,c_nationkey,c_name,c_comment] | 124239 | -| hive | tpch100 | supplier | [s_comment,s_phone,s_nationkey,s_name,s_address,s_acctbal,s_suppkey] | 124249 | -| hive | tpch100 | nation | [n_comment,n_nationkey,n_regionkey,n_name] | 124258 | -| hive | tpch100 | region | [r_regionkey,r_comment,r_name] | 124264 | +| hive | tpch100 | supplier | [s_comment,s_phone,s_nationkey,s_name,s_address,s_acctbal,s_suppkey] | 17018 | +| hive | tpch100 | nation | [n_comment,n_nationkey,n_regionkey,n_name] | 17027 | +| hive | tpch100 | region | [r_regionkey,r_comment,r_name] | 17033 | +| hive | tpch100 | partsupp | [ps_suppkey,ps_availqty,ps_comment,ps_partkey,ps_supplycost] | 17038 | +| hive | tpch100 | orders | [o_orderstatus,o_clerk,o_orderdate,o_shippriority,o_custkey,o_totalprice,o_orderkey,o_comment,o_orderpriority] | 17045 | +| hive | tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | 17056 | +| hive | tpch100 | part | [p_partkey,p_container,p_name,p_comment,p_brand,p_type,p_retailprice,p_mfgr,p_size] | 17074 | +| hive | tpch100 | customer | [c_custkey,c_phone,c_acctbal,c_mktsegment,c_address,c_nationkey,c_name,c_comment] | 17085 | +--------------+---------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ 8 rows in set (0.29 sec) ``` @@ -140,28 +140,38 @@ The method of job management is also the same as that of the internal table, inc ``` mysql> SHOW ANALYZE; -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------+ -| job_id | catalog_name | db_name | tbl_name | col_name | job_type | analysis_type | message | last_exec_time_in_ms | state | schedule_type | -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------+ -| 12152 | hive | tpch100 | partsupp | [ps_suppkey,ps_availqty,ps_comment,ps_partkey,ps_supplycost] | MANUAL | FUNDAMENTALS | | 2023-07-11 15:57:16 | FINISHED | ONCE | -| 12159 | hive | tpch100 | orders | [o_orderstatus,o_clerk,o_orderdate,o_shippriority,o_custkey,o_totalprice,o_orderkey,o_comment,o_orderpriority] | MANUAL | FUNDAMENTALS | | 2023-07-11 15:57:24 | FINISHED | ONCE | -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------+ ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------------------------------------+---------------+ +| job_id | catalog_name | db_name | tbl_name | col_name | job_type | analysis_type | message | last_exec_time_in_ms | state | progress | schedule_type | ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------------------------------------+---------------+ +| 16990 | hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | MANUAL | FUNDAMENTALS | | 2023-07-27 16:05:02 | FINISHED | 17 Finished/0 Failed/0 In Progress/17 Total | ONCE | ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------------------------------------+---------------+ ``` - Show all tasks status of a job ``` -mysql> SHOW ANALYZE TASK STATUS 12152; -+---------+--------------------------------------------------------------+---------+----------------------+----------+ -| task_id | col_name | message | last_exec_time_in_ms | state | -+---------+--------------------------------------------------------------+---------+----------------------+----------+ -| 12153 | ps_availqty | | 2023-07-11 15:56:58 | FINISHED | -| 12154 | ps_suppkey | | 2023-07-11 15:56:57 | FINISHED | -| 12155 | ps_comment | | 2023-07-11 15:57:16 | FINISHED | -| 12156 | ps_supplycost | | 2023-07-11 15:56:57 | FINISHED | -| 12157 | ps_partkey | | 2023-07-11 15:56:58 | FINISHED | -| 12158 | [ps_suppkey,ps_availqty,ps_comment,ps_partkey,ps_supplycost] | | 2023-07-11 15:56:57 | FINISHED | -+---------+--------------------------------------------------------------+---------+----------------------+----------+ +mysql> SHOW ANALYZE TASK STATUS 16990; ++---------+-----------------+---------+------------------------+-----------------+----------+ +| task_id | col_name | message | last_state_change_time | time_cost_in_ms | state | ++---------+-----------------+---------+------------------------+-----------------+----------+ +| 16991 | l_receiptdate | | 2023-07-27 16:05:02 | 9560 | FINISHED | +| 16992 | l_returnflag | | 2023-07-27 16:01:44 | 14394 | FINISHED | +| 16993 | l_tax | | 2023-07-27 16:01:52 | 7975 | FINISHED | +| 16994 | l_shipmode | | 2023-07-27 16:02:11 | 18961 | FINISHED | +| 16995 | l_suppkey | | 2023-07-27 16:02:17 | 6684 | FINISHED | +| 16996 | l_shipdate | | 2023-07-27 16:02:26 | 8518 | FINISHED | +| 16997 | l_commitdate | | 2023-07-27 16:02:34 | 8380 | FINISHED | +| 16998 | l_partkey | | 2023-07-27 16:02:40 | 6060 | FINISHED | +| 16999 | l_quantity | | 2023-07-27 16:02:50 | 9768 | FINISHED | +| 17000 | l_orderkey | | 2023-07-27 16:02:57 | 7200 | FINISHED | +| 17001 | l_comment | | 2023-07-27 16:03:36 | 38468 | FINISHED | +| 17002 | l_linestatus | | 2023-07-27 16:03:51 | 15226 | FINISHED | +| 17003 | l_extendedprice | | 2023-07-27 16:04:00 | 8713 | FINISHED | +| 17004 | l_linenumber | | 2023-07-27 16:04:06 | 6659 | FINISHED | +| 17005 | l_shipinstruct | | 2023-07-27 16:04:36 | 29777 | FINISHED | +| 17006 | l_discount | | 2023-07-27 16:04:45 | 9212 | FINISHED | +| 17007 | TableRowCount | | 2023-07-27 16:04:52 | 6974 | FINISHED | ++---------+-----------------+---------+------------------------+-----------------+----------+ ``` - Terminate unfinished jobs diff --git a/docs/zh-CN/docs/lakehouse/external_statistics.md b/docs/zh-CN/docs/lakehouse/external_statistics.md index 586571f5b6..0b47ed5329 100644 --- a/docs/zh-CN/docs/lakehouse/external_statistics.md +++ b/docs/zh-CN/docs/lakehouse/external_statistics.md @@ -53,45 +53,45 @@ mysql> ANALYZE TABLE hive.tpch100.lineitem; +--------------+-------------------------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ | Catalog_Name | DB_Name | Table_Name | Columns | Job_Id | +--------------+-------------------------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ -| hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | 126039 | +| hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | 16990 | +--------------+-------------------------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ -1 row in set (0.03 sec) +1 row in set (0.06 sec) ``` 此操作是异步执行，会在后台创建收集任务，可以通过job_id查看任务进度。 ``` -mysql> SHOW ANALYZE 126039; -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------+ -| job_id | catalog_name | db_name | tbl_name | col_name | job_type | analysis_type | message | last_exec_time_in_ms | state | schedule_type | -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------+ -| 126039 | hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | MANUAL | FUNDAMENTALS | | 2023-07-13 10:33:44 | PENDING | ONCE | -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------+ +mysql> SHOW ANALYZE 16990; ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------------------------------------+---------------+ +| job_id | catalog_name | db_name | tbl_name | col_name | job_type | analysis_type | message | last_exec_time_in_ms | state | progress | schedule_type | ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------------------------------------+---------------+ +| 16990 | hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | MANUAL | FUNDAMENTALS | | 2023-07-27 16:01:52 | RUNNING | 2 Finished/0 Failed/15 In Progress/17 Total | ONCE | ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------------------------------------+---------------+ 1 row in set (0.00 sec) ``` 以及查看每一列的task状态。 ``` -mysql> SHOW ANALYZE TASK STATUS 126039; -+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+----------------------+----------+ -| task_id | col_name | message | last_exec_time_in_ms | state | -+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+----------------------+----------+ -| 126040 | l_receiptdate | | 2023-07-13 10:33:44 | RUNNING | -| 126041 | l_returnflag | | 2023-07-13 10:33:44 | RUNNING | -| 126042 | l_tax | | 2023-07-13 10:33:44 | RUNNING | -| 126043 | l_shipmode | | 2023-07-13 10:33:44 | RUNNING | -| 126044 | l_suppkey | | 2023-07-13 10:33:44 | RUNNING | -| 126045 | l_shipdate | | 2023-07-13 10:33:44 | RUNNING | -| 126046 | l_commitdate | | 2023-07-13 10:33:44 | RUNNING | -| 126047 | l_partkey | | 2023-07-13 10:33:44 | RUNNING | -| 126048 | l_quantity | | 2023-07-13 10:33:44 | RUNNING | -| 126049 | l_orderkey | | 2023-07-13 10:33:44 | RUNNING | -| 126050 | l_comment | | 2023-07-13 10:33:44 | RUNNING | -| 126051 | l_linestatus | | 2023-07-13 10:33:44 | RUNNING | -| 126052 | l_extendedprice | | 2023-07-13 10:33:44 | RUNNING | -| 126053 | l_linenumber | | 2023-07-13 10:33:44 | RUNNING | -| 126054 | l_shipinstruct | | 2023-07-13 10:33:44 | RUNNING | -| 126055 | l_discount | | 2023-07-13 10:33:44 | RUNNING | -| 126056 | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | | 2023-07-13 10:33:56 | FINISHED | -+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+----------------------+----------+ -17 rows in set (0.01 sec) +mysql> SHOW ANALYZE TASK STATUS 16990; ++---------+-----------------+---------+------------------------+-----------------+----------+ +| task_id | col_name | message | last_state_change_time | time_cost_in_ms | state | ++---------+-----------------+---------+------------------------+-----------------+----------+ +| 16991 | l_receiptdate | | 2023-07-27 16:01:29 | 0 | PENDING | +| 16992 | l_returnflag | | 2023-07-27 16:01:44 | 14394 | FINISHED | +| 16993 | l_tax | | 2023-07-27 16:01:52 | 7975 | FINISHED | +| 16994 | l_shipmode | | 2023-07-27 16:02:11 | 18961 | FINISHED | +| 16995 | l_suppkey | | 2023-07-27 16:02:17 | 6684 | FINISHED | +| 16996 | l_shipdate | | 2023-07-27 16:02:26 | 8518 | FINISHED | +| 16997 | l_commitdate | | 2023-07-27 16:02:26 | 0 | RUNNING | +| 16998 | l_partkey | | 2023-07-27 16:01:29 | 0 | PENDING | +| 16999 | l_quantity | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17000 | l_orderkey | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17001 | l_comment | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17002 | l_linestatus | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17003 | l_extendedprice | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17004 | l_linenumber | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17005 | l_shipinstruct | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17006 | l_discount | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17007 | TableRowCount | | 2023-07-27 16:01:29 | 0 | PENDING | ++---------+-----------------+---------+------------------------+-----------------+----------+ +17 rows in set (0.00 sec) ``` - 收集tpch100数据库所有表的信息 @@ -101,14 +101,14 @@ mysql> ANALYZE DATABASE hive.tpch100; +--------------+---------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ | Catalog_Name | DB_Name | Table_Name | Columns | Job_Id | +--------------+---------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ -| hive | tpch100 | partsupp | [ps_suppkey,ps_availqty,ps_comment,ps_partkey,ps_supplycost] | 124192 | -| hive | tpch100 | orders | [o_orderstatus,o_clerk,o_orderdate,o_shippriority,o_custkey,o_totalprice,o_orderkey,o_comment,o_orderpriority] | 124199 | -| hive | tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | 124210 | -| hive | tpch100 | part | [p_partkey,p_container,p_name,p_comment,p_brand,p_type,p_retailprice,p_mfgr,p_size] | 124228 | -| hive | tpch100 | customer | [c_custkey,c_phone,c_acctbal,c_mktsegment,c_address,c_nationkey,c_name,c_comment] | 124239 | -| hive | tpch100 | supplier | [s_comment,s_phone,s_nationkey,s_name,s_address,s_acctbal,s_suppkey] | 124249 | -| hive | tpch100 | nation | [n_comment,n_nationkey,n_regionkey,n_name] | 124258 | -| hive | tpch100 | region | [r_regionkey,r_comment,r_name] | 124264 | +| hive | tpch100 | supplier | [s_comment,s_phone,s_nationkey,s_name,s_address,s_acctbal,s_suppkey] | 17018 | +| hive | tpch100 | nation | [n_comment,n_nationkey,n_regionkey,n_name] | 17027 | +| hive | tpch100 | region | [r_regionkey,r_comment,r_name] | 17033 | +| hive | tpch100 | partsupp | [ps_suppkey,ps_availqty,ps_comment,ps_partkey,ps_supplycost] | 17038 | +| hive | tpch100 | orders | [o_orderstatus,o_clerk,o_orderdate,o_shippriority,o_custkey,o_totalprice,o_orderkey,o_comment,o_orderpriority] | 17045 | +| hive | tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | 17056 | +| hive | tpch100 | part | [p_partkey,p_container,p_name,p_comment,p_brand,p_type,p_retailprice,p_mfgr,p_size] | 17074 | +| hive | tpch100 | customer | [c_custkey,c_phone,c_acctbal,c_mktsegment,c_address,c_nationkey,c_name,c_comment] | 17085 | +--------------+---------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ 8 rows in set (0.29 sec) ``` @@ -140,28 +140,38 @@ Query OK, 0 rows affected (33.19 sec) ``` mysql> SHOW ANALYZE; -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------+ -| job_id | catalog_name | db_name | tbl_name | col_name | job_type | analysis_type | message | last_exec_time_in_ms | state | schedule_type | -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------+ -| 12152 | hive | tpch100 | partsupp | [ps_suppkey,ps_availqty,ps_comment,ps_partkey,ps_supplycost] | MANUAL | FUNDAMENTALS | | 2023-07-11 15:57:16 | FINISHED | ONCE | -| 12159 | hive | tpch100 | orders | [o_orderstatus,o_clerk,o_orderdate,o_shippriority,o_custkey,o_totalprice,o_orderkey,o_comment,o_orderpriority] | MANUAL | FUNDAMENTALS | | 2023-07-11 15:57:24 | FINISHED | ONCE | -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------+ ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------------------------------------+---------------+ +| job_id | catalog_name | db_name | tbl_name | col_name | job_type | analysis_type | message | last_exec_time_in_ms | state | progress | schedule_type | ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------------------------------------+---------------+ +| 16990 | hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | MANUAL | FUNDAMENTALS | | 2023-07-27 16:05:02 | FINISHED | 17 Finished/0 Failed/0 In Progress/17 Total | ONCE | ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------------------------------------+---------------+ ``` - 查看一个job的所有task状态 ``` -mysql> SHOW ANALYZE TASK STATUS 12152; -+---------+--------------------------------------------------------------+---------+----------------------+----------+ -| task_id | col_name | message | last_exec_time_in_ms | state | -+---------+--------------------------------------------------------------+---------+----------------------+----------+ -| 12153 | ps_availqty | | 2023-07-11 15:56:58 | FINISHED | -| 12154 | ps_suppkey | | 2023-07-11 15:56:57 | FINISHED | -| 12155 | ps_comment | | 2023-07-11 15:57:16 | FINISHED | -| 12156 | ps_supplycost | | 2023-07-11 15:56:57 | FINISHED | -| 12157 | ps_partkey | | 2023-07-11 15:56:58 | FINISHED | -| 12158 | [ps_suppkey,ps_availqty,ps_comment,ps_partkey,ps_supplycost] | | 2023-07-11 15:56:57 | FINISHED | -+---------+--------------------------------------------------------------+---------+----------------------+----------+ +mysql> SHOW ANALYZE TASK STATUS 16990; ++---------+-----------------+---------+------------------------+-----------------+----------+ +| task_id | col_name | message | last_state_change_time | time_cost_in_ms | state | ++---------+-----------------+---------+------------------------+-----------------+----------+ +| 16991 | l_receiptdate | | 2023-07-27 16:05:02 | 9560 | FINISHED | +| 16992 | l_returnflag | | 2023-07-27 16:01:44 | 14394 | FINISHED | +| 16993 | l_tax | | 2023-07-27 16:01:52 | 7975 | FINISHED | +| 16994 | l_shipmode | | 2023-07-27 16:02:11 | 18961 | FINISHED | +| 16995 | l_suppkey | | 2023-07-27 16:02:17 | 6684 | FINISHED | +| 16996 | l_shipdate | | 2023-07-27 16:02:26 | 8518 | FINISHED | +| 16997 | l_commitdate | | 2023-07-27 16:02:34 | 8380 | FINISHED | +| 16998 | l_partkey | | 2023-07-27 16:02:40 | 6060 | FINISHED | +| 16999 | l_quantity | | 2023-07-27 16:02:50 | 9768 | FINISHED | +| 17000 | l_orderkey | | 2023-07-27 16:02:57 | 7200 | FINISHED | +| 17001 | l_comment | | 2023-07-27 16:03:36 | 38468 | FINISHED | +| 17002 | l_linestatus | | 2023-07-27 16:03:51 | 15226 | FINISHED | +| 17003 | l_extendedprice | | 2023-07-27 16:04:00 | 8713 | FINISHED | +| 17004 | l_linenumber | | 2023-07-27 16:04:06 | 6659 | FINISHED | +| 17005 | l_shipinstruct | | 2023-07-27 16:04:36 | 29777 | FINISHED | +| 17006 | l_discount | | 2023-07-27 16:04:45 | 9212 | FINISHED | +| 17007 | TableRowCount | | 2023-07-27 16:04:52 | 6974 | FINISHED | ++---------+-----------------+---------+------------------------+-----------------+----------+ ``` - 终止未完成的job diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java index f9c6241c40..da08f45bee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java @@ -84,6 +84,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt { private final TableName tableName; private List columnNames; private List partitionNames; + private boolean isAllColumns; // after analyzed private long dbId; @@ -98,6 +99,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt { this.partitionNames = partitionNames == null ? null : partitionNames.getPartitionNames(); this.columnNames = columnNames; this.analyzeProperties = properties; + this.isAllColumns = columnNames == null; } public AnalyzeTblStmt(AnalyzeProperties analyzeProperties, TableName tableName, List columnNames, long dbId, @@ -107,6 +109,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt { this.columnNames = columnNames; this.dbId = dbId; this.table = table; + this.isAllColumns = columnNames == null; } @Override @@ -128,6 +131,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt { DatabaseIf db = catalog.getDbOrAnalysisException(dbName); dbId = db.getId(); table = db.getTableOrAnalysisException(tblName); + isAllColumns = columnNames == null; check(); } @@ -301,4 +305,8 @@ public class AnalyzeTblStmt extends AnalyzeStmt { public Database getDb() throws AnalysisException { return analyzer.getEnv().getInternalCatalog().getDbOrAnalysisException(dbId); } + + public boolean isAllColumns() { + return isAllColumns; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java index 73bf77b23c..3a0aa05a7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java @@ -64,6 +64,7 @@ public class ShowAnalyzeStmt extends ShowStmt { .add("message") .add("last_exec_time_in_ms") .add("state") + .add("progress") .add("schedule_type") .build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeTaskStatus.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeTaskStatus.java index 03d304f393..927a56d19d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeTaskStatus.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeTaskStatus.java @@ -33,7 +33,8 @@ public class ShowAnalyzeTaskStatus extends ShowStmt { .addColumn(new Column("task_id", ScalarType.createVarchar(100))) .addColumn(new Column("col_name", ScalarType.createVarchar(1000))) .addColumn(new Column("message", ScalarType.createVarchar(1000))) - .addColumn(new Column("last_exec_time_in_ms", ScalarType.createVarchar(1000))) + .addColumn(new Column("last_state_change_time", ScalarType.createVarchar(1000))) + .addColumn(new Column("time_cost_in_ms", ScalarType.createVarchar(1000))) .addColumn(new Column("state", ScalarType.createVarchar(1000))).build(); private final long jobId; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 7f5d197723..59bfbada21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -2562,6 +2562,7 @@ public class ShowExecutor { LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.lastExecTimeInMs), ZoneId.systemDefault()))); row.add(analysisInfo.state.toString()); + row.add(Env.getCurrentEnv().getAnalysisManager().getJobProgress(analysisInfo.jobId)); row.add(analysisInfo.scheduleType.toString()); resultRows.add(row); } @@ -2762,6 +2763,7 @@ public class ShowExecutor { row.add(TimeUtils.DATETIME_FORMAT.format( LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.lastExecTimeInMs), ZoneId.systemDefault()))); + row.add(String.valueOf(analysisInfo.timeCostInMs)); row.add(analysisInfo.state.toString()); rows.add(row); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 9f39a82ad9..4d5d5e8851 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -20,6 +20,7 @@ package org.apache.doris.qe; import org.apache.doris.analysis.AddPartitionLikeClause; import org.apache.doris.analysis.AlterClause; import org.apache.doris.analysis.AlterTableStmt; +import org.apache.doris.analysis.AnalyzeDBStmt; import org.apache.doris.analysis.AnalyzeStmt; import org.apache.doris.analysis.AnalyzeTblStmt; import org.apache.doris.analysis.Analyzer; @@ -1149,7 +1150,7 @@ public class StmtExecutor { if (mysqlLoadId != null) { Env.getCurrentEnv().getLoadManager().getMysqlLoadManager().cancelMySqlLoad(mysqlLoadId); } - if (parsedStmt instanceof AnalyzeTblStmt) { + if (parsedStmt instanceof AnalyzeTblStmt || parsedStmt instanceof AnalyzeDBStmt) { Env.getCurrentEnv().getAnalysisManager().cancelSyncTask(context); } } @@ -2485,7 +2486,7 @@ public class StmtExecutor { analyze(context.getSessionVariable().toThrift()); } } catch (Exception e) { - throw new RuntimeException("Failed to execute internal SQL", e); + throw new RuntimeException("Failed to execute internal SQL. " + Util.getRootCauseMessage(e), e); } planner.getFragments(); RowBatch batch; @@ -2495,7 +2496,7 @@ public class StmtExecutor { QeProcessorImpl.INSTANCE.registerQuery(context.queryId(), new QeProcessorImpl.QueryInfo(context, originStmt.originStmt, coord)); } catch (UserException e) { - throw new RuntimeException("Failed to execute internal SQL", e); + throw new RuntimeException("Failed to execute internal SQL. " + Util.getRootCauseMessage(e), e); } Span queryScheduleSpan = context.getTracer() @@ -2504,7 +2505,7 @@ public class StmtExecutor { coord.exec(); } catch (Exception e) { queryScheduleSpan.recordException(e); - throw new RuntimeException("Failed to execute internal SQL", e); + throw new RuntimeException("Failed to execute internal SQL. " + Util.getRootCauseMessage(e), e); } finally { queryScheduleSpan.end(); } @@ -2521,7 +2522,7 @@ public class StmtExecutor { } } catch (Exception e) { fetchResultSpan.recordException(e); - throw new RuntimeException("Failed to execute internal SQL", e); + throw new RuntimeException("Failed to execute internal SQL. " + Util.getRootCauseMessage(e), e); } finally { fetchResultSpan.end(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java index 0f95848331..53032778cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java @@ -131,6 +131,10 @@ public class AnalysisInfo implements Writable { @SerializedName("lastExecTimeInMs") public long lastExecTimeInMs; + // finished or failed + @SerializedName("timeCostInMs") + public long timeCostInMs; + @SerializedName("state") public AnalysisState state; @@ -161,8 +165,9 @@ public class AnalysisInfo implements Writable { Map> colToPartitions, Set partitionNames, String colName, Long indexId, JobType jobType, AnalysisMode analysisMode, AnalysisMethod analysisMethod, AnalysisType analysisType, int samplePercent, int sampleRows, int maxBucketNum, long periodTimeInMs, String message, - long lastExecTimeInMs, AnalysisState state, ScheduleType scheduleType, boolean isExternalTableLevelTask, - boolean partitionOnly, boolean samplingPartition, CronExpression cronExpression) { + long lastExecTimeInMs, long timeCostInMs, AnalysisState state, ScheduleType scheduleType, + boolean isExternalTableLevelTask, boolean partitionOnly, boolean samplingPartition, + CronExpression cronExpression) { this.jobId = jobId; this.taskId = taskId; this.catalogName = catalogName; @@ -182,6 +187,7 @@ public class AnalysisInfo implements Writable { this.periodTimeInMs = periodTimeInMs; this.message = message; this.lastExecTimeInMs = lastExecTimeInMs; + this.timeCostInMs = timeCostInMs; this.state = state; this.scheduleType = scheduleType; this.externalTableLevelTask = isExternalTableLevelTask; @@ -218,6 +224,9 @@ public class AnalysisInfo implements Writable { if (lastExecTimeInMs > 0) { sj.add("LastExecTime: " + StatisticsUtil.getReadableTime(lastExecTimeInMs)); } + if (timeCostInMs > 0) { + sj.add("timeCost: " + timeCostInMs); + } if (periodTimeInMs > 0) { sj.add("periodTimeInMs: " + StatisticsUtil.getReadableTime(periodTimeInMs)); } @@ -275,6 +284,8 @@ public class AnalysisInfo implements Writable { analysisInfoBuilder.setPeriodTimeInMs(StatisticsUtil.convertStrToInt(periodTimeInMs)); String lastExecTimeInMs = resultRow.getColumnValue("last_exec_time_in_ms"); analysisInfoBuilder.setLastExecTimeInMs(StatisticsUtil.convertStrToLong(lastExecTimeInMs)); + String timeCostInMs = resultRow.getColumnValue("time_cost_in_ms"); + analysisInfoBuilder.setTimeCostInMs(StatisticsUtil.convertStrToLong(timeCostInMs)); String message = resultRow.getColumnValue("message"); analysisInfoBuilder.setMessage(message); return analysisInfoBuilder.build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java index bff4d7d69f..5efce203de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java @@ -47,6 +47,7 @@ public class AnalysisInfoBuilder { private int sampleRows; private long periodTimeInMs; private long lastExecTimeInMs; + private long timeCostInMs; private AnalysisState state; private ScheduleType scheduleType; private String message = ""; @@ -79,6 +80,7 @@ public class AnalysisInfoBuilder { maxBucketNum = info.maxBucketNum; message = info.message; lastExecTimeInMs = info.lastExecTimeInMs; + timeCostInMs = info.timeCostInMs; state = info.state; scheduleType = info.scheduleType; externalTableLevelTask = info.externalTableLevelTask; @@ -181,6 +183,11 @@ public class AnalysisInfoBuilder { return this; } + public AnalysisInfoBuilder setTimeCostInMs(long timeCostInMs) { + this.timeCostInMs = timeCostInMs; + return this; + } + public AnalysisInfoBuilder setState(AnalysisState state) { this.state = state; return this; @@ -213,7 +220,7 @@ public class AnalysisInfoBuilder { public AnalysisInfo build() { return new AnalysisInfo(jobId, taskId, catalogName, dbName, tblName, colToPartitions, partitionNames, colName, indexId, jobType, analysisMode, analysisMethod, analysisType, samplePercent, - sampleRows, maxBucketNum, periodTimeInMs, message, lastExecTimeInMs, state, scheduleType, + sampleRows, maxBucketNum, periodTimeInMs, message, lastExecTimeInMs, timeCostInMs, state, scheduleType, externalTableLevelTask, partitionOnly, samplingPartition, cronExpression); } @@ -237,6 +244,7 @@ public class AnalysisInfoBuilder { .setMaxBucketNum(maxBucketNum) .setMessage(message) .setLastExecTimeInMs(lastExecTimeInMs) + .setTimeCostInMs(timeCostInMs) .setState(state) .setScheduleType(scheduleType) .setExternalTableLevelTask(externalTableLevelTask); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index 373cc57184..cb6a3dfe5c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -43,6 +43,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.Daemon; +import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.persist.AnalyzeDeletionLog; @@ -184,10 +185,9 @@ public class AnalysisManager extends Daemon implements Writable { } TableName tableName = new TableName(db.getCatalog().getName(), db.getFullName(), table.getName()); + // columnNames null means to add all visitable columns. AnalyzeTblStmt analyzeTblStmt = new AnalyzeTblStmt(analyzeProperties, tableName, - table.getBaseSchema().stream().map( - Column::getName).collect( - Collectors.toList()), db.getId(), table); + null, db.getId(), table); try { analyzeTblStmt.check(); } catch (AnalysisException analysisException) { @@ -229,8 +229,9 @@ public class AnalysisManager extends Daemon implements Writable { Map analysisTaskInfos = new HashMap<>(); createTaskForEachColumns(jobInfo, analysisTaskInfos, isSync); createTaskForMVIdx(jobInfo, analysisTaskInfos, isSync); - createTaskForExternalTable(jobInfo, analysisTaskInfos, isSync); - + if (stmt.isAllColumns()) { + createTaskForExternalTable(jobInfo, analysisTaskInfos, isSync); + } if (!isSync) { persistAnalysisJob(jobInfo); analysisJobIdToTaskMap.put(jobInfo.jobId, analysisTaskInfos); @@ -513,12 +514,12 @@ public class AnalysisManager extends Daemon implements Writable { long taskId = Env.getCurrentEnv().getNextId(); AnalysisInfoBuilder indexTaskInfoBuilder = new AnalysisInfoBuilder(jobInfo); AnalysisInfo analysisInfo = indexTaskInfoBuilder.setIndexId(indexId) - .setTaskId(taskId).build(); + .setTaskId(taskId).setLastExecTimeInMs(System.currentTimeMillis()).build(); if (isSync) { return; } analysisTasks.put(taskId, createTask(analysisInfo)); - logCreateAnalysisJob(analysisInfo); + logCreateAnalysisTask(analysisInfo); } } finally { olapTable.readUnlock(); @@ -538,7 +539,7 @@ public class AnalysisManager extends Daemon implements Writable { colTaskInfoBuilder.setColToPartitions(Collections.singletonMap(colName, entry.getValue())); } AnalysisInfo analysisInfo = colTaskInfoBuilder.setColName(colName).setIndexId(indexId) - .setTaskId(taskId).build(); + .setTaskId(taskId).setLastExecTimeInMs(System.currentTimeMillis()).build(); analysisTasks.put(taskId, createTask(analysisInfo)); if (isSync) { continue; @@ -553,13 +554,15 @@ public class AnalysisManager extends Daemon implements Writable { } } - private void logCreateAnalysisTask(AnalysisInfo analysisInfo) { - analysisTaskInfoMap.put(analysisInfo.taskId, analysisInfo); + // Change to public for unit test. + public void logCreateAnalysisTask(AnalysisInfo analysisInfo) { + replayCreateAnalysisTask(analysisInfo); Env.getCurrentEnv().getEditLog().logCreateAnalysisTasks(analysisInfo); } - private void logCreateAnalysisJob(AnalysisInfo analysisJob) { - analysisJobInfoMap.put(analysisJob.jobId, analysisJob); + // Change to public for unit test. + public void logCreateAnalysisJob(AnalysisInfo analysisJob) { + replayCreateAnalysisJob(analysisJob); Env.getCurrentEnv().getEditLog().logCreateAnalysisJob(analysisJob); } @@ -578,63 +581,71 @@ public class AnalysisManager extends Daemon implements Writable { } AnalysisInfoBuilder colTaskInfoBuilder = new AnalysisInfoBuilder(jobInfo); long taskId = Env.getCurrentEnv().getNextId(); - AnalysisInfo analysisInfo = colTaskInfoBuilder.setIndexId(-1L) - .setTaskId(taskId).setExternalTableLevelTask(true).build(); + AnalysisInfo analysisInfo = colTaskInfoBuilder.setIndexId(-1L).setLastExecTimeInMs(System.currentTimeMillis()) + .setTaskId(taskId).setColName("TableRowCount").setExternalTableLevelTask(true).build(); analysisTasks.put(taskId, createTask(analysisInfo)); if (isSync) { // For sync job, don't need to persist, return here and execute it immediately. return; } try { - logCreateAnalysisJob(analysisInfo); + logCreateAnalysisTask(analysisInfo); } catch (Exception e) { throw new DdlException("Failed to create analysis task", e); } } - public void updateTaskStatus(AnalysisInfo info, AnalysisState jobState, String message, long time) { + public void updateTaskStatus(AnalysisInfo info, AnalysisState taskState, String message, long time) { if (analysisJobIdToTaskMap.get(info.jobId) == null) { return; } - info.state = jobState; + info.state = taskState; info.message = message; + // Update the task cost time when task finished or failed. And only log the final state. + if (taskState.equals(AnalysisState.FINISHED) || taskState.equals(AnalysisState.FAILED)) { + info.timeCostInMs = time - info.lastExecTimeInMs; + info.lastExecTimeInMs = time; + logCreateAnalysisTask(info); + } info.lastExecTimeInMs = time; - logCreateAnalysisTask(info); - AnalysisInfo job = analysisJobInfoMap.get(info.jobId); - job.lastExecTimeInMs = time; - if (info.state.equals(AnalysisState.RUNNING) && !job.state.equals(AnalysisState.PENDING)) { - job.state = AnalysisState.RUNNING; - Env.getCurrentEnv().getEditLog().logCreateAnalysisTasks(job); - } - boolean allFinished = true; - boolean hasFailure = false; - for (BaseAnalysisTask task : analysisJobIdToTaskMap.get(info.jobId).values()) { - AnalysisInfo taskInfo = task.info; - if (taskInfo.state.equals(AnalysisState.RUNNING) || taskInfo.state.equals(AnalysisState.PENDING)) { - allFinished = false; - break; + // Synchronize the job state change in job level. + synchronized (job) { + job.lastExecTimeInMs = time; + // Set the job state to RUNNING when its first task becomes RUNNING. + if (info.state.equals(AnalysisState.RUNNING) && job.state.equals(AnalysisState.PENDING)) { + job.state = AnalysisState.RUNNING; + replayCreateAnalysisJob(job); } - if (taskInfo.state.equals(AnalysisState.FAILED)) { - hasFailure = true; - } - } - if (allFinished) { - if (hasFailure) { - job.state = AnalysisState.FAILED; - logCreateAnalysisJob(job); - } else { - job.state = AnalysisState.FINISHED; - if (job.jobType.equals(JobType.SYSTEM)) { - try { - updateTableStats(job); - } catch (Throwable e) { - LOG.warn("Failed to update Table statistics in job: {}", info.toString()); - } + boolean allFinished = true; + boolean hasFailure = false; + for (BaseAnalysisTask task : analysisJobIdToTaskMap.get(info.jobId).values()) { + AnalysisInfo taskInfo = task.info; + if (taskInfo.state.equals(AnalysisState.RUNNING) || taskInfo.state.equals(AnalysisState.PENDING)) { + allFinished = false; + break; + } + if (taskInfo.state.equals(AnalysisState.FAILED)) { + hasFailure = true; } - logCreateAnalysisJob(job); } - analysisJobIdToTaskMap.remove(job.jobId); + if (allFinished) { + if (hasFailure) { + job.state = AnalysisState.FAILED; + logCreateAnalysisJob(job); + } else { + job.state = AnalysisState.FINISHED; + if (job.jobType.equals(JobType.SYSTEM)) { + try { + updateTableStats(job); + } catch (Throwable e) { + LOG.warn("Failed to update Table statistics in job: {}", info.toString(), e); + } + } + logCreateAnalysisJob(job); + } + analysisJobIdToTaskMap.remove(job.jobId); + } } } @@ -700,6 +711,28 @@ public class AnalysisManager extends Daemon implements Writable { .collect(Collectors.toList()); } + public String getJobProgress(long jobId) { + List tasks = findTasks(jobId); + int finished = 0; + int failed = 0; + int inProgress = 0; + int total = tasks.size(); + for (AnalysisInfo info : tasks) { + switch (info.state) { + case FINISHED: + finished++; + break; + case FAILED: + failed++; + break; + default: + inProgress++; + break; + } + } + return String.format("%d Finished/%d Failed/%d In Progress/%d Total", finished, failed, inProgress, total); + } + private void syncExecute(Collection tasks) { SyncTaskCollection syncTaskCollection = new SyncTaskCollection(tasks); ConnectContext ctx = ConnectContext.get(); @@ -813,8 +846,11 @@ public class AnalysisManager extends Daemon implements Writable { public void execute() { List colNames = new ArrayList<>(); + List errorMessages = new ArrayList<>(); for (BaseAnalysisTask task : tasks) { if (cancelled) { + colNames.add(task.info.colName); + errorMessages.add("Cancelled"); continue; } try { @@ -822,12 +858,14 @@ public class AnalysisManager extends Daemon implements Writable { updateSyncTaskStatus(task, AnalysisState.FINISHED); } catch (Throwable t) { colNames.add(task.info.colName); + errorMessages.add(Util.getRootCauseMessage(t)); updateSyncTaskStatus(task, AnalysisState.FAILED); LOG.warn("Failed to analyze, info: {}", task, t); } } if (!colNames.isEmpty()) { - throw new RuntimeException("Failed to analyze following columns: " + String.join(",", colNames)); + throw new RuntimeException("Failed to analyze following columns:[" + String.join(",", colNames) + + "] Reasons: " + String.join(",", errorMessages)); } } @@ -920,4 +958,9 @@ public class AnalysisManager extends Daemon implements Writable { entry.getValue().write(out); } } + + // For unit test use only. + public void addToJobIdTasksMap(long jobId, Map tasks) { + analysisJobIdToTaskMap.put(jobId, tasks); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java index 11f244afd8..b5ec7aeb87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java @@ -17,7 +17,6 @@ package org.apache.doris.statistics; -import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.ThreadPoolManager.BlockedPolicy; @@ -101,9 +100,6 @@ public class AnalysisTaskExecutor extends Thread { BaseAnalysisTask task = taskScheduler.getPendingTasks(); AnalysisTaskWrapper taskWrapper = new AnalysisTaskWrapper(this, task); executors.submit(taskWrapper); - Env.getCurrentEnv().getAnalysisManager() - .updateTaskStatus(task.info, - AnalysisState.RUNNING, "", System.currentTimeMillis()); } public void putJob(AnalysisTaskWrapper wrapper) throws Exception { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java index 9a0d6ec2fc..7f55469f53 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java @@ -18,6 +18,7 @@ package org.apache.doris.statistics; import org.apache.doris.catalog.Env; +import org.apache.doris.common.util.Util; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -62,16 +63,17 @@ public class AnalysisTaskWrapper extends FutureTask { } finally { if (!task.killed) { if (except != null) { - LOG.warn("Failed to execute task", except); + LOG.warn("Analyze {} failed.", task.toString(), except); Env.getCurrentEnv().getAnalysisManager() .updateTaskStatus(task.info, - AnalysisState.FAILED, except.getMessage(), System.currentTimeMillis()); + AnalysisState.FAILED, Util.getRootCauseMessage(except), System.currentTimeMillis()); } else { + LOG.debug("Analyze {} finished, cost time:{}", task.toString(), + System.currentTimeMillis() - startTime); Env.getCurrentEnv().getAnalysisManager() .updateTaskStatus(task.info, AnalysisState.FINISHED, "", System.currentTimeMillis()); } - LOG.warn("{} finished, cost time:{}", task.toString(), System.currentTimeMillis() - startTime); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java index c486bf36d1..e146a2e8e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java @@ -171,12 +171,20 @@ public abstract class BaseAnalysisTask { break; } catch (Throwable t) { LOG.warn("Failed to execute analysis task, retried times: {}", retriedTimes++, t); + if (retriedTimes > StatisticConstants.ANALYZE_TASK_RETRY_TIMES) { + throw new RuntimeException(t); + } } } } public abstract void doExecute() throws Exception; + protected void setTaskStateToRunning() { + Env.getCurrentEnv().getAnalysisManager() + .updateTaskStatus(info, AnalysisState.RUNNING, "", System.currentTimeMillis()); + } + public void cancel() { killed = true; if (stmtExecutor != null) { @@ -184,7 +192,7 @@ public abstract class BaseAnalysisTask { } Env.getCurrentEnv().getAnalysisManager() .updateTaskStatus(info, AnalysisState.FAILED, - String.format("Job has been cancelled: %s", info.toString()), -1); + String.format("Job has been cancelled: %s", info.message), System.currentTimeMillis()); } public long getLastExecTime() { @@ -218,4 +226,11 @@ public abstract class BaseAnalysisTask { return String.format("TABLESAMPLE(%d ROWS)", info.sampleRows); } } + + @Override + public String toString() { + return String.format("Job id [%d], Task id [%d], catalog [%s], db [%s], table [%s], column [%s]", + info.jobId, info.taskId, catalog.getName(), db.getFullName(), tbl.getName(), + col == null ? "TableRowCount" : col.getName()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java index 87e7bbe1b5..03840f091d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.external.HMSExternalTable; import org.apache.doris.common.FeConstants; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.qe.AutoCloseConnectContext; +import org.apache.doris.qe.QueryState; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.statistics.util.InternalQueryResult; import org.apache.doris.statistics.util.StatisticsUtil; @@ -105,6 +106,7 @@ public class HMSAnalysisTask extends BaseAnalysisTask { } public void doExecute() throws Exception { + setTaskStateToRunning(); if (isTableLevelTask) { getTableStats(); } else { @@ -190,11 +192,7 @@ public class HMSAnalysisTask extends BaseAnalysisTask { params.put("dataSizeFunction", getDataSizeFunction(col)); StringSubstitutor stringSubstitutor = new StringSubstitutor(params); String sql = stringSubstitutor.replace(sb.toString()); - try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) { - r.connectContext.getSessionVariable().disableNereidsPlannerOnce(); - this.stmtExecutor = new StmtExecutor(r.connectContext, sql); - this.stmtExecutor.execute(); - } + executeInsertSql(sql); } } else { StringBuilder sb = new StringBuilder(); @@ -233,12 +231,27 @@ public class HMSAnalysisTask extends BaseAnalysisTask { params.put("dataSizeFunction", getDataSizeFunction(col)); StringSubstitutor stringSubstitutor = new StringSubstitutor(params); String sql = stringSubstitutor.replace(sb.toString()); - try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) { - r.connectContext.getSessionVariable().disableNereidsPlannerOnce(); - this.stmtExecutor = new StmtExecutor(r.connectContext, sql); - this.stmtExecutor.execute(); + executeInsertSql(sql); + Env.getCurrentEnv().getStatisticsCache().refreshColStatsSync( + catalog.getId(), db.getId(), tbl.getId(), -1, col.getName()); + } + } + + private void executeInsertSql(String sql) throws Exception { + long startTime = System.currentTimeMillis(); + try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) { + r.connectContext.getSessionVariable().disableNereidsPlannerOnce(); + this.stmtExecutor = new StmtExecutor(r.connectContext, sql); + r.connectContext.setExecutor(stmtExecutor); + this.stmtExecutor.execute(); + QueryState queryState = r.connectContext.getState(); + if (queryState.getStateType().equals(QueryState.MysqlStateType.ERR)) { + LOG.warn(String.format("Failed to analyze %s.%s.%s, sql: [%s], error: [%s]", + info.catalogName, info.dbName, info.colName, sql, queryState.getErrorMessage())); + throw new RuntimeException(queryState.getErrorMessage()); } - Env.getCurrentEnv().getStatisticsCache().refreshColStatsSync(tbl.getId(), -1, col.getName()); + LOG.debug(String.format("Analyze %s.%s.%s done. SQL: [%s]. Cost %d ms.", + info.catalogName, info.dbName, info.colName, sql, (System.currentTimeMillis() - startTime))); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java index a0b6e6d9a5..a1b53cb3be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java @@ -59,6 +59,7 @@ public class HistogramTask extends BaseAnalysisTask { @Override public void doExecute() throws Exception { + setTaskStateToRunning(); Map params = new HashMap<>(); params.put("internalDB", FeConstants.INTERNAL_DB_NAME); params.put("histogramStatTbl", StatisticConstants.HISTOGRAM_TBL_NAME); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java index 31b3b76d5d..ed33252172 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java @@ -87,6 +87,7 @@ public class MVAnalysisTask extends BaseAnalysisTask { @Override public void doExecute() throws Exception { + setTaskStateToRunning(); for (Column column : meta.getSchema()) { SelectStmt selectOne = (SelectStmt) selectStmt.clone(); TableRef tableRef = selectOne.getTableRefs().get(0); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java index fab9f68197..a980385bde 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java @@ -60,6 +60,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask { } public void doExecute() throws Exception { + setTaskStateToRunning(); Map params = new HashMap<>(); params.put("internalDB", FeConstants.INTERNAL_DB_NAME); params.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java index 284083ac01..1149ecdd5a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java @@ -169,7 +169,11 @@ public class StatisticsCache { } public void refreshColStatsSync(long tblId, long idxId, String colName) { - columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(tblId, idxId, colName)); + columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(-1, -1, tblId, idxId, colName)); + } + + public void refreshColStatsSync(long catalogId, long dbId, long tblId, long idxId, String colName) { + columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(catalogId, dbId, tblId, idxId, colName)); } public void refreshHistogramSync(long tblId, long idxId, String colName) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index e7c40a9ce4..207c34fee2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -174,6 +174,8 @@ public class StatisticsUtil { sessionVariable.parallelPipelineTaskNum = Config.statistics_sql_parallel_exec_instance_num; sessionVariable.setEnableNereidsPlanner(false); sessionVariable.enableProfile = false; + sessionVariable.queryTimeoutS = Config.analyze_task_timeout_in_hours * 60 * 60; + sessionVariable.insertTimeoutS = Config.analyze_task_timeout_in_hours * 60 * 60; connectContext.setEnv(Env.getCurrentEnv()); connectContext.setDatabase(FeConstants.INTERNAL_DB_NAME); connectContext.setQualifiedUser(UserIdentity.ROOT.getQualifiedUser()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java new file mode 100644 index 0000000000..253f9c9332 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics; + +import mockit.Mock; +import mockit.MockUp; +import mockit.Mocked; +import org.junit.Test; +import org.junit.jupiter.api.Assertions; + +import java.util.HashMap; +import java.util.Map; + +public class AnalysisManagerTest { + @Test + public void testUpdateTaskStatus(@Mocked BaseAnalysisTask task1, + @Mocked BaseAnalysisTask task2) { + + new MockUp() { + @Mock + public void logCreateAnalysisTask(AnalysisInfo job) { + } + + @Mock + public void logCreateAnalysisJob(AnalysisInfo job) { + } + + }; + + AnalysisInfo job = new AnalysisInfoBuilder().setJobId(1) + .setState(AnalysisState.PENDING).setJobType(AnalysisInfo.JobType.MANUAL).build(); + AnalysisInfo taskInfo1 = new AnalysisInfoBuilder().setJobId(1) + .setTaskId(2).setState(AnalysisState.PENDING).build(); + AnalysisInfo taskInfo2 = new AnalysisInfoBuilder().setJobId(1) + .setTaskId(3).setState(AnalysisState.PENDING).build(); + AnalysisManager manager = new AnalysisManager(); + manager.replayCreateAnalysisJob(job); + manager.replayCreateAnalysisTask(taskInfo1); + manager.replayCreateAnalysisTask(taskInfo2); + Map tasks = new HashMap<>(); + + task1.info = taskInfo1; + task2.info = taskInfo2; + tasks.put(2L, task1); + tasks.put(3L, task2); + manager.addToJobIdTasksMap(1, tasks); + + Assertions.assertEquals(job.state, AnalysisState.PENDING); + manager.updateTaskStatus(taskInfo1, AnalysisState.RUNNING, "", 0); + Assertions.assertEquals(job.state, AnalysisState.RUNNING); + manager.updateTaskStatus(taskInfo2, AnalysisState.RUNNING, "", 0); + Assertions.assertEquals(job.state, AnalysisState.RUNNING); + manager.updateTaskStatus(taskInfo1, AnalysisState.FINISHED, "", 0); + Assertions.assertEquals(job.state, AnalysisState.RUNNING); + manager.updateTaskStatus(taskInfo2, AnalysisState.FINISHED, "", 0); + Assertions.assertEquals(job.state, AnalysisState.FINISHED); + } +} diff --git a/regression-test/data/external_table_emr_p2/hive/test_hive_analyze_db.out b/regression-test/data/external_table_emr_p2/hive/test_hive_analyze_db.out deleted file mode 100644 index 0b75316476..0000000000 --- a/regression-test/data/external_table_emr_p2/hive/test_hive_analyze_db.out +++ /dev/null @@ -1,20 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !1 -- -lo_orderpriority 100.0 5.0 0.0 888.8000000000001 8.8 '1-URGENT' '5-LOW' -lo_custkey 100.0 26.0 0.0 404.0 4.0 67423 2735521 -lo_partkey 100.0 100.0 0.0 404.0 4.0 2250 989601 -lo_revenue 100.0 100.0 0.0 404.0 4.0 101171 8703450 -lo_commitdate 100.0 95.0 0.0 404.0 4.0 19920515 19981016 -lo_quantity 100.0 46.0 0.0 404.0 4.0 1 50 -lo_orderkey 100.0 26.0 0.0 404.0 4.0 1 98 -lo_suppkey 100.0 100.0 0.0 404.0 4.0 4167 195845 -lo_supplycost 100.0 100.0 0.0 404.0 4.0 58023 121374 -lo_shipmode 100.0 7.0 0.0 425.21 4.21 'AIR' 'TRUCK' -lo_orderdate 100.0 26.0 0.0 404.0 4.0 19920221 19980721 -lo_linenumber 100.0 7.0 0.0 404.0 4.0 1 7 -lo_shippriority 100.0 1.0 0.0 404.0 4.0 0 0 -lo_ordtotalprice 100.0 26.0 0.0 404.0 4.0 3428256 36771805 -lo_extendedprice 100.0 100.0 0.0 404.0 4.0 104300 9066094 -lo_tax 100.0 9.0 0.0 404.0 4.0 0 8 -lo_discount 100.0 11.0 0.0 404.0 4.0 0 10 - diff --git a/regression-test/data/external_table_emr_p2/hive/test_hive_statistic.out b/regression-test/data/external_table_emr_p2/hive/test_hive_statistic.out deleted file mode 100644 index ad8b494c00..0000000000 --- a/regression-test/data/external_table_emr_p2/hive/test_hive_statistic.out +++ /dev/null @@ -1,57 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !1 -- -lo_quantity 100.0 46.0 0.0 404.0 4.0 1 50 - --- !2 -- -lo_orderkey 100.0 26.0 0.0 404.0 4.0 1 98 - --- !3 -- -lo_linenumber 100.0 7.0 0.0 404.0 4.0 1 7 - --- !4 -- -lo_custkey 100.0 26.0 0.0 404.0 4.0 67423 2735521 - --- !5 -- -lo_partkey 100.0 100.0 0.0 404.0 4.0 2250 989601 - --- !6 -- -lo_suppkey 100.0 100.0 0.0 404.0 4.0 4167 195845 - --- !7 -- -lo_orderdate 100.0 26.0 0.0 404.0 4.0 19920221 19980721 - --- !8 -- -lo_orderpriority 100.0 5.0 0.0 888.8000000000001 8.8 '1-URGENT' '5-LOW' - --- !9 -- -lo_shippriority 100.0 1.0 0.0 404.0 4.0 0 0 - --- !10 -- -lo_extendedprice 100.0 100.0 0.0 404.0 4.0 104300 9066094 - --- !11 -- -lo_ordtotalprice 100.0 26.0 0.0 404.0 4.0 3428256 36771805 - --- !12 -- -lo_discount 100.0 11.0 0.0 404.0 4.0 0 10 - --- !13 -- -lo_revenue 100.0 100.0 0.0 404.0 4.0 101171 8703450 - --- !14 -- -lo_supplycost 100.0 100.0 0.0 404.0 4.0 58023 121374 - --- !15 -- -lo_tax 100.0 9.0 0.0 404.0 4.0 0 8 - --- !16 -- -lo_commitdate 100.0 95.0 0.0 404.0 4.0 19920515 19981016 - --- !17 -- -lo_shipmode 100.0 7.0 0.0 425.21 4.21 'AIR' 'TRUCK' - --- !18 -- -lo_shipmode 6001215.0 0.0 0.0 0.0 0.0 'NULL' 'NULL' - --- !19 -- - diff --git a/regression-test/data/external_table_emr_p2/hive/test_hive_statistic_cache.out b/regression-test/data/external_table_emr_p2/hive/test_hive_statistic_cache.out deleted file mode 100644 index dc5ded3837..0000000000 --- a/regression-test/data/external_table_emr_p2/hive/test_hive_statistic_cache.out +++ /dev/null @@ -1,58 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !1 -- -100 - --- !2 -- -lo_orderkey 100.0 26.0 0.0 404.0 4.0 1 98 - --- !3 -- -lo_linenumber 100.0 6.0 0.0 404.0 4.0 1 7 - --- !4 -- -lo_custkey 100.0 30.0 0.0 404.0 4.0 67423 2735521 - --- !5 -- -lo_partkey 100.0 107.0 0.0 404.0 4.0 2250 989601 - --- !6 -- -lo_suppkey 100.0 90.0 0.0 404.0 4.0 4167 195845 - --- !7 -- -lo_orderdate 100.0 22.0 0.0 404.0 4.0 19920221 19980721 - --- !8 -- -lo_orderpriority 100.0 5.0 0.0 888.8000000000001 8.8 N/A N/A - --- !9 -- -lo_shippriority 100.0 2.0 0.0 404.0 4.0 0 0 - --- !10 -- -lo_extendedprice 100.0 112.0 0.0 404.0 4.0 104300 9066094 - --- !11 -- -lo_ordtotalprice 100.0 31.0 0.0 404.0 4.0 3428256 36771805 - --- !12 -- -lo_discount 100.0 11.0 0.0 404.0 4.0 0 10 - --- !13 -- -lo_revenue 100.0 127.0 0.0 404.0 4.0 101171 8703450 - --- !14 -- -lo_supplycost 100.0 112.0 0.0 404.0 4.0 58023 121374 - --- !15 -- -lo_tax 100.0 9.0 0.0 404.0 4.0 0 8 - --- !16 -- -lo_commitdate 100.0 122.0 0.0 404.0 4.0 19920515 19981016 - --- !17 -- -lo_shipmode 100.0 7.0 0.0 425.21 4.21 N/A N/A - --- !18 -- -lo_orderkey 100.0 26.0 0.0 404.0 4.0 1 98 - --- !19 -- -lo_quantity 100.0 34.0 0.0 404.0 4.0 1 50 - diff --git a/regression-test/suites/external_table_emr_p2/hive/test_hive_analyze_db.groovy b/regression-test/suites/external_table_emr_p2/hive/test_hive_analyze_db.groovy index a30c87275b..fd724a67f8 100644 --- a/regression-test/suites/external_table_emr_p2/hive/test_hive_analyze_db.groovy +++ b/regression-test/suites/external_table_emr_p2/hive/test_hive_analyze_db.groovy @@ -33,8 +33,163 @@ suite("test_hive_analyze_db", "p2") { sql """switch ${catalog_name};""" logger.info("switched to catalog " + catalog_name) sql """use statistics;""" + sql """set query_timeout=300""" sql """analyze database statistics with sync""" - qt_1 "show column stats statistics" + def result = sql """show column stats statistics""" + assertTrue(result.size() == 17) + + assertTrue(result[0][0] == "lo_orderpriority") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "5.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "888.8000000000001") + assertTrue(result[0][5] == "8.8") + assertTrue(result[0][6] == "'1-URGENT'") + assertTrue(result[0][7] == "'5-LOW'") + + assertTrue(result[1][0] == "lo_custkey") + assertTrue(result[1][1] == "100.0") + assertTrue(result[1][2] == "26.0") + assertTrue(result[1][3] == "0.0") + assertTrue(result[1][4] == "404.0") + assertTrue(result[1][5] == "4.0") + assertTrue(result[1][6] == "67423") + assertTrue(result[1][7] == "2735521") + + assertTrue(result[2][0] == "lo_partkey") + assertTrue(result[2][1] == "100.0") + assertTrue(result[2][2] == "100.0") + assertTrue(result[2][3] == "0.0") + assertTrue(result[2][4] == "404.0") + assertTrue(result[2][5] == "4.0") + assertTrue(result[2][6] == "2250") + assertTrue(result[2][7] == "989601") + + assertTrue(result[3][0] == "lo_revenue") + assertTrue(result[3][1] == "100.0") + assertTrue(result[3][2] == "100.0") + assertTrue(result[3][3] == "0.0") + assertTrue(result[3][4] == "404.0") + assertTrue(result[3][5] == "4.0") + assertTrue(result[3][6] == "101171") + assertTrue(result[3][7] == "8703450") + + assertTrue(result[4][0] == "lo_commitdate") + assertTrue(result[4][1] == "100.0") + assertTrue(result[4][2] == "95.0") + assertTrue(result[4][3] == "0.0") + assertTrue(result[4][4] == "404.0") + assertTrue(result[4][5] == "4.0") + assertTrue(result[4][6] == "19920515") + assertTrue(result[4][7] == "19981016") + + assertTrue(result[5][0] == "lo_quantity") + assertTrue(result[5][1] == "100.0") + assertTrue(result[5][2] == "46.0") + assertTrue(result[5][3] == "0.0") + assertTrue(result[5][4] == "404.0") + assertTrue(result[5][5] == "4.0") + assertTrue(result[5][6] == "1") + assertTrue(result[5][7] == "50") + + assertTrue(result[6][0] == "lo_orderkey") + assertTrue(result[6][1] == "100.0") + assertTrue(result[6][2] == "26.0") + assertTrue(result[6][3] == "0.0") + assertTrue(result[6][4] == "404.0") + assertTrue(result[6][5] == "4.0") + assertTrue(result[6][6] == "1") + assertTrue(result[6][7] == "98") + + assertTrue(result[7][0] == "lo_suppkey") + assertTrue(result[7][1] == "100.0") + assertTrue(result[7][2] == "100.0") + assertTrue(result[7][3] == "0.0") + assertTrue(result[7][4] == "404.0") + assertTrue(result[7][5] == "4.0") + assertTrue(result[7][6] == "4167") + assertTrue(result[7][7] == "195845") + + assertTrue(result[8][0] == "lo_supplycost") + assertTrue(result[8][1] == "100.0") + assertTrue(result[8][2] == "100.0") + assertTrue(result[8][3] == "0.0") + assertTrue(result[8][4] == "404.0") + assertTrue(result[8][5] == "4.0") + assertTrue(result[8][6] == "58023") + assertTrue(result[8][7] == "121374") + + assertTrue(result[9][0] == "lo_shipmode") + assertTrue(result[9][1] == "100.0") + assertTrue(result[9][2] == "7.0") + assertTrue(result[9][3] == "0.0") + assertTrue(result[9][4] == "425.21") + assertTrue(result[9][5] == "4.21") + assertTrue(result[9][6] == "'AIR'") + assertTrue(result[9][7] == "'TRUCK'") + + assertTrue(result[10][0] == "lo_orderdate") + assertTrue(result[10][1] == "100.0") + assertTrue(result[10][2] == "26.0") + assertTrue(result[10][3] == "0.0") + assertTrue(result[10][4] == "404.0") + assertTrue(result[10][5] == "4.0") + assertTrue(result[10][6] == "19920221") + assertTrue(result[10][7] == "19980721") + + assertTrue(result[11][0] == "lo_linenumber") + assertTrue(result[11][1] == "100.0") + assertTrue(result[11][2] == "7.0") + assertTrue(result[11][3] == "0.0") + assertTrue(result[11][4] == "404.0") + assertTrue(result[11][5] == "4.0") + assertTrue(result[11][6] == "1") + assertTrue(result[11][7] == "7") + + assertTrue(result[12][0] == "lo_shippriority") + assertTrue(result[12][1] == "100.0") + assertTrue(result[12][2] == "1.0") + assertTrue(result[12][3] == "0.0") + assertTrue(result[12][4] == "404.0") + assertTrue(result[12][5] == "4.0") + assertTrue(result[12][6] == "0") + assertTrue(result[12][7] == "0") + + assertTrue(result[13][0] == "lo_ordtotalprice") + assertTrue(result[13][1] == "100.0") + assertTrue(result[13][2] == "26.0") + assertTrue(result[13][3] == "0.0") + assertTrue(result[13][4] == "404.0") + assertTrue(result[13][5] == "4.0") + assertTrue(result[13][6] == "3428256") + assertTrue(result[13][7] == "36771805") + + assertTrue(result[14][0] == "lo_extendedprice") + assertTrue(result[14][1] == "100.0") + assertTrue(result[14][2] == "100.0") + assertTrue(result[14][3] == "0.0") + assertTrue(result[14][4] == "404.0") + assertTrue(result[14][5] == "4.0") + assertTrue(result[14][6] == "104300") + assertTrue(result[14][7] == "9066094") + + assertTrue(result[15][0] == "lo_tax") + assertTrue(result[15][1] == "100.0") + assertTrue(result[15][2] == "9.0") + assertTrue(result[15][3] == "0.0") + assertTrue(result[15][4] == "404.0") + assertTrue(result[15][5] == "4.0") + assertTrue(result[15][6] == "0") + assertTrue(result[15][7] == "8") + + assertTrue(result[16][0] == "lo_discount") + assertTrue(result[16][1] == "100.0") + assertTrue(result[16][2] == "11.0") + assertTrue(result[16][3] == "0.0") + assertTrue(result[16][4] == "404.0") + assertTrue(result[16][5] == "4.0") + assertTrue(result[16][6] == "0") + assertTrue(result[16][7] == "10") } } diff --git a/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic.groovy b/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic.groovy index ff184e4506..c6df0b0eaa 100644 --- a/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic.groovy +++ b/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic.groovy @@ -34,29 +34,202 @@ suite("test_hive_statistic", "p2") { logger.info("switched to catalog " + catalog_name) sql """use statistics;""" sql """analyze table `statistics` with sync""" - qt_1 "show column stats `statistics` (lo_quantity)" - qt_2 "show column stats `statistics` (lo_orderkey)" - qt_3 "show column stats `statistics` (lo_linenumber)" - qt_4 "show column stats `statistics` (lo_custkey)" - qt_5 "show column stats `statistics` (lo_partkey)" - qt_6 "show column stats `statistics` (lo_suppkey)" - qt_7 "show column stats `statistics` (lo_orderdate)" - qt_8 "show column stats `statistics` (lo_orderpriority)" - qt_9 "show column stats `statistics` (lo_shippriority)" - qt_10 "show column stats `statistics` (lo_extendedprice)" - qt_11 "show column stats `statistics` (lo_ordtotalprice)" - qt_12 "show column stats `statistics` (lo_discount)" - qt_13 "show column stats `statistics` (lo_revenue)" - qt_14 "show column stats `statistics` (lo_supplycost)" - qt_15 "show column stats `statistics` (lo_tax)" - qt_16 "show column stats `statistics` (lo_commitdate)" - qt_17 "show column stats `statistics` (lo_shipmode)" + def result = sql """show column stats `statistics` (lo_quantity)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_quantity") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "46.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "1") + assertTrue(result[0][7] == "50") - sql """ALTER TABLE statistics MODIFY COLUMN lo_shipmode SET STATS ('row_count'='6001215')""" - qt_18 "show column stats `statistics` (lo_shipmode)" + result = sql """show column stats `statistics` (lo_orderkey)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_orderkey") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "26.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "1") + assertTrue(result[0][7] == "98") + + result = sql """show column stats `statistics` (lo_linenumber)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_linenumber") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "7.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "1") + assertTrue(result[0][7] == "7") + + result = sql """show column stats `statistics` (lo_custkey)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_custkey") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "26.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "67423") + assertTrue(result[0][7] == "2735521") + + result = sql """show column stats `statistics` (lo_partkey)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_partkey") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "2250") + assertTrue(result[0][7] == "989601") + + result = sql """show column stats `statistics` (lo_suppkey)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_suppkey") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "4167") + assertTrue(result[0][7] == "195845") + + result = sql """show column stats `statistics` (lo_orderdate)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_orderdate") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "26.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "19920221") + assertTrue(result[0][7] == "19980721") + + result = sql """show column stats `statistics` (lo_orderpriority)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_orderpriority") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "5.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "888.8000000000001") + assertTrue(result[0][5] == "8.8") + assertTrue(result[0][6] == "'1-URGENT'") + assertTrue(result[0][7] == "'5-LOW'") + + result = sql """show column stats `statistics` (lo_shippriority)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_shippriority") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "1.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "0") + assertTrue(result[0][7] == "0") + + result = sql """show column stats `statistics` (lo_extendedprice)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_extendedprice") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "104300") + assertTrue(result[0][7] == "9066094") + + result = sql """show column stats `statistics` (lo_ordtotalprice)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_ordtotalprice") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "26.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "3428256") + assertTrue(result[0][7] == "36771805") + + result = sql """show column stats `statistics` (lo_discount)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_discount") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "11.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "0") + assertTrue(result[0][7] == "10") + + result = sql """show column stats `statistics` (lo_revenue)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_revenue") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "101171") + assertTrue(result[0][7] == "8703450") + + result = sql """show column stats `statistics` (lo_supplycost)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_supplycost") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "58023") + assertTrue(result[0][7] == "121374") + + result = sql """show column stats `statistics` (lo_tax)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_tax") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "9.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "0") + assertTrue(result[0][7] == "8") + + result = sql """show column stats `statistics` (lo_commitdate)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_commitdate") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "95.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "19920515") + assertTrue(result[0][7] == "19981016") + + result = sql """show column stats `statistics` (lo_shipmode)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_shipmode") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "7.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "425.21") + assertTrue(result[0][5] == "4.21") + assertTrue(result[0][6] == "'AIR'") + assertTrue(result[0][7] == "'TRUCK'") + + // sql """ALTER TABLE statistics MODIFY COLUMN lo_shipmode SET STATS ('row_count'='6001215')""" + // result = sql "show column stats `statistics` (lo_shipmode)" + // assertTrue(result.size() == 1) + // assertTrue(result[0][0] == "lo_shipmode") + // assertTrue(result[0][1] == "6001215.0") sql """drop stats statistics""" - qt_19 "show column stats statistics" + result = sql """show column stats statistics""" + assertTrue(result.size() == 0) } } diff --git a/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic_cache.groovy b/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic_cache.groovy index 2fb574a0da..d1399ef49b 100644 --- a/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic_cache.groovy +++ b/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic_cache.groovy @@ -33,26 +33,179 @@ suite("test_hive_statistic_cache", "p2") { sql """switch ${catalog_name};""" logger.info("switched to catalog " + catalog_name) sql """use statistics;""" - qt_1 "select count(*) from stats" + sql """set query_timeout=300;""" + sql """analyze table `stats` with sync;""" + sql """select count(*) from stats""" Thread.sleep(5000); - qt_2 "show column cached stats `stats` (lo_orderkey)" - qt_3 "show column cached stats `stats` (lo_linenumber)" - qt_4 "show column cached stats `stats` (lo_custkey)" - qt_5 "show column cached stats `stats` (lo_partkey)" - qt_6 "show column cached stats `stats` (lo_suppkey)" - qt_7 "show column cached stats `stats` (lo_orderdate)" - qt_8 "show column cached stats `stats` (lo_orderpriority)" - qt_9 "show column cached stats `stats` (lo_shippriority)" - qt_10 "show column cached stats `stats` (lo_extendedprice)" - qt_11 "show column cached stats `stats` (lo_ordtotalprice)" - qt_12 "show column cached stats `stats` (lo_discount)" - qt_13 "show column cached stats `stats` (lo_revenue)" - qt_14 "show column cached stats `stats` (lo_supplycost)" - qt_15 "show column cached stats `stats` (lo_tax)" - qt_16 "show column cached stats `stats` (lo_commitdate)" - qt_17 "show column cached stats `stats` (lo_shipmode)" - qt_18 "show column cached stats `stats` (lo_orderkey)" - qt_19 "show column cached stats `stats` (lo_quantity)" + def result = sql """show column cached stats `stats` (lo_orderkey)""" + assertTrue(result[0][0] == "lo_orderkey") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "26.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "1") + assertTrue(result[0][7] == "98") + + result = sql """show column cached stats `stats` (lo_linenumber)""" + assertTrue(result[0][0] == "lo_linenumber") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "7.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "1") + assertTrue(result[0][7] == "7") + + result = sql """show column cached stats `stats` (lo_custkey)""" + assertTrue(result[0][0] == "lo_custkey") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "26.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "67423") + assertTrue(result[0][7] == "2735521") + + result = sql """show column cached stats `stats` (lo_partkey)""" + assertTrue(result[0][0] == "lo_partkey") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "2250") + assertTrue(result[0][7] == "989601") + + result = sql """show column cached stats `stats` (lo_suppkey)""" + assertTrue(result[0][0] == "lo_suppkey") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "4167") + assertTrue(result[0][7] == "195845") + + result = sql """show column cached stats `stats` (lo_orderdate)""" + assertTrue(result[0][0] == "lo_orderdate") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "26.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "19920221") + assertTrue(result[0][7] == "19980721") + + result = sql """show column cached stats `stats` (lo_orderpriority)""" + assertTrue(result[0][0] == "lo_orderpriority") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "5.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "888.8000000000001") + assertTrue(result[0][5] == "8.8") + assertTrue(result[0][6] == "'1-URGENT'") + assertTrue(result[0][7] == "'5-LOW'") + + result = sql """show column cached stats `stats` (lo_shippriority)""" + assertTrue(result[0][0] == "lo_shippriority") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "1.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "0") + assertTrue(result[0][7] == "0") + + result = sql """show column cached stats `stats` (lo_extendedprice)""" + assertTrue(result[0][0] == "lo_extendedprice") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "104300") + assertTrue(result[0][7] == "9066094") + + result = sql """show column cached stats `stats` (lo_ordtotalprice)""" + assertTrue(result[0][0] == "lo_ordtotalprice") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "26.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "3428256") + assertTrue(result[0][7] == "36771805") + + result = sql """show column cached stats `stats` (lo_discount)""" + assertTrue(result[0][0] == "lo_discount") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "11.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "0") + assertTrue(result[0][7] == "10") + + result = sql """show column cached stats `stats` (lo_revenue)""" + assertTrue(result[0][0] == "lo_revenue") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "101171") + assertTrue(result[0][7] == "8703450") + + result = sql """show column cached stats `stats` (lo_supplycost)""" + assertTrue(result[0][0] == "lo_supplycost") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "58023") + assertTrue(result[0][7] == "121374") + + result = sql """show column cached stats `stats` (lo_tax)""" + assertTrue(result[0][0] == "lo_tax") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "9.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "0") + assertTrue(result[0][7] == "8") + + result = sql """show column cached stats `stats` (lo_commitdate)""" + assertTrue(result[0][0] == "lo_commitdate") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "95.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "19920515") + assertTrue(result[0][7] == "19981016") + + result = sql """show column cached stats `stats` (lo_shipmode)""" + assertTrue(result[0][0] == "lo_shipmode") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "7.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "425.21") + assertTrue(result[0][5] == "4.21") + assertTrue(result[0][6] == "'AIR'") + assertTrue(result[0][7] == "'TRUCK'") + + result = sql """show column cached stats `stats` (lo_quantity)""" + assertTrue(result[0][0] == "lo_quantity") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "46.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "1") + assertTrue(result[0][7] == "50") } }