diff --git a/docs/en/docs/lakehouse/external_statistics.md b/docs/en/docs/lakehouse/external_statistics.md index 7329310f0b..6c469961a8 100644 --- a/docs/en/docs/lakehouse/external_statistics.md +++ b/docs/en/docs/lakehouse/external_statistics.md @@ -53,45 +53,45 @@ mysql> ANALYZE TABLE hive.tpch100.lineitem; +--------------+-------------------------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ | Catalog_Name | DB_Name | Table_Name | Columns | Job_Id | +--------------+-------------------------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ -| hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | 126039 | +| hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | 16990 | +--------------+-------------------------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ -1 row in set (0.03 sec) +1 row in set (0.06 sec) ``` This operation is performed asynchronously, a collection job will be created in the background, and the progress of the job can be viewed using job_id ``` -mysql> SHOW ANALYZE 126039; -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------+ -| job_id | catalog_name | db_name | tbl_name | col_name | job_type | analysis_type | message | last_exec_time_in_ms | state | schedule_type | -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------+ -| 126039 | hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | MANUAL | FUNDAMENTALS | | 2023-07-13 10:33:44 | PENDING | ONCE | -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------+ +mysql> SHOW ANALYZE 16990; ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------------------------------------+---------------+ +| job_id | catalog_name | db_name | tbl_name | col_name | job_type | analysis_type | message | last_exec_time_in_ms | state | progress | schedule_type | ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------------------------------------+---------------+ +| 16990 | hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | MANUAL | FUNDAMENTALS | | 2023-07-27 16:01:52 | RUNNING | 2 Finished/0 Failed/15 In Progress/17 Total | ONCE | ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------------------------------------+---------------+ 1 row in set (0.00 sec) ``` And view the task status of each column. ``` -mysql> SHOW ANALYZE TASK STATUS 126039; -+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+----------------------+----------+ -| task_id | col_name | message | last_exec_time_in_ms | state | -+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+----------------------+----------+ -| 126040 | l_receiptdate | | 2023-07-13 10:33:44 | RUNNING | -| 126041 | l_returnflag | | 2023-07-13 10:33:44 | RUNNING | -| 126042 | l_tax | | 2023-07-13 10:33:44 | RUNNING | -| 126043 | l_shipmode | | 2023-07-13 10:33:44 | RUNNING | -| 126044 | l_suppkey | | 2023-07-13 10:33:44 | RUNNING | -| 126045 | l_shipdate | | 2023-07-13 10:33:44 | RUNNING | -| 126046 | l_commitdate | | 2023-07-13 10:33:44 | RUNNING | -| 126047 | l_partkey | | 2023-07-13 10:33:44 | RUNNING | -| 126048 | l_quantity | | 2023-07-13 10:33:44 | RUNNING | -| 126049 | l_orderkey | | 2023-07-13 10:33:44 | RUNNING | -| 126050 | l_comment | | 2023-07-13 10:33:44 | RUNNING | -| 126051 | l_linestatus | | 2023-07-13 10:33:44 | RUNNING | -| 126052 | l_extendedprice | | 2023-07-13 10:33:44 | RUNNING | -| 126053 | l_linenumber | | 2023-07-13 10:33:44 | RUNNING | -| 126054 | l_shipinstruct | | 2023-07-13 10:33:44 | RUNNING | -| 126055 | l_discount | | 2023-07-13 10:33:44 | RUNNING | -| 126056 | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | | 2023-07-13 10:33:56 | FINISHED | -+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+----------------------+----------+ -17 rows in set (0.01 sec) +mysql> SHOW ANALYZE TASK STATUS 16990; ++---------+-----------------+---------+------------------------+-----------------+----------+ +| task_id | col_name | message | last_state_change_time | time_cost_in_ms | state | ++---------+-----------------+---------+------------------------+-----------------+----------+ +| 16991 | l_receiptdate | | 2023-07-27 16:01:29 | 0 | PENDING | +| 16992 | l_returnflag | | 2023-07-27 16:01:44 | 14394 | FINISHED | +| 16993 | l_tax | | 2023-07-27 16:01:52 | 7975 | FINISHED | +| 16994 | l_shipmode | | 2023-07-27 16:02:11 | 18961 | FINISHED | +| 16995 | l_suppkey | | 2023-07-27 16:02:17 | 6684 | FINISHED | +| 16996 | l_shipdate | | 2023-07-27 16:02:26 | 8518 | FINISHED | +| 16997 | l_commitdate | | 2023-07-27 16:02:26 | 0 | RUNNING | +| 16998 | l_partkey | | 2023-07-27 16:01:29 | 0 | PENDING | +| 16999 | l_quantity | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17000 | l_orderkey | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17001 | l_comment | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17002 | l_linestatus | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17003 | l_extendedprice | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17004 | l_linenumber | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17005 | l_shipinstruct | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17006 | l_discount | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17007 | TableRowCount | | 2023-07-27 16:01:29 | 0 | PENDING | ++---------+-----------------+---------+------------------------+-----------------+----------+ +17 rows in set (0.00 sec) ``` - Collect statistics about all tables in the tpch100 database @@ -101,14 +101,14 @@ mysql> ANALYZE DATABASE hive.tpch100; +--------------+---------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ | Catalog_Name | DB_Name | Table_Name | Columns | Job_Id | +--------------+---------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ -| hive | tpch100 | partsupp | [ps_suppkey,ps_availqty,ps_comment,ps_partkey,ps_supplycost] | 124192 | -| hive | tpch100 | orders | [o_orderstatus,o_clerk,o_orderdate,o_shippriority,o_custkey,o_totalprice,o_orderkey,o_comment,o_orderpriority] | 124199 | -| hive | tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | 124210 | -| hive | tpch100 | part | [p_partkey,p_container,p_name,p_comment,p_brand,p_type,p_retailprice,p_mfgr,p_size] | 124228 | -| hive | tpch100 | customer | [c_custkey,c_phone,c_acctbal,c_mktsegment,c_address,c_nationkey,c_name,c_comment] | 124239 | -| hive | tpch100 | supplier | [s_comment,s_phone,s_nationkey,s_name,s_address,s_acctbal,s_suppkey] | 124249 | -| hive | tpch100 | nation | [n_comment,n_nationkey,n_regionkey,n_name] | 124258 | -| hive | tpch100 | region | [r_regionkey,r_comment,r_name] | 124264 | +| hive | tpch100 | supplier | [s_comment,s_phone,s_nationkey,s_name,s_address,s_acctbal,s_suppkey] | 17018 | +| hive | tpch100 | nation | [n_comment,n_nationkey,n_regionkey,n_name] | 17027 | +| hive | tpch100 | region | [r_regionkey,r_comment,r_name] | 17033 | +| hive | tpch100 | partsupp | [ps_suppkey,ps_availqty,ps_comment,ps_partkey,ps_supplycost] | 17038 | +| hive | tpch100 | orders | [o_orderstatus,o_clerk,o_orderdate,o_shippriority,o_custkey,o_totalprice,o_orderkey,o_comment,o_orderpriority] | 17045 | +| hive | tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | 17056 | +| hive | tpch100 | part | [p_partkey,p_container,p_name,p_comment,p_brand,p_type,p_retailprice,p_mfgr,p_size] | 17074 | +| hive | tpch100 | customer | [c_custkey,c_phone,c_acctbal,c_mktsegment,c_address,c_nationkey,c_name,c_comment] | 17085 | +--------------+---------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ 8 rows in set (0.29 sec) ``` @@ -140,28 +140,38 @@ The method of job management is also the same as that of the internal table, inc ``` mysql> SHOW ANALYZE; -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------+ -| job_id | catalog_name | db_name | tbl_name | col_name | job_type | analysis_type | message | last_exec_time_in_ms | state | schedule_type | -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------+ -| 12152 | hive | tpch100 | partsupp | [ps_suppkey,ps_availqty,ps_comment,ps_partkey,ps_supplycost] | MANUAL | FUNDAMENTALS | | 2023-07-11 15:57:16 | FINISHED | ONCE | -| 12159 | hive | tpch100 | orders | [o_orderstatus,o_clerk,o_orderdate,o_shippriority,o_custkey,o_totalprice,o_orderkey,o_comment,o_orderpriority] | MANUAL | FUNDAMENTALS | | 2023-07-11 15:57:24 | FINISHED | ONCE | -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------+ ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------------------------------------+---------------+ +| job_id | catalog_name | db_name | tbl_name | col_name | job_type | analysis_type | message | last_exec_time_in_ms | state | progress | schedule_type | ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------------------------------------+---------------+ +| 16990 | hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | MANUAL | FUNDAMENTALS | | 2023-07-27 16:05:02 | FINISHED | 17 Finished/0 Failed/0 In Progress/17 Total | ONCE | ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------------------------------------+---------------+ ``` - Show all tasks status of a job ``` -mysql> SHOW ANALYZE TASK STATUS 12152; -+---------+--------------------------------------------------------------+---------+----------------------+----------+ -| task_id | col_name | message | last_exec_time_in_ms | state | -+---------+--------------------------------------------------------------+---------+----------------------+----------+ -| 12153 | ps_availqty | | 2023-07-11 15:56:58 | FINISHED | -| 12154 | ps_suppkey | | 2023-07-11 15:56:57 | FINISHED | -| 12155 | ps_comment | | 2023-07-11 15:57:16 | FINISHED | -| 12156 | ps_supplycost | | 2023-07-11 15:56:57 | FINISHED | -| 12157 | ps_partkey | | 2023-07-11 15:56:58 | FINISHED | -| 12158 | [ps_suppkey,ps_availqty,ps_comment,ps_partkey,ps_supplycost] | | 2023-07-11 15:56:57 | FINISHED | -+---------+--------------------------------------------------------------+---------+----------------------+----------+ +mysql> SHOW ANALYZE TASK STATUS 16990; ++---------+-----------------+---------+------------------------+-----------------+----------+ +| task_id | col_name | message | last_state_change_time | time_cost_in_ms | state | ++---------+-----------------+---------+------------------------+-----------------+----------+ +| 16991 | l_receiptdate | | 2023-07-27 16:05:02 | 9560 | FINISHED | +| 16992 | l_returnflag | | 2023-07-27 16:01:44 | 14394 | FINISHED | +| 16993 | l_tax | | 2023-07-27 16:01:52 | 7975 | FINISHED | +| 16994 | l_shipmode | | 2023-07-27 16:02:11 | 18961 | FINISHED | +| 16995 | l_suppkey | | 2023-07-27 16:02:17 | 6684 | FINISHED | +| 16996 | l_shipdate | | 2023-07-27 16:02:26 | 8518 | FINISHED | +| 16997 | l_commitdate | | 2023-07-27 16:02:34 | 8380 | FINISHED | +| 16998 | l_partkey | | 2023-07-27 16:02:40 | 6060 | FINISHED | +| 16999 | l_quantity | | 2023-07-27 16:02:50 | 9768 | FINISHED | +| 17000 | l_orderkey | | 2023-07-27 16:02:57 | 7200 | FINISHED | +| 17001 | l_comment | | 2023-07-27 16:03:36 | 38468 | FINISHED | +| 17002 | l_linestatus | | 2023-07-27 16:03:51 | 15226 | FINISHED | +| 17003 | l_extendedprice | | 2023-07-27 16:04:00 | 8713 | FINISHED | +| 17004 | l_linenumber | | 2023-07-27 16:04:06 | 6659 | FINISHED | +| 17005 | l_shipinstruct | | 2023-07-27 16:04:36 | 29777 | FINISHED | +| 17006 | l_discount | | 2023-07-27 16:04:45 | 9212 | FINISHED | +| 17007 | TableRowCount | | 2023-07-27 16:04:52 | 6974 | FINISHED | ++---------+-----------------+---------+------------------------+-----------------+----------+ ``` - Terminate unfinished jobs diff --git a/docs/zh-CN/docs/lakehouse/external_statistics.md b/docs/zh-CN/docs/lakehouse/external_statistics.md index 586571f5b6..0b47ed5329 100644 --- a/docs/zh-CN/docs/lakehouse/external_statistics.md +++ b/docs/zh-CN/docs/lakehouse/external_statistics.md @@ -53,45 +53,45 @@ mysql> ANALYZE TABLE hive.tpch100.lineitem; +--------------+-------------------------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ | Catalog_Name | DB_Name | Table_Name | Columns | Job_Id | +--------------+-------------------------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ -| hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | 126039 | +| hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | 16990 | +--------------+-------------------------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ -1 row in set (0.03 sec) +1 row in set (0.06 sec) ``` 此操作是异步执行，会在后台创建收集任务，可以通过job_id查看任务进度。 ``` -mysql> SHOW ANALYZE 126039; -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------+ -| job_id | catalog_name | db_name | tbl_name | col_name | job_type | analysis_type | message | last_exec_time_in_ms | state | schedule_type | -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------+ -| 126039 | hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | MANUAL | FUNDAMENTALS | | 2023-07-13 10:33:44 | PENDING | ONCE | -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------+ +mysql> SHOW ANALYZE 16990; ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------------------------------------+---------------+ +| job_id | catalog_name | db_name | tbl_name | col_name | job_type | analysis_type | message | last_exec_time_in_ms | state | progress | schedule_type | ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------------------------------------+---------------+ +| 16990 | hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | MANUAL | FUNDAMENTALS | | 2023-07-27 16:01:52 | RUNNING | 2 Finished/0 Failed/15 In Progress/17 Total | ONCE | ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+---------+---------------------------------------------+---------------+ 1 row in set (0.00 sec) ``` 以及查看每一列的task状态。 ``` -mysql> SHOW ANALYZE TASK STATUS 126039; -+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+----------------------+----------+ -| task_id | col_name | message | last_exec_time_in_ms | state | -+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+----------------------+----------+ -| 126040 | l_receiptdate | | 2023-07-13 10:33:44 | RUNNING | -| 126041 | l_returnflag | | 2023-07-13 10:33:44 | RUNNING | -| 126042 | l_tax | | 2023-07-13 10:33:44 | RUNNING | -| 126043 | l_shipmode | | 2023-07-13 10:33:44 | RUNNING | -| 126044 | l_suppkey | | 2023-07-13 10:33:44 | RUNNING | -| 126045 | l_shipdate | | 2023-07-13 10:33:44 | RUNNING | -| 126046 | l_commitdate | | 2023-07-13 10:33:44 | RUNNING | -| 126047 | l_partkey | | 2023-07-13 10:33:44 | RUNNING | -| 126048 | l_quantity | | 2023-07-13 10:33:44 | RUNNING | -| 126049 | l_orderkey | | 2023-07-13 10:33:44 | RUNNING | -| 126050 | l_comment | | 2023-07-13 10:33:44 | RUNNING | -| 126051 | l_linestatus | | 2023-07-13 10:33:44 | RUNNING | -| 126052 | l_extendedprice | | 2023-07-13 10:33:44 | RUNNING | -| 126053 | l_linenumber | | 2023-07-13 10:33:44 | RUNNING | -| 126054 | l_shipinstruct | | 2023-07-13 10:33:44 | RUNNING | -| 126055 | l_discount | | 2023-07-13 10:33:44 | RUNNING | -| 126056 | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | | 2023-07-13 10:33:56 | FINISHED | -+---------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------+----------------------+----------+ -17 rows in set (0.01 sec) +mysql> SHOW ANALYZE TASK STATUS 16990; ++---------+-----------------+---------+------------------------+-----------------+----------+ +| task_id | col_name | message | last_state_change_time | time_cost_in_ms | state | ++---------+-----------------+---------+------------------------+-----------------+----------+ +| 16991 | l_receiptdate | | 2023-07-27 16:01:29 | 0 | PENDING | +| 16992 | l_returnflag | | 2023-07-27 16:01:44 | 14394 | FINISHED | +| 16993 | l_tax | | 2023-07-27 16:01:52 | 7975 | FINISHED | +| 16994 | l_shipmode | | 2023-07-27 16:02:11 | 18961 | FINISHED | +| 16995 | l_suppkey | | 2023-07-27 16:02:17 | 6684 | FINISHED | +| 16996 | l_shipdate | | 2023-07-27 16:02:26 | 8518 | FINISHED | +| 16997 | l_commitdate | | 2023-07-27 16:02:26 | 0 | RUNNING | +| 16998 | l_partkey | | 2023-07-27 16:01:29 | 0 | PENDING | +| 16999 | l_quantity | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17000 | l_orderkey | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17001 | l_comment | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17002 | l_linestatus | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17003 | l_extendedprice | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17004 | l_linenumber | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17005 | l_shipinstruct | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17006 | l_discount | | 2023-07-27 16:01:29 | 0 | PENDING | +| 17007 | TableRowCount | | 2023-07-27 16:01:29 | 0 | PENDING | ++---------+-----------------+---------+------------------------+-----------------+----------+ +17 rows in set (0.00 sec) ``` - 收集tpch100数据库所有表的信息 @@ -101,14 +101,14 @@ mysql> ANALYZE DATABASE hive.tpch100; +--------------+---------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ | Catalog_Name | DB_Name | Table_Name | Columns | Job_Id | +--------------+---------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ -| hive | tpch100 | partsupp | [ps_suppkey,ps_availqty,ps_comment,ps_partkey,ps_supplycost] | 124192 | -| hive | tpch100 | orders | [o_orderstatus,o_clerk,o_orderdate,o_shippriority,o_custkey,o_totalprice,o_orderkey,o_comment,o_orderpriority] | 124199 | -| hive | tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | 124210 | -| hive | tpch100 | part | [p_partkey,p_container,p_name,p_comment,p_brand,p_type,p_retailprice,p_mfgr,p_size] | 124228 | -| hive | tpch100 | customer | [c_custkey,c_phone,c_acctbal,c_mktsegment,c_address,c_nationkey,c_name,c_comment] | 124239 | -| hive | tpch100 | supplier | [s_comment,s_phone,s_nationkey,s_name,s_address,s_acctbal,s_suppkey] | 124249 | -| hive | tpch100 | nation | [n_comment,n_nationkey,n_regionkey,n_name] | 124258 | -| hive | tpch100 | region | [r_regionkey,r_comment,r_name] | 124264 | +| hive | tpch100 | supplier | [s_comment,s_phone,s_nationkey,s_name,s_address,s_acctbal,s_suppkey] | 17018 | +| hive | tpch100 | nation | [n_comment,n_nationkey,n_regionkey,n_name] | 17027 | +| hive | tpch100 | region | [r_regionkey,r_comment,r_name] | 17033 | +| hive | tpch100 | partsupp | [ps_suppkey,ps_availqty,ps_comment,ps_partkey,ps_supplycost] | 17038 | +| hive | tpch100 | orders | [o_orderstatus,o_clerk,o_orderdate,o_shippriority,o_custkey,o_totalprice,o_orderkey,o_comment,o_orderpriority] | 17045 | +| hive | tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | 17056 | +| hive | tpch100 | part | [p_partkey,p_container,p_name,p_comment,p_brand,p_type,p_retailprice,p_mfgr,p_size] | 17074 | +| hive | tpch100 | customer | [c_custkey,c_phone,c_acctbal,c_mktsegment,c_address,c_nationkey,c_name,c_comment] | 17085 | +--------------+---------+------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+ 8 rows in set (0.29 sec) ``` @@ -140,28 +140,38 @@ Query OK, 0 rows affected (33.19 sec) ``` mysql> SHOW ANALYZE; -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------+ -| job_id | catalog_name | db_name | tbl_name | col_name | job_type | analysis_type | message | last_exec_time_in_ms | state | schedule_type | -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------+ -| 12152 | hive | tpch100 | partsupp | [ps_suppkey,ps_availqty,ps_comment,ps_partkey,ps_supplycost] | MANUAL | FUNDAMENTALS | | 2023-07-11 15:57:16 | FINISHED | ONCE | -| 12159 | hive | tpch100 | orders | [o_orderstatus,o_clerk,o_orderdate,o_shippriority,o_custkey,o_totalprice,o_orderkey,o_comment,o_orderpriority] | MANUAL | FUNDAMENTALS | | 2023-07-11 15:57:24 | FINISHED | ONCE | -+--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------+ ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------------------------------------+---------------+ +| job_id | catalog_name | db_name | tbl_name | col_name | job_type | analysis_type | message | last_exec_time_in_ms | state | progress | schedule_type | ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------------------------------------+---------------+ +| 16990 | hive | default_cluster:tpch100 | lineitem | [l_returnflag,l_receiptdate,l_tax,l_shipmode,l_suppkey,l_shipdate,l_commitdate,l_partkey,l_orderkey,l_quantity,l_linestatus,l_comment,l_extendedprice,l_linenumber,l_discount,l_shipinstruct] | MANUAL | FUNDAMENTALS | | 2023-07-27 16:05:02 | FINISHED | 17 Finished/0 Failed/0 In Progress/17 Total | ONCE | ++--------+--------------+-------------------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------+---------------+---------+----------------------+----------+---------------------------------------------+---------------+ ``` - 查看一个job的所有task状态 ``` -mysql> SHOW ANALYZE TASK STATUS 12152; -+---------+--------------------------------------------------------------+---------+----------------------+----------+ -| task_id | col_name | message | last_exec_time_in_ms | state | -+---------+--------------------------------------------------------------+---------+----------------------+----------+ -| 12153 | ps_availqty | | 2023-07-11 15:56:58 | FINISHED | -| 12154 | ps_suppkey | | 2023-07-11 15:56:57 | FINISHED | -| 12155 | ps_comment | | 2023-07-11 15:57:16 | FINISHED | -| 12156 | ps_supplycost | | 2023-07-11 15:56:57 | FINISHED | -| 12157 | ps_partkey | | 2023-07-11 15:56:58 | FINISHED | -| 12158 | [ps_suppkey,ps_availqty,ps_comment,ps_partkey,ps_supplycost] | | 2023-07-11 15:56:57 | FINISHED | -+---------+--------------------------------------------------------------+---------+----------------------+----------+ +mysql> SHOW ANALYZE TASK STATUS 16990; ++---------+-----------------+---------+------------------------+-----------------+----------+ +| task_id | col_name | message | last_state_change_time | time_cost_in_ms | state | ++---------+-----------------+---------+------------------------+-----------------+----------+ +| 16991 | l_receiptdate | | 2023-07-27 16:05:02 | 9560 | FINISHED | +| 16992 | l_returnflag | | 2023-07-27 16:01:44 | 14394 | FINISHED | +| 16993 | l_tax | | 2023-07-27 16:01:52 | 7975 | FINISHED | +| 16994 | l_shipmode | | 2023-07-27 16:02:11 | 18961 | FINISHED | +| 16995 | l_suppkey | | 2023-07-27 16:02:17 | 6684 | FINISHED | +| 16996 | l_shipdate | | 2023-07-27 16:02:26 | 8518 | FINISHED | +| 16997 | l_commitdate | | 2023-07-27 16:02:34 | 8380 | FINISHED | +| 16998 | l_partkey | | 2023-07-27 16:02:40 | 6060 | FINISHED | +| 16999 | l_quantity | | 2023-07-27 16:02:50 | 9768 | FINISHED | +| 17000 | l_orderkey | | 2023-07-27 16:02:57 | 7200 | FINISHED | +| 17001 | l_comment | | 2023-07-27 16:03:36 | 38468 | FINISHED | +| 17002 | l_linestatus | | 2023-07-27 16:03:51 | 15226 | FINISHED | +| 17003 | l_extendedprice | | 2023-07-27 16:04:00 | 8713 | FINISHED | +| 17004 | l_linenumber | | 2023-07-27 16:04:06 | 6659 | FINISHED | +| 17005 | l_shipinstruct | | 2023-07-27 16:04:36 | 29777 | FINISHED | +| 17006 | l_discount | | 2023-07-27 16:04:45 | 9212 | FINISHED | +| 17007 | TableRowCount | | 2023-07-27 16:04:52 | 6974 | FINISHED | ++---------+-----------------+---------+------------------------+-----------------+----------+ ``` - 终止未完成的job diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java index f9c6241c40..da08f45bee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java @@ -84,6 +84,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt { private final TableName tableName; private List columnNames; private List partitionNames; + private boolean isAllColumns; // after analyzed private long dbId; @@ -98,6 +99,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt { this.partitionNames = partitionNames == null ? null : partitionNames.getPartitionNames(); this.columnNames = columnNames; this.analyzeProperties = properties; + this.isAllColumns = columnNames == null; } public AnalyzeTblStmt(AnalyzeProperties analyzeProperties, TableName tableName, List columnNames, long dbId, @@ -107,6 +109,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt { this.columnNames = columnNames; this.dbId = dbId; this.table = table; + this.isAllColumns = columnNames == null; } @Override @@ -128,6 +131,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt { DatabaseIf db = catalog.getDbOrAnalysisException(dbName); dbId = db.getId(); table = db.getTableOrAnalysisException(tblName); + isAllColumns = columnNames == null; check(); } @@ -301,4 +305,8 @@ public class AnalyzeTblStmt extends AnalyzeStmt { public Database getDb() throws AnalysisException { return analyzer.getEnv().getInternalCatalog().getDbOrAnalysisException(dbId); } + + public boolean isAllColumns() { + return isAllColumns; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java index 73bf77b23c..3a0aa05a7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java @@ -64,6 +64,7 @@ public class ShowAnalyzeStmt extends ShowStmt { .add("message") .add("last_exec_time_in_ms") .add("state") + .add("progress") .add("schedule_type") .build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeTaskStatus.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeTaskStatus.java index 03d304f393..927a56d19d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeTaskStatus.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeTaskStatus.java @@ -33,7 +33,8 @@ public class ShowAnalyzeTaskStatus extends ShowStmt { .addColumn(new Column("task_id", ScalarType.createVarchar(100))) .addColumn(new Column("col_name", ScalarType.createVarchar(1000))) .addColumn(new Column("message", ScalarType.createVarchar(1000))) - .addColumn(new Column("last_exec_time_in_ms", ScalarType.createVarchar(1000))) + .addColumn(new Column("last_state_change_time", ScalarType.createVarchar(1000))) + .addColumn(new Column("time_cost_in_ms", ScalarType.createVarchar(1000))) .addColumn(new Column("state", ScalarType.createVarchar(1000))).build(); private final long jobId; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 7f5d197723..59bfbada21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -2562,6 +2562,7 @@ public class ShowExecutor { LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.lastExecTimeInMs), ZoneId.systemDefault()))); row.add(analysisInfo.state.toString()); + row.add(Env.getCurrentEnv().getAnalysisManager().getJobProgress(analysisInfo.jobId)); row.add(analysisInfo.scheduleType.toString()); resultRows.add(row); } @@ -2762,6 +2763,7 @@ public class ShowExecutor { row.add(TimeUtils.DATETIME_FORMAT.format( LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.lastExecTimeInMs), ZoneId.systemDefault()))); + row.add(String.valueOf(analysisInfo.timeCostInMs)); row.add(analysisInfo.state.toString()); rows.add(row); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 9f39a82ad9..4d5d5e8851 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -20,6 +20,7 @@ package org.apache.doris.qe; import org.apache.doris.analysis.AddPartitionLikeClause; import org.apache.doris.analysis.AlterClause; import org.apache.doris.analysis.AlterTableStmt; +import org.apache.doris.analysis.AnalyzeDBStmt; import org.apache.doris.analysis.AnalyzeStmt; import org.apache.doris.analysis.AnalyzeTblStmt; import org.apache.doris.analysis.Analyzer; @@ -1149,7 +1150,7 @@ public class StmtExecutor { if (mysqlLoadId != null) { Env.getCurrentEnv().getLoadManager().getMysqlLoadManager().cancelMySqlLoad(mysqlLoadId); } - if (parsedStmt instanceof AnalyzeTblStmt) { + if (parsedStmt instanceof AnalyzeTblStmt || parsedStmt instanceof AnalyzeDBStmt) { Env.getCurrentEnv().getAnalysisManager().cancelSyncTask(context); } } @@ -2485,7 +2486,7 @@ public class StmtExecutor { analyze(context.getSessionVariable().toThrift()); } } catch (Exception e) { - throw new RuntimeException("Failed to execute internal SQL", e); + throw new RuntimeException("Failed to execute internal SQL. " + Util.getRootCauseMessage(e), e); } planner.getFragments(); RowBatch batch; @@ -2495,7 +2496,7 @@ public class StmtExecutor { QeProcessorImpl.INSTANCE.registerQuery(context.queryId(), new QeProcessorImpl.QueryInfo(context, originStmt.originStmt, coord)); } catch (UserException e) { - throw new RuntimeException("Failed to execute internal SQL", e); + throw new RuntimeException("Failed to execute internal SQL. " + Util.getRootCauseMessage(e), e); } Span queryScheduleSpan = context.getTracer() @@ -2504,7 +2505,7 @@ public class StmtExecutor { coord.exec(); } catch (Exception e) { queryScheduleSpan.recordException(e); - throw new RuntimeException("Failed to execute internal SQL", e); + throw new RuntimeException("Failed to execute internal SQL. " + Util.getRootCauseMessage(e), e); } finally { queryScheduleSpan.end(); } @@ -2521,7 +2522,7 @@ public class StmtExecutor { } } catch (Exception e) { fetchResultSpan.recordException(e); - throw new RuntimeException("Failed to execute internal SQL", e); + throw new RuntimeException("Failed to execute internal SQL. " + Util.getRootCauseMessage(e), e); } finally { fetchResultSpan.end(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java index 0f95848331..53032778cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java @@ -131,6 +131,10 @@ public class AnalysisInfo implements Writable { @SerializedName("lastExecTimeInMs") public long lastExecTimeInMs; + // finished or failed + @SerializedName("timeCostInMs") + public long timeCostInMs; + @SerializedName("state") public AnalysisState state; @@ -161,8 +165,9 @@ public class AnalysisInfo implements Writable { Map> colToPartitions, Set partitionNames, String colName, Long indexId, JobType jobType, AnalysisMode analysisMode, AnalysisMethod analysisMethod, AnalysisType analysisType, int samplePercent, int sampleRows, int maxBucketNum, long periodTimeInMs, String message, - long lastExecTimeInMs, AnalysisState state, ScheduleType scheduleType, boolean isExternalTableLevelTask, - boolean partitionOnly, boolean samplingPartition, CronExpression cronExpression) { + long lastExecTimeInMs, long timeCostInMs, AnalysisState state, ScheduleType scheduleType, + boolean isExternalTableLevelTask, boolean partitionOnly, boolean samplingPartition, + CronExpression cronExpression) { this.jobId = jobId; this.taskId = taskId; this.catalogName = catalogName; @@ -182,6 +187,7 @@ public class AnalysisInfo implements Writable { this.periodTimeInMs = periodTimeInMs; this.message = message; this.lastExecTimeInMs = lastExecTimeInMs; + this.timeCostInMs = timeCostInMs; this.state = state; this.scheduleType = scheduleType; this.externalTableLevelTask = isExternalTableLevelTask; @@ -218,6 +224,9 @@ public class AnalysisInfo implements Writable { if (lastExecTimeInMs > 0) { sj.add("LastExecTime: " + StatisticsUtil.getReadableTime(lastExecTimeInMs)); } + if (timeCostInMs > 0) { + sj.add("timeCost: " + timeCostInMs); + } if (periodTimeInMs > 0) { sj.add("periodTimeInMs: " + StatisticsUtil.getReadableTime(periodTimeInMs)); } @@ -275,6 +284,8 @@ public class AnalysisInfo implements Writable { analysisInfoBuilder.setPeriodTimeInMs(StatisticsUtil.convertStrToInt(periodTimeInMs)); String lastExecTimeInMs = resultRow.getColumnValue("last_exec_time_in_ms"); analysisInfoBuilder.setLastExecTimeInMs(StatisticsUtil.convertStrToLong(lastExecTimeInMs)); + String timeCostInMs = resultRow.getColumnValue("time_cost_in_ms"); + analysisInfoBuilder.setTimeCostInMs(StatisticsUtil.convertStrToLong(timeCostInMs)); String message = resultRow.getColumnValue("message"); analysisInfoBuilder.setMessage(message); return analysisInfoBuilder.build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java index bff4d7d69f..5efce203de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java @@ -47,6 +47,7 @@ public class AnalysisInfoBuilder { private int sampleRows; private long periodTimeInMs; private long lastExecTimeInMs; + private long timeCostInMs; private AnalysisState state; private ScheduleType scheduleType; private String message = ""; @@ -79,6 +80,7 @@ public class AnalysisInfoBuilder { maxBucketNum = info.maxBucketNum; message = info.message; lastExecTimeInMs = info.lastExecTimeInMs; + timeCostInMs = info.timeCostInMs; state = info.state; scheduleType = info.scheduleType; externalTableLevelTask = info.externalTableLevelTask; @@ -181,6 +183,11 @@ public class AnalysisInfoBuilder { return this; } + public AnalysisInfoBuilder setTimeCostInMs(long timeCostInMs) { + this.timeCostInMs = timeCostInMs; + return this; + } + public AnalysisInfoBuilder setState(AnalysisState state) { this.state = state; return this; @@ -213,7 +220,7 @@ public class AnalysisInfoBuilder { public AnalysisInfo build() { return new AnalysisInfo(jobId, taskId, catalogName, dbName, tblName, colToPartitions, partitionNames, colName, indexId, jobType, analysisMode, analysisMethod, analysisType, samplePercent, - sampleRows, maxBucketNum, periodTimeInMs, message, lastExecTimeInMs, state, scheduleType, + sampleRows, maxBucketNum, periodTimeInMs, message, lastExecTimeInMs, timeCostInMs, state, scheduleType, externalTableLevelTask, partitionOnly, samplingPartition, cronExpression); } @@ -237,6 +244,7 @@ public class AnalysisInfoBuilder { .setMaxBucketNum(maxBucketNum) .setMessage(message) .setLastExecTimeInMs(lastExecTimeInMs) + .setTimeCostInMs(timeCostInMs) .setState(state) .setScheduleType(scheduleType) .setExternalTableLevelTask(externalTableLevelTask); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index 373cc57184..cb6a3dfe5c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -43,6 +43,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.Daemon; +import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.persist.AnalyzeDeletionLog; @@ -184,10 +185,9 @@ public class AnalysisManager extends Daemon implements Writable { } TableName tableName = new TableName(db.getCatalog().getName(), db.getFullName(), table.getName()); + // columnNames null means to add all visitable columns. AnalyzeTblStmt analyzeTblStmt = new AnalyzeTblStmt(analyzeProperties, tableName, - table.getBaseSchema().stream().map( - Column::getName).collect( - Collectors.toList()), db.getId(), table); + null, db.getId(), table); try { analyzeTblStmt.check(); } catch (AnalysisException analysisException) { @@ -229,8 +229,9 @@ public class AnalysisManager extends Daemon implements Writable { Map analysisTaskInfos = new HashMap<>(); createTaskForEachColumns(jobInfo, analysisTaskInfos, isSync); createTaskForMVIdx(jobInfo, analysisTaskInfos, isSync); - createTaskForExternalTable(jobInfo, analysisTaskInfos, isSync); - + if (stmt.isAllColumns()) { + createTaskForExternalTable(jobInfo, analysisTaskInfos, isSync); + } if (!isSync) { persistAnalysisJob(jobInfo); analysisJobIdToTaskMap.put(jobInfo.jobId, analysisTaskInfos); @@ -513,12 +514,12 @@ public class AnalysisManager extends Daemon implements Writable { long taskId = Env.getCurrentEnv().getNextId(); AnalysisInfoBuilder indexTaskInfoBuilder = new AnalysisInfoBuilder(jobInfo); AnalysisInfo analysisInfo = indexTaskInfoBuilder.setIndexId(indexId) - .setTaskId(taskId).build(); + .setTaskId(taskId).setLastExecTimeInMs(System.currentTimeMillis()).build(); if (isSync) { return; } analysisTasks.put(taskId, createTask(analysisInfo)); - logCreateAnalysisJob(analysisInfo); + logCreateAnalysisTask(analysisInfo); } } finally { olapTable.readUnlock(); @@ -538,7 +539,7 @@ public class AnalysisManager extends Daemon implements Writable { colTaskInfoBuilder.setColToPartitions(Collections.singletonMap(colName, entry.getValue())); } AnalysisInfo analysisInfo = colTaskInfoBuilder.setColName(colName).setIndexId(indexId) - .setTaskId(taskId).build(); + .setTaskId(taskId).setLastExecTimeInMs(System.currentTimeMillis()).build(); analysisTasks.put(taskId, createTask(analysisInfo)); if (isSync) { continue; @@ -553,13 +554,15 @@ public class AnalysisManager extends Daemon implements Writable { } } - private void logCreateAnalysisTask(AnalysisInfo analysisInfo) { - analysisTaskInfoMap.put(analysisInfo.taskId, analysisInfo); + // Change to public for unit test. + public void logCreateAnalysisTask(AnalysisInfo analysisInfo) { + replayCreateAnalysisTask(analysisInfo); Env.getCurrentEnv().getEditLog().logCreateAnalysisTasks(analysisInfo); } - private void logCreateAnalysisJob(AnalysisInfo analysisJob) { - analysisJobInfoMap.put(analysisJob.jobId, analysisJob); + // Change to public for unit test. + public void logCreateAnalysisJob(AnalysisInfo analysisJob) { + replayCreateAnalysisJob(analysisJob); Env.getCurrentEnv().getEditLog().logCreateAnalysisJob(analysisJob); } @@ -578,63 +581,71 @@ public class AnalysisManager extends Daemon implements Writable { } AnalysisInfoBuilder colTaskInfoBuilder = new AnalysisInfoBuilder(jobInfo); long taskId = Env.getCurrentEnv().getNextId(); - AnalysisInfo analysisInfo = colTaskInfoBuilder.setIndexId(-1L) - .setTaskId(taskId).setExternalTableLevelTask(true).build(); + AnalysisInfo analysisInfo = colTaskInfoBuilder.setIndexId(-1L).setLastExecTimeInMs(System.currentTimeMillis()) + .setTaskId(taskId).setColName("TableRowCount").setExternalTableLevelTask(true).build(); analysisTasks.put(taskId, createTask(analysisInfo)); if (isSync) { // For sync job, don't need to persist, return here and execute it immediately. return; } try { - logCreateAnalysisJob(analysisInfo); + logCreateAnalysisTask(analysisInfo); } catch (Exception e) { throw new DdlException("Failed to create analysis task", e); } } - public void updateTaskStatus(AnalysisInfo info, AnalysisState jobState, String message, long time) { + public void updateTaskStatus(AnalysisInfo info, AnalysisState taskState, String message, long time) { if (analysisJobIdToTaskMap.get(info.jobId) == null) { return; } - info.state = jobState; + info.state = taskState; info.message = message; + // Update the task cost time when task finished or failed. And only log the final state. + if (taskState.equals(AnalysisState.FINISHED) || taskState.equals(AnalysisState.FAILED)) { + info.timeCostInMs = time - info.lastExecTimeInMs; + info.lastExecTimeInMs = time; + logCreateAnalysisTask(info); + } info.lastExecTimeInMs = time; - logCreateAnalysisTask(info); - AnalysisInfo job = analysisJobInfoMap.get(info.jobId); - job.lastExecTimeInMs = time; - if (info.state.equals(AnalysisState.RUNNING) && !job.state.equals(AnalysisState.PENDING)) { - job.state = AnalysisState.RUNNING; - Env.getCurrentEnv().getEditLog().logCreateAnalysisTasks(job); - } - boolean allFinished = true; - boolean hasFailure = false; - for (BaseAnalysisTask task : analysisJobIdToTaskMap.get(info.jobId).values()) { - AnalysisInfo taskInfo = task.info; - if (taskInfo.state.equals(AnalysisState.RUNNING) || taskInfo.state.equals(AnalysisState.PENDING)) { - allFinished = false; - break; + // Synchronize the job state change in job level. + synchronized (job) { + job.lastExecTimeInMs = time; + // Set the job state to RUNNING when its first task becomes RUNNING. + if (info.state.equals(AnalysisState.RUNNING) && job.state.equals(AnalysisState.PENDING)) { + job.state = AnalysisState.RUNNING; + replayCreateAnalysisJob(job); } - if (taskInfo.state.equals(AnalysisState.FAILED)) { - hasFailure = true; - } - } - if (allFinished) { - if (hasFailure) { - job.state = AnalysisState.FAILED; - logCreateAnalysisJob(job); - } else { - job.state = AnalysisState.FINISHED; - if (job.jobType.equals(JobType.SYSTEM)) { - try { - updateTableStats(job); - } catch (Throwable e) { - LOG.warn("Failed to update Table statistics in job: {}", info.toString()); - } + boolean allFinished = true; + boolean hasFailure = false; + for (BaseAnalysisTask task : analysisJobIdToTaskMap.get(info.jobId).values()) { + AnalysisInfo taskInfo = task.info; + if (taskInfo.state.equals(AnalysisState.RUNNING) || taskInfo.state.equals(AnalysisState.PENDING)) { + allFinished = false; + break; + } + if (taskInfo.state.equals(AnalysisState.FAILED)) { + hasFailure = true; } - logCreateAnalysisJob(job); } - analysisJobIdToTaskMap.remove(job.jobId); + if (allFinished) { + if (hasFailure) { + job.state = AnalysisState.FAILED; + logCreateAnalysisJob(job); + } else { + job.state = AnalysisState.FINISHED; + if (job.jobType.equals(JobType.SYSTEM)) { + try { + updateTableStats(job); + } catch (Throwable e) { + LOG.warn("Failed to update Table statistics in job: {}", info.toString(), e); + } + } + logCreateAnalysisJob(job); + } + analysisJobIdToTaskMap.remove(job.jobId); + } } } @@ -700,6 +711,28 @@ public class AnalysisManager extends Daemon implements Writable { .collect(Collectors.toList()); } + public String getJobProgress(long jobId) { + List tasks = findTasks(jobId); + int finished = 0; + int failed = 0; + int inProgress = 0; + int total = tasks.size(); + for (AnalysisInfo info : tasks) { + switch (info.state) { + case FINISHED: + finished++; + break; + case FAILED: + failed++; + break; + default: + inProgress++; + break; + } + } + return String.format("%d Finished/%d Failed/%d In Progress/%d Total", finished, failed, inProgress, total); + } + private void syncExecute(Collection tasks) { SyncTaskCollection syncTaskCollection = new SyncTaskCollection(tasks); ConnectContext ctx = ConnectContext.get(); @@ -813,8 +846,11 @@ public class AnalysisManager extends Daemon implements Writable { public void execute() { List colNames = new ArrayList<>(); + List errorMessages = new ArrayList<>(); for (BaseAnalysisTask task : tasks) { if (cancelled) { + colNames.add(task.info.colName); + errorMessages.add("Cancelled"); continue; } try { @@ -822,12 +858,14 @@ public class AnalysisManager extends Daemon implements Writable { updateSyncTaskStatus(task, AnalysisState.FINISHED); } catch (Throwable t) { colNames.add(task.info.colName); + errorMessages.add(Util.getRootCauseMessage(t)); updateSyncTaskStatus(task, AnalysisState.FAILED); LOG.warn("Failed to analyze, info: {}", task, t); } } if (!colNames.isEmpty()) { - throw new RuntimeException("Failed to analyze following columns: " + String.join(",", colNames)); + throw new RuntimeException("Failed to analyze following columns:[" + String.join(",", colNames) + + "] Reasons: " + String.join(",", errorMessages)); } } @@ -920,4 +958,9 @@ public class AnalysisManager extends Daemon implements Writable { entry.getValue().write(out); } } + + // For unit test use only. + public void addToJobIdTasksMap(long jobId, Map tasks) { + analysisJobIdToTaskMap.put(jobId, tasks); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java index 11f244afd8..b5ec7aeb87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java @@ -17,7 +17,6 @@ package org.apache.doris.statistics; -import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.ThreadPoolManager.BlockedPolicy; @@ -101,9 +100,6 @@ public class AnalysisTaskExecutor extends Thread { BaseAnalysisTask task = taskScheduler.getPendingTasks(); AnalysisTaskWrapper taskWrapper = new AnalysisTaskWrapper(this, task); executors.submit(taskWrapper); - Env.getCurrentEnv().getAnalysisManager() - .updateTaskStatus(task.info, - AnalysisState.RUNNING, "", System.currentTimeMillis()); } public void putJob(AnalysisTaskWrapper wrapper) throws Exception { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java index 9a0d6ec2fc..7f55469f53 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java @@ -18,6 +18,7 @@ package org.apache.doris.statistics; import org.apache.doris.catalog.Env; +import org.apache.doris.common.util.Util; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -62,16 +63,17 @@ public class AnalysisTaskWrapper extends FutureTask { } finally { if (!task.killed) { if (except != null) { - LOG.warn("Failed to execute task", except); + LOG.warn("Analyze {} failed.", task.toString(), except); Env.getCurrentEnv().getAnalysisManager() .updateTaskStatus(task.info, - AnalysisState.FAILED, except.getMessage(), System.currentTimeMillis()); + AnalysisState.FAILED, Util.getRootCauseMessage(except), System.currentTimeMillis()); } else { + LOG.debug("Analyze {} finished, cost time:{}", task.toString(), + System.currentTimeMillis() - startTime); Env.getCurrentEnv().getAnalysisManager() .updateTaskStatus(task.info, AnalysisState.FINISHED, "", System.currentTimeMillis()); } - LOG.warn("{} finished, cost time:{}", task.toString(), System.currentTimeMillis() - startTime); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java index c486bf36d1..e146a2e8e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java @@ -171,12 +171,20 @@ public abstract class BaseAnalysisTask { break; } catch (Throwable t) { LOG.warn("Failed to execute analysis task, retried times: {}", retriedTimes++, t); + if (retriedTimes > StatisticConstants.ANALYZE_TASK_RETRY_TIMES) { + throw new RuntimeException(t); + } } } } public abstract void doExecute() throws Exception; + protected void setTaskStateToRunning() { + Env.getCurrentEnv().getAnalysisManager() + .updateTaskStatus(info, AnalysisState.RUNNING, "", System.currentTimeMillis()); + } + public void cancel() { killed = true; if (stmtExecutor != null) { @@ -184,7 +192,7 @@ public abstract class BaseAnalysisTask { } Env.getCurrentEnv().getAnalysisManager() .updateTaskStatus(info, AnalysisState.FAILED, - String.format("Job has been cancelled: %s", info.toString()), -1); + String.format("Job has been cancelled: %s", info.message), System.currentTimeMillis()); } public long getLastExecTime() { @@ -218,4 +226,11 @@ public abstract class BaseAnalysisTask { return String.format("TABLESAMPLE(%d ROWS)", info.sampleRows); } } + + @Override + public String toString() { + return String.format("Job id [%d], Task id [%d], catalog [%s], db [%s], table [%s], column [%s]", + info.jobId, info.taskId, catalog.getName(), db.getFullName(), tbl.getName(), + col == null ? "TableRowCount" : col.getName()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java index 87e7bbe1b5..03840f091d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.external.HMSExternalTable; import org.apache.doris.common.FeConstants; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.qe.AutoCloseConnectContext; +import org.apache.doris.qe.QueryState; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.statistics.util.InternalQueryResult; import org.apache.doris.statistics.util.StatisticsUtil; @@ -105,6 +106,7 @@ public class HMSAnalysisTask extends BaseAnalysisTask { } public void doExecute() throws Exception { + setTaskStateToRunning(); if (isTableLevelTask) { getTableStats(); } else { @@ -190,11 +192,7 @@ public class HMSAnalysisTask extends BaseAnalysisTask { params.put("dataSizeFunction", getDataSizeFunction(col)); StringSubstitutor stringSubstitutor = new StringSubstitutor(params); String sql = stringSubstitutor.replace(sb.toString()); - try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) { - r.connectContext.getSessionVariable().disableNereidsPlannerOnce(); - this.stmtExecutor = new StmtExecutor(r.connectContext, sql); - this.stmtExecutor.execute(); - } + executeInsertSql(sql); } } else { StringBuilder sb = new StringBuilder(); @@ -233,12 +231,27 @@ public class HMSAnalysisTask extends BaseAnalysisTask { params.put("dataSizeFunction", getDataSizeFunction(col)); StringSubstitutor stringSubstitutor = new StringSubstitutor(params); String sql = stringSubstitutor.replace(sb.toString()); - try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) { - r.connectContext.getSessionVariable().disableNereidsPlannerOnce(); - this.stmtExecutor = new StmtExecutor(r.connectContext, sql); - this.stmtExecutor.execute(); + executeInsertSql(sql); + Env.getCurrentEnv().getStatisticsCache().refreshColStatsSync( + catalog.getId(), db.getId(), tbl.getId(), -1, col.getName()); + } + } + + private void executeInsertSql(String sql) throws Exception { + long startTime = System.currentTimeMillis(); + try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext()) { + r.connectContext.getSessionVariable().disableNereidsPlannerOnce(); + this.stmtExecutor = new StmtExecutor(r.connectContext, sql); + r.connectContext.setExecutor(stmtExecutor); + this.stmtExecutor.execute(); + QueryState queryState = r.connectContext.getState(); + if (queryState.getStateType().equals(QueryState.MysqlStateType.ERR)) { + LOG.warn(String.format("Failed to analyze %s.%s.%s, sql: [%s], error: [%s]", + info.catalogName, info.dbName, info.colName, sql, queryState.getErrorMessage())); + throw new RuntimeException(queryState.getErrorMessage()); } - Env.getCurrentEnv().getStatisticsCache().refreshColStatsSync(tbl.getId(), -1, col.getName()); + LOG.debug(String.format("Analyze %s.%s.%s done. SQL: [%s]. Cost %d ms.", + info.catalogName, info.dbName, info.colName, sql, (System.currentTimeMillis() - startTime))); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java index a0b6e6d9a5..a1b53cb3be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java @@ -59,6 +59,7 @@ public class HistogramTask extends BaseAnalysisTask { @Override public void doExecute() throws Exception { + setTaskStateToRunning(); Map params = new HashMap<>(); params.put("internalDB", FeConstants.INTERNAL_DB_NAME); params.put("histogramStatTbl", StatisticConstants.HISTOGRAM_TBL_NAME); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java index 31b3b76d5d..ed33252172 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java @@ -87,6 +87,7 @@ public class MVAnalysisTask extends BaseAnalysisTask { @Override public void doExecute() throws Exception { + setTaskStateToRunning(); for (Column column : meta.getSchema()) { SelectStmt selectOne = (SelectStmt) selectStmt.clone(); TableRef tableRef = selectOne.getTableRefs().get(0); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java index fab9f68197..a980385bde 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java @@ -60,6 +60,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask { } public void doExecute() throws Exception { + setTaskStateToRunning(); Map params = new HashMap<>(); params.put("internalDB", FeConstants.INTERNAL_DB_NAME); params.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java index 284083ac01..1149ecdd5a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java @@ -169,7 +169,11 @@ public class StatisticsCache { } public void refreshColStatsSync(long tblId, long idxId, String colName) { - columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(tblId, idxId, colName)); + columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(-1, -1, tblId, idxId, colName)); + } + + public void refreshColStatsSync(long catalogId, long dbId, long tblId, long idxId, String colName) { + columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(catalogId, dbId, tblId, idxId, colName)); } public void refreshHistogramSync(long tblId, long idxId, String colName) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index e7c40a9ce4..207c34fee2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -174,6 +174,8 @@ public class StatisticsUtil { sessionVariable.parallelPipelineTaskNum = Config.statistics_sql_parallel_exec_instance_num; sessionVariable.setEnableNereidsPlanner(false); sessionVariable.enableProfile = false; + sessionVariable.queryTimeoutS = Config.analyze_task_timeout_in_hours * 60 * 60; + sessionVariable.insertTimeoutS = Config.analyze_task_timeout_in_hours * 60 * 60; connectContext.setEnv(Env.getCurrentEnv()); connectContext.setDatabase(FeConstants.INTERNAL_DB_NAME); connectContext.setQualifiedUser(UserIdentity.ROOT.getQualifiedUser()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java new file mode 100644 index 0000000000..253f9c9332 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics; + +import mockit.Mock; +import mockit.MockUp; +import mockit.Mocked; +import org.junit.Test; +import org.junit.jupiter.api.Assertions; + +import java.util.HashMap; +import java.util.Map; + +public class AnalysisManagerTest { + @Test + public void testUpdateTaskStatus(@Mocked BaseAnalysisTask task1, + @Mocked BaseAnalysisTask task2) { + + new MockUp() { + @Mock + public void logCreateAnalysisTask(AnalysisInfo job) { + } + + @Mock + public void logCreateAnalysisJob(AnalysisInfo job) { + } + + }; + + AnalysisInfo job = new AnalysisInfoBuilder().setJobId(1) + .setState(AnalysisState.PENDING).setJobType(AnalysisInfo.JobType.MANUAL).build(); + AnalysisInfo taskInfo1 = new AnalysisInfoBuilder().setJobId(1) + .setTaskId(2).setState(AnalysisState.PENDING).build(); + AnalysisInfo taskInfo2 = new AnalysisInfoBuilder().setJobId(1) + .setTaskId(3).setState(AnalysisState.PENDING).build(); + AnalysisManager manager = new AnalysisManager(); + manager.replayCreateAnalysisJob(job); + manager.replayCreateAnalysisTask(taskInfo1); + manager.replayCreateAnalysisTask(taskInfo2); + Map tasks = new HashMap<>(); + + task1.info = taskInfo1; + task2.info = taskInfo2; + tasks.put(2L, task1); + tasks.put(3L, task2); + manager.addToJobIdTasksMap(1, tasks); + + Assertions.assertEquals(job.state, AnalysisState.PENDING); + manager.updateTaskStatus(taskInfo1, AnalysisState.RUNNING, "", 0); + Assertions.assertEquals(job.state, AnalysisState.RUNNING); + manager.updateTaskStatus(taskInfo2, AnalysisState.RUNNING, "", 0); + Assertions.assertEquals(job.state, AnalysisState.RUNNING); + manager.updateTaskStatus(taskInfo1, AnalysisState.FINISHED, "", 0); + Assertions.assertEquals(job.state, AnalysisState.RUNNING); + manager.updateTaskStatus(taskInfo2, AnalysisState.FINISHED, "", 0); + Assertions.assertEquals(job.state, AnalysisState.FINISHED); + } +} diff --git a/regression-test/data/external_table_emr_p2/hive/test_hive_analyze_db.out b/regression-test/data/external_table_emr_p2/hive/test_hive_analyze_db.out deleted file mode 100644 index 0b75316476..0000000000 --- a/regression-test/data/external_table_emr_p2/hive/test_hive_analyze_db.out +++ /dev/null @@ -1,20 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !1 -- -lo_orderpriority 100.0 5.0 0.0 888.8000000000001 8.8 '1-URGENT' '5-LOW' -lo_custkey 100.0 26.0 0.0 404.0 4.0 67423 2735521 -lo_partkey 100.0 100.0 0.0 404.0 4.0 2250 989601 -lo_revenue 100.0 100.0 0.0 404.0 4.0 101171 8703450 -lo_commitdate 100.0 95.0 0.0 404.0 4.0 19920515 19981016 -lo_quantity 100.0 46.0 0.0 404.0 4.0 1 50 -lo_orderkey 100.0 26.0 0.0 404.0 4.0 1 98 -lo_suppkey 100.0 100.0 0.0 404.0 4.0 4167 195845 -lo_supplycost 100.0 100.0 0.0 404.0 4.0 58023 121374 -lo_shipmode 100.0 7.0 0.0 425.21 4.21 'AIR' 'TRUCK' -lo_orderdate 100.0 26.0 0.0 404.0 4.0 19920221 19980721 -lo_linenumber 100.0 7.0 0.0 404.0 4.0 1 7 -lo_shippriority 100.0 1.0 0.0 404.0 4.0 0 0 -lo_ordtotalprice 100.0 26.0 0.0 404.0 4.0 3428256 36771805 -lo_extendedprice 100.0 100.0 0.0 404.0 4.0 104300 9066094 -lo_tax 100.0 9.0 0.0 404.0 4.0 0 8 -lo_discount 100.0 11.0 0.0 404.0 4.0 0 10 - diff --git a/regression-test/data/external_table_emr_p2/hive/test_hive_statistic.out b/regression-test/data/external_table_emr_p2/hive/test_hive_statistic.out deleted file mode 100644 index ad8b494c00..0000000000 --- a/regression-test/data/external_table_emr_p2/hive/test_hive_statistic.out +++ /dev/null @@ -1,57 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !1 -- -lo_quantity 100.0 46.0 0.0 404.0 4.0 1 50 - --- !2 -- -lo_orderkey 100.0 26.0 0.0 404.0 4.0 1 98 - --- !3 -- -lo_linenumber 100.0 7.0 0.0 404.0 4.0 1 7 - --- !4 -- -lo_custkey 100.0 26.0 0.0 404.0 4.0 67423 2735521 - --- !5 -- -lo_partkey 100.0 100.0 0.0 404.0 4.0 2250 989601 - --- !6 -- -lo_suppkey 100.0 100.0 0.0 404.0 4.0 4167 195845 - --- !7 -- -lo_orderdate 100.0 26.0 0.0 404.0 4.0 19920221 19980721 - --- !8 -- -lo_orderpriority 100.0 5.0 0.0 888.8000000000001 8.8 '1-URGENT' '5-LOW' - --- !9 -- -lo_shippriority 100.0 1.0 0.0 404.0 4.0 0 0 - --- !10 -- -lo_extendedprice 100.0 100.0 0.0 404.0 4.0 104300 9066094 - --- !11 -- -lo_ordtotalprice 100.0 26.0 0.0 404.0 4.0 3428256 36771805 - --- !12 -- -lo_discount 100.0 11.0 0.0 404.0 4.0 0 10 - --- !13 -- -lo_revenue 100.0 100.0 0.0 404.0 4.0 101171 8703450 - --- !14 -- -lo_supplycost 100.0 100.0 0.0 404.0 4.0 58023 121374 - --- !15 -- -lo_tax 100.0 9.0 0.0 404.0 4.0 0 8 - --- !16 -- -lo_commitdate 100.0 95.0 0.0 404.0 4.0 19920515 19981016 - --- !17 -- -lo_shipmode 100.0 7.0 0.0 425.21 4.21 'AIR' 'TRUCK' - --- !18 -- -lo_shipmode 6001215.0 0.0 0.0 0.0 0.0 'NULL' 'NULL' - --- !19 -- - diff --git a/regression-test/data/external_table_emr_p2/hive/test_hive_statistic_cache.out b/regression-test/data/external_table_emr_p2/hive/test_hive_statistic_cache.out deleted file mode 100644 index dc5ded3837..0000000000 --- a/regression-test/data/external_table_emr_p2/hive/test_hive_statistic_cache.out +++ /dev/null @@ -1,58 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !1 -- -100 - --- !2 -- -lo_orderkey 100.0 26.0 0.0 404.0 4.0 1 98 - --- !3 -- -lo_linenumber 100.0 6.0 0.0 404.0 4.0 1 7 - --- !4 -- -lo_custkey 100.0 30.0 0.0 404.0 4.0 67423 2735521 - --- !5 -- -lo_partkey 100.0 107.0 0.0 404.0 4.0 2250 989601 - --- !6 -- -lo_suppkey 100.0 90.0 0.0 404.0 4.0 4167 195845 - --- !7 -- -lo_orderdate 100.0 22.0 0.0 404.0 4.0 19920221 19980721 - --- !8 -- -lo_orderpriority 100.0 5.0 0.0 888.8000000000001 8.8 N/A N/A - --- !9 -- -lo_shippriority 100.0 2.0 0.0 404.0 4.0 0 0 - --- !10 -- -lo_extendedprice 100.0 112.0 0.0 404.0 4.0 104300 9066094 - --- !11 -- -lo_ordtotalprice 100.0 31.0 0.0 404.0 4.0 3428256 36771805 - --- !12 -- -lo_discount 100.0 11.0 0.0 404.0 4.0 0 10 - --- !13 -- -lo_revenue 100.0 127.0 0.0 404.0 4.0 101171 8703450 - --- !14 -- -lo_supplycost 100.0 112.0 0.0 404.0 4.0 58023 121374 - --- !15 -- -lo_tax 100.0 9.0 0.0 404.0 4.0 0 8 - --- !16 -- -lo_commitdate 100.0 122.0 0.0 404.0 4.0 19920515 19981016 - --- !17 -- -lo_shipmode 100.0 7.0 0.0 425.21 4.21 N/A N/A - --- !18 -- -lo_orderkey 100.0 26.0 0.0 404.0 4.0 1 98 - --- !19 -- -lo_quantity 100.0 34.0 0.0 404.0 4.0 1 50 - diff --git a/regression-test/suites/external_table_emr_p2/hive/test_hive_analyze_db.groovy b/regression-test/suites/external_table_emr_p2/hive/test_hive_analyze_db.groovy index a30c87275b..fd724a67f8 100644 --- a/regression-test/suites/external_table_emr_p2/hive/test_hive_analyze_db.groovy +++ b/regression-test/suites/external_table_emr_p2/hive/test_hive_analyze_db.groovy @@ -33,8 +33,163 @@ suite("test_hive_analyze_db", "p2") { sql """switch ${catalog_name};""" logger.info("switched to catalog " + catalog_name) sql """use statistics;""" + sql """set query_timeout=300""" sql """analyze database statistics with sync""" - qt_1 "show column stats statistics" + def result = sql """show column stats statistics""" + assertTrue(result.size() == 17) + + assertTrue(result[0][0] == "lo_orderpriority") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "5.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "888.8000000000001") + assertTrue(result[0][5] == "8.8") + assertTrue(result[0][6] == "'1-URGENT'") + assertTrue(result[0][7] == "'5-LOW'") + + assertTrue(result[1][0] == "lo_custkey") + assertTrue(result[1][1] == "100.0") + assertTrue(result[1][2] == "26.0") + assertTrue(result[1][3] == "0.0") + assertTrue(result[1][4] == "404.0") + assertTrue(result[1][5] == "4.0") + assertTrue(result[1][6] == "67423") + assertTrue(result[1][7] == "2735521") + + assertTrue(result[2][0] == "lo_partkey") + assertTrue(result[2][1] == "100.0") + assertTrue(result[2][2] == "100.0") + assertTrue(result[2][3] == "0.0") + assertTrue(result[2][4] == "404.0") + assertTrue(result[2][5] == "4.0") + assertTrue(result[2][6] == "2250") + assertTrue(result[2][7] == "989601") + + assertTrue(result[3][0] == "lo_revenue") + assertTrue(result[3][1] == "100.0") + assertTrue(result[3][2] == "100.0") + assertTrue(result[3][3] == "0.0") + assertTrue(result[3][4] == "404.0") + assertTrue(result[3][5] == "4.0") + assertTrue(result[3][6] == "101171") + assertTrue(result[3][7] == "8703450") + + assertTrue(result[4][0] == "lo_commitdate") + assertTrue(result[4][1] == "100.0") + assertTrue(result[4][2] == "95.0") + assertTrue(result[4][3] == "0.0") + assertTrue(result[4][4] == "404.0") + assertTrue(result[4][5] == "4.0") + assertTrue(result[4][6] == "19920515") + assertTrue(result[4][7] == "19981016") + + assertTrue(result[5][0] == "lo_quantity") + assertTrue(result[5][1] == "100.0") + assertTrue(result[5][2] == "46.0") + assertTrue(result[5][3] == "0.0") + assertTrue(result[5][4] == "404.0") + assertTrue(result[5][5] == "4.0") + assertTrue(result[5][6] == "1") + assertTrue(result[5][7] == "50") + + assertTrue(result[6][0] == "lo_orderkey") + assertTrue(result[6][1] == "100.0") + assertTrue(result[6][2] == "26.0") + assertTrue(result[6][3] == "0.0") + assertTrue(result[6][4] == "404.0") + assertTrue(result[6][5] == "4.0") + assertTrue(result[6][6] == "1") + assertTrue(result[6][7] == "98") + + assertTrue(result[7][0] == "lo_suppkey") + assertTrue(result[7][1] == "100.0") + assertTrue(result[7][2] == "100.0") + assertTrue(result[7][3] == "0.0") + assertTrue(result[7][4] == "404.0") + assertTrue(result[7][5] == "4.0") + assertTrue(result[7][6] == "4167") + assertTrue(result[7][7] == "195845") + + assertTrue(result[8][0] == "lo_supplycost") + assertTrue(result[8][1] == "100.0") + assertTrue(result[8][2] == "100.0") + assertTrue(result[8][3] == "0.0") + assertTrue(result[8][4] == "404.0") + assertTrue(result[8][5] == "4.0") + assertTrue(result[8][6] == "58023") + assertTrue(result[8][7] == "121374") + + assertTrue(result[9][0] == "lo_shipmode") + assertTrue(result[9][1] == "100.0") + assertTrue(result[9][2] == "7.0") + assertTrue(result[9][3] == "0.0") + assertTrue(result[9][4] == "425.21") + assertTrue(result[9][5] == "4.21") + assertTrue(result[9][6] == "'AIR'") + assertTrue(result[9][7] == "'TRUCK'") + + assertTrue(result[10][0] == "lo_orderdate") + assertTrue(result[10][1] == "100.0") + assertTrue(result[10][2] == "26.0") + assertTrue(result[10][3] == "0.0") + assertTrue(result[10][4] == "404.0") + assertTrue(result[10][5] == "4.0") + assertTrue(result[10][6] == "19920221") + assertTrue(result[10][7] == "19980721") + + assertTrue(result[11][0] == "lo_linenumber") + assertTrue(result[11][1] == "100.0") + assertTrue(result[11][2] == "7.0") + assertTrue(result[11][3] == "0.0") + assertTrue(result[11][4] == "404.0") + assertTrue(result[11][5] == "4.0") + assertTrue(result[11][6] == "1") + assertTrue(result[11][7] == "7") + + assertTrue(result[12][0] == "lo_shippriority") + assertTrue(result[12][1] == "100.0") + assertTrue(result[12][2] == "1.0") + assertTrue(result[12][3] == "0.0") + assertTrue(result[12][4] == "404.0") + assertTrue(result[12][5] == "4.0") + assertTrue(result[12][6] == "0") + assertTrue(result[12][7] == "0") + + assertTrue(result[13][0] == "lo_ordtotalprice") + assertTrue(result[13][1] == "100.0") + assertTrue(result[13][2] == "26.0") + assertTrue(result[13][3] == "0.0") + assertTrue(result[13][4] == "404.0") + assertTrue(result[13][5] == "4.0") + assertTrue(result[13][6] == "3428256") + assertTrue(result[13][7] == "36771805") + + assertTrue(result[14][0] == "lo_extendedprice") + assertTrue(result[14][1] == "100.0") + assertTrue(result[14][2] == "100.0") + assertTrue(result[14][3] == "0.0") + assertTrue(result[14][4] == "404.0") + assertTrue(result[14][5] == "4.0") + assertTrue(result[14][6] == "104300") + assertTrue(result[14][7] == "9066094") + + assertTrue(result[15][0] == "lo_tax") + assertTrue(result[15][1] == "100.0") + assertTrue(result[15][2] == "9.0") + assertTrue(result[15][3] == "0.0") + assertTrue(result[15][4] == "404.0") + assertTrue(result[15][5] == "4.0") + assertTrue(result[15][6] == "0") + assertTrue(result[15][7] == "8") + + assertTrue(result[16][0] == "lo_discount") + assertTrue(result[16][1] == "100.0") + assertTrue(result[16][2] == "11.0") + assertTrue(result[16][3] == "0.0") + assertTrue(result[16][4] == "404.0") + assertTrue(result[16][5] == "4.0") + assertTrue(result[16][6] == "0") + assertTrue(result[16][7] == "10") } } diff --git a/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic.groovy b/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic.groovy index ff184e4506..c6df0b0eaa 100644 --- a/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic.groovy +++ b/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic.groovy @@ -34,29 +34,202 @@ suite("test_hive_statistic", "p2") { logger.info("switched to catalog " + catalog_name) sql """use statistics;""" sql """analyze table `statistics` with sync""" - qt_1 "show column stats `statistics` (lo_quantity)" - qt_2 "show column stats `statistics` (lo_orderkey)" - qt_3 "show column stats `statistics` (lo_linenumber)" - qt_4 "show column stats `statistics` (lo_custkey)" - qt_5 "show column stats `statistics` (lo_partkey)" - qt_6 "show column stats `statistics` (lo_suppkey)" - qt_7 "show column stats `statistics` (lo_orderdate)" - qt_8 "show column stats `statistics` (lo_orderpriority)" - qt_9 "show column stats `statistics` (lo_shippriority)" - qt_10 "show column stats `statistics` (lo_extendedprice)" - qt_11 "show column stats `statistics` (lo_ordtotalprice)" - qt_12 "show column stats `statistics` (lo_discount)" - qt_13 "show column stats `statistics` (lo_revenue)" - qt_14 "show column stats `statistics` (lo_supplycost)" - qt_15 "show column stats `statistics` (lo_tax)" - qt_16 "show column stats `statistics` (lo_commitdate)" - qt_17 "show column stats `statistics` (lo_shipmode)" + def result = sql """show column stats `statistics` (lo_quantity)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_quantity") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "46.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "1") + assertTrue(result[0][7] == "50") - sql """ALTER TABLE statistics MODIFY COLUMN lo_shipmode SET STATS ('row_count'='6001215')""" - qt_18 "show column stats `statistics` (lo_shipmode)" + result = sql """show column stats `statistics` (lo_orderkey)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_orderkey") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "26.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "1") + assertTrue(result[0][7] == "98") + + result = sql """show column stats `statistics` (lo_linenumber)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_linenumber") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "7.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "1") + assertTrue(result[0][7] == "7") + + result = sql """show column stats `statistics` (lo_custkey)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_custkey") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "26.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "67423") + assertTrue(result[0][7] == "2735521") + + result = sql """show column stats `statistics` (lo_partkey)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_partkey") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "2250") + assertTrue(result[0][7] == "989601") + + result = sql """show column stats `statistics` (lo_suppkey)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_suppkey") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "4167") + assertTrue(result[0][7] == "195845") + + result = sql """show column stats `statistics` (lo_orderdate)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_orderdate") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "26.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "19920221") + assertTrue(result[0][7] == "19980721") + + result = sql """show column stats `statistics` (lo_orderpriority)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_orderpriority") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "5.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "888.8000000000001") + assertTrue(result[0][5] == "8.8") + assertTrue(result[0][6] == "'1-URGENT'") + assertTrue(result[0][7] == "'5-LOW'") + + result = sql """show column stats `statistics` (lo_shippriority)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_shippriority") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "1.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "0") + assertTrue(result[0][7] == "0") + + result = sql """show column stats `statistics` (lo_extendedprice)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_extendedprice") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "104300") + assertTrue(result[0][7] == "9066094") + + result = sql """show column stats `statistics` (lo_ordtotalprice)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_ordtotalprice") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "26.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "3428256") + assertTrue(result[0][7] == "36771805") + + result = sql """show column stats `statistics` (lo_discount)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_discount") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "11.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "0") + assertTrue(result[0][7] == "10") + + result = sql """show column stats `statistics` (lo_revenue)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_revenue") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "101171") + assertTrue(result[0][7] == "8703450") + + result = sql """show column stats `statistics` (lo_supplycost)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_supplycost") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "58023") + assertTrue(result[0][7] == "121374") + + result = sql """show column stats `statistics` (lo_tax)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_tax") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "9.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "0") + assertTrue(result[0][7] == "8") + + result = sql """show column stats `statistics` (lo_commitdate)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_commitdate") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "95.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "19920515") + assertTrue(result[0][7] == "19981016") + + result = sql """show column stats `statistics` (lo_shipmode)""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == "lo_shipmode") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "7.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "425.21") + assertTrue(result[0][5] == "4.21") + assertTrue(result[0][6] == "'AIR'") + assertTrue(result[0][7] == "'TRUCK'") + + // sql """ALTER TABLE statistics MODIFY COLUMN lo_shipmode SET STATS ('row_count'='6001215')""" + // result = sql "show column stats `statistics` (lo_shipmode)" + // assertTrue(result.size() == 1) + // assertTrue(result[0][0] == "lo_shipmode") + // assertTrue(result[0][1] == "6001215.0") sql """drop stats statistics""" - qt_19 "show column stats statistics" + result = sql """show column stats statistics""" + assertTrue(result.size() == 0) } } diff --git a/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic_cache.groovy b/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic_cache.groovy index 2fb574a0da..d1399ef49b 100644 --- a/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic_cache.groovy +++ b/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic_cache.groovy @@ -33,26 +33,179 @@ suite("test_hive_statistic_cache", "p2") { sql """switch ${catalog_name};""" logger.info("switched to catalog " + catalog_name) sql """use statistics;""" - qt_1 "select count(*) from stats" + sql """set query_timeout=300;""" + sql """analyze table `stats` with sync;""" + sql """select count(*) from stats""" Thread.sleep(5000); - qt_2 "show column cached stats `stats` (lo_orderkey)" - qt_3 "show column cached stats `stats` (lo_linenumber)" - qt_4 "show column cached stats `stats` (lo_custkey)" - qt_5 "show column cached stats `stats` (lo_partkey)" - qt_6 "show column cached stats `stats` (lo_suppkey)" - qt_7 "show column cached stats `stats` (lo_orderdate)" - qt_8 "show column cached stats `stats` (lo_orderpriority)" - qt_9 "show column cached stats `stats` (lo_shippriority)" - qt_10 "show column cached stats `stats` (lo_extendedprice)" - qt_11 "show column cached stats `stats` (lo_ordtotalprice)" - qt_12 "show column cached stats `stats` (lo_discount)" - qt_13 "show column cached stats `stats` (lo_revenue)" - qt_14 "show column cached stats `stats` (lo_supplycost)" - qt_15 "show column cached stats `stats` (lo_tax)" - qt_16 "show column cached stats `stats` (lo_commitdate)" - qt_17 "show column cached stats `stats` (lo_shipmode)" - qt_18 "show column cached stats `stats` (lo_orderkey)" - qt_19 "show column cached stats `stats` (lo_quantity)" + def result = sql """show column cached stats `stats` (lo_orderkey)""" + assertTrue(result[0][0] == "lo_orderkey") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "26.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "1") + assertTrue(result[0][7] == "98") + + result = sql """show column cached stats `stats` (lo_linenumber)""" + assertTrue(result[0][0] == "lo_linenumber") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "7.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "1") + assertTrue(result[0][7] == "7") + + result = sql """show column cached stats `stats` (lo_custkey)""" + assertTrue(result[0][0] == "lo_custkey") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "26.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "67423") + assertTrue(result[0][7] == "2735521") + + result = sql """show column cached stats `stats` (lo_partkey)""" + assertTrue(result[0][0] == "lo_partkey") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "2250") + assertTrue(result[0][7] == "989601") + + result = sql """show column cached stats `stats` (lo_suppkey)""" + assertTrue(result[0][0] == "lo_suppkey") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "4167") + assertTrue(result[0][7] == "195845") + + result = sql """show column cached stats `stats` (lo_orderdate)""" + assertTrue(result[0][0] == "lo_orderdate") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "26.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "19920221") + assertTrue(result[0][7] == "19980721") + + result = sql """show column cached stats `stats` (lo_orderpriority)""" + assertTrue(result[0][0] == "lo_orderpriority") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "5.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "888.8000000000001") + assertTrue(result[0][5] == "8.8") + assertTrue(result[0][6] == "'1-URGENT'") + assertTrue(result[0][7] == "'5-LOW'") + + result = sql """show column cached stats `stats` (lo_shippriority)""" + assertTrue(result[0][0] == "lo_shippriority") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "1.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "0") + assertTrue(result[0][7] == "0") + + result = sql """show column cached stats `stats` (lo_extendedprice)""" + assertTrue(result[0][0] == "lo_extendedprice") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "104300") + assertTrue(result[0][7] == "9066094") + + result = sql """show column cached stats `stats` (lo_ordtotalprice)""" + assertTrue(result[0][0] == "lo_ordtotalprice") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "26.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "3428256") + assertTrue(result[0][7] == "36771805") + + result = sql """show column cached stats `stats` (lo_discount)""" + assertTrue(result[0][0] == "lo_discount") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "11.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "0") + assertTrue(result[0][7] == "10") + + result = sql """show column cached stats `stats` (lo_revenue)""" + assertTrue(result[0][0] == "lo_revenue") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "101171") + assertTrue(result[0][7] == "8703450") + + result = sql """show column cached stats `stats` (lo_supplycost)""" + assertTrue(result[0][0] == "lo_supplycost") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "100.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "58023") + assertTrue(result[0][7] == "121374") + + result = sql """show column cached stats `stats` (lo_tax)""" + assertTrue(result[0][0] == "lo_tax") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "9.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "0") + assertTrue(result[0][7] == "8") + + result = sql """show column cached stats `stats` (lo_commitdate)""" + assertTrue(result[0][0] == "lo_commitdate") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "95.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "19920515") + assertTrue(result[0][7] == "19981016") + + result = sql """show column cached stats `stats` (lo_shipmode)""" + assertTrue(result[0][0] == "lo_shipmode") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "7.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "425.21") + assertTrue(result[0][5] == "4.21") + assertTrue(result[0][6] == "'AIR'") + assertTrue(result[0][7] == "'TRUCK'") + + result = sql """show column cached stats `stats` (lo_quantity)""" + assertTrue(result[0][0] == "lo_quantity") + assertTrue(result[0][1] == "100.0") + assertTrue(result[0][2] == "46.0") + assertTrue(result[0][3] == "0.0") + assertTrue(result[0][4] == "404.0") + assertTrue(result[0][5] == "4.0") + assertTrue(result[0][6] == "1") + assertTrue(result[0][7] == "50") } }