From 642f1c7d8498506378160f964cc0fe524464f5e2 Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 28 Apr 2023 13:11:58 +0000 Subject: [PATCH] [FEAT MERGE] optimizer statistics gather enhance Co-authored-by: Larry955 <1412857955@qq.com> Co-authored-by: wangt1xiuyi <13547954130@163.com> --- deps/oblib/src/lib/stat/ob_latch_define.h | 1 + deps/oblib/src/lib/wait_event/ob_wait_event.h | 1 + src/observer/CMakeLists.txt | 1 + src/observer/table/ob_table_context.cpp | 5 +- .../table_load/ob_table_load_coordinator.cpp | 12 +- ...ob_all_virtual_opt_stat_gather_monitor.cpp | 178 ++++ .../ob_all_virtual_opt_stat_gather_monitor.h | 67 ++ .../ob_virtual_table_iterator_factory.cpp | 12 + src/pl/sys_package/ob_dbms_stats.cpp | 464 ++++++---- src/pl/sys_package/ob_dbms_stats.h | 42 +- src/share/CMakeLists.txt | 3 +- .../ob_inner_table_schema.12351_12400.cpp | 826 ++++++++++++++++++ .../ob_inner_table_schema.15251_15300.cpp | 817 +++++++++++++++++ .../ob_inner_table_schema.21251_21300.cpp | 2 +- .../ob_inner_table_schema.21351_21400.cpp | 200 +++++ .../ob_inner_table_schema.21401_21450.cpp | 50 ++ .../ob_inner_table_schema.25151_25200.cpp | 6 +- .../ob_inner_table_schema.25201_25250.cpp | 150 ++++ .../ob_inner_table_schema.28151_28200.cpp | 100 +++ .../ob_inner_table_schema.451_500.cpp | 593 +++++++++++++ .../ob_inner_table_schema.50451_50500.cpp | 300 +++++++ .../ob_inner_table_schema.60451_60500.cpp | 210 +++++ src/share/inner_table/ob_inner_table_schema.h | 125 ++- .../inner_table/ob_inner_table_schema.lob.cpp | 2 +- .../ob_inner_table_schema_constants.h | 44 + .../inner_table/ob_inner_table_schema_def.py | 701 ++++++++++++++- .../ob_inner_table_schema_misc.ipp | 40 + src/share/schema/ob_schema_struct.cpp | 15 +- src/share/schema/ob_schema_struct.h | 20 +- src/share/schema/ob_table_schema.cpp | 11 +- src/share/schema/ob_table_schema.h | 37 +- src/share/stat/ob_basic_stats_estimator.cpp | 174 +++- src/share/stat/ob_basic_stats_estimator.h | 10 + src/share/stat/ob_dbms_stats_executor.cpp | 77 +- src/share/stat/ob_dbms_stats_executor.h | 6 +- .../stat/ob_dbms_stats_export_import.cpp | 2 +- .../stat/ob_dbms_stats_history_manager.cpp | 49 +- .../stat/ob_dbms_stats_history_manager.h | 1 - src/share/stat/ob_dbms_stats_utils.cpp | 77 +- src/share/stat/ob_dbms_stats_utils.h | 2 +- src/share/stat/ob_hybrid_hist_estimator.cpp | 110 +++ src/share/stat/ob_hybrid_hist_estimator.h | 16 + .../stat/ob_incremental_stat_estimator.cpp | 65 +- .../stat/ob_incremental_stat_estimator.h | 26 +- src/share/stat/ob_index_stats_estimator.cpp | 2 +- src/share/stat/ob_opt_column_stat.cpp | 289 +++--- src/share/stat/ob_opt_column_stat.h | 66 +- src/share/stat/ob_opt_osg_column_stat.cpp | 304 +++++++ src/share/stat/ob_opt_osg_column_stat.h | 111 +++ src/share/stat/ob_opt_stat_gather_stat.cpp | 328 +++++++ src/share/stat/ob_opt_stat_gather_stat.h | 234 +++++ src/share/stat/ob_opt_stat_manager.cpp | 10 + src/share/stat/ob_opt_stat_manager.h | 3 + src/share/stat/ob_opt_stat_service.h | 1 + src/share/stat/ob_opt_stat_sql_service.cpp | 147 +++- src/share/stat/ob_opt_stat_sql_service.h | 10 + src/share/stat/ob_stat_define.cpp | 69 ++ src/share/stat/ob_stat_define.h | 62 +- src/share/stat/ob_stat_item.cpp | 20 +- src/share/stat/ob_stat_item.h | 7 +- src/share/stat/ob_topk_hist_estimator.cpp | 14 +- src/share/stat/ob_topk_hist_estimator.h | 10 +- src/share/table/ob_table_load_define.cpp | 39 +- src/share/table/ob_table_load_define.h | 80 +- src/sql/CMakeLists.txt | 1 + .../code_generator/ob_static_engine_cg.cpp | 5 +- src/sql/das/ob_das_context.cpp | 18 +- src/sql/das/ob_das_context.h | 4 +- src/sql/das/ob_das_define.cpp | 6 +- src/sql/das/ob_das_define.h | 12 +- src/sql/das/ob_das_location_router.cpp | 97 +- src/sql/das/ob_das_location_router.h | 28 +- .../aggregate/ob_aggregate_processor.cpp | 470 ++++++++-- .../engine/aggregate/ob_aggregate_processor.h | 65 +- src/sql/engine/aggregate/ob_groupby_op.h | 3 +- src/sql/engine/basic/ob_chunk_datum_store.cpp | 66 +- src/sql/engine/basic/ob_chunk_datum_store.h | 6 + src/sql/engine/basic/ob_material_op.cpp | 178 +--- src/sql/engine/basic/ob_material_op.h | 23 +- src/sql/engine/basic/ob_material_op_impl.cpp | 309 +++++++ src/sql/engine/basic/ob_material_op_impl.h | 124 +++ src/sql/engine/cmd/ob_analyze_executor.cpp | 37 +- src/sql/engine/expr/ob_expr_uuid.cpp | 5 - .../ob_optimizer_stats_gathering_op.cpp | 388 ++++---- .../ob_optimizer_stats_gathering_op.h | 26 +- .../components/ob_dh_opt_stats_gather.cpp | 39 +- .../components/ob_dh_opt_stats_gather.h | 11 +- src/sql/engine/table/ob_table_scan_op.cpp | 61 +- src/sql/engine/table/ob_table_scan_op.h | 1 + .../window_function/ob_window_function_op.h | 2 +- src/sql/ob_sql.cpp | 5 + src/sql/optimizer/ob_insert_log_plan.cpp | 54 +- src/sql/optimizer/ob_insert_log_plan.h | 6 +- src/sql/optimizer/ob_log_del_upd.cpp | 3 - .../ob_log_optimizer_stats_gathering.cpp | 6 +- .../ob_log_optimizer_stats_gathering.h | 7 - src/sql/optimizer/ob_log_plan.cpp | 48 + src/sql/optimizer/ob_log_plan.h | 7 + src/sql/optimizer/ob_log_table_scan.h | 6 + src/sql/optimizer/ob_logical_operator.cpp | 2 +- src/sql/optimizer/ob_logical_operator.h | 6 +- src/sql/optimizer/ob_opt_selectivity.cpp | 34 +- src/sql/optimizer/ob_opt_selectivity.h | 4 - src/sql/optimizer/ob_optimizer.cpp | 6 +- src/sql/optimizer/ob_optimizer_util.cpp | 78 +- src/sql/optimizer/ob_optimizer_util.h | 7 +- .../optimizer/ob_phy_table_location_info.cpp | 13 +- .../optimizer/ob_phy_table_location_info.h | 13 +- src/sql/optimizer/ob_table_location.cpp | 80 +- src/sql/optimizer/ob_table_location.h | 2 + src/sql/optimizer/ob_table_partition_info.cpp | 24 - src/sql/optimizer/ob_table_partition_info.h | 6 - src/sql/plan_cache/ob_plan_cache_util.cpp | 2 + src/sql/resolver/dml/ob_select_resolver.cpp | 2 +- src/sql/resolver/dml/ob_view_table_resolver.h | 16 +- src/sql/resolver/ob_resolver_define.h | 1 + src/sql/rewrite/ob_expand_aggregate_utils.cpp | 2 +- src/storage/blocksstable/ob_datum_row.h | 1 + .../ob_direct_load_fast_heap_table.cpp | 43 +- .../ob_direct_load_fast_heap_table.h | 7 +- ...ob_direct_load_fast_heap_table_builder.cpp | 39 +- .../ob_direct_load_fast_heap_table_builder.h | 6 +- .../direct_load/ob_direct_load_merge_ctx.cpp | 78 +- .../direct_load/ob_direct_load_merge_ctx.h | 3 +- .../ob_direct_load_partition_merge_task.cpp | 50 +- .../ob_direct_load_partition_merge_task.h | 7 +- .../ob_direct_load_table_store.cpp | 1 + .../r/mysql/inner_table_overall.result | 10 + unittest/share/test_ob_column_stat.cpp | 1 - unittest/sql/optimizer/ob_mock_stat_manager.h | 8 +- 130 files changed, 8572 insertions(+), 1563 deletions(-) create mode 100644 src/observer/virtual_table/ob_all_virtual_opt_stat_gather_monitor.cpp create mode 100644 src/observer/virtual_table/ob_all_virtual_opt_stat_gather_monitor.h create mode 100644 src/share/inner_table/ob_inner_table_schema.451_500.cpp create mode 100644 src/share/inner_table/ob_inner_table_schema.50451_50500.cpp create mode 100644 src/share/inner_table/ob_inner_table_schema.60451_60500.cpp create mode 100644 src/share/stat/ob_opt_osg_column_stat.cpp create mode 100644 src/share/stat/ob_opt_osg_column_stat.h create mode 100644 src/share/stat/ob_opt_stat_gather_stat.cpp create mode 100644 src/share/stat/ob_opt_stat_gather_stat.h create mode 100644 src/sql/engine/basic/ob_material_op_impl.cpp create mode 100644 src/sql/engine/basic/ob_material_op_impl.h diff --git a/deps/oblib/src/lib/stat/ob_latch_define.h b/deps/oblib/src/lib/stat/ob_latch_define.h index b8c7348f85..832496f0ef 100644 --- a/deps/oblib/src/lib/stat/ob_latch_define.h +++ b/deps/oblib/src/lib/stat/ob_latch_define.h @@ -311,6 +311,7 @@ LATCH_DEF(CDC_SERVICE_LS_CTX_LOCK, 298, "cdcservice clientlsctx lock", LATCH_FIF LATCH_DEF(MAJOR_FREEZE_DIAGNOSE_LOCK, 299, "major freeze diagnose lock", LATCH_READ_PREFER, 2000, 0, MAJOR_FREEZE_DIAGNOSE_LOCK_WAIT, "major freeze diagnose lock") LATCH_DEF(HB_RESPONSES_LOCK, 300, "hb responses lock", LATCH_READ_PREFER, 2000, 0, HB_RESPONSES_LOCK_WAIT, "hb responses lock") LATCH_DEF(ALL_SERVERS_INFO_IN_TABLE_LOCK, 301, "all servers info in table lock", LATCH_READ_PREFER, 2000, 0, ALL_SERVERS_INFO_IN_TABLE_LOCK_WAIT, "all servers info in table lock") +LATCH_DEF(OPT_STAT_GATHER_STAT_LOCK, 302, "optimizer stat gather stat lock", LATCH_FIFO, 2000, 0, OPT_STAT_GATHER_STAT_LOCK_WAIT, "optimizer stat gather stat lock") LATCH_DEF(LATCH_END, 99999, "latch end", LATCH_FIFO, 2000, 0, WAIT_EVENT_END, "latch end") #endif diff --git a/deps/oblib/src/lib/wait_event/ob_wait_event.h b/deps/oblib/src/lib/wait_event/ob_wait_event.h index dbd0e5f06f..d78e084aa4 100644 --- a/deps/oblib/src/lib/wait_event/ob_wait_event.h +++ b/deps/oblib/src/lib/wait_event/ob_wait_event.h @@ -302,6 +302,7 @@ WAIT_EVENT_DEF(SQL_WF_PARTICIPATOR_COND_WAIT, 15256, "mutex: window function par WAIT_EVENT_DEF(MAJOR_FREEZE_DIAGNOSE_LOCK_WAIT, 15257, "latch: major_freeze diagnose lock wait", "address", "number", "tries", CONCURRENCY, "latch: major_freeze diagnose lock wait", true) WAIT_EVENT_DEF(HB_RESPONSES_LOCK_WAIT, 15258, "latch: hb responses lock wait", "address", "number", "tries", CONCURRENCY, "latch: hb responses lock wait", true) WAIT_EVENT_DEF(ALL_SERVERS_INFO_IN_TABLE_LOCK_WAIT, 15259, "latch: all servers info in table lock wait", "address", "number", "tries", CONCURRENCY, "latch: all servers info in table lock wait", true) +WAIT_EVENT_DEF(OPT_STAT_GATHER_STAT_LOCK_WAIT, 15260, "latch: optimizer stat gather stat lock wait", "address", "number", "tries", CONCURRENCY, "latch: optimizer stat gather stat lock wait", true) //transaction WAIT_EVENT_DEF(END_TRANS_WAIT, 16001, "wait end trans", "rollback", "trans_hash_value", "participant_count", COMMIT,"wait end trans", false) diff --git a/src/observer/CMakeLists.txt b/src/observer/CMakeLists.txt index 6b2afb279c..4d7799ce1f 100644 --- a/src/observer/CMakeLists.txt +++ b/src/observer/CMakeLists.txt @@ -363,6 +363,7 @@ ob_set_subtarget(ob_server virtual_table virtual_table/ob_virtual_span_info.cpp virtual_table/ob_virtual_show_trace.cpp virtual_table/ob_all_virtual_sql_plan.cpp + virtual_table/ob_all_virtual_opt_stat_gather_monitor.cpp virtual_table/ob_all_virtual_thread.cpp ) diff --git a/src/observer/table/ob_table_context.cpp b/src/observer/table/ob_table_context.cpp index ccb59fe1ea..39f786e6a5 100644 --- a/src/observer/table/ob_table_context.cpp +++ b/src/observer/table/ob_table_context.cpp @@ -983,6 +983,7 @@ int ObTableCtx::init_das_context(ObDASCtx &das_ctx) ObTableID related_table_id = related_index_ids_.at(i); const ObSimpleTableSchemaV2 *relative_table_schema = nullptr; ObObjectID related_part_id = OB_INVALID_ID; + ObObjectID related_first_level_part_id = OB_INVALID_ID; ObTabletID related_tablet_id; if (OB_FAIL(schema_guard_.get_simple_table_schema(tenant_id_, related_table_id, @@ -994,12 +995,14 @@ int ObTableCtx::init_das_context(ObDASCtx &das_ctx) } else if (OB_FAIL(relative_table_schema->get_part_id_and_tablet_id_by_idx(part_idx, subpart_idx, related_part_id, + related_first_level_part_id, related_tablet_id))) { LOG_WARN("get part by idx failed", K(ret), K(part_idx), K(subpart_idx), K(related_table_id)); } else if (OB_FAIL(related_tablet_map.add_related_tablet_id(tablet_id_, related_table_id, related_tablet_id, - related_part_id))) { + related_part_id, + related_first_level_part_id))) { LOG_WARN("fail to add related tablet id", K(ret), K(tablet_id_), K(related_table_id), K(related_part_id), K(related_tablet_id)); } diff --git a/src/observer/table_load/ob_table_load_coordinator.cpp b/src/observer/table_load/ob_table_load_coordinator.cpp index 9d0925b199..0aa348a230 100644 --- a/src/observer/table_load/ob_table_load_coordinator.cpp +++ b/src/observer/table_load/ob_table_load_coordinator.cpp @@ -742,11 +742,13 @@ int ObTableLoadCoordinator::drive_sql_stat(ObExecContext &ctx, ObTableLoadSqlSta ctx.get_sql_ctx()->schema_guard_ = &schema_guard; ctx.get_das_ctx().get_schema_guard() = &schema_guard; } - if (OB_SUCC(ret)) { - if (OB_FAIL(ObIncrementalStatEstimator::drive_global_stat_by_direct_load( - ctx, sql_statistics.table_stat_array_, sql_statistics.col_stat_array_))) { - LOG_WARN("fail to drive global stat by direct load", KR(ret)); - } + ObSEArray part_column_stats; + if (OB_FAIL(ret)) { + } else if (OB_FAIL(sql_statistics.get_col_stat_array(part_column_stats))) { + LOG_WARN("failed to get column stat array"); + } else if (OB_FAIL(ObIncrementalStatEstimator::derive_global_stat_by_direct_load( + ctx, sql_statistics.table_stat_array_, part_column_stats))) { + LOG_WARN("fail to drive global stat by direct load", KR(ret)); } ctx.get_sql_ctx()->schema_guard_ = tmp_schema_guard; ctx.get_das_ctx().get_schema_guard() = tmp_schema_guard2; diff --git a/src/observer/virtual_table/ob_all_virtual_opt_stat_gather_monitor.cpp b/src/observer/virtual_table/ob_all_virtual_opt_stat_gather_monitor.cpp new file mode 100644 index 0000000000..8bf3bbfee4 --- /dev/null +++ b/src/observer/virtual_table/ob_all_virtual_opt_stat_gather_monitor.cpp @@ -0,0 +1,178 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "observer/virtual_table/ob_all_virtual_opt_stat_gather_monitor.h" +#include "observer/ob_server_utils.h" + +namespace oceanbase +{ +using namespace common; +using namespace sql; +namespace observer +{ + +ObAllVirtualOptStatGatherMonitor::ObAllVirtualOptStatGatherMonitor(): + addr_(NULL), + start_to_read_(false), + stat_array_(), + index_(0), + ipstr_(), + port_(0) +{ + MEMSET(svr_ip_, 0, sizeof(svr_ip_)); +} + +ObAllVirtualOptStatGatherMonitor::~ObAllVirtualOptStatGatherMonitor() +{ + reset(); +} + +int ObAllVirtualOptStatGatherMonitor::set_ip() +{ + int ret = OB_SUCCESS; + MEMSET(svr_ip_, 0, sizeof(svr_ip_)); + if (NULL == addr_) { + ret = OB_ENTRY_NOT_EXIST; + } else if (!addr_->ip_to_string(svr_ip_, sizeof(svr_ip_))) { + SERVER_LOG(ERROR, "ip to string failed"); + ret = OB_ERR_UNEXPECTED; + } else { + ipstr_ = ObString::make_string(svr_ip_); + port_ = addr_->get_port(); + } + return ret; +} + +void ObAllVirtualOptStatGatherMonitor::reset() +{ + addr_ = NULL; + start_to_read_ = false; + stat_array_.reset(); + index_ = 0; + ipstr_.reset(); + port_ = 0; + MEMSET(svr_ip_, 0, sizeof(svr_ip_)); +} + +int ObAllVirtualOptStatGatherMonitor::inner_get_next_row(ObNewRow *&row) +{ + int ret = OB_SUCCESS; + ObObj *cells = cur_row_.cells_; + if (OB_ISNULL(allocator_) || OB_ISNULL(addr_)) { + ret = OB_NOT_INIT; + SERVER_LOG(WARN, "allocator_ or addr_ is null", K_(allocator), K_(addr), K(ret)); + } else if (OB_ISNULL(cells)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "cur row cell is NULL", K(ret)); + } else { + if (!start_to_read_) { + if (OB_FAIL(set_ip())) { + SERVER_LOG(WARN, "failed to set ip", K(ret)); + } else if (OB_FAIL(ObOptStatGatherStatList::instance().list_to_array(*allocator_, + effective_tenant_id_, + stat_array_))) { + SERVER_LOG(WARN, "failed to list to array", K(ret)); + } + } + if (OB_SUCC(ret) && index_ >= stat_array_.size()) { + ret = OB_ITER_END; + } + for (int64_t cell_idx = 0; + OB_SUCC(ret) && cell_idx < output_column_ids_.count(); + ++cell_idx) { + const uint64_t column_id = output_column_ids_.at(cell_idx); + switch(column_id) { + case TENANT_ID: { + cells[cell_idx].set_int(stat_array_.at(index_).get_tenant_id()); + break; + } + case SVR_IP: { + cells[cell_idx].set_varchar(ipstr_); + cells[cell_idx].set_collation_type( + ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + } + case SVR_PORT: { + cells[cell_idx].set_int(port_); + break; + } + case SESSION_ID: { + cells[cell_idx].set_int(stat_array_.at(index_).get_session_id()); + break; + } + case TRACE_ID: { + cells[cell_idx].set_varchar(stat_array_.at(index_).get_trace_id()); + cells[cell_idx].set_collation_type( + ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + } + case TASK_ID: { + cells[cell_idx].set_varchar(stat_array_.at(index_).get_task_id()); + cells[cell_idx].set_collation_type( + ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + } + case TYPE: { + cells[cell_idx].set_int(stat_array_.at(index_).get_type()); + break; + } + case TASK_START_TIME: { + cells[cell_idx].set_timestamp(stat_array_.at(index_).get_task_start_time()); + break; + } + case TASK_TABLE_COUNT: { + cells[cell_idx].set_int(stat_array_.at(index_).get_task_table_count()); + break; + } + case TASK_DURATION_TIME: { + cells[cell_idx].set_int(stat_array_.at(index_).get_task_duration_time()); + break; + } + case COMPLETED_TABLE_COUNT: { + cells[cell_idx].set_int(stat_array_.at(index_).get_completed_table_count()); + break; + } + case RUNNING_TABLE_OWNER: { + cells[cell_idx].set_varchar(stat_array_.at(index_).get_database_name()); + cells[cell_idx].set_collation_type( + ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + } + case RUNNING_TABLE_NAME: { + cells[cell_idx].set_varchar(stat_array_.at(index_).get_table_name()); + cells[cell_idx].set_collation_type( + ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + } + case RUNNING_TABLE_DURATION_TIME: { + cells[cell_idx].set_int(stat_array_.at(index_).get_running_table_duration_time()); + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "invalid column id", K(cell_idx), + K_(output_column_ids), K(ret)); + break; + } + } + } + ++index_; + } + if (OB_SUCC(ret)) { + start_to_read_ = true; + row = &cur_row_; + } + return ret; +} + +}/* ns observer*/ +}/* ns oceanbase */ \ No newline at end of file diff --git a/src/observer/virtual_table/ob_all_virtual_opt_stat_gather_monitor.h b/src/observer/virtual_table/ob_all_virtual_opt_stat_gather_monitor.h new file mode 100644 index 0000000000..71238b748d --- /dev/null +++ b/src/observer/virtual_table/ob_all_virtual_opt_stat_gather_monitor.h @@ -0,0 +1,67 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_ALL_VIRTUAL_OPT_STAT_GATHER_STAT_H +#define OB_ALL_VIRTUAL_OPT_STAT_GATHER_STAT_H + +#include "share/ob_virtual_table_scanner_iterator.h" +#include "lib/net/ob_addr.h" +#include "share/stat/ob_opt_stat_gather_stat.h" + +namespace oceanbase +{ + +namespace observer +{ + +class ObAllVirtualOptStatGatherMonitor : public common::ObVirtualTableScannerIterator +{ +public: + ObAllVirtualOptStatGatherMonitor(); + virtual ~ObAllVirtualOptStatGatherMonitor(); + virtual void reset(); + virtual int inner_get_next_row(common::ObNewRow *&row); + inline void set_addr(common::ObAddr &addr) { addr_ = &addr; } + int set_ip(); +private: + common::ObAddr *addr_; + bool start_to_read_; + common::ObArray stat_array_; + int64_t index_; + common::ObString ipstr_; + int32_t port_; + char svr_ip_[common::MAX_IP_ADDR_LENGTH + 2]; + enum COLUMNS + { + TENANT_ID = common::OB_APP_MIN_COLUMN_ID, + SVR_IP, + SVR_PORT, + SESSION_ID, + TRACE_ID, + TASK_ID, + TYPE, + TASK_START_TIME, + TASK_TABLE_COUNT, + TASK_DURATION_TIME, + COMPLETED_TABLE_COUNT, + RUNNING_TABLE_OWNER, + RUNNING_TABLE_NAME, + RUNNING_TABLE_DURATION_TIME + }; +private: + DISALLOW_COPY_AND_ASSIGN(ObAllVirtualOptStatGatherMonitor); +}; + +}// namespace observer +}// namespace oceanbase + +#endif /* !OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_H */ diff --git a/src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp b/src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp index 329bc3f137..99ef38c742 100644 --- a/src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp +++ b/src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp @@ -196,6 +196,7 @@ #include "observer/virtual_table/ob_all_virtual_archive_dest_status.h" #include "observer/virtual_table/ob_virtual_show_trace.h" #include "observer/virtual_table/ob_all_virtual_sql_plan.h" +#include "observer/virtual_table/ob_all_virtual_opt_stat_gather_monitor.h" #include "observer/virtual_table/ob_all_virtual_thread.h" namespace oceanbase @@ -2324,6 +2325,17 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam ¶ms, } break; } + case OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_TID: { + ObAllVirtualOptStatGatherMonitor *opt_stats_gather_stat = NULL; + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObAllVirtualOptStatGatherMonitor, opt_stats_gather_stat))) { + SERVER_LOG(ERROR, "failed to init ObAllVirtualOptStatGatherMonitor", K(ret)); + } else { + opt_stats_gather_stat->set_allocator(&allocator); + opt_stats_gather_stat->set_addr(addr_); + vt_iter = static_cast(opt_stats_gather_stat); + } + break; + } END_CREATE_VT_ITER_SWITCH_LAMBDA #define AGENT_VIRTUAL_TABLE_CREATE_ITER diff --git a/src/pl/sys_package/ob_dbms_stats.cpp b/src/pl/sys_package/ob_dbms_stats.cpp index 4188780d2f..ef34b80d71 100644 --- a/src/pl/sys_package/ob_dbms_stats.cpp +++ b/src/pl/sys_package/ob_dbms_stats.cpp @@ -29,6 +29,8 @@ #include "sql/das/ob_das_location_router.h" #include "sql/ob_sql_utils.h" #include "storage/ob_locality_manager.h" +#include "share/stat/ob_opt_stat_gather_stat.h" +#include "sql/engine/expr/ob_expr_uuid.h" namespace oceanbase { @@ -36,7 +38,6 @@ using namespace sql; using namespace common; using namespace share::schema; namespace pl { - /** * @brief ObDbmsStat::gather_table_stats * @param ctx @@ -69,51 +70,72 @@ int ObDbmsStats::gather_table_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb stat_param.allocator_ = &ctx.get_allocator(); bool is_all_fast_gather = false; ObSEArray no_gather_index_ids; + ObOptStatTaskInfo task_info; + int64_t task_cnt = 1; + int64_t start_time = ObTimeUtility::current_time(); if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); } else if (OB_ISNULL(ctx.get_my_session()) || OB_ISNULL(ctx.get_task_executor_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(ctx.get_my_session()), K(ctx.get_task_executor_ctx())); - } else if (OB_FAIL(parse_table_part_info(ctx, - params.at(0), - params.at(1), - params.at(2), - stat_param))) { - LOG_WARN("failed to parse owner", K(ret)); - } else if (OB_FAIL(parse_gather_stat_options(ctx, - params.at(3), - params.at(4), - params.at(5), - params.at(6), - params.at(7), - params.at(8), - params.at(12), - params.at(14), - stat_param))) { - LOG_WARN("failed to parse stat optitions", K(ret)); - } else if (stat_param.force_ && - OB_FAIL(ObDbmsStatsLockUnlock::fill_stat_locked(ctx, stat_param))) { - LOG_WARN("failed fill stat locked", K(ret)); - } else if (!stat_param.force_ && - OB_FAIL(ObDbmsStatsLockUnlock::check_stat_locked(ctx, stat_param))) { - LOG_WARN("failed check stat locked", K(ret)); - } else if (OB_FAIL(ObOptStatMonitorManager::flush_database_monitoring_info(ctx, false, true))) { - LOG_WARN("failed to do flush database monitoring info", K(ret)); - } else if (OB_FAIL(ObDbmsStatsExecutor::gather_table_stats(ctx, stat_param))) { - LOG_WARN("failed to gather table stats", K(ret)); - } else if (OB_FAIL(update_stat_cache(ctx.get_my_session()->get_rpc_tenant_id(), stat_param))) { - LOG_WARN("failed to update stat cache", K(ret)); - } else if (is_virtual_table(stat_param.table_id_)) {//not gather virtual table index. - //do nothing - } else if (stat_param.cascade_ && - OB_FAIL(fast_gather_index_stats(ctx, stat_param, - is_all_fast_gather, no_gather_index_ids))) { - LOG_WARN("failed to fast gather index stats", K(ret)); - } else if (stat_param.cascade_ && !is_all_fast_gather && - OB_FAIL(gather_table_index_stats(ctx, stat_param, no_gather_index_ids))) { - LOG_WARN("failed to gather table index stats", K(ret)); + } else if (OB_FAIL(init_gather_task_info(ctx, ObOptStatGatherType::MANUAL_GATHER, start_time, task_cnt, task_info))) { + LOG_WARN("failed to init gather task info", K(ret)); } else { - LOG_TRACE("Succeed to gather table stats", K(stat_param)); + ObOptStatGatherStat gather_stat(task_info); + ObOptStatGatherStatList::instance().push(gather_stat); + ObOptStatRunningMonitor running_monitor(ctx.get_allocator(), start_time, stat_param.allocator_->used(), gather_stat); + if (OB_FAIL(parse_table_part_info(ctx, + params.at(0), + params.at(1), + params.at(2), + stat_param))) { + LOG_WARN("failed to parse owner", K(ret)); + } else if (OB_FAIL(parse_gather_stat_options(ctx, + params.at(3), + params.at(4), + params.at(5), + params.at(6), + params.at(7), + params.at(8), + params.at(12), + params.at(14), + stat_param))) { + LOG_WARN("failed to parse stat optitions", K(ret)); + } else if (OB_FAIL(running_monitor.add_table_info(stat_param))) { + LOG_WARN("failed to add table info", K(ret)); + } else if (stat_param.force_ && + OB_FAIL(ObDbmsStatsLockUnlock::fill_stat_locked(ctx, stat_param))) { + LOG_WARN("failed fill stat locked", K(ret)); + } else if (!stat_param.force_ && + OB_FAIL(ObDbmsStatsLockUnlock::check_stat_locked(ctx, stat_param))) { + LOG_WARN("failed check stat locked", K(ret)); + } else if (OB_FAIL(ObOptStatMonitorManager::flush_database_monitoring_info(ctx, false, true))) { + LOG_WARN("failed to do flush database monitoring info", K(ret)); + } else if (OB_FAIL(ObDbmsStatsExecutor::gather_table_stats(ctx, stat_param))) { + LOG_WARN("failed to gather table stats", K(ret)); + } else if (OB_FAIL(update_stat_cache(ctx.get_my_session()->get_rpc_tenant_id(), + stat_param, + &running_monitor))) { + LOG_WARN("failed to update stat cache", K(ret)); + } else if (is_virtual_table(stat_param.table_id_)) {//not gather virtual table index. + //do nothing + } else if (stat_param.cascade_ && + OB_FAIL(fast_gather_index_stats(ctx, stat_param, + is_all_fast_gather, no_gather_index_ids))) { + LOG_WARN("failed to fast gather index stats", K(ret)); + } else if (stat_param.cascade_ && !is_all_fast_gather && + OB_FAIL(gather_table_index_stats(ctx, stat_param, no_gather_index_ids))) { + LOG_WARN("failed to gather table index stats", K(ret)); + } else { + LOG_TRACE("Succeed to gather table stats", K(stat_param)); + } + running_monitor.set_monitor_result(ret, ObTimeUtility::current_time(), stat_param.allocator_->used()); + task_info.task_end_time_ = ObTimeUtility::current_time(); + task_info.ret_code_ = ret; + task_info.failed_count_ = ret == OB_SUCCESS ? 0 : 1; + ObOptStatManager::get_instance().update_opt_stat_task_stat(task_info); + ObOptStatManager::get_instance().update_opt_stat_gather_stat(gather_stat); + ObOptStatGatherStatList::instance().remove(gather_stat); } return ret; } @@ -145,6 +167,8 @@ int ObDbmsStats::gather_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO ObTableStatParam global_param; global_param.allocator_ = &ctx.get_allocator(); ObSEArray table_ids; + ObOptStatTaskInfo task_info; + int64_t start_time = ObTimeUtility::current_time(); if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); } else if (OB_ISNULL(ctx.get_my_session()) || OB_ISNULL(ctx.get_task_executor_ctx())) { @@ -154,34 +178,45 @@ int ObDbmsStats::gather_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO LOG_WARN("failed to do flush database monitoring info", K(ret)); } else if (OB_FAIL(get_all_table_ids_in_database(ctx, params.at(0), global_param, table_ids))) { LOG_WARN("failed to get all table ids in database", K(ret)); + } else if (table_ids.empty()) { + //do nothing + } else if (OB_FAIL(init_gather_task_info(ctx, ObOptStatGatherType::MANUAL_GATHER, start_time, table_ids.count(), task_info))) { + LOG_WARN("failed to init gather task info", K(ret)); } else { - ObArenaAllocator tmp_alloc("OptStatGather", OB_MALLOC_NORMAL_BLOCK_SIZE, global_param.tenant_id_); - for (int64_t i = 0; OB_SUCC(ret) && i < table_ids.count(); ++i) { + int64_t i = 0; + for (; OB_SUCC(ret) && i < table_ids.count(); ++i) { StatTable stat_table; stat_table.database_id_ = global_param.db_id_; stat_table.table_id_ = table_ids.at(i); ObTableStatParam stat_param = global_param; + ObArenaAllocator tmp_alloc("OptStatGather", OB_MALLOC_NORMAL_BLOCK_SIZE, global_param.tenant_id_); stat_param.allocator_ = &tmp_alloc;//use the temp allocator to free memory after gather stats. bool is_all_fast_gather = false; ObSEArray no_gather_index_ids; + int64_t start_time = ObTimeUtility::current_time(); + ObOptStatGatherStat gather_stat(task_info); + ObOptStatGatherStatList::instance().push(gather_stat); + ObOptStatRunningMonitor running_monitor(ctx.get_allocator(), start_time, stat_param.allocator_->used(), gather_stat); if (OB_FAIL(parse_table_part_info(ctx, stat_table, stat_param))) { LOG_WARN("failed to parse table part info", K(ret)); } else if (OB_FAIL(parse_gather_stat_options(ctx, - params.at(1), - params.at(2), - params.at(3), - params.at(4), - params.at(5), - params.at(6), - params.at(10), - params.at(12), - stat_param))) { + params.at(1), + params.at(2), + params.at(3), + params.at(4), + params.at(5), + params.at(6), + params.at(10), + params.at(12), + stat_param))) { LOG_WARN("failed to parse stat optitions", K(ret)); + } else if (OB_FAIL(running_monitor.add_table_info(stat_param))) { + LOG_WARN("failed to add table info", K(ret)); } else if (stat_param.force_ && - OB_FAIL(ObDbmsStatsLockUnlock::fill_stat_locked(ctx, stat_param))) { - LOG_WARN("failed fill stat locked", K(ret)); + OB_FAIL(ObDbmsStatsLockUnlock::fill_stat_locked(ctx, stat_param))) { + LOG_WARN("failed fill stat locked", K(ret)); } else if (!stat_param.force_ && - OB_FAIL(ObDbmsStatsLockUnlock::check_stat_locked(ctx, stat_param))) { + OB_FAIL(ObDbmsStatsLockUnlock::check_stat_locked(ctx, stat_param))) { if (OB_ERR_DBMS_STATS_PL == ret) { // all table/partition locked, just skip ret = OB_SUCCESS; @@ -191,22 +226,31 @@ int ObDbmsStats::gather_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO } } else if (OB_FAIL(ObDbmsStatsExecutor::gather_table_stats(ctx, stat_param))) { LOG_WARN("failed to gather table stats", K(ret)); - } else if (OB_FAIL(update_stat_cache(ctx.get_my_session()->get_rpc_tenant_id(), stat_param))) { + } else if (OB_FAIL(update_stat_cache(ctx.get_my_session()->get_rpc_tenant_id(), + stat_param, + &running_monitor))) { LOG_WARN("failed to update stat cache", K(ret)); } else if (is_virtual_table(stat_param.table_id_)) {//not gather virtual table index. tmp_alloc.reset(); } else if (stat_param.cascade_ && - OB_FAIL(fast_gather_index_stats(ctx, stat_param, - is_all_fast_gather, no_gather_index_ids))) { + OB_FAIL(fast_gather_index_stats(ctx, stat_param, + is_all_fast_gather, no_gather_index_ids))) { LOG_WARN("failed to fast gather index stats", K(ret)); } else if (stat_param.cascade_ && !is_all_fast_gather && - OB_FAIL(gather_table_index_stats(ctx, stat_param, no_gather_index_ids))) { + OB_FAIL(gather_table_index_stats(ctx, stat_param, no_gather_index_ids))) { LOG_WARN("failed to gather table index stats", K(ret)); } else { - tmp_alloc.reset(); - LOG_TRACE("Succeed to gather table stats", K(stat_param)); + LOG_TRACE("Succeed to gather table stats", K(stat_param), K(running_monitor)); } + running_monitor.set_monitor_result(ret, ObTimeUtility::current_time(), stat_param.allocator_->used()); + ObOptStatManager::get_instance().update_opt_stat_gather_stat(gather_stat); + ObOptStatGatherStatList::instance().remove(gather_stat); + task_info.completed_table_count_ ++; } + task_info.task_end_time_ = ObTimeUtility::current_time(); + task_info.ret_code_ = ret; + task_info.failed_count_ = ret == OB_SUCCESS ? 0 : table_ids.count() - i + 1; + ObOptStatManager::get_instance().update_opt_stat_task_stat(task_info); } return ret; } @@ -2899,7 +2943,8 @@ int ObDbmsStats::delete_table_prefs(sql::ObExecContext &ctx, } int ObDbmsStats::update_stat_cache(const uint64_t rpc_tenant_id, - const ObTableStatParam ¶m) + const ObTableStatParam ¶m, + ObOptStatRunningMonitor *running_monitor/*default null*/) { int ret = OB_SUCCESS; obrpc::ObUpdateStatCacheArg stat_arg; @@ -2956,13 +3001,37 @@ int ObDbmsStats::update_stat_cache(const uint64_t rpc_tenant_id, .update_local_stat_cache(stat_arg))) { LOG_WARN("failed to update local stat cache caused by unknow error", K(ret), K(all_server_arr.at(i).get_addr()), K(stat_arg)); - //ignore flush cache failed, TODO @jiangxiu.wt can aduit it and flush cache manually later if (OB_FAIL(failed_server_arr.push_back(all_server_arr.at(i)))) { LOG_WARN("failed to push back", K(ret)); } } } LOG_TRACE("update stat cache", K(param), K(stat_arg), K(failed_server_arr), K(all_server_arr)); + if (OB_SUCC(ret) && !failed_server_arr.empty() && running_monitor != NULL) { + ObSqlString tmp_str; + char *buf = NULL; + if (failed_server_arr.count() * (common::MAX_IP_ADDR_LENGTH + 1) <= common::MAX_VALUE_LENGTH) { + for (int64_t i = 0; OB_SUCC(ret) && i < failed_server_arr.count(); ++i) { + char svr_buf[common::MAX_IP_ADDR_LENGTH] = {0}; + failed_server_arr.at(i).get_addr().to_string(svr_buf, common::MAX_IP_ADDR_LENGTH); + if (OB_FAIL(tmp_str.append_fmt("%s%s", svr_buf, i == 0 ? "" : ","))) { + LOG_WARN("failed to append fmt", K(ret)); + } + } + } else if (OB_FAIL(tmp_str.append_fmt("more than %ld servers refresh stat cache failed", + failed_server_arr.count()))) { + LOG_WARN("failed to append fmt", K(ret)); + } + if (OB_FAIL(ret)) { + //do nothing + } else if (OB_ISNULL(buf = static_cast(running_monitor->allocator_.alloc(tmp_str.length())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("memory is not enough", K(ret), K(tmp_str)); + } else { + MEMCPY(buf, tmp_str.ptr(), tmp_str.length()); + running_monitor->opt_stat_gather_stat_.set_stat_refresh_failed_list(buf, tmp_str.length()); + } + } } } return ret; @@ -5063,72 +5132,44 @@ int ObDbmsStats::gather_database_stats_job_proc(sql::ObExecContext &ctx, common::ObObj &result) { int ret = OB_SUCCESS; - UNUSED(params); UNUSED(result); - ObArray user_missing_tables; - ObArray user_stale_tables; - ObArray sys_missing_tables; - ObArray sys_stale_tables; - number::ObNumber num_duration; - //duration_time to is used to mark the gather database stats job can use max time. default value - //is -1, it's meaning gather until all table have been gathered. - int64_t duration_time = -1; const int64_t start_time = ObTimeUtility::current_time(); - int64_t failed_count = 0; - int64_t succeed_count = 0; + ObOptStatTaskInfo task_info; + ObGatherTableStatsHelper helper; if (OB_FAIL(check_statistic_table_writeable(ctx))) { ret = OB_SUCCESS; LOG_INFO("auto gather database statistics abort because of statistic table is unwriteable"); - } else if (lib::is_oracle_mode() && !params.at(0).is_null() && - OB_FAIL(params.at(0).get_number(num_duration))) { - LOG_WARN("failed to get duration", K(ret), K(params.at(0))); - } else if (lib::is_oracle_mode() && !params.at(0).is_null() && - OB_FAIL(num_duration.extract_valid_int64_with_trunc(duration_time))) { - LOG_WARN("extract_valid_int64_with_trunc failed", K(ret), K(num_duration)); - } else if (lib::is_mysql_mode() && !params.at(0).is_null() && - OB_FAIL(params.at(0).get_int(duration_time))) { - LOG_WARN("failed to get duration", K(ret), K(params.at(0))); } else if (OB_FAIL(ObOptStatMonitorManager::flush_database_monitoring_info(ctx))) { LOG_WARN("failed to flush database monitoring info", K(ret)); - } else if (OB_FAIL(get_need_statistics_tables(ctx, user_missing_tables, user_stale_tables, - sys_missing_tables, sys_stale_tables))) { + } else if (OB_FAIL(helper.get_duration_time(params))) { + LOG_WARN("failed to get duration time"); + } else if (OB_FAIL(get_need_statistics_tables(ctx, helper))) { LOG_WARN("failed to get need statistics tables", K(ret)); - } else if (OB_FAIL(gather_tables_stats_with_default_param(ctx, false, start_time, duration_time, - user_missing_tables, succeed_count, - failed_count))) { - LOG_WARN("failed to gather tables stats with default param", K(ret)); - } else if (OB_FAIL(gather_tables_stats_with_default_param(ctx, true, start_time, duration_time, - user_stale_tables, succeed_count, - failed_count))) { - LOG_WARN("failed to gather tables stats with default param", K(ret)); - } else if (OB_FAIL(gather_tables_stats_with_default_param(ctx, false, start_time, duration_time, - sys_missing_tables, succeed_count, - failed_count))) { - LOG_WARN("failed to gather tables stats with default param", K(ret)); - } else if (OB_FAIL(gather_tables_stats_with_default_param(ctx, true, start_time, duration_time, - sys_stale_tables, succeed_count, - failed_count))) { - LOG_WARN("failed to gather tables stats with default param", K(ret)); - } else {/*do nothing*/} - const int64_t exe_time = ObTimeUtility::current_time() - start_time; - const int64_t total_cnt = user_missing_tables.count() + user_stale_tables.count() + - sys_missing_tables.count() + sys_stale_tables.count(); - LOG_INFO("have been gathered database stats job", - "the total used time:", exe_time, - "the duration time:", duration_time, - "the toatal gather table cnt:", total_cnt, - "the succeed to gather table cnt:", succeed_count, - "the failed to gather table cnt:", failed_count, K(ret)); - //reset the error code, the reason is that the total gather time is reach the duration time. - ret = ret == OB_TIMEOUT ? OB_SUCCESS : ret; + } else if (helper.need_gather_table_stats()) { + int64_t total_cnt = helper.stat_tables_.count(); + if (OB_FAIL(init_gather_task_info(ctx, ObOptStatGatherType::AUTO_GATHER, start_time, total_cnt, task_info))) { + LOG_WARN("failed to init gather task info", K(ret)); + } else if (OB_FAIL(gather_tables_stats_with_default_param(ctx, helper, task_info))) { + LOG_WARN("failed to gather tables tats with default param"); + } else {/*do nothing*/} + const int64_t exe_time = ObTimeUtility::current_time() - start_time; + LOG_INFO("have been gathered database stats job", + "the total used time:", exe_time, + "the duration time:", helper.duration_time_, + "the toatal gather table cnt:", total_cnt, + "the succeed to gather table cnt:", helper.succeed_count_, + "the failed to gather table cnt:", helper.failed_count_, K(ret)); + //reset the error code, the reason is that the total gather time is reach the duration time. + ret = ret == OB_TIMEOUT ? OB_SUCCESS : ret; + task_info.task_end_time_ = ObTimeUtility::current_time(); + task_info.ret_code_ = ret; + task_info.failed_count_ = helper.failed_count_; + ObOptStatManager::get_instance().update_opt_stat_task_stat(task_info); + } return ret; } -int ObDbmsStats::get_need_statistics_tables(sql::ObExecContext &ctx, - ObIArray &user_missing_tables, - ObIArray &user_stale_tables, - ObIArray &sys_missing_tables, - ObIArray &sys_stale_tables) +int ObDbmsStats::get_need_statistics_tables(sql::ObExecContext &ctx, ObGatherTableStatsHelper &helper) { int ret = OB_SUCCESS; ObSchemaGetterGuard *schema_guard = ctx.get_virtual_table_ctx().schema_guard_; @@ -5159,8 +5200,8 @@ int ObDbmsStats::get_need_statistics_tables(sql::ObExecContext &ctx, table_schemas))) { LOG_WARN("failed to get table schema in database", K(ret)); } else { - bool is_valid = false; for (int64_t j = 0; OB_SUCC(ret) && j < table_schemas.count(); ++j) { + bool is_valid = false; if (OB_ISNULL(table_schema = table_schemas.at(j))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(table_schema)); @@ -5175,39 +5216,52 @@ int ObDbmsStats::get_need_statistics_tables(sql::ObExecContext &ctx, // 2. valid sys table // 3. virtual table } else { + ObStatTableWrapper wrapper; StatTable stat_table(database_id, table_schema->get_table_id()); - ObIArray *target_array = NULL; double stale_percent_threshold = OPT_DEFAULT_STALE_PERCENT; + bool is_big_table = false; if (OB_FAIL(get_table_stale_percent_threshold(ctx, tenant_id, stat_table.table_id_, stale_percent_threshold))) { LOG_WARN("failed to get table stale percent threshold", K(ret)); - } else if (OB_FAIL(get_table_stale_percent(ctx, tenant_id, *table_schema, - stale_percent_threshold, stat_table))) { + } else if (OB_FAIL(get_table_stale_percent(ctx, tenant_id, + *table_schema, + stale_percent_threshold, + stat_table, + is_big_table))) { LOG_WARN("failed to get table stale percent", K(ret)); - } else if (table_schema->is_user_table()) { - if (stat_table.stale_percent_ < 0) { - target_array = &user_missing_tables; - } else if (stat_table.stale_percent_ > stale_percent_threshold) { - target_array = &user_stale_tables; - } + } else if (OB_FAIL(ObBasicStatsEstimator::get_gather_table_duration(ctx, + tenant_id, + table_schema->get_table_id(), + wrapper.last_gather_duration_))) { + LOG_WARN("failed to get gather table duration"); + } else if (stat_table.stale_percent_ < 0) { + wrapper.stat_type_ = ObStatType::ObFirstTimeToGather; + } else if (stat_table.stale_percent_ > stale_percent_threshold) { + wrapper.stat_type_ = ObStatType::ObStale; } else { - if (stat_table.stale_percent_ < 0) { - target_array = &sys_missing_tables; - } else if (stat_table.stale_percent_ > stale_percent_threshold) { - target_array = &sys_stale_tables; - } + wrapper.stat_type_ = ObStatType::ObNotStale; } - if (OB_SUCC(ret) && NULL != target_array) { - if (OB_FAIL(target_array->push_back(stat_table))) { - LOG_WARN("failed to push back stat table", K(ret)); + if (OB_SUCC(ret)) { + wrapper.table_type_ = table_schema->is_user_table() ? + ObStatTableType::ObUserTable : ObStatTableType::ObSysTable; + wrapper.is_big_table_ = is_big_table; + if (wrapper.stat_type_ == ObStatType::ObNotStale) { + // do nothing + } else if (OB_FAIL(wrapper.stat_table_.assign(stat_table))) { + LOG_WARN("failed to assign stat table"); + } else if (OB_FAIL(helper.stat_tables_.push_back(wrapper))) { + LOG_WARN("failed to push back stat tables"); } } } } } } + if (OB_SUCC(ret) && !helper.stat_tables_.empty()) { + std::sort(&helper.stat_tables_.at(0), &helper.stat_tables_.at(0) + helper.stat_tables_.count()); + } } return ret; } @@ -5216,13 +5270,15 @@ int ObDbmsStats::get_table_stale_percent(sql::ObExecContext &ctx, const uint64_t tenant_id, const ObTableSchema &table_schema, const double stale_percent_threshold, - StatTable &stat_table) + StatTable &stat_table, + bool &is_big_table) { int ret = OB_SUCCESS; uint64_t table_id = table_schema.get_table_id(); const int64_t part_id = PARTITION_LEVEL_ZERO == table_schema.get_part_level() ? table_id : -1; ObSEArray partition_stat_infos; bool is_locked = false; + is_big_table = false; if (OB_FAIL(ObBasicStatsEstimator::check_table_statistics_state(ctx, tenant_id, table_id, @@ -5238,11 +5294,12 @@ int ObDbmsStats::get_table_stale_percent(sql::ObExecContext &ctx, if (OB_FAIL(get_user_partition_table_stale_percent(ctx, tenant_id, table_schema, stale_percent_threshold, partition_stat_infos, - stat_table))) { + stat_table, + is_big_table))) { LOG_WARN("faild to get user partition table stale percent", K(ret)); } else {/*do nothing*/} } else if (OB_FAIL(get_common_table_stale_percent(ctx, tenant_id, table_schema, - partition_stat_infos, stat_table))) { + partition_stat_infos, stat_table, is_big_table))) { LOG_WARN("failed to get common table stale percent", K(ret)); } else {/*do nothing*/} return ret; @@ -5257,9 +5314,11 @@ int ObDbmsStats::get_common_table_stale_percent(sql::ObExecContext &ctx, const uint64_t tenant_id, const ObTableSchema &table_schema, const ObIArray &partition_stat_infos, - StatTable &stat_table) + StatTable &stat_table, + bool &is_big_table) { int ret = OB_SUCCESS; + is_big_table = false; //if this is virtual table real agent, we need see the real table id modifed count uint64_t table_id = share::is_oracle_mapping_real_virtual_table(table_schema.get_table_id()) ? share::get_real_table_mappings_tid(table_schema.get_table_id()) : @@ -5274,6 +5333,15 @@ int ObDbmsStats::get_common_table_stale_percent(sql::ObExecContext &ctx, LOG_WARN("get unexpected error", K(ret), K(table_schema.is_user_table()), K(part_id)); } else if (!is_table_gather_global_stats(part_id, partition_stat_infos, row_cnt)) { stat_table.stale_percent_ = -1.0; + if (OB_FAIL(ObBasicStatsEstimator::estimate_row_count(ctx, + tenant_id, + table_id, + row_cnt))) { + LOG_WARN("failed to estimate row count"); + } else { + // currently we only regard tables whose rows >= 1000w as big table + is_big_table = row_cnt >= OPT_STATS_BIG_TABLE_ROWS; + } } else if (is_virtual_table(table_id)) {//virtual table doesn't see the modfiy count, no need regather stat_table.stale_percent_ = 0.0; } else if (OB_FAIL(ObBasicStatsEstimator::estimate_modified_count(ctx, @@ -5290,7 +5358,7 @@ int ObDbmsStats::get_common_table_stale_percent(sql::ObExecContext &ctx, } else { stat_table.stale_percent_ = row_cnt == 0 ? 1.0 : 1.0 * (inc_modified_count) / row_cnt; } - LOG_TRACE("succeed to get common table stale percent", K(stat_table), K(partition_stat_infos)); + LOG_TRACE("succeed to get common table stale percent", K(stat_table), K(partition_stat_infos), K(is_big_table)); return ret; } @@ -5321,9 +5389,11 @@ int ObDbmsStats::get_user_partition_table_stale_percent( const ObTableSchema &table_schema, const double stale_percent_threshold, const ObIArray &partition_stat_infos, - StatTable &stat_table) + StatTable &stat_table, + bool &is_big_table) { int ret = OB_SUCCESS; + is_big_table = false; uint64_t table_id = table_schema.get_table_id(); const int64_t part_id = PARTITION_LEVEL_ZERO == table_schema.get_part_level() ? table_id : -1; int64_t inc_modified_count = 0; @@ -5338,10 +5408,19 @@ int ObDbmsStats::get_user_partition_table_stale_percent( // do not have any statistics stat_table.stale_percent_ = -1.0; stat_table.incremental_stat_ = false; + if (OB_FAIL(ObBasicStatsEstimator::estimate_row_count(ctx, + tenant_id, + table_id, + row_cnt))) { + LOG_WARN("failed to estimate row count"); + } else { + is_big_table = row_cnt >= OPT_STATS_BIG_TABLE_ROWS; + } } else if (OB_FAIL(get_table_partition_infos(table_schema, partition_infos))) { LOG_WARN("failed to get table subpart infos", K(ret)); } else if (!is_table_gather_global_stats(part_id, partition_stat_infos, row_cnt)) { //do not have global statistics, but have part level statistics + stat_table.stale_percent_ = -1.0; if (OB_FAIL(ObBasicStatsEstimator::estimate_stale_partition(ctx, tenant_id, table_id, @@ -5355,12 +5434,10 @@ int ObDbmsStats::get_user_partition_table_stale_percent( if (OB_FAIL(append(stat_table.no_regather_partition_ids_, no_regather_partition_ids))) { LOG_WARN("failed to append table ids"); } else { - stat_table.stale_percent_ = -1.0; stat_table.incremental_stat_ = true; stat_table.need_gather_subpart_ = PARTITION_LEVEL_TWO == table_schema.get_part_level(); } } else { - stat_table.stale_percent_ = -1.0; stat_table.incremental_stat_ = false; } //have global statistics @@ -5429,32 +5506,29 @@ int ObDbmsStats::get_user_partition_table_stale_percent( } } } - LOG_TRACE("succeed to get user partition table stale percent", K(stat_table), - K(partition_stat_infos)); + LOG_TRACE("succeed to get user partition table stale percent", + K(stat_table), K(partition_stat_infos), K(is_big_table)); return ret; } int ObDbmsStats::gather_tables_stats_with_default_param(ObExecContext &ctx, - const bool need_sort, - const int64_t start_time, - const int64_t duration_time, - ObIArray &stat_tables, - int64_t &succeed_count, - int64_t &failed_count) + ObGatherTableStatsHelper &helper, + ObOptStatTaskInfo &task_info) { int ret = OB_SUCCESS; - if (need_sort && stat_tables.count() > 0) { - std::sort(&stat_tables.at(0), &stat_tables.at(0) + stat_tables.count()); - } - for (int64_t i = 0; OB_SUCC(ret) && i < stat_tables.count(); ++i) { - if (is_oceanbase_sys_database_id(stat_tables.at(i).database_id_)) { + for (int64_t i = 0; OB_SUCC(ret) && i < helper.stat_tables_.count(); ++i) { + if (is_oceanbase_sys_database_id(helper.stat_tables_.at(i).stat_table_.database_id_)) { lib::CompatModeGuard compat_guard(lib::Worker::CompatMode::MYSQL); - if (OB_FAIL(gather_table_stats_with_default_param(ctx, start_time, - duration_time, stat_tables.at(i)))) { + if (OB_FAIL(gather_table_stats_with_default_param(ctx, + helper.duration_time_, + helper.stat_tables_.at(i).stat_table_, + task_info))) { LOG_WARN("failed to gather table stats with default param", K(ret)); } - } else if (OB_FAIL(gather_table_stats_with_default_param(ctx, start_time, - duration_time, stat_tables.at(i)))) { + } else if (OB_FAIL(gather_table_stats_with_default_param(ctx, + helper.duration_time_, + helper.stat_tables_.at(i).stat_table_, + task_info))) { LOG_WARN("failed to gather table stats with default param", K(ret)); } if (OB_FAIL(ret)) { @@ -5462,23 +5536,24 @@ int ObDbmsStats::gather_tables_stats_with_default_param(ObExecContext &ctx, LOG_WARN("query interrupted", K(ret)); } else if (OB_TABLE_NOT_EXIST == ret || OB_TIMEOUT == ret) { // do nothing + ++helper.failed_count_; ret = OB_SUCCESS; } else { - ++failed_count; + ++helper.failed_count_; LOG_WARN("failed to gather table stats with some unknown reason", K(ret)); ret = OB_SUCCESS; } } else { - ++ succeed_count; + ++helper.succeed_count_; } } return ret; } int ObDbmsStats::gather_table_stats_with_default_param(ObExecContext &ctx, - const int64_t start_time, const int64_t duration_time, - const StatTable &stat_table) + const StatTable &stat_table, + ObOptStatTaskInfo &task_info) { int ret = OB_SUCCESS; ObArenaAllocator tmp_alloc("OptStatGather", OB_MALLOC_NORMAL_BLOCK_SIZE, ctx.get_my_session()->get_effective_tenant_id()); @@ -5487,7 +5562,11 @@ int ObDbmsStats::gather_table_stats_with_default_param(ObExecContext &ctx, stat_param.db_id_ = stat_table.database_id_; bool is_all_fast_gather = false; ObSEArray no_gather_index_ids; - if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(start_time, + ObOptStatGatherStat gather_stat(task_info); + gather_stat.set_table_id(stat_table.table_id_); + ObOptStatGatherStatList::instance().push(gather_stat); + ObOptStatRunningMonitor running_monitor(ctx.get_allocator(), ObTimeUtility::current_time(), stat_param.allocator_->used(), gather_stat); + if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(task_info.task_start_time_, duration_time, stat_param.duration_time_))) { LOG_WARN("failed to get valid duration time", K(ret)); @@ -5499,15 +5578,19 @@ int ObDbmsStats::gather_table_stats_with_default_param(ObExecContext &ctx, stat_param.subpart_stat_param_.set_gather_stat(); } if (OB_SUCC(ret)) { - if (OB_FAIL(ObDbmsStatsLockUnlock::adjust_table_stat_param(stat_table.no_regather_partition_ids_, - stat_param))) { + if (OB_FAIL(running_monitor.add_table_info(stat_param, stat_table.stale_percent_))) { + LOG_WARN("failed to add table info", K(ret)); + } else if (OB_FAIL(ObDbmsStatsLockUnlock::adjust_table_stat_param(stat_table.no_regather_partition_ids_, + stat_param))) { LOG_WARN("failed to adjust table stat param", K(ret)); } else if (OB_FAIL(ObDbmsStatsExecutor::gather_table_stats(ctx, stat_param))) { LOG_WARN("failed to gather table stats", K(ret)); - } else if (OB_FAIL(update_stat_cache(ctx.get_my_session()->get_rpc_tenant_id(), stat_param))) { + } else if (OB_FAIL(update_stat_cache(ctx.get_my_session()->get_rpc_tenant_id(), + stat_param, + &running_monitor))) { LOG_WARN("failed to update stat cache", K(ret)); //refresh duration time - } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(start_time, + } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(task_info.task_start_time_, duration_time, stat_param.duration_time_))) { LOG_WARN("failed to get valid duration time", K(ret)); @@ -5518,7 +5601,7 @@ int ObDbmsStats::gather_table_stats_with_default_param(ObExecContext &ctx, is_all_fast_gather, no_gather_index_ids))) { LOG_WARN("failed to fast gather index stats", K(ret)); //refresh duration time - } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(start_time, + } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(task_info.task_start_time_, duration_time, stat_param.duration_time_))) { LOG_WARN("failed to get valid duration time", K(ret)); @@ -5529,6 +5612,10 @@ int ObDbmsStats::gather_table_stats_with_default_param(ObExecContext &ctx, LOG_TRACE("Succeed to gather table stats", K(stat_param)); } } + running_monitor.set_monitor_result(ret, ObTimeUtility::current_time(), stat_param.allocator_->used()); + ObOptStatManager::get_instance().update_opt_stat_gather_stat(gather_stat); + ObOptStatGatherStatList::instance().remove(gather_stat); + task_info.completed_table_count_ ++; return ret; } @@ -5911,7 +5998,6 @@ int ObDbmsStats::get_table_partition_map(const ObTableSchema &table_schema, LOG_WARN("failed to get table part infos", K(ret)); } } - return ret; } @@ -6012,5 +6098,41 @@ void ObDbmsStats::decide_modified_part(ObTableStatParam ¶m, const bool casca } } +int ObDbmsStats::init_gather_task_info(ObExecContext &ctx, + ObOptStatGatherType type, + int64_t start_time, + int64_t task_table_count, + ObOptStatTaskInfo &task_info) +{ + int ret = OB_SUCCESS; + ObString task_id; + char *server_uuid = NULL; + int64_t length_uuid = 36; + if (OB_ISNULL(server_uuid = static_cast(ctx.get_allocator().alloc(length_uuid)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("memory is not enough", K(ret), K(length_uuid)); + } else if (OB_FAIL(ObExprUuid::gen_server_uuid(server_uuid, length_uuid))) { + LOG_WARN("failed to gen server uuid", K(ret)); + } else { + task_id.assign_ptr(server_uuid, length_uuid); + if (OB_ISNULL(ctx.get_my_session())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx.get_my_session())); + } else if (OB_FAIL(task_info.init(ctx.get_allocator(), + ctx.get_my_session()->get_effective_tenant_id(), + ctx.get_my_session()->get_sessid(), + ctx.get_my_session()->get_current_trace_id(), + task_id, + type, + start_time, + task_table_count))) { + LOG_WARN("failed to init", K(ret)); + } else { + LOG_TRACE("Succeed to init gather task info", K(task_info)); + } + } + return ret; +} + } } diff --git a/src/pl/sys_package/ob_dbms_stats.h b/src/pl/sys_package/ob_dbms_stats.h index a00a1b383a..b86e95898f 100644 --- a/src/pl/sys_package/ob_dbms_stats.h +++ b/src/pl/sys_package/ob_dbms_stats.h @@ -18,6 +18,7 @@ #include "sql/engine/ob_exec_context.h" #include "pl/ob_pl_type.h" #include "share/stat/ob_dbms_stats_preferences.h" +#include "share/stat/ob_opt_stat_gather_stat.h" namespace oceanbase { @@ -371,7 +372,8 @@ public: common::ObIArray &target_part_ids); static int update_stat_cache(const uint64_t rpc_tenant_id, - const ObTableStatParam ¶m); + const ObTableStatParam ¶m, + ObOptStatRunningMonitor *running_monitor = NULL); static int parse_set_table_stat_options(ObExecContext &ctx, const ObObjParam &stattab, @@ -434,29 +436,23 @@ public: common::ObObj &result); static int get_need_statistics_tables(sql::ObExecContext &ctx, - ObIArray &user_missing_tables, - ObIArray &user_stale_tables, - ObIArray &sys_missing_tables, - ObIArray &sys_stale_tables); + ObGatherTableStatsHelper &helper); static int get_table_stale_percent(sql::ObExecContext &ctx, const uint64_t tenant_id, const share::schema::ObTableSchema &table_schema, const double stale_percent_threshold, - StatTable &stat_table); + StatTable &stat_table, + bool &is_big_table); static int gather_tables_stats_with_default_param(ObExecContext &ctx, - const bool need_sort, - const int64_t start_time, - const int64_t duration_time, - ObIArray &stat_tables, - int64_t &succeed_count, - int64_t &failed_count); + ObGatherTableStatsHelper &helper, + ObOptStatTaskInfo &task_info); static int gather_table_stats_with_default_param(ObExecContext &ctx, - const int64_t start_time, const int64_t duration_time, - const StatTable &stat_table); + const StatTable &stat_table, + ObOptStatTaskInfo &task_info); static int set_param_global_part_id(ObExecContext &ctx, ObTableStatParam ¶m, @@ -468,6 +464,12 @@ public: static int get_table_partition_map(const ObTableSchema &table_schema, OSGPartMap &part_map); + static int init_gather_task_info(ObExecContext &ctx, + ObOptStatGatherType type, + int64_t start_time, + int64_t task_table_count, + ObOptStatTaskInfo &task_info); + static int get_table_stale_percent_threshold(sql::ObExecContext &ctx, const uint64_t tenant_id, const uint64_t table_id, @@ -506,14 +508,16 @@ private: const uint64_t tenant_id, const share::schema::ObTableSchema &table_schema, const ObIArray &partition_stat_infos, - StatTable &stat_table); + StatTable &stat_table, + bool &is_big_table); static int get_user_partition_table_stale_percent(sql::ObExecContext &ctx, const uint64_t tenant_id, const share::schema::ObTableSchema &table_schema, const double stale_percent_threshold, const ObIArray &partition_stat_infos, - StatTable &stat_table); + StatTable &stat_table, + bool &is_big_table); static bool is_table_gather_global_stats(const int64_t global_id, const ObIArray &partition_stat_infos, @@ -533,6 +537,9 @@ private: const int64_t table_id, ObIArray &index_infos); + static int get_table_partition_infos(const ObTableSchema &table_schema, + ObIArray &partition_infos); + static int get_index_schema(sql::ObExecContext &ctx, common::ObIAllocator &allocator, const int64_t data_table_id, @@ -540,9 +547,6 @@ private: ObString &index_name, const share::schema::ObTableSchema *&index_schema); - static int get_table_partition_infos(const ObTableSchema &table_schema, - ObIArray &partition_infos); - static bool is_func_index(const ObTableStatParam &index_param); static int resovle_granularity(ObGranularityType granu_type, ObTableStatParam ¶m); diff --git a/src/share/CMakeLists.txt b/src/share/CMakeLists.txt index a10f3e91ce..52412ce4b6 100644 --- a/src/share/CMakeLists.txt +++ b/src/share/CMakeLists.txt @@ -421,7 +421,6 @@ ob_set_subtarget(ob_share schema ob_set_subtarget(ob_share stat stat/ob_basic_stats_estimator.cpp - stat/ob_column_stat.cpp stat/ob_opt_stat_monitor_manager.cpp stat/ob_dbms_stats_executor.cpp stat/ob_dbms_stats_export_import.cpp @@ -432,6 +431,7 @@ ob_set_subtarget(ob_share stat stat/ob_hybrid_hist_estimator.cpp stat/ob_incremental_stat_estimator.cpp stat/ob_opt_column_stat.cpp + stat/ob_opt_osg_column_stat.cpp stat/ob_opt_column_stat_cache.cpp stat/ob_opt_stat_manager.cpp stat/ob_opt_stat_service.cpp @@ -444,6 +444,7 @@ ob_set_subtarget(ob_share stat stat/ob_topk_hist_estimator.cpp stat/ob_index_stats_estimator.cpp stat/ob_dbms_stats_maintenance_window.cpp + stat/ob_opt_stat_gather_stat.cpp stat/ob_opt_ds_stat.cpp stat/ob_opt_ds_stat_cache.cpp ) diff --git a/src/share/inner_table/ob_inner_table_schema.12351_12400.cpp b/src/share/inner_table/ob_inner_table_schema.12351_12400.cpp index 8c3183eb58..344168b182 100644 --- a/src/share/inner_table/ob_inner_table_schema.12351_12400.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12351_12400.cpp @@ -3514,6 +3514,832 @@ int ObInnerTableSchema::all_virtual_io_scheduler_schema(ObTableSchema &table_sch return ret; } +int ObInnerTableSchema::all_virtual_task_opt_stat_gather_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 36, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ret_code", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_count", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("failed_count", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("start_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("end_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare1", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare2", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare3", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare4", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_virtual_table_opt_stat_gather_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 36, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + ++column_id, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ret_code", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("start_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("end_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("memory_used", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("stat_refresh_failed_list", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("properties", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare1", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare2", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare3", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare4", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_virtual_opt_stat_gather_monitor_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_ip", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 1, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("svr_port", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 2, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("session_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("trace_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TRACE_ID_BUFFER_SIZE, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 36, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("task_start_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_table_count", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_duration_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("completed_table_count", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("running_table_owner", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_DATABASE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("running_table_name", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MAX_TABLE_NAME_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("running_table_duration_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare1", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare2", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + if (OB_SUCC(ret)) { + table_schema.get_part_option().set_part_num(1); + table_schema.set_part_level(PARTITION_LEVEL_ONE); + table_schema.get_part_option().set_part_func_type(PARTITION_FUNC_TYPE_LIST_COLUMNS); + if (OB_FAIL(table_schema.get_part_option().set_part_expr("svr_ip, svr_port"))) { + LOG_WARN("set_part_expr failed", K(ret)); + } else if (OB_FAIL(table_schema.mock_list_partition_array())) { + LOG_WARN("mock list partition array failed", K(ret)); + } + } + table_schema.set_index_using_type(USING_HASH); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_virtual_thread_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.15251_15300.cpp b/src/share/inner_table/ob_inner_table_schema.15251_15300.cpp index de73bd6307..6ec48fb47e 100644 --- a/src/share/inner_table/ob_inner_table_schema.15251_15300.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15251_15300.cpp @@ -9246,6 +9246,823 @@ int ObInnerTableSchema::all_virtual_archive_dest_status_ora_schema(ObTableSchema return ret; } +int ObInnerTableSchema::all_virtual_task_opt_stat_gather_history_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TASK_ID", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + 36, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("GMT_CREATE", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("GMT_MODIFIED", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TYPE", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("RET_CODE", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_COUNT", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("FAILED_COUNT", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("START_TIME", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("END_TIME", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SPARE1", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SPARE2", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SPARE3", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_VALUE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SPARE4", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_VALUE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_virtual_table_opt_stat_gather_history_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TASK_ID", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + 36, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TABLE_ID", //column_name + ++column_id, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("GMT_CREATE", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("GMT_MODIFIED", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("RET_CODE", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("START_TIME", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("END_TIME", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("MEMORY_USED", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("STAT_REFRESH_FAILED_LIST", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_VALUE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PROPERTIES", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_VALUE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SPARE1", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SPARE2", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SPARE3", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_VALUE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SPARE4", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_VALUE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_virtual_opt_stat_gather_monitor_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(VIRTUAL_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_BIN); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TENANT_ID", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SVR_IP", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 1, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_IP_ADDR_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SVR_PORT", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 2, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SESSION_ID", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TRACE_ID", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_TRACE_ID_BUFFER_SIZE, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TASK_ID", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + 36, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TYPE", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TASK_START_TIME", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampLTZType, //column_type + CS_TYPE_INVALID, //column_collation_type + 0, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TASK_TABLE_COUNT", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("TASK_DURATION_TIME", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("COMPLETED_TABLE_COUNT", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("RUNNING_TABLE_OWNER", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_DATABASE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("RUNNING_TABLE_NAME", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MAX_TABLE_NAME_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("RUNNING_TABLE_DURATION_TIME", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SPARE1", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SPARE2", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + MAX_VALUE_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + if (OB_SUCC(ret)) { + table_schema.get_part_option().set_part_num(1); + table_schema.set_part_level(PARTITION_LEVEL_ONE); + table_schema.get_part_option().set_part_func_type(PARTITION_FUNC_TYPE_LIST); + if (OB_FAIL(table_schema.get_part_option().set_part_expr("SVR_IP, SVR_PORT"))) { + LOG_WARN("set_part_expr failed", K(ret)); + } else if (OB_FAIL(table_schema.mock_list_partition_array())) { + LOG_WARN("mock list partition array failed", K(ret)); + } + } + table_schema.set_index_using_type(USING_HASH); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::all_virtual_long_ops_status_sys_agent_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.21251_21300.cpp b/src/share/inner_table/ob_inner_table_schema.21251_21300.cpp index e0710deaf0..67203b08dc 100644 --- a/src/share/inner_table/ob_inner_table_schema.21251_21300.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21251_21300.cpp @@ -1110,7 +1110,7 @@ int ObInnerTableSchema::dba_tab_modifications_schema(ObTableSchema &table_schema table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(DB.DATABASE_NAME AS CHAR(128)) AS TABLE_OWNER, CAST(T.TABLE_NAME AS CHAR(128)) AS TABLE_NAME, CAST(P.PART_NAME AS CHAR(128)) AS PARTITION_NAME, CAST(SP.SUB_PART_NAME AS CHAR(128)) AS SUBPARTITION_NAME, CAST(V.INSERTS AS SIGNED) AS INSERTS, CAST(V.UPDATES AS SIGNED) AS UPDATES, CAST(V.DELETES AS SIGNED) AS DELETES, CAST(V.MODIFIED_TIME AS DATE) AS TIMESTAMP, CAST(NULL AS CHAR(3)) AS TRUNCATED, CAST(NULL AS SIGNED) AS DROP_SEGMENTS FROM (SELECT CASE WHEN T.TENANT_ID IS NOT NULL THEN T.TENANT_ID ELSE 0 END AS TENANT_ID, CASE WHEN T.TABLE_ID IS NOT NULL THEN T.TABLE_ID ELSE VT.TABLE_ID END AS TABLE_ID, CASE WHEN T.TABLET_ID IS NOT NULL THEN T.TABLET_ID ELSE VT.TABLET_ID END AS TABLET_ID, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.INSERTS + VT.INSERT_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.INSERTS ELSE VT.INSERT_ROW_COUNT END) END AS INSERTS, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.UPDATES + VT.UPDATE_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.UPDATES ELSE VT.UPDATE_ROW_COUNT END) END AS UPDATES, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.DELETES + VT.DELETE_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.DELETES ELSE VT.DELETE_ROW_COUNT END) END AS DELETES, CASE WHEN T.GMT_MODIFIED IS NOT NULL THEN T.GMT_MODIFIED ELSE NULL END AS MODIFIED_TIME FROM OCEANBASE.__ALL_MONITOR_MODIFIED T FULL JOIN OCEANBASE.__ALL_VIRTUAL_DML_STATS VT ON T.TABLET_ID = VT.TABLET_ID AND VT.TENANT_ID = EFFECTIVE_TENANT_ID() )V JOIN OCEANBASE.__ALL_TABLE T ON V.TENANT_ID = T.TENANT_ID AND V.TABLE_ID = T.TABLE_ID JOIN OCEANBASE.__ALL_DATABASE DB ON T.TENANT_ID = DB.TENANT_ID AND DB.DATABASE_ID = T.DATABASE_ID LEFT JOIN OCEANBASE.__ALL_PART P ON V.TENANT_ID = P.TENANT_ID AND V.TABLE_ID = P.TABLE_ID AND V.TABLET_ID = P.TABLET_ID LEFT JOIN OCEANBASE.__ALL_SUB_PART SP ON V.TENANT_ID = SP.TENANT_ID AND V.TABLE_ID = SP.TABLE_ID AND V.TABLET_ID = SP.TABLET_ID )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(DB.DATABASE_NAME AS CHAR(128)) AS TABLE_OWNER, CAST(T.TABLE_NAME AS CHAR(128)) AS TABLE_NAME, CAST(P.PART_NAME AS CHAR(128)) AS PARTITION_NAME, CAST(SP.SUB_PART_NAME AS CHAR(128)) AS SUBPARTITION_NAME, CAST(V.INSERTS AS SIGNED) AS INSERTS, CAST(V.UPDATES AS SIGNED) AS UPDATES, CAST(V.DELETES AS SIGNED) AS DELETES, CAST(V.MODIFIED_TIME AS DATE) AS TIMESTAMP, CAST(NULL AS CHAR(3)) AS TRUNCATED, CAST(NULL AS SIGNED) AS DROP_SEGMENTS FROM (SELECT CASE WHEN T.TENANT_ID IS NOT NULL THEN T.TENANT_ID ELSE 0 END AS TENANT_ID, CASE WHEN T.TABLE_ID IS NOT NULL THEN T.TABLE_ID ELSE VT.TABLE_ID END AS TABLE_ID, CASE WHEN T.TABLET_ID IS NOT NULL THEN T.TABLET_ID ELSE VT.TABLET_ID END AS TABLET_ID, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.INSERTS + VT.INSERT_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.INSERTS ELSE VT.INSERT_ROW_COUNT END) END AS INSERTS, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.UPDATES + VT.UPDATE_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.UPDATES ELSE VT.UPDATE_ROW_COUNT END) END AS UPDATES, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.DELETES + VT.DELETE_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.DELETES ELSE VT.DELETE_ROW_COUNT END) END AS DELETES, CASE WHEN T.GMT_MODIFIED IS NOT NULL THEN T.GMT_MODIFIED ELSE NULL END AS MODIFIED_TIME FROM OCEANBASE.__ALL_MONITOR_MODIFIED T FULL JOIN OCEANBASE.__ALL_VIRTUAL_DML_STATS VT ON T.TABLET_ID = VT.TABLET_ID AND VT.TENANT_ID = EFFECTIVE_TENANT_ID() )V JOIN OCEANBASE.__ALL_TABLE T ON V.TENANT_ID = T.TENANT_ID AND V.TABLE_ID = T.TABLE_ID AND T.TABLE_TYPE in (0, 3, 6) JOIN OCEANBASE.__ALL_DATABASE DB ON T.TENANT_ID = DB.TENANT_ID AND DB.DATABASE_ID = T.DATABASE_ID LEFT JOIN OCEANBASE.__ALL_PART P ON V.TENANT_ID = P.TENANT_ID AND V.TABLE_ID = P.TABLE_ID AND V.TABLET_ID = P.TABLET_ID LEFT JOIN OCEANBASE.__ALL_SUB_PART SP ON V.TENANT_ID = SP.TENANT_ID AND V.TABLE_ID = SP.TABLE_ID AND V.TABLET_ID = SP.TABLET_ID )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.21351_21400.cpp b/src/share/inner_table/ob_inner_table_schema.21351_21400.cpp index e478a0634d..58b2291da3 100644 --- a/src/share/inner_table/ob_inner_table_schema.21351_21400.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21351_21400.cpp @@ -825,6 +825,206 @@ int ObInnerTableSchema::dba_ob_rsrc_io_directives_schema(ObTableSchema &table_sc return ret; } +int ObInnerTableSchema::gv_ob_opt_stat_gather_monitor_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_GV_OB_OPT_STAT_GATHER_MONITOR_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_GV_OB_OPT_STAT_GATHER_MONITOR_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(TENANT_ID AS SIGNED) AS TENANT_ID, CAST(SVR_IP AS CHAR(46)) AS SVR_IP, CAST(SVR_PORT AS SIGNED) AS SVR_PORT, CAST(SESSION_ID AS SIGNED) AS SESSION_ID, CAST(TRACE_ID AS CHAR(64)) AS TRACE_ID, CAST(TASK_ID AS CHAR(36)) AS TASK_ID, CAST((CASE WHEN TYPE = 0 THEN 'MANUAL GATHER' ELSE (CASE WHEN TYPE = 1 THEN 'AUTO GATHER' ELSE 'UNDEFINED GATHER' END) END) AS CHAR(16)) AS TYPE, CAST(TASK_START_TIME AS DATETIME(6)) AS TASK_START_TIME, CAST(TASK_DURATION_TIME AS SIGNED) AS TASK_DURATION_TIME, CAST(TASK_TABLE_COUNT AS SIGNED) AS TASK_TABLE_COUNT, CAST(COMPLETED_TABLE_COUNT AS SIGNED) AS COMPLETED_TABLE_COUNT, CAST(RUNNING_TABLE_OWNER AS CHAR(128)) AS RUNNING_TABLE_OWNER, CAST(RUNNING_TABLE_NAME AS CHAR(256)) AS RUNNING_TABLE_NAME, CAST(RUNNING_TABLE_DURATION_TIME AS SIGNED) AS RUNNING_TABLE_DURATION_TIME FROM oceanbase.__all_virtual_opt_stat_gather_monitor )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::v_ob_opt_stat_gather_monitor_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_V_OB_OPT_STAT_GATHER_MONITOR_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_V_OB_OPT_STAT_GATHER_MONITOR_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT * FROM oceanbase.GV$OB_OPT_STAT_GATHER_MONITOR WHERE SVR_IP=HOST_IP() AND SVR_PORT=RPC_PORT() )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_ob_task_opt_stat_gather_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_OB_TASK_OPT_STAT_GATHER_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_OB_TASK_OPT_STAT_GATHER_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(TENANT_ID AS SIGNED) AS TENANT_ID, CAST(TASK_ID AS CHAR(36)) AS TASK_ID, CAST((CASE WHEN type = 0 THEN 'MANUAL GATHER' ELSE ( CASE WHEN type = 1 THEN 'AUTO GATHER' ELSE ( CASE WHEN type IS NULL THEN NULL ELSE 'UNDEFINED GATHER' END )END ) END ) AS CHAR(16)) AS TYPE, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE 'FAILED' END) AS CHAR(8)) AS STATUS, CAST(TABLE_COUNT AS SIGNED) AS TABLE_COUNT, CAST(FAILED_COUNT AS SIGNED) AS FAILED_COUNT, CAST(START_TIME AS DATETIME(6)) AS START_TIME, CAST(END_TIME AS DATETIME(6)) AS END_TIME FROM oceanbase.__all_virtual_task_opt_stat_gather_history WHERE TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_ob_table_opt_stat_gather_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(DB.DATABASE_NAME AS CHAR(128)) AS OWNER, CAST(V.TABLE_NAME AS CHAR(256)) AS TABLE_NAME, CAST(STAT.TASK_ID AS CHAR(36)) AS TASK_ID, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE 'FAILED' END) AS CHAR(8)) AS STATUS, CAST(STAT.START_TIME AS DATETIME(6)) AS START_TIME, CAST(STAT.END_TIME AS DATETIME(6)) AS END_TIME, CAST(STAT.MEMORY_USED AS SIGNED) AS MEMORY_USED, CAST(STAT.STAT_REFRESH_FAILED_LIST AS CHAR(4096)) AS STAT_REFRESH_FAILED_LIST, CAST(STAT.PROPERTIES AS CHAR(4096)) AS PROPERTIES FROM ( (SELECT CAST(0 AS SIGNED) AS TENANT_ID, DATABASE_ID, TABLE_ID, TABLE_NAME FROM OCEANBASE.__ALL_VIRTUAL_CORE_ALL_TABLE WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID, DATABASE_ID, TABLE_ID, TABLE_NAME FROM oceanbase.__all_table T WHERE T.TABLE_TYPE IN (0,2,3,6)) ) V JOIN oceanbase.__all_database DB ON DB.TENANT_ID = V.TENANT_ID AND DB.DATABASE_ID = V.DATABASE_ID AND V.TENANT_ID = 0 LEFT JOIN oceanbase.__all_virtual_table_opt_stat_gather_history STAT ON STAT.TENANT_ID = EFFECTIVE_TENANT_ID() AND V.TABLE_ID = STAT.TABLE_ID )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::gv_ob_thread_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.21401_21450.cpp b/src/share/inner_table/ob_inner_table_schema.21401_21450.cpp index 8f474163bb..2e38edf620 100644 --- a/src/share/inner_table/ob_inner_table_schema.21401_21450.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21401_21450.cpp @@ -75,6 +75,56 @@ int ObInnerTableSchema::v_ob_timestamp_service_schema(ObTableSchema &table_schem return ret; } +int ObInnerTableSchema::dba_ob_table_stat_stale_info_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_OB_TABLE_STAT_STALE_INFO_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_OB_TABLE_STAT_STALE_INFO_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( WITH V AS (SELECT NVL(T.TENANT_ID, 0) AS TENANT_ID, NVL(T.TABLE_ID, VT.TABLE_ID) AS TABLE_ID, NVL(T.TABLET_ID, VT.TABLET_ID) AS TABLET_ID, NVL(T.INSERTS, 0) + NVL(VT.INSERT_ROW_COUNT, 0) - NVL(T.LAST_INSERTS, 0) AS INSERTS, NVL(T.UPDATES, 0) + NVL(VT.UPDATE_ROW_COUNT, 0) - NVL(T.LAST_UPDATES, 0) AS UPDATES, NVL(T.DELETES, 0) + NVL(VT.DELETE_ROW_COUNT, 0) - NVL(T.LAST_DELETES, 0) AS DELETES FROM OCEANBASE.__ALL_MONITOR_MODIFIED T FULL JOIN OCEANBASE.__ALL_VIRTUAL_DML_STATS VT ON T.TABLE_ID = VT.TABLE_ID AND T.TABLET_ID = VT.TABLET_ID AND VT.TENANT_ID = EFFECTIVE_TENANT_ID() ) SELECT CAST(TM.DATABASE_NAME AS CHAR(128)) AS DATABASE_NAME, CAST(TM.TABLE_NAME AS CHAR(128)) AS TABLE_NAME, CAST(TM.PART_NAME AS CHAR(128)) AS PARTITION_NAME, CAST(TM.SUB_PART_NAME AS CHAR(128)) AS SUBPARTITION_NAME, CAST(TS.ROW_CNT AS SIGNED) AS LAST_ANALYZED_ROWS, TS.LAST_ANALYZED AS LAST_ANALYZED_TIME, CAST(TM.INSERTS AS SIGNED) AS INSERTS, CAST(TM.UPDATES AS SIGNED) AS UPDATES, CAST(TM.DELETES AS SIGNED) AS DELETES, CAST(NVL(CAST(UP.VALCHAR AS SIGNED), CAST(GP.SPARE4 AS SIGNED)) AS SIGNED) STALE_PERCENT, CAST(CASE NVL((TM.INSERTS + TM.UPDATES + TM.DELETES) > TS.ROW_CNT * NVL(CAST(UP.VALCHAR AS SIGNED), CAST(GP.SPARE4 AS SIGNED)) / 100, (TM.INSERTS + TM.UPDATES + TM.DELETES) > 0) WHEN 0 THEN 'NO' WHEN 1 THEN 'YES' END AS CHAR(3)) AS IS_STALE FROM (SELECT T.TENANT_ID, T.TABLE_ID, CASE T.PART_LEVEL WHEN 0 THEN T.TABLE_ID WHEN 1 THEN P.PART_ID WHEN 2 THEN SP.SUB_PART_ID END AS PARTITION_ID, DB.DATABASE_NAME, T.TABLE_NAME, P.PART_NAME, SP.SUB_PART_NAME, NVL(V.INSERTS, 0) AS INSERTS, NVL(V.UPDATES, 0) AS UPDATES, NVL(V.DELETES, 0) AS DELETES FROM OCEANBASE.__ALL_TABLE T JOIN OCEANBASE.__ALL_DATABASE DB ON T.TENANT_ID = DB.TENANT_ID AND DB.DATABASE_ID = T.DATABASE_ID LEFT JOIN OCEANBASE.__ALL_PART P ON T.TENANT_ID = P.TENANT_ID AND T.TABLE_ID = P.TABLE_ID LEFT JOIN OCEANBASE.__ALL_SUB_PART SP ON T.TENANT_ID = SP.TENANT_ID AND T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID LEFT JOIN V ON T.TENANT_ID = V.TENANT_ID AND T.TABLE_ID = V.TABLE_ID AND V.TABLET_ID = CASE T.PART_LEVEL WHEN 0 THEN T.TABLET_ID WHEN 1 THEN P.TABLET_ID WHEN 2 THEN SP.TABLET_ID END WHERE T.TABLE_TYPE IN (0, 3, 6) UNION ALL SELECT MIN(T.TENANT_ID), MIN(T.TABLE_ID), -1 AS PARTITION_ID, DB.DATABASE_NAME, T.TABLE_NAME, NULL AS PART_NAME, NULL AS SUB_PART_NAME, SUM(NVL(V.INSERTS, 0)) AS INSERTS, SUM(NVL(V.UPDATES, 0)) AS UPDATES, SUM(NVL(V.DELETES, 0)) AS DELETES FROM OCEANBASE.__ALL_TABLE T JOIN OCEANBASE.__ALL_DATABASE DB ON T.TENANT_ID = DB.TENANT_ID AND DB.DATABASE_ID = T.DATABASE_ID JOIN OCEANBASE.__ALL_PART P ON T.TENANT_ID = P.TENANT_ID AND T.TABLE_ID = P.TABLE_ID LEFT JOIN V ON T.TENANT_ID = V.TENANT_ID AND T.TABLE_ID = V.TABLE_ID AND V.TABLET_ID = P.TABLET_ID WHERE T.TABLE_TYPE IN (0, 3, 6) AND T.PART_LEVEL = 1 GROUP BY DB.DATABASE_NAME, T.TABLE_NAME UNION ALL SELECT MIN(T.TENANT_ID), MIN(T.TABLE_ID), MIN(P.PART_ID) AS PARTITION_ID, DB.DATABASE_NAME, T.TABLE_NAME, P.PART_NAME, NULL AS SUB_PART_NAME, SUM(NVL(V.INSERTS, 0)) AS INSERTS, SUM(NVL(V.UPDATES, 0)) AS UPDATES, SUM(NVL(V.DELETES, 0)) AS DELETES FROM OCEANBASE.__ALL_TABLE T JOIN OCEANBASE.__ALL_DATABASE DB ON T.TENANT_ID = DB.TENANT_ID AND DB.DATABASE_ID = T.DATABASE_ID JOIN OCEANBASE.__ALL_PART P ON T.TENANT_ID = P.TENANT_ID AND T.TABLE_ID = P.TABLE_ID JOIN OCEANBASE.__ALL_SUB_PART SP ON T.TENANT_ID = SP.TENANT_ID AND T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID LEFT JOIN V ON T.TENANT_ID = V.TENANT_ID AND T.TABLE_ID = V.TABLE_ID AND V.TABLET_ID = SP.TABLET_ID WHERE T.TABLE_TYPE IN (0, 3, 6) AND T.PART_LEVEL = 2 GROUP BY DB.DATABASE_NAME, T.TABLE_NAME, P.PART_NAME UNION ALL SELECT MIN(T.TENANT_ID), MIN(T.TABLE_ID), -1 AS PARTITION_ID, DB.DATABASE_NAME, T.TABLE_NAME, NULL AS PART_NAME, NULL AS SUB_PART_NAME, SUM(NVL(V.INSERTS, 0)) AS INSERTS, SUM(NVL(V.UPDATES, 0)) AS UPDATES, SUM(NVL(V.DELETES, 0)) AS DELETES FROM OCEANBASE.__ALL_TABLE T JOIN OCEANBASE.__ALL_DATABASE DB ON T.TENANT_ID = DB.TENANT_ID AND DB.DATABASE_ID = T.DATABASE_ID JOIN OCEANBASE.__ALL_PART P ON T.TENANT_ID = P.TENANT_ID AND T.TABLE_ID = P.TABLE_ID JOIN OCEANBASE.__ALL_SUB_PART SP ON T.TENANT_ID = SP.TENANT_ID AND T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID LEFT JOIN V ON T.TENANT_ID = V.TENANT_ID AND T.TABLE_ID = V.TABLE_ID AND V.TABLET_ID = SP.TABLET_ID WHERE T.TABLE_TYPE IN (0, 3, 6) AND T.PART_LEVEL = 2 GROUP BY DB.DATABASE_NAME, T.TABLE_NAME ) TM LEFT JOIN OCEANBASE.__ALL_TABLE_STAT TS ON TM.TENANT_ID = TS.TENANT_ID AND TM.TABLE_ID = TS.TABLE_ID AND TM.PARTITION_ID = TS.PARTITION_ID LEFT JOIN OCEANBASE.__ALL_OPTSTAT_USER_PREFS UP ON TM.TENANT_ID = UP.TENANT_ID AND TM.TABLE_ID = UP.TABLE_ID AND UP.PNAME = 'STALE_PERCENT' JOIN OCEANBASE.__ALL_OPTSTAT_GLOBAL_PREFS GP ON GP.SNAME = 'STALE_PERCENT' )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.25151_25200.cpp b/src/share/inner_table/ob_inner_table_schema.25151_25200.cpp index 04a9bf608d..67f15761da 100644 --- a/src/share/inner_table/ob_inner_table_schema.25151_25200.cpp +++ b/src/share/inner_table/ob_inner_table_schema.25151_25200.cpp @@ -1610,7 +1610,7 @@ int ObInnerTableSchema::all_tab_modifications_ora_schema(ObTableSchema &table_sc table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(DB.DATABASE_NAME AS VARCHAR2(128)) AS TABLE_OWNER, CAST(T.TABLE_NAME AS VARCHAR2(128)) AS TABLE_NAME, CAST(P.PART_NAME AS VARCHAR2(128)) AS PARTITION_NAME, CAST(SP.SUB_PART_NAME AS VARCHAR2(128)) AS SUBPARTITION_NAME, CAST(V.INSERTS AS NUMBER) AS INSERTS, CAST(V.UPDATES AS NUMBER) AS UPDATES, CAST(V.DELETES AS NUMBER) AS DELETES, CAST(V.MODIFIED_TIME AS DATE) AS TIMESTAMP, CAST(NULL AS VARCHAR2(3)) AS TRUNCATED, CAST(NULL AS NUMBER) AS DROP_SEGMENTS FROM (SELECT CASE WHEN T.TENANT_ID IS NOT NULL THEN T.TENANT_ID ELSE VT.TENANT_ID END AS TENANT_ID, CASE WHEN T.TABLE_ID IS NOT NULL THEN T.TABLE_ID ELSE VT.TABLE_ID END AS TABLE_ID, CASE WHEN T.TABLET_ID IS NOT NULL THEN T.TABLET_ID ELSE VT.TABLET_ID END AS TABLET_ID, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.INSERTS + VT.INSERT_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.INSERTS ELSE VT.INSERT_ROW_COUNT END) END AS INSERTS, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.UPDATES + VT.UPDATE_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.UPDATES ELSE VT.UPDATE_ROW_COUNT END) END AS UPDATES, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.DELETES + VT.DELETE_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.DELETES ELSE VT.DELETE_ROW_COUNT END) END AS DELETES, CASE WHEN T.GMT_MODIFIED IS NOT NULL THEN T.GMT_MODIFIED ELSE NULL END AS MODIFIED_TIME FROM SYS.ALL_VIRTUAL_MONITOR_MODIFIED_REAL_AGENT T FULL JOIN SYS.ALL_VIRTUAL_DML_STATS VT ON T.TENANT_ID = VT.TENANT_ID AND T.TABLE_ID = VT.TABLE_ID AND T.TABLET_ID = VT.TABLET_ID )V JOIN SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T ON T.TABLE_ID = V.TABLE_ID AND T.TENANT_ID = V.TENANT_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB ON DB.TENANT_ID = V.TENANT_ID AND DB.DATABASE_ID = T.DATABASE_ID AND (DB.DATABASE_ID = USERENV('SCHEMAID') OR USER_CAN_ACCESS_OBJ(1, V.TABLE_ID, DB.DATABASE_ID) = 1) LEFT JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P ON V.TENANT_ID = P.TENANT_ID AND V.TABLE_ID = P.TABLE_ID AND V.TABLET_ID = P.TABLET_ID LEFT JOIN SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SP ON V.TENANT_ID = SP.TENANT_ID AND V.TABLE_ID = SP.TABLE_ID AND V.TABLET_ID = SP.TABLET_ID )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(DB.DATABASE_NAME AS VARCHAR2(128)) AS TABLE_OWNER, CAST(T.TABLE_NAME AS VARCHAR2(128)) AS TABLE_NAME, CAST(P.PART_NAME AS VARCHAR2(128)) AS PARTITION_NAME, CAST(SP.SUB_PART_NAME AS VARCHAR2(128)) AS SUBPARTITION_NAME, CAST(V.INSERTS AS NUMBER) AS INSERTS, CAST(V.UPDATES AS NUMBER) AS UPDATES, CAST(V.DELETES AS NUMBER) AS DELETES, CAST(V.MODIFIED_TIME AS DATE) AS TIMESTAMP, CAST(NULL AS VARCHAR2(3)) AS TRUNCATED, CAST(NULL AS NUMBER) AS DROP_SEGMENTS FROM (SELECT CASE WHEN T.TENANT_ID IS NOT NULL THEN T.TENANT_ID ELSE VT.TENANT_ID END AS TENANT_ID, CASE WHEN T.TABLE_ID IS NOT NULL THEN T.TABLE_ID ELSE VT.TABLE_ID END AS TABLE_ID, CASE WHEN T.TABLET_ID IS NOT NULL THEN T.TABLET_ID ELSE VT.TABLET_ID END AS TABLET_ID, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.INSERTS + VT.INSERT_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.INSERTS ELSE VT.INSERT_ROW_COUNT END) END AS INSERTS, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.UPDATES + VT.UPDATE_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.UPDATES ELSE VT.UPDATE_ROW_COUNT END) END AS UPDATES, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.DELETES + VT.DELETE_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.DELETES ELSE VT.DELETE_ROW_COUNT END) END AS DELETES, CASE WHEN T.GMT_MODIFIED IS NOT NULL THEN T.GMT_MODIFIED ELSE NULL END AS MODIFIED_TIME FROM SYS.ALL_VIRTUAL_MONITOR_MODIFIED_REAL_AGENT T FULL JOIN SYS.ALL_VIRTUAL_DML_STATS VT ON T.TENANT_ID = VT.TENANT_ID AND T.TABLE_ID = VT.TABLE_ID AND T.TABLET_ID = VT.TABLET_ID )V JOIN SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T ON T.TABLE_ID = V.TABLE_ID AND T.TENANT_ID = V.TENANT_ID AND T.TABLE_TYPE IN (0, 3, 8, 9) AND T.TENANT_ID = EFFECTIVE_TENANT_ID() JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB ON DB.TENANT_ID = V.TENANT_ID AND DB.DATABASE_ID = T.DATABASE_ID AND (DB.DATABASE_ID = USERENV('SCHEMAID') OR USER_CAN_ACCESS_OBJ(1, V.TABLE_ID, DB.DATABASE_ID) = 1) LEFT JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P ON V.TENANT_ID = P.TENANT_ID AND V.TABLE_ID = P.TABLE_ID AND V.TABLET_ID = P.TABLET_ID LEFT JOIN SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SP ON V.TENANT_ID = SP.TENANT_ID AND V.TABLE_ID = SP.TABLE_ID AND V.TABLET_ID = SP.TABLET_ID )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1660,7 +1660,7 @@ int ObInnerTableSchema::dba_tab_modifications_ora_schema(ObTableSchema &table_sc table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(DB.DATABASE_NAME AS VARCHAR2(128)) AS TABLE_OWNER, CAST(T.TABLE_NAME AS VARCHAR2(128)) AS TABLE_NAME, CAST(P.PART_NAME AS VARCHAR2(128)) AS PARTITION_NAME, CAST(SP.SUB_PART_NAME AS VARCHAR2(128)) AS SUBPARTITION_NAME, CAST(V.INSERTS AS NUMBER) AS INSERTS, CAST(V.UPDATES AS NUMBER) AS UPDATES, CAST(V.DELETES AS NUMBER) AS DELETES, CAST(V.MODIFIED_TIME AS DATE) AS TIMESTAMP, CAST(NULL AS VARCHAR2(3)) AS TRUNCATED, CAST(NULL AS NUMBER) AS DROP_SEGMENTS FROM (SELECT CASE WHEN T.TENANT_ID IS NOT NULL THEN T.TENANT_ID ELSE VT.TENANT_ID END AS TENANT_ID, CASE WHEN T.TABLE_ID IS NOT NULL THEN T.TABLE_ID ELSE VT.TABLE_ID END AS TABLE_ID, CASE WHEN T.TABLET_ID IS NOT NULL THEN T.TABLET_ID ELSE VT.TABLET_ID END AS TABLET_ID, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.INSERTS + VT.INSERT_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.INSERTS ELSE VT.INSERT_ROW_COUNT END) END AS INSERTS, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.UPDATES + VT.UPDATE_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.UPDATES ELSE VT.UPDATE_ROW_COUNT END) END AS UPDATES, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.DELETES + VT.DELETE_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.DELETES ELSE VT.DELETE_ROW_COUNT END) END AS DELETES, CASE WHEN T.GMT_MODIFIED IS NOT NULL THEN T.GMT_MODIFIED ELSE NULL END AS MODIFIED_TIME FROM SYS.ALL_VIRTUAL_MONITOR_MODIFIED_REAL_AGENT T FULL JOIN SYS.ALL_VIRTUAL_DML_STATS VT ON T.TENANT_ID = VT.TENANT_ID AND T.TABLE_ID = VT.TABLE_ID AND T.TABLET_ID = VT.TABLET_ID )V JOIN SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T ON T.TABLE_ID = V.TABLE_ID AND T.TENANT_ID = V.TENANT_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB ON DB.TENANT_ID = V.TENANT_ID AND DB.DATABASE_ID = T.DATABASE_ID LEFT JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P ON V.TENANT_ID = P.TENANT_ID AND V.TABLE_ID = P.TABLE_ID AND V.TABLET_ID = P.TABLET_ID LEFT JOIN SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SP ON V.TENANT_ID = SP.TENANT_ID AND V.TABLE_ID = SP.TABLE_ID AND V.TABLET_ID = SP.TABLET_ID )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(DB.DATABASE_NAME AS VARCHAR2(128)) AS TABLE_OWNER, CAST(T.TABLE_NAME AS VARCHAR2(128)) AS TABLE_NAME, CAST(P.PART_NAME AS VARCHAR2(128)) AS PARTITION_NAME, CAST(SP.SUB_PART_NAME AS VARCHAR2(128)) AS SUBPARTITION_NAME, CAST(V.INSERTS AS NUMBER) AS INSERTS, CAST(V.UPDATES AS NUMBER) AS UPDATES, CAST(V.DELETES AS NUMBER) AS DELETES, CAST(V.MODIFIED_TIME AS DATE) AS TIMESTAMP, CAST(NULL AS VARCHAR2(3)) AS TRUNCATED, CAST(NULL AS NUMBER) AS DROP_SEGMENTS FROM (SELECT CASE WHEN T.TENANT_ID IS NOT NULL THEN T.TENANT_ID ELSE VT.TENANT_ID END AS TENANT_ID, CASE WHEN T.TABLE_ID IS NOT NULL THEN T.TABLE_ID ELSE VT.TABLE_ID END AS TABLE_ID, CASE WHEN T.TABLET_ID IS NOT NULL THEN T.TABLET_ID ELSE VT.TABLET_ID END AS TABLET_ID, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.INSERTS + VT.INSERT_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.INSERTS ELSE VT.INSERT_ROW_COUNT END) END AS INSERTS, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.UPDATES + VT.UPDATE_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.UPDATES ELSE VT.UPDATE_ROW_COUNT END) END AS UPDATES, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.DELETES + VT.DELETE_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.DELETES ELSE VT.DELETE_ROW_COUNT END) END AS DELETES, CASE WHEN T.GMT_MODIFIED IS NOT NULL THEN T.GMT_MODIFIED ELSE NULL END AS MODIFIED_TIME FROM SYS.ALL_VIRTUAL_MONITOR_MODIFIED_REAL_AGENT T FULL JOIN SYS.ALL_VIRTUAL_DML_STATS VT ON T.TENANT_ID = VT.TENANT_ID AND T.TABLE_ID = VT.TABLE_ID AND T.TABLET_ID = VT.TABLET_ID )V JOIN SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T ON T.TABLE_ID = V.TABLE_ID AND T.TENANT_ID = V.TENANT_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND T.TABLE_TYPE IN (0, 3, 8, 9) JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB ON DB.TENANT_ID = V.TENANT_ID AND DB.DATABASE_ID = T.DATABASE_ID LEFT JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P ON V.TENANT_ID = P.TENANT_ID AND V.TABLE_ID = P.TABLE_ID AND V.TABLET_ID = P.TABLET_ID LEFT JOIN SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SP ON V.TENANT_ID = SP.TENANT_ID AND V.TABLE_ID = SP.TABLE_ID AND V.TABLET_ID = SP.TABLET_ID )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1710,7 +1710,7 @@ int ObInnerTableSchema::user_tab_modifications_ora_schema(ObTableSchema &table_s table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(T.TABLE_NAME AS VARCHAR2(128)) AS TABLE_NAME, CAST(P.PART_NAME AS VARCHAR2(128)) AS PARTITION_NAME, CAST(SP.SUB_PART_NAME AS VARCHAR2(128)) AS SUBPARTITION_NAME, CAST(V.INSERTS AS NUMBER) AS INSERTS, CAST(V.UPDATES AS NUMBER) AS UPDATES, CAST(V.DELETES AS NUMBER) AS DELETES, CAST(V.MODIFIED_TIME AS DATE) AS TIMESTAMP, CAST(NULL AS VARCHAR2(3)) AS TRUNCATED, CAST(NULL AS NUMBER) AS DROP_SEGMENTS FROM (SELECT CASE WHEN T.TENANT_ID IS NOT NULL THEN T.TENANT_ID ELSE VT.TENANT_ID END AS TENANT_ID, CASE WHEN T.TABLE_ID IS NOT NULL THEN T.TABLE_ID ELSE VT.TABLE_ID END AS TABLE_ID, CASE WHEN T.TABLET_ID IS NOT NULL THEN T.TABLET_ID ELSE VT.TABLET_ID END AS TABLET_ID, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.INSERTS + VT.INSERT_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.INSERTS ELSE VT.INSERT_ROW_COUNT END) END AS INSERTS, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.UPDATES + VT.UPDATE_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.UPDATES ELSE VT.UPDATE_ROW_COUNT END) END AS UPDATES, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.DELETES + VT.DELETE_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.DELETES ELSE VT.DELETE_ROW_COUNT END) END AS DELETES, CASE WHEN T.GMT_MODIFIED IS NOT NULL THEN T.GMT_MODIFIED ELSE NULL END AS MODIFIED_TIME FROM SYS.ALL_VIRTUAL_MONITOR_MODIFIED_REAL_AGENT T FULL JOIN SYS.ALL_VIRTUAL_DML_STATS VT ON T.TENANT_ID = VT.TENANT_ID AND T.TABLE_ID = VT.TABLE_ID AND T.TABLET_ID = VT.TABLET_ID )V JOIN SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T ON T.TABLE_ID = V.TABLE_ID AND T.TENANT_ID = V.TENANT_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND T.DATABASE_ID = USERENV('SCHEMAID') JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB ON DB.TENANT_ID = V.TENANT_ID AND DB.DATABASE_ID = T.DATABASE_ID LEFT JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P ON V.TENANT_ID = P.TENANT_ID AND V.TABLE_ID = P.TABLE_ID AND V.TABLET_ID = P.TABLET_ID LEFT JOIN SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SP ON V.TENANT_ID = SP.TENANT_ID AND V.TABLE_ID = SP.TABLE_ID AND V.TABLET_ID = SP.TABLET_ID )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(T.TABLE_NAME AS VARCHAR2(128)) AS TABLE_NAME, CAST(P.PART_NAME AS VARCHAR2(128)) AS PARTITION_NAME, CAST(SP.SUB_PART_NAME AS VARCHAR2(128)) AS SUBPARTITION_NAME, CAST(V.INSERTS AS NUMBER) AS INSERTS, CAST(V.UPDATES AS NUMBER) AS UPDATES, CAST(V.DELETES AS NUMBER) AS DELETES, CAST(V.MODIFIED_TIME AS DATE) AS TIMESTAMP, CAST(NULL AS VARCHAR2(3)) AS TRUNCATED, CAST(NULL AS NUMBER) AS DROP_SEGMENTS FROM (SELECT CASE WHEN T.TENANT_ID IS NOT NULL THEN T.TENANT_ID ELSE VT.TENANT_ID END AS TENANT_ID, CASE WHEN T.TABLE_ID IS NOT NULL THEN T.TABLE_ID ELSE VT.TABLE_ID END AS TABLE_ID, CASE WHEN T.TABLET_ID IS NOT NULL THEN T.TABLET_ID ELSE VT.TABLET_ID END AS TABLET_ID, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.INSERTS + VT.INSERT_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.INSERTS ELSE VT.INSERT_ROW_COUNT END) END AS INSERTS, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.UPDATES + VT.UPDATE_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.UPDATES ELSE VT.UPDATE_ROW_COUNT END) END AS UPDATES, CASE WHEN T.TABLET_ID IS NOT NULL AND VT.TABLET_ID IS NOT NULL THEN T.DELETES + VT.DELETE_ROW_COUNT ELSE (CASE WHEN T.TABLET_ID IS NOT NULL THEN T.DELETES ELSE VT.DELETE_ROW_COUNT END) END AS DELETES, CASE WHEN T.GMT_MODIFIED IS NOT NULL THEN T.GMT_MODIFIED ELSE NULL END AS MODIFIED_TIME FROM SYS.ALL_VIRTUAL_MONITOR_MODIFIED_REAL_AGENT T FULL JOIN SYS.ALL_VIRTUAL_DML_STATS VT ON T.TENANT_ID = VT.TENANT_ID AND T.TABLE_ID = VT.TABLE_ID AND T.TABLET_ID = VT.TABLET_ID )V JOIN SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T ON T.TABLE_ID = V.TABLE_ID AND T.TENANT_ID = V.TENANT_ID AND T.TABLE_TYPE IN (0, 3, 8, 9) AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND T.DATABASE_ID = USERENV('SCHEMAID') JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB ON DB.TENANT_ID = V.TENANT_ID AND DB.DATABASE_ID = T.DATABASE_ID LEFT JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P ON V.TENANT_ID = P.TENANT_ID AND V.TABLE_ID = P.TABLE_ID AND V.TABLET_ID = P.TABLET_ID LEFT JOIN SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SP ON V.TENANT_ID = SP.TENANT_ID AND V.TABLE_ID = SP.TABLE_ID AND V.TABLET_ID = SP.TABLET_ID )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.25201_25250.cpp b/src/share/inner_table/ob_inner_table_schema.25201_25250.cpp index e6cec41a47..d5f4de36c4 100644 --- a/src/share/inner_table/ob_inner_table_schema.25201_25250.cpp +++ b/src/share/inner_table/ob_inner_table_schema.25201_25250.cpp @@ -1275,6 +1275,156 @@ int ObInnerTableSchema::user_db_links_ora_schema(ObTableSchema &table_schema) return ret; } +int ObInnerTableSchema::dba_ob_task_opt_stat_gather_history_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_OB_TASK_OPT_STAT_GATHER_HISTORY_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_OB_TASK_OPT_STAT_GATHER_HISTORY_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(TENANT_ID AS NUMBER) AS TENANT_ID, CAST(TASK_ID AS VARCHAR2(36)) AS TASK_ID, CAST((CASE WHEN type = 0 THEN 'MANUAL GATHER' ELSE ( CASE WHEN type = 1 THEN 'AUTO GATHER' ELSE ( CASE WHEN type IS NULL THEN NULL ELSE 'UNDEFINED GATHER' END )END ) END ) AS VARCHAR2(16)) AS TYPE, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE 'FAILED' END) AS VARCHAR2(8)) AS STATUS, CAST(TABLE_COUNT AS NUMBER) AS TASK_TABLE_COUNT, CAST(FAILED_COUNT AS NUMBER) AS FAILED_COUNT, CAST(START_TIME AS TIMESTAMP(6)) AS TASK_START_TIME, CAST(END_TIME AS TIMESTAMP(6)) AS TASK_END_TIME FROM SYS.ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_ob_table_opt_stat_gather_history_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(DB.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, CAST(V.TABLE_NAME AS VARCHAR2(256)) AS TABLE_NAME, CAST(STAT.TASK_ID AS VARCHAR2(36)) AS TASK_ID, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE 'FAILED' END) AS VARCHAR2(8)) AS STATUS, CAST(STAT.START_TIME AS TIMESTAMP(6)) AS START_TIME, CAST(STAT.END_TIME AS TIMESTAMP(6)) AS END_TIME, CAST(STAT.MEMORY_USED AS NUMBER) AS MEMORY_USED, CAST(STAT.STAT_REFRESH_FAILED_LIST AS VARCHAR2(4096)) AS STAT_REFRESH_FAILED_LIST, CAST(STAT.PROPERTIES AS VARCHAR2(4096)) AS PROPERTIES FROM ( (SELECT TENANT_ID, DATABASE_ID, TABLE_ID, TABLE_ID AS PARTITION_ID, TABLE_NAME, NULL AS PARTITION_NAME, NULL AS SUBPARTITION_NAME, NULL AS PARTITION_POSITION, NULL AS SUBPARTITION_POSITION, 'TABLE' AS OBJECT_TYPE FROM SYS.ALL_VIRTUAL_CORE_ALL_TABLE UNION ALL SELECT TENANT_ID, DATABASE_ID, TABLE_ID, CASE WHEN PART_LEVEL = 0 THEN TABLE_ID ELSE -1 END AS PARTITION_ID, TABLE_NAME, NULL AS PARTITION_NAME, NULL AS SUBPARTITION_NAME, NULL AS PARTITION_POSITION, NULL AS SUBPARTITION_POSITION, 'TABLE' AS OBJECT_TYPE FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T WHERE T.TABLE_TYPE IN (0,2,3,8,9)) ) V JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT db ON db.tenant_id = V.tenant_id AND db.database_id = V.database_id AND V.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY STAT ON V.TENANT_ID = STAT.TENANT_ID AND V.TABLE_ID = STAT.TABLE_ID AND STAT.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_ob_table_stat_stale_info_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_OB_TABLE_STAT_STALE_INFO_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_OB_TABLE_STAT_STALE_INFO_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( WITH V AS (SELECT NVL(T.TABLE_ID, VT.TABLE_ID) AS TABLE_ID, NVL(T.TABLET_ID, VT.TABLET_ID) AS TABLET_ID, NVL(T.INSERTS, 0) + NVL(VT.INSERT_ROW_COUNT, 0) - NVL(T.LAST_INSERTS, 0) AS INSERTS, NVL(T.UPDATES, 0) + NVL(VT.UPDATE_ROW_COUNT, 0) - NVL(T.LAST_UPDATES, 0) AS UPDATES, NVL(T.DELETES, 0) + NVL(VT.DELETE_ROW_COUNT, 0) - NVL(T.LAST_DELETES, 0) AS DELETES FROM SYS.ALL_VIRTUAL_MONITOR_MODIFIED_REAL_AGENT T FULL JOIN SYS.ALL_VIRTUAL_DML_STATS VT ON T.TABLE_ID = VT.TABLE_ID AND T.TABLET_ID = VT.TABLET_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND VT.TENANT_ID = EFFECTIVE_TENANT_ID() ) SELECT CAST(TM.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, CAST(TM.TABLE_NAME AS VARCHAR2(128)) AS TABLE_NAME, CAST(TM.PART_NAME AS VARCHAR2(128)) AS PARTITION_NAME, CAST(TM.SUB_PART_NAME AS VARCHAR2(128)) AS SUBPARTITION_NAME, CAST(TS.ROW_CNT AS NUMBER) AS LAST_ANALYZED_ROWS, TS.LAST_ANALYZED AS LAST_ANALYZED_TIME, CAST(TM.INSERTS AS NUMBER) AS INSERTS, CAST(TM.UPDATES AS NUMBER) AS UPDATES, CAST(TM.DELETES AS NUMBER) AS DELETES, CAST(NVL(CAST(UP.VALCHAR AS NUMBER), CAST(GP.SPARE4 AS NUMBER)) AS NUMBER) STALE_PERCENT, CAST(CASE WHEN TS.ROW_CNT IS NOT NULL THEN CASE WHEN (TM.INSERTS + TM.UPDATES + TM.DELETES) > TS.ROW_CNT * NVL(CAST(UP.VALCHAR AS NUMBER), CAST(GP.SPARE4 AS NUMBER)) / 100 THEN 'YES' ELSE 'NO' END ELSE CASE WHEN (TM.INSERTS + TM.UPDATES + TM.DELETES) > 0 THEN 'YES' ELSE 'NO' END END AS VARCHAR2(3)) AS IS_STALE FROM (SELECT T.TENANT_ID, T.TABLE_ID, CASE T.PART_LEVEL WHEN 0 THEN T.TABLE_ID WHEN 1 THEN P.PART_ID WHEN 2 THEN SP.SUB_PART_ID END AS PARTITION_ID, DB.DATABASE_NAME, T.TABLE_NAME, P.PART_NAME, SP.SUB_PART_NAME, NVL(V.INSERTS, 0) AS INSERTS, NVL(V.UPDATES, 0) AS UPDATES, NVL(V.DELETES, 0) AS DELETES FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB ON DB.DATABASE_ID = T.DATABASE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P ON T.TABLE_ID = P.TABLE_ID AND P.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SP ON T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID AND SP.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN V ON T.TABLE_ID = V.TABLE_ID AND V.TABLET_ID = CASE T.PART_LEVEL WHEN 0 THEN T.TABLET_ID WHEN 1 THEN P.TABLET_ID WHEN 2 THEN SP.TABLET_ID END WHERE T.TABLE_TYPE IN (0, 3, 8, 9) UNION ALL SELECT MIN(T.TENANT_ID), MIN(T.TABLE_ID), -1 AS PARTITION_ID, DB.DATABASE_NAME, T.TABLE_NAME, NULL AS PART_NAME, NULL AS SUB_PART_NAME, SUM(NVL(V.INSERTS, 0)) AS INSERTS, SUM(NVL(V.UPDATES, 0)) AS UPDATES, SUM(NVL(V.DELETES, 0)) AS DELETES FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB ON DB.DATABASE_ID = T.DATABASE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P ON T.TABLE_ID = P.TABLE_ID AND P.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN V ON T.TABLE_ID = V.TABLE_ID AND V.TABLET_ID = P.TABLET_ID WHERE T.TABLE_TYPE IN (0, 3, 8, 9) AND T.PART_LEVEL = 1 GROUP BY DB.DATABASE_NAME, T.TABLE_NAME UNION ALL SELECT MIN(T.TENANT_ID), MIN(T.TABLE_ID), MIN(P.PART_ID) AS PARTITION_ID, DB.DATABASE_NAME, T.TABLE_NAME, P.PART_NAME, NULL AS SUB_PART_NAME, SUM(NVL(V.INSERTS, 0)) AS INSERTS, SUM(NVL(V.UPDATES, 0)) AS UPDATES, SUM(NVL(V.DELETES, 0)) AS DELETES FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB ON DB.DATABASE_ID = T.DATABASE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P ON T.TENANT_ID = P.TENANT_ID AND T.TABLE_ID = P.TABLE_ID JOIN SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SP ON T.TENANT_ID = SP.TENANT_ID AND T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID LEFT JOIN V ON T.TABLE_ID = V.TABLE_ID AND V.TABLET_ID = SP.TABLET_ID WHERE T.TABLE_TYPE IN (0, 3, 8, 9) AND T.PART_LEVEL = 2 GROUP BY DB.DATABASE_NAME, T.TABLE_NAME, P.PART_NAME UNION ALL SELECT MIN(T.TENANT_ID), MIN(T.TABLE_ID), -1 AS PARTITION_ID, DB.DATABASE_NAME, T.TABLE_NAME, NULL AS PART_NAME, NULL AS SUB_PART_NAME, SUM(NVL(V.INSERTS, 0)) AS INSERTS, SUM(NVL(V.UPDATES, 0)) AS UPDATES, SUM(NVL(V.DELETES, 0)) AS DELETES FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB ON DB.DATABASE_ID = T.DATABASE_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P ON T.TABLE_ID = P.TABLE_ID AND P.TENANT_ID = EFFECTIVE_TENANT_ID() JOIN SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SP ON T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID AND SP.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN V ON T.TABLE_ID = V.TABLE_ID AND V.TABLET_ID = SP.TABLET_ID WHERE T.TABLE_TYPE IN (0, 3, 8, 9) AND T.PART_LEVEL = 2 GROUP BY DB.DATABASE_NAME, T.TABLE_NAME ) TM LEFT JOIN SYS.ALL_VIRTUAL_TABLE_STAT_REAL_AGENT TS ON TM.TABLE_ID = TS.TABLE_ID AND TM.PARTITION_ID = TS.PARTITION_ID AND TM.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_OPTSTAT_USER_PREFS_REAL_AGENT UP ON TM.TABLE_ID = UP.TABLE_ID AND UP.PNAME = 'STALE_PERCENT' AND UP.TENANT_ID = EFFECTIVE_TENANT_ID() JOIN SYS.ALL_VIRTUAL_OPTSTAT_GLOBAL_PREFS_REAL_AGENT GP ON GP.SNAME = 'STALE_PERCENT' )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp b/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp index e437844f46..77a5b937ab 100644 --- a/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp +++ b/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp @@ -675,6 +675,106 @@ int ObInnerTableSchema::dba_ob_ls_log_archive_progress_ora_schema(ObTableSchema return ret; } +int ObInnerTableSchema::gv_ob_opt_stat_gather_monitor_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_GV_OB_OPT_STAT_GATHER_MONITOR_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_GV_OB_OPT_STAT_GATHER_MONITOR_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(TENANT_ID AS NUMBER) AS TENANT_ID, CAST(SVR_IP AS VARCHAR2(46)) AS SVR_IP, CAST(SVR_PORT AS NUMBER) AS SVR_PORT, CAST(SESSION_ID AS NUMBER) AS SESSION_ID, CAST(TRACE_ID AS VARCHAR2(64)) AS TRACE_ID, CAST(TASK_ID AS VARCHAR(36)) AS TASK_ID, CAST(DECODE(TYPE, 0, 'MANUAL GATHER', 1, 'AUTO GATHER', 'UNDEFINED GATHER') AS VARCHAR2(16)) AS TYPE, CAST(TASK_START_TIME AS TIMESTAMP(6)) AS TASK_START_TIME, CAST(TASK_DURATION_TIME AS NUMBER) AS TASK_DURATION_TIME, CAST(TASK_TABLE_COUNT AS NUMBER) AS TASK_TABLE_COUNT, CAST(COMPLETED_TABLE_COUNT AS NUMBER) AS COMPLETED_TABLE_COUNT, CAST(RUNNING_TABLE_OWNER AS VARCHAR2(128)) AS RUNNING_TABLE_OWNER, CAST(RUNNING_TABLE_NAME AS VARCHAR2(256)) AS RUNNING_TABLE_NAME, CAST(RUNNING_TABLE_DURATION_TIME AS VARCHAR2(256)) AS RUNNING_TABLE_DURATION_TIME FROM SYS.ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::v_ob_opt_stat_gather_monitor_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_V_OB_OPT_STAT_GATHER_MONITOR_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_V_OB_OPT_STAT_GATHER_MONITOR_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT * FROM SYS.GV$OB_OPT_STAT_GATHER_MONITOR WHERE SVR_IP=HOST_IP() AND SVR_PORT=RPC_PORT() )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + int ObInnerTableSchema::gv_session_longops_ora_schema(ObTableSchema &table_schema) { int ret = OB_SUCCESS; diff --git a/src/share/inner_table/ob_inner_table_schema.451_500.cpp b/src/share/inner_table/ob_inner_table_schema.451_500.cpp new file mode 100644 index 0000000000..1121712e69 --- /dev/null +++ b/src/share/inner_table/ob_inner_table_schema.451_500.cpp @@ -0,0 +1,593 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SHARE_SCHEMA +#include "ob_inner_table_schema.h" + +#include "share/schema/ob_schema_macro_define.h" +#include "share/schema/ob_schema_service_sql_impl.h" +#include "share/schema/ob_table_schema.h" +#include "share/scn.h" + +namespace oceanbase +{ +using namespace share::schema; +using namespace common; +namespace share +{ + +int ObInnerTableSchema::all_task_opt_stat_gather_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ObObj gmt_create_default; + ObObj gmt_create_default_null; + + gmt_create_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_create_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_create_default_null, + gmt_create_default) + } + + if (OB_SUCC(ret)) { + ObObj gmt_modified_default; + ObObj gmt_modified_default_null; + + gmt_modified_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_modified_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + true, //is_on_update_for_timestamp + gmt_modified_default_null, + gmt_modified_default) + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 36, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ret_code", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_count", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("failed_count", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("start_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("end_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare1", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare2", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare3", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare4", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_TID); + table_schema.set_aux_lob_meta_tid(OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TID); + table_schema.set_aux_lob_piece_tid(OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_table_opt_stat_gather_history_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(3); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_TABLE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ObObj gmt_create_default; + ObObj gmt_create_default_null; + + gmt_create_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_create_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_create", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + false, //is_on_update_for_timestamp + gmt_create_default_null, + gmt_create_default) + } + + if (OB_SUCC(ret)) { + ObObj gmt_modified_default; + ObObj gmt_modified_default_null; + + gmt_modified_default.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); + gmt_modified_default_null.set_null(); + ADD_COLUMN_SCHEMA_TS_T("gmt_modified", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_BINARY,//collation_type + 0, //column length + -1, //column_precision + 6, //column_scale + true,//is nullable + false, //is_autoincrement + true, //is_on_update_for_timestamp + gmt_modified_default_null, + gmt_modified_default) + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("tenant_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("task_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 36, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("table_id", //column_name + ++column_id, //column_id + 3, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ret_code", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("start_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA_TS("end_time", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTimestampType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(ObPreciseDateTime), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false, //is_autoincrement + false); //is_on_update_for_timestamp + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("memory_used", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("stat_refresh_failed_list", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("properties", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare1", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare2", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare3", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("spare4", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + MAX_VALUE_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_TID); + table_schema.set_aux_lob_meta_tid(OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TID); + table_schema.set_aux_lob_piece_tid(OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + + +} // end namespace share +} // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.50451_50500.cpp b/src/share/inner_table/ob_inner_table_schema.50451_50500.cpp new file mode 100644 index 0000000000..5b1e379278 --- /dev/null +++ b/src/share/inner_table/ob_inner_table_schema.50451_50500.cpp @@ -0,0 +1,300 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SHARE_SCHEMA +#include "ob_inner_table_schema.h" + +#include "share/schema/ob_schema_macro_define.h" +#include "share/schema/ob_schema_service_sql_impl.h" +#include "share/schema/ob_table_schema.h" +#include "share/scn.h" + +namespace oceanbase +{ +using namespace share::schema; +using namespace common; +namespace share +{ + +int ObInnerTableSchema::all_task_opt_stat_gather_history_aux_lob_meta_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_META); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 16, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("seq_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 8192, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("binary_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("char_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 262144, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TID); + table_schema.set_data_table_id(OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_table_opt_stat_gather_history_aux_lob_meta_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(2); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_META); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 16, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("seq_id", //column_name + ++column_id, //column_id + 2, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 8192, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("binary_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("char_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 262144, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TID); + table_schema.set_data_table_id(OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + + +} // end namespace share +} // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.60451_60500.cpp b/src/share/inner_table/ob_inner_table_schema.60451_60500.cpp new file mode 100644 index 0000000000..32f5bad90f --- /dev/null +++ b/src/share/inner_table/ob_inner_table_schema.60451_60500.cpp @@ -0,0 +1,210 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SHARE_SCHEMA +#include "ob_inner_table_schema.h" + +#include "share/schema/ob_schema_macro_define.h" +#include "share/schema/ob_schema_service_sql_impl.h" +#include "share/schema/ob_table_schema.h" +#include "share/scn.h" + +namespace oceanbase +{ +using namespace share::schema; +using namespace common; +namespace share +{ + +int ObInnerTableSchema::all_task_opt_stat_gather_history_aux_lob_piece_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_PIECE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 32, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TID); + table_schema.set_data_table_id(OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_table_opt_stat_gather_history_aux_lob_piece_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(1); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(AUX_LOB_PIECE); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("piece_id", //column_name + ++column_id, //column_id + 1, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("data_len", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt32Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint32_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("lob_data", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_BINARY, //column_collation_type + 32, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TID); + table_schema.set_data_table_id(OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_TID); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + + +} // end namespace share +} // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.h b/src/share/inner_table/ob_inner_table_schema.h index cc02afc545..e90a43a96a 100644 --- a/src/share/inner_table/ob_inner_table_schema.h +++ b/src/share/inner_table/ob_inner_table_schema.h @@ -519,6 +519,8 @@ public: static int all_tenant_rewrite_rules_schema(share::schema::ObTableSchema &table_schema); static int all_reserved_snapshot_schema(share::schema::ObTableSchema &table_schema); static int all_cluster_event_history_schema(share::schema::ObTableSchema &table_schema); + static int all_task_opt_stat_gather_history_schema(share::schema::ObTableSchema &table_schema); + static int all_table_opt_stat_gather_history_schema(share::schema::ObTableSchema &table_schema); static int tenant_virtual_all_table_schema(share::schema::ObTableSchema &table_schema); static int tenant_virtual_table_column_schema(share::schema::ObTableSchema &table_schema); static int tenant_virtual_table_index_schema(share::schema::ObTableSchema &table_schema); @@ -871,6 +873,9 @@ public: static int all_virtual_ls_arb_replica_task_history_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_archive_dest_status_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_io_scheduler_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_task_opt_stat_gather_history_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_table_opt_stat_gather_history_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_opt_stat_gather_monitor_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_thread_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_virtual_long_ops_status_mysql_sys_agent_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_timestamp_service_schema(share::schema::ObTableSchema &table_schema); @@ -1082,6 +1087,9 @@ public: static int all_virtual_ls_arb_replica_task_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_ls_arb_replica_task_history_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_archive_dest_status_ora_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_task_opt_stat_gather_history_ora_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_table_opt_stat_gather_history_ora_schema(share::schema::ObTableSchema &table_schema); + static int all_virtual_opt_stat_gather_monitor_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_long_ops_status_sys_agent_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_thread_ora_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_timestamp_service_ora_schema(share::schema::ObTableSchema &table_schema); @@ -1374,9 +1382,14 @@ public: static int dba_ob_ls_log_archive_progress_schema(share::schema::ObTableSchema &table_schema); static int cdb_ob_ls_log_archive_progress_schema(share::schema::ObTableSchema &table_schema); static int dba_ob_rsrc_io_directives_schema(share::schema::ObTableSchema &table_schema); + static int gv_ob_opt_stat_gather_monitor_schema(share::schema::ObTableSchema &table_schema); + static int v_ob_opt_stat_gather_monitor_schema(share::schema::ObTableSchema &table_schema); + static int dba_ob_task_opt_stat_gather_history_schema(share::schema::ObTableSchema &table_schema); + static int dba_ob_table_opt_stat_gather_history_schema(share::schema::ObTableSchema &table_schema); static int gv_ob_thread_schema(share::schema::ObTableSchema &table_schema); static int v_ob_thread_schema(share::schema::ObTableSchema &table_schema); static int v_ob_timestamp_service_schema(share::schema::ObTableSchema &table_schema); + static int dba_ob_table_stat_stale_info_schema(share::schema::ObTableSchema &table_schema); static int dba_synonyms_schema(share::schema::ObTableSchema &table_schema); static int dba_objects_ora_schema(share::schema::ObTableSchema &table_schema); static int all_objects_schema(share::schema::ObTableSchema &table_schema); @@ -1599,6 +1612,9 @@ public: static int all_db_links_ora_schema(share::schema::ObTableSchema &table_schema); static int dba_db_links_ora_schema(share::schema::ObTableSchema &table_schema); static int user_db_links_ora_schema(share::schema::ObTableSchema &table_schema); + static int dba_ob_task_opt_stat_gather_history_ora_schema(share::schema::ObTableSchema &table_schema); + static int dba_ob_table_opt_stat_gather_history_ora_schema(share::schema::ObTableSchema &table_schema); + static int dba_ob_table_stat_stale_info_ora_schema(share::schema::ObTableSchema &table_schema); static int gv_ob_sql_audit_ora_schema(share::schema::ObTableSchema &table_schema); static int v_ob_sql_audit_ora_schema(share::schema::ObTableSchema &table_schema); static int gv_instance_schema(share::schema::ObTableSchema &table_schema); @@ -1749,6 +1765,8 @@ public: static int v_ob_sql_plan_ora_schema(share::schema::ObTableSchema &table_schema); static int v_ob_archive_dest_status_ora_schema(share::schema::ObTableSchema &table_schema); static int dba_ob_ls_log_archive_progress_ora_schema(share::schema::ObTableSchema &table_schema); + static int gv_ob_opt_stat_gather_monitor_ora_schema(share::schema::ObTableSchema &table_schema); + static int v_ob_opt_stat_gather_monitor_ora_schema(share::schema::ObTableSchema &table_schema); static int gv_session_longops_ora_schema(share::schema::ObTableSchema &table_schema); static int v_session_longops_ora_schema(share::schema::ObTableSchema &table_schema); static int gv_ob_thread_ora_schema(share::schema::ObTableSchema &table_schema); @@ -1987,6 +2005,8 @@ public: static int all_tenant_rewrite_rules_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_reserved_snapshot_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_cluster_event_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); + static int all_task_opt_stat_gather_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); + static int all_table_opt_stat_gather_history_aux_lob_meta_schema(share::schema::ObTableSchema &table_schema); static int all_table_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_column_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_ddl_operation_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); @@ -2220,6 +2240,8 @@ public: static int all_tenant_rewrite_rules_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_reserved_snapshot_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_cluster_event_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); + static int all_task_opt_stat_gather_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); + static int all_table_opt_stat_gather_history_aux_lob_piece_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_sql_plan_monitor_all_virtual_sql_plan_monitor_i1_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_sql_audit_all_virtual_sql_audit_i1_schema(share::schema::ObTableSchema &table_schema); static int all_virtual_sysstat_all_virtual_sysstat_i1_schema(share::schema::ObTableSchema &table_schema); @@ -2649,6 +2671,8 @@ const schema_create_func sys_table_schema_creators [] = { ObInnerTableSchema::all_tenant_rewrite_rules_schema, ObInnerTableSchema::all_reserved_snapshot_schema, ObInnerTableSchema::all_cluster_event_history_schema, + ObInnerTableSchema::all_task_opt_stat_gather_history_schema, + ObInnerTableSchema::all_table_opt_stat_gather_history_schema, NULL,}; const schema_create_func virtual_table_schema_creators [] = { @@ -3004,6 +3028,9 @@ const schema_create_func virtual_table_schema_creators [] = { ObInnerTableSchema::all_virtual_ls_arb_replica_task_history_schema, ObInnerTableSchema::all_virtual_archive_dest_status_schema, ObInnerTableSchema::all_virtual_io_scheduler_schema, + ObInnerTableSchema::all_virtual_task_opt_stat_gather_history_schema, + ObInnerTableSchema::all_virtual_table_opt_stat_gather_history_schema, + ObInnerTableSchema::all_virtual_opt_stat_gather_monitor_schema, ObInnerTableSchema::all_virtual_thread_schema, ObInnerTableSchema::all_virtual_virtual_long_ops_status_mysql_sys_agent_schema, ObInnerTableSchema::all_virtual_timestamp_service_schema, @@ -3224,6 +3251,9 @@ const schema_create_func virtual_table_schema_creators [] = { ObInnerTableSchema::all_virtual_ls_arb_replica_task_ora_schema, ObInnerTableSchema::all_virtual_ls_arb_replica_task_history_ora_schema, ObInnerTableSchema::all_virtual_archive_dest_status_ora_schema, + ObInnerTableSchema::all_virtual_task_opt_stat_gather_history_ora_schema, + ObInnerTableSchema::all_virtual_table_opt_stat_gather_history_ora_schema, + ObInnerTableSchema::all_virtual_opt_stat_gather_monitor_ora_schema, ObInnerTableSchema::all_virtual_long_ops_status_sys_agent_schema, ObInnerTableSchema::all_virtual_thread_ora_schema, ObInnerTableSchema::all_virtual_timestamp_service_ora_schema, @@ -3597,9 +3627,14 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::dba_ob_ls_log_archive_progress_schema, ObInnerTableSchema::cdb_ob_ls_log_archive_progress_schema, ObInnerTableSchema::dba_ob_rsrc_io_directives_schema, + ObInnerTableSchema::gv_ob_opt_stat_gather_monitor_schema, + ObInnerTableSchema::v_ob_opt_stat_gather_monitor_schema, + ObInnerTableSchema::dba_ob_task_opt_stat_gather_history_schema, + ObInnerTableSchema::dba_ob_table_opt_stat_gather_history_schema, ObInnerTableSchema::gv_ob_thread_schema, ObInnerTableSchema::v_ob_thread_schema, ObInnerTableSchema::v_ob_timestamp_service_schema, + ObInnerTableSchema::dba_ob_table_stat_stale_info_schema, ObInnerTableSchema::dba_synonyms_schema, ObInnerTableSchema::dba_objects_ora_schema, ObInnerTableSchema::all_objects_schema, @@ -3822,6 +3857,9 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::all_db_links_ora_schema, ObInnerTableSchema::dba_db_links_ora_schema, ObInnerTableSchema::user_db_links_ora_schema, + ObInnerTableSchema::dba_ob_task_opt_stat_gather_history_ora_schema, + ObInnerTableSchema::dba_ob_table_opt_stat_gather_history_ora_schema, + ObInnerTableSchema::dba_ob_table_stat_stale_info_ora_schema, ObInnerTableSchema::gv_ob_sql_audit_ora_schema, ObInnerTableSchema::v_ob_sql_audit_ora_schema, ObInnerTableSchema::gv_instance_schema, @@ -3972,6 +4010,8 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::v_ob_sql_plan_ora_schema, ObInnerTableSchema::v_ob_archive_dest_status_ora_schema, ObInnerTableSchema::dba_ob_ls_log_archive_progress_ora_schema, + ObInnerTableSchema::gv_ob_opt_stat_gather_monitor_ora_schema, + ObInnerTableSchema::v_ob_opt_stat_gather_monitor_ora_schema, ObInnerTableSchema::gv_session_longops_ora_schema, ObInnerTableSchema::v_session_longops_ora_schema, ObInnerTableSchema::gv_ob_thread_ora_schema, @@ -4299,6 +4339,8 @@ const uint64_t tenant_space_tables [] = { OB_ALL_RLS_ATTRIBUTE_HISTORY_TID, OB_ALL_TENANT_REWRITE_RULES_TID, OB_ALL_RESERVED_SNAPSHOT_TID, + OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_TID, + OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_TID, OB_TENANT_VIRTUAL_ALL_TABLE_TID, OB_TENANT_VIRTUAL_TABLE_COLUMN_TID, OB_TENANT_VIRTUAL_TABLE_INDEX_TID, @@ -4456,6 +4498,9 @@ const uint64_t tenant_space_tables [] = { OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_TID, OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_HISTORY_TID, OB_ALL_VIRTUAL_ARCHIVE_DEST_STATUS_TID, + OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_TID, + OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_TID, + OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_TID, OB_ALL_VIRTUAL_THREAD_TID, OB_ALL_VIRTUAL_VIRTUAL_LONG_OPS_STATUS_MYSQL_SYS_AGENT_TID, OB_ALL_VIRTUAL_TIMESTAMP_SERVICE_TID, @@ -4675,6 +4720,9 @@ const uint64_t tenant_space_tables [] = { OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_ORA_TID, OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_HISTORY_ORA_TID, OB_ALL_VIRTUAL_ARCHIVE_DEST_STATUS_ORA_TID, + OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_ORA_TID, + OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_ORA_TID, + OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_ORA_TID, OB_ALL_VIRTUAL_LONG_OPS_STATUS_SYS_AGENT_TID, OB_ALL_VIRTUAL_THREAD_ORA_TID, OB_ALL_VIRTUAL_TIMESTAMP_SERVICE_ORA_TID, @@ -4890,9 +4938,14 @@ const uint64_t tenant_space_tables [] = { OB_V_OB_ARCHIVE_DEST_STATUS_TID, OB_DBA_OB_LS_LOG_ARCHIVE_PROGRESS_TID, OB_DBA_OB_RSRC_IO_DIRECTIVES_TID, + OB_GV_OB_OPT_STAT_GATHER_MONITOR_TID, + OB_V_OB_OPT_STAT_GATHER_MONITOR_TID, + OB_DBA_OB_TASK_OPT_STAT_GATHER_HISTORY_TID, + OB_DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY_TID, OB_GV_OB_THREAD_TID, OB_V_OB_THREAD_TID, OB_V_OB_TIMESTAMP_SERVICE_TID, + OB_DBA_OB_TABLE_STAT_STALE_INFO_TID, OB_DBA_SYNONYMS_TID, OB_DBA_OBJECTS_ORA_TID, OB_ALL_OBJECTS_TID, @@ -5115,6 +5168,9 @@ const uint64_t tenant_space_tables [] = { OB_ALL_DB_LINKS_ORA_TID, OB_DBA_DB_LINKS_ORA_TID, OB_USER_DB_LINKS_ORA_TID, + OB_DBA_OB_TASK_OPT_STAT_GATHER_HISTORY_ORA_TID, + OB_DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY_ORA_TID, + OB_DBA_OB_TABLE_STAT_STALE_INFO_ORA_TID, OB_GV_OB_SQL_AUDIT_ORA_TID, OB_V_OB_SQL_AUDIT_ORA_TID, OB_GV_INSTANCE_TID, @@ -5265,6 +5321,8 @@ const uint64_t tenant_space_tables [] = { OB_V_OB_SQL_PLAN_ORA_TID, OB_V_OB_ARCHIVE_DEST_STATUS_ORA_TID, OB_DBA_OB_LS_LOG_ARCHIVE_PROGRESS_ORA_TID, + OB_GV_OB_OPT_STAT_GATHER_MONITOR_ORA_TID, + OB_V_OB_OPT_STAT_GATHER_MONITOR_ORA_TID, OB_GV_SESSION_LONGOPS_ORA_TID, OB_V_SESSION_LONGOPS_ORA_TID, OB_GV_OB_THREAD_ORA_TID, @@ -5640,6 +5698,8 @@ const uint64_t tenant_space_tables [] = { OB_ALL_RLS_ATTRIBUTE_HISTORY_AUX_LOB_META_TID, OB_ALL_TENANT_REWRITE_RULES_AUX_LOB_META_TID, OB_ALL_RESERVED_SNAPSHOT_AUX_LOB_META_TID, + OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TID, + OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TID, OB_ALL_TABLE_AUX_LOB_PIECE_TID, OB_ALL_COLUMN_AUX_LOB_PIECE_TID, OB_ALL_DDL_OPERATION_AUX_LOB_PIECE_TID, @@ -5851,7 +5911,9 @@ const uint64_t tenant_space_tables [] = { OB_ALL_RLS_ATTRIBUTE_AUX_LOB_PIECE_TID, OB_ALL_RLS_ATTRIBUTE_HISTORY_AUX_LOB_PIECE_TID, OB_ALL_TENANT_REWRITE_RULES_AUX_LOB_PIECE_TID, - OB_ALL_RESERVED_SNAPSHOT_AUX_LOB_PIECE_TID, }; + OB_ALL_RESERVED_SNAPSHOT_AUX_LOB_PIECE_TID, + OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TID, + OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TID, }; const uint64_t all_ora_mapping_virtual_table_org_tables [] = { OB_ALL_VIRTUAL_SQL_AUDIT_TID, @@ -5965,6 +6027,9 @@ const uint64_t all_ora_mapping_virtual_table_org_tables [] = { OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_TID, OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_HISTORY_TID, OB_ALL_VIRTUAL_ARCHIVE_DEST_STATUS_TID, + OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_TID, + OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_TID, + OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_TID, OB_ALL_VIRTUAL_THREAD_TID, OB_ALL_VIRTUAL_TIMESTAMP_SERVICE_TID, }; @@ -6079,6 +6144,9 @@ const uint64_t all_ora_mapping_virtual_tables [] = { OB_ALL_VIRTUAL_SQL_AUDIT_O , OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_ORA_TID , OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_HISTORY_ORA_TID , OB_ALL_VIRTUAL_ARCHIVE_DEST_STATUS_ORA_TID +, OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_ORA_TID +, OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_ORA_TID +, OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_ORA_TID , OB_ALL_VIRTUAL_THREAD_ORA_TID , OB_ALL_VIRTUAL_TIMESTAMP_SERVICE_ORA_TID , }; @@ -6308,6 +6376,8 @@ const char* const tenant_space_table_names [] = { OB_ALL_RLS_ATTRIBUTE_HISTORY_TNAME, OB_ALL_TENANT_REWRITE_RULES_TNAME, OB_ALL_RESERVED_SNAPSHOT_TNAME, + OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_TNAME, + OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_TNAME, OB_TENANT_VIRTUAL_ALL_TABLE_TNAME, OB_TENANT_VIRTUAL_TABLE_COLUMN_TNAME, OB_TENANT_VIRTUAL_TABLE_INDEX_TNAME, @@ -6465,6 +6535,9 @@ const char* const tenant_space_table_names [] = { OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_TNAME, OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_HISTORY_TNAME, OB_ALL_VIRTUAL_ARCHIVE_DEST_STATUS_TNAME, + OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_TNAME, + OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_TNAME, + OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_TNAME, OB_ALL_VIRTUAL_THREAD_TNAME, OB_ALL_VIRTUAL_VIRTUAL_LONG_OPS_STATUS_MYSQL_SYS_AGENT_TNAME, OB_ALL_VIRTUAL_TIMESTAMP_SERVICE_TNAME, @@ -6684,6 +6757,9 @@ const char* const tenant_space_table_names [] = { OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_ORA_TNAME, OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_HISTORY_ORA_TNAME, OB_ALL_VIRTUAL_ARCHIVE_DEST_STATUS_ORA_TNAME, + OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_ORA_TNAME, + OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_ORA_TNAME, + OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_ORA_TNAME, OB_ALL_VIRTUAL_LONG_OPS_STATUS_SYS_AGENT_TNAME, OB_ALL_VIRTUAL_THREAD_ORA_TNAME, OB_ALL_VIRTUAL_TIMESTAMP_SERVICE_ORA_TNAME, @@ -6899,9 +6975,14 @@ const char* const tenant_space_table_names [] = { OB_V_OB_ARCHIVE_DEST_STATUS_TNAME, OB_DBA_OB_LS_LOG_ARCHIVE_PROGRESS_TNAME, OB_DBA_OB_RSRC_IO_DIRECTIVES_TNAME, + OB_GV_OB_OPT_STAT_GATHER_MONITOR_TNAME, + OB_V_OB_OPT_STAT_GATHER_MONITOR_TNAME, + OB_DBA_OB_TASK_OPT_STAT_GATHER_HISTORY_TNAME, + OB_DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY_TNAME, OB_GV_OB_THREAD_TNAME, OB_V_OB_THREAD_TNAME, OB_V_OB_TIMESTAMP_SERVICE_TNAME, + OB_DBA_OB_TABLE_STAT_STALE_INFO_TNAME, OB_DBA_SYNONYMS_TNAME, OB_DBA_OBJECTS_ORA_TNAME, OB_ALL_OBJECTS_TNAME, @@ -7124,6 +7205,9 @@ const char* const tenant_space_table_names [] = { OB_ALL_DB_LINKS_ORA_TNAME, OB_DBA_DB_LINKS_ORA_TNAME, OB_USER_DB_LINKS_ORA_TNAME, + OB_DBA_OB_TASK_OPT_STAT_GATHER_HISTORY_ORA_TNAME, + OB_DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY_ORA_TNAME, + OB_DBA_OB_TABLE_STAT_STALE_INFO_ORA_TNAME, OB_GV_OB_SQL_AUDIT_ORA_TNAME, OB_V_OB_SQL_AUDIT_ORA_TNAME, OB_GV_INSTANCE_TNAME, @@ -7274,6 +7358,8 @@ const char* const tenant_space_table_names [] = { OB_V_OB_SQL_PLAN_ORA_TNAME, OB_V_OB_ARCHIVE_DEST_STATUS_ORA_TNAME, OB_DBA_OB_LS_LOG_ARCHIVE_PROGRESS_ORA_TNAME, + OB_GV_OB_OPT_STAT_GATHER_MONITOR_ORA_TNAME, + OB_V_OB_OPT_STAT_GATHER_MONITOR_ORA_TNAME, OB_GV_SESSION_LONGOPS_ORA_TNAME, OB_V_SESSION_LONGOPS_ORA_TNAME, OB_GV_OB_THREAD_ORA_TNAME, @@ -7649,6 +7735,8 @@ const char* const tenant_space_table_names [] = { OB_ALL_RLS_ATTRIBUTE_HISTORY_AUX_LOB_META_TNAME, OB_ALL_TENANT_REWRITE_RULES_AUX_LOB_META_TNAME, OB_ALL_RESERVED_SNAPSHOT_AUX_LOB_META_TNAME, + OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TNAME, + OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TNAME, OB_ALL_TABLE_AUX_LOB_PIECE_TNAME, OB_ALL_COLUMN_AUX_LOB_PIECE_TNAME, OB_ALL_DDL_OPERATION_AUX_LOB_PIECE_TNAME, @@ -7860,7 +7948,9 @@ const char* const tenant_space_table_names [] = { OB_ALL_RLS_ATTRIBUTE_AUX_LOB_PIECE_TNAME, OB_ALL_RLS_ATTRIBUTE_HISTORY_AUX_LOB_PIECE_TNAME, OB_ALL_TENANT_REWRITE_RULES_AUX_LOB_PIECE_TNAME, - OB_ALL_RESERVED_SNAPSHOT_AUX_LOB_PIECE_TNAME, }; + OB_ALL_RESERVED_SNAPSHOT_AUX_LOB_PIECE_TNAME, + OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TNAME, + OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TNAME, }; const uint64_t only_rs_vtables [] = { OB_ALL_VIRTUAL_CORE_META_TABLE_TID, @@ -7989,6 +8079,7 @@ const uint64_t tenant_distributed_vtables [] = { OB_ALL_VIRTUAL_TABLET_COMPACTION_INFO_TID, OB_ALL_VIRTUAL_SQL_PLAN_TID, OB_ALL_VIRTUAL_MALLOC_SAMPLE_INFO_TID, + OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_TID, OB_ALL_VIRTUAL_THREAD_TID, OB_ALL_VIRTUAL_TIMESTAMP_SERVICE_TID, OB_ALL_VIRTUAL_SQL_AUDIT_ORA_TID, @@ -8048,6 +8139,7 @@ const uint64_t tenant_distributed_vtables [] = { OB_ALL_VIRTUAL_ARCHIVE_STAT_ORA_TID, OB_ALL_VIRTUAL_SQL_PLAN_ORA_TID, OB_ALL_VIRTUAL_TRANS_SCHEDULER_ORA_TID, + OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_ORA_TID, OB_ALL_VIRTUAL_THREAD_ORA_TID, OB_ALL_VIRTUAL_TIMESTAMP_SERVICE_ORA_TID, }; @@ -8137,6 +8229,9 @@ const uint64_t restrict_access_virtual_tables[] = { OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_ORA_TID, OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_HISTORY_ORA_TID, OB_ALL_VIRTUAL_ARCHIVE_DEST_STATUS_ORA_TID, + OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_ORA_TID, + OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_ORA_TID, + OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_ORA_TID, OB_ALL_VIRTUAL_THREAD_ORA_TID, OB_ALL_VIRTUAL_TIMESTAMP_SERVICE_ORA_TID }; @@ -10149,6 +10244,22 @@ LOBMapping const lob_aux_table_mappings [] = { ObInnerTableSchema::all_cluster_event_history_aux_lob_piece_schema }, + { + OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_TID, + OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TID, + OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TID, + ObInnerTableSchema::all_task_opt_stat_gather_history_aux_lob_meta_schema, + ObInnerTableSchema::all_task_opt_stat_gather_history_aux_lob_piece_schema + }, + + { + OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_TID, + OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TID, + OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TID, + ObInnerTableSchema::all_table_opt_stat_gather_history_aux_lob_meta_schema, + ObInnerTableSchema::all_table_opt_stat_gather_history_aux_lob_piece_schema + }, + }; static inline bool get_sys_table_lob_aux_table_id(const uint64_t tid, uint64_t& meta_tid, uint64_t& piece_tid) @@ -10186,12 +10297,12 @@ static inline int get_sys_table_lob_aux_schema(const uint64_t tid, } const int64_t OB_CORE_TABLE_COUNT = 4; -const int64_t OB_SYS_TABLE_COUNT = 230; -const int64_t OB_VIRTUAL_TABLE_COUNT = 653; -const int64_t OB_SYS_VIEW_COUNT = 669; -const int64_t OB_SYS_TENANT_TABLE_COUNT = 1557; +const int64_t OB_SYS_TABLE_COUNT = 232; +const int64_t OB_VIRTUAL_TABLE_COUNT = 659; +const int64_t OB_SYS_VIEW_COUNT = 679; +const int64_t OB_SYS_TENANT_TABLE_COUNT = 1575; const int64_t OB_CORE_SCHEMA_VERSION = 1; -const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 1560; +const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 1578; } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.lob.cpp b/src/share/inner_table/ob_inner_table_schema.lob.cpp index 818186b1e3..9c3148e284 100644 --- a/src/share/inner_table/ob_inner_table_schema.lob.cpp +++ b/src/share/inner_table/ob_inner_table_schema.lob.cpp @@ -21,7 +21,7 @@ inner_lob_map_t inner_lob_map; bool lob_mapping_init() { int ret = OB_SUCCESS; - if (OB_FAIL(inner_lob_map.create(233, ObModIds::OB_INNER_LOB_HASH_SET))) { + if (OB_FAIL(inner_lob_map.create(235, ObModIds::OB_INNER_LOB_HASH_SET))) { SERVER_LOG(WARN, "fail to create inner lob map", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(lob_aux_table_mappings); ++i) { diff --git a/src/share/inner_table/ob_inner_table_schema_constants.h b/src/share/inner_table/ob_inner_table_schema_constants.h index 78cfe99440..415e3a0a8a 100644 --- a/src/share/inner_table/ob_inner_table_schema_constants.h +++ b/src/share/inner_table/ob_inner_table_schema_constants.h @@ -261,6 +261,8 @@ const uint64_t OB_ALL_RLS_ATTRIBUTE_HISTORY_TID = 442; // "__all_rls_attribute_h const uint64_t OB_ALL_TENANT_REWRITE_RULES_TID = 443; // "__all_tenant_rewrite_rules" const uint64_t OB_ALL_RESERVED_SNAPSHOT_TID = 444; // "__all_reserved_snapshot" const uint64_t OB_ALL_CLUSTER_EVENT_HISTORY_TID = 445; // "__all_cluster_event_history" +const uint64_t OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_TID = 451; // "__all_task_opt_stat_gather_history" +const uint64_t OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_TID = 452; // "__all_table_opt_stat_gather_history" const uint64_t OB_TENANT_VIRTUAL_ALL_TABLE_TID = 10001; // "__tenant_virtual_all_table" const uint64_t OB_TENANT_VIRTUAL_TABLE_COLUMN_TID = 10002; // "__tenant_virtual_table_column" const uint64_t OB_TENANT_VIRTUAL_TABLE_INDEX_TID = 10003; // "__tenant_virtual_table_index" @@ -613,6 +615,9 @@ const uint64_t OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_TID = 12364; // "__all_virtual const uint64_t OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_HISTORY_TID = 12365; // "__all_virtual_ls_arb_replica_task_history" const uint64_t OB_ALL_VIRTUAL_ARCHIVE_DEST_STATUS_TID = 12366; // "__all_virtual_archive_dest_status" const uint64_t OB_ALL_VIRTUAL_IO_SCHEDULER_TID = 12369; // "__all_virtual_io_scheduler" +const uint64_t OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_TID = 12381; // "__all_virtual_task_opt_stat_gather_history" +const uint64_t OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_TID = 12382; // "__all_virtual_table_opt_stat_gather_history" +const uint64_t OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_TID = 12383; // "__all_virtual_opt_stat_gather_monitor" const uint64_t OB_ALL_VIRTUAL_THREAD_TID = 12384; // "__all_virtual_thread" const uint64_t OB_ALL_VIRTUAL_VIRTUAL_LONG_OPS_STATUS_MYSQL_SYS_AGENT_TID = 12393; // "__all_virtual_virtual_long_ops_status_mysql_sys_agent" const uint64_t OB_ALL_VIRTUAL_TIMESTAMP_SERVICE_TID = 12395; // "__all_virtual_timestamp_service" @@ -824,6 +829,9 @@ const uint64_t OB_ALL_VIRTUAL_TRANS_SCHEDULER_ORA_TID = 15287; // "ALL_VIRTUAL_T const uint64_t OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_ORA_TID = 15288; // "ALL_VIRTUAL_LS_ARB_REPLICA_TASK_ORA" const uint64_t OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_HISTORY_ORA_TID = 15289; // "ALL_VIRTUAL_LS_ARB_REPLICA_TASK_HISTORY_ORA" const uint64_t OB_ALL_VIRTUAL_ARCHIVE_DEST_STATUS_ORA_TID = 15290; // "ALL_VIRTUAL_ARCHIVE_DEST_STATUS_ORA" +const uint64_t OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_ORA_TID = 15294; // "ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_ORA" +const uint64_t OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_ORA_TID = 15295; // "ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_ORA" +const uint64_t OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_ORA_TID = 15296; // "ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_ORA" const uint64_t OB_ALL_VIRTUAL_LONG_OPS_STATUS_SYS_AGENT_TID = 15297; // "ALL_VIRTUAL_LONG_OPS_STATUS_SYS_AGENT" const uint64_t OB_ALL_VIRTUAL_THREAD_ORA_TID = 15298; // "ALL_VIRTUAL_THREAD_ORA" const uint64_t OB_ALL_VIRTUAL_TIMESTAMP_SERVICE_ORA_TID = 15385; // "ALL_VIRTUAL_TIMESTAMP_SERVICE_ORA" @@ -1116,9 +1124,14 @@ const uint64_t OB_V_OB_ARCHIVE_DEST_STATUS_TID = 21362; // "V$OB_ARCHIVE_DEST_ST const uint64_t OB_DBA_OB_LS_LOG_ARCHIVE_PROGRESS_TID = 21363; // "DBA_OB_LS_LOG_ARCHIVE_PROGRESS" const uint64_t OB_CDB_OB_LS_LOG_ARCHIVE_PROGRESS_TID = 21364; // "CDB_OB_LS_LOG_ARCHIVE_PROGRESS" const uint64_t OB_DBA_OB_RSRC_IO_DIRECTIVES_TID = 21369; // "DBA_OB_RSRC_IO_DIRECTIVES" +const uint64_t OB_GV_OB_OPT_STAT_GATHER_MONITOR_TID = 21376; // "GV$OB_OPT_STAT_GATHER_MONITOR" +const uint64_t OB_V_OB_OPT_STAT_GATHER_MONITOR_TID = 21377; // "V$OB_OPT_STAT_GATHER_MONITOR" +const uint64_t OB_DBA_OB_TASK_OPT_STAT_GATHER_HISTORY_TID = 21378; // "DBA_OB_TASK_OPT_STAT_GATHER_HISTORY" +const uint64_t OB_DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY_TID = 21379; // "DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY" const uint64_t OB_GV_OB_THREAD_TID = 21380; // "GV$OB_THREAD" const uint64_t OB_V_OB_THREAD_TID = 21381; // "V$OB_THREAD" const uint64_t OB_V_OB_TIMESTAMP_SERVICE_TID = 21404; // "V$OB_TIMESTAMP_SERVICE" +const uint64_t OB_DBA_OB_TABLE_STAT_STALE_INFO_TID = 21423; // "DBA_OB_TABLE_STAT_STALE_INFO" const uint64_t OB_DBA_SYNONYMS_TID = 25001; // "DBA_SYNONYMS" const uint64_t OB_DBA_OBJECTS_ORA_TID = 25002; // "DBA_OBJECTS_ORA" const uint64_t OB_ALL_OBJECTS_TID = 25003; // "ALL_OBJECTS" @@ -1341,6 +1354,9 @@ const uint64_t OB_DBA_OB_RSRC_IO_DIRECTIVES_ORA_TID = 25223; // "DBA_OB_RSRC_IO_ const uint64_t OB_ALL_DB_LINKS_ORA_TID = 25224; // "ALL_DB_LINKS_ORA" const uint64_t OB_DBA_DB_LINKS_ORA_TID = 25225; // "DBA_DB_LINKS_ORA" const uint64_t OB_USER_DB_LINKS_ORA_TID = 25226; // "USER_DB_LINKS_ORA" +const uint64_t OB_DBA_OB_TASK_OPT_STAT_GATHER_HISTORY_ORA_TID = 25227; // "DBA_OB_TASK_OPT_STAT_GATHER_HISTORY_ORA" +const uint64_t OB_DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY_ORA_TID = 25228; // "DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY_ORA" +const uint64_t OB_DBA_OB_TABLE_STAT_STALE_INFO_ORA_TID = 25247; // "DBA_OB_TABLE_STAT_STALE_INFO_ORA" const uint64_t OB_GV_OB_SQL_AUDIT_ORA_TID = 28002; // "GV$OB_SQL_AUDIT_ORA" const uint64_t OB_V_OB_SQL_AUDIT_ORA_TID = 28003; // "V$OB_SQL_AUDIT_ORA" const uint64_t OB_GV_INSTANCE_TID = 28004; // "GV$INSTANCE" @@ -1491,6 +1507,8 @@ const uint64_t OB_GV_OB_SQL_PLAN_ORA_TID = 28172; // "GV$OB_SQL_PLAN_ORA" const uint64_t OB_V_OB_SQL_PLAN_ORA_TID = 28173; // "V$OB_SQL_PLAN_ORA" const uint64_t OB_V_OB_ARCHIVE_DEST_STATUS_ORA_TID = 28176; // "V$OB_ARCHIVE_DEST_STATUS_ORA" const uint64_t OB_DBA_OB_LS_LOG_ARCHIVE_PROGRESS_ORA_TID = 28177; // "DBA_OB_LS_LOG_ARCHIVE_PROGRESS_ORA" +const uint64_t OB_GV_OB_OPT_STAT_GATHER_MONITOR_ORA_TID = 28183; // "GV$OB_OPT_STAT_GATHER_MONITOR_ORA" +const uint64_t OB_V_OB_OPT_STAT_GATHER_MONITOR_ORA_TID = 28184; // "V$OB_OPT_STAT_GATHER_MONITOR_ORA" const uint64_t OB_GV_SESSION_LONGOPS_ORA_TID = 28185; // "GV$SESSION_LONGOPS_ORA" const uint64_t OB_V_SESSION_LONGOPS_ORA_TID = 28186; // "V$SESSION_LONGOPS_ORA" const uint64_t OB_GV_OB_THREAD_ORA_TID = 28187; // "GV$OB_THREAD_ORA" @@ -1729,6 +1747,8 @@ const uint64_t OB_ALL_RLS_ATTRIBUTE_HISTORY_AUX_LOB_META_TID = 50442; // "__all_ const uint64_t OB_ALL_TENANT_REWRITE_RULES_AUX_LOB_META_TID = 50443; // "__all_tenant_rewrite_rules_aux_lob_meta" const uint64_t OB_ALL_RESERVED_SNAPSHOT_AUX_LOB_META_TID = 50444; // "__all_reserved_snapshot_aux_lob_meta" const uint64_t OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_META_TID = 50445; // "__all_cluster_event_history_aux_lob_meta" +const uint64_t OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TID = 50451; // "__all_task_opt_stat_gather_history_aux_lob_meta" +const uint64_t OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TID = 50452; // "__all_table_opt_stat_gather_history_aux_lob_meta" const uint64_t OB_ALL_TABLE_AUX_LOB_PIECE_TID = 60003; // "__all_table_aux_lob_piece" const uint64_t OB_ALL_COLUMN_AUX_LOB_PIECE_TID = 60004; // "__all_column_aux_lob_piece" const uint64_t OB_ALL_DDL_OPERATION_AUX_LOB_PIECE_TID = 60005; // "__all_ddl_operation_aux_lob_piece" @@ -1962,6 +1982,8 @@ const uint64_t OB_ALL_RLS_ATTRIBUTE_HISTORY_AUX_LOB_PIECE_TID = 60442; // "__all const uint64_t OB_ALL_TENANT_REWRITE_RULES_AUX_LOB_PIECE_TID = 60443; // "__all_tenant_rewrite_rules_aux_lob_piece" const uint64_t OB_ALL_RESERVED_SNAPSHOT_AUX_LOB_PIECE_TID = 60444; // "__all_reserved_snapshot_aux_lob_piece" const uint64_t OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_PIECE_TID = 60445; // "__all_cluster_event_history_aux_lob_piece" +const uint64_t OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TID = 60451; // "__all_task_opt_stat_gather_history_aux_lob_piece" +const uint64_t OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TID = 60452; // "__all_table_opt_stat_gather_history_aux_lob_piece" const uint64_t OB_ALL_VIRTUAL_PLAN_CACHE_STAT_ALL_VIRTUAL_PLAN_CACHE_STAT_I1_TID = 14999; // "__all_virtual_plan_cache_stat" const uint64_t OB_ALL_VIRTUAL_SESSION_EVENT_ALL_VIRTUAL_SESSION_EVENT_I1_TID = 14998; // "__all_virtual_session_event" const uint64_t OB_ALL_VIRTUAL_SESSION_WAIT_ALL_VIRTUAL_SESSION_WAIT_I1_TID = 14997; // "__all_virtual_session_wait" @@ -2378,6 +2400,8 @@ const char *const OB_ALL_RLS_ATTRIBUTE_HISTORY_TNAME = "__all_rls_attribute_hist const char *const OB_ALL_TENANT_REWRITE_RULES_TNAME = "__all_tenant_rewrite_rules"; const char *const OB_ALL_RESERVED_SNAPSHOT_TNAME = "__all_reserved_snapshot"; const char *const OB_ALL_CLUSTER_EVENT_HISTORY_TNAME = "__all_cluster_event_history"; +const char *const OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_TNAME = "__all_task_opt_stat_gather_history"; +const char *const OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_TNAME = "__all_table_opt_stat_gather_history"; const char *const OB_TENANT_VIRTUAL_ALL_TABLE_TNAME = "__tenant_virtual_all_table"; const char *const OB_TENANT_VIRTUAL_TABLE_COLUMN_TNAME = "__tenant_virtual_table_column"; const char *const OB_TENANT_VIRTUAL_TABLE_INDEX_TNAME = "__tenant_virtual_table_index"; @@ -2730,6 +2754,9 @@ const char *const OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_TNAME = "__all_virtual_ls_a const char *const OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_HISTORY_TNAME = "__all_virtual_ls_arb_replica_task_history"; const char *const OB_ALL_VIRTUAL_ARCHIVE_DEST_STATUS_TNAME = "__all_virtual_archive_dest_status"; const char *const OB_ALL_VIRTUAL_IO_SCHEDULER_TNAME = "__all_virtual_io_scheduler"; +const char *const OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_TNAME = "__all_virtual_task_opt_stat_gather_history"; +const char *const OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_TNAME = "__all_virtual_table_opt_stat_gather_history"; +const char *const OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_TNAME = "__all_virtual_opt_stat_gather_monitor"; const char *const OB_ALL_VIRTUAL_THREAD_TNAME = "__all_virtual_thread"; const char *const OB_ALL_VIRTUAL_VIRTUAL_LONG_OPS_STATUS_MYSQL_SYS_AGENT_TNAME = "__all_virtual_virtual_long_ops_status_mysql_sys_agent"; const char *const OB_ALL_VIRTUAL_TIMESTAMP_SERVICE_TNAME = "__all_virtual_timestamp_service"; @@ -2941,6 +2968,9 @@ const char *const OB_ALL_VIRTUAL_TRANS_SCHEDULER_ORA_TNAME = "ALL_VIRTUAL_TRANS_ const char *const OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_ORA_TNAME = "ALL_VIRTUAL_LS_ARB_REPLICA_TASK"; const char *const OB_ALL_VIRTUAL_LS_ARB_REPLICA_TASK_HISTORY_ORA_TNAME = "ALL_VIRTUAL_LS_ARB_REPLICA_TASK_HISTORY"; const char *const OB_ALL_VIRTUAL_ARCHIVE_DEST_STATUS_ORA_TNAME = "ALL_VIRTUAL_ARCHIVE_DEST_STATUS"; +const char *const OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_ORA_TNAME = "ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY"; +const char *const OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_ORA_TNAME = "ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY"; +const char *const OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_ORA_TNAME = "ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR"; const char *const OB_ALL_VIRTUAL_LONG_OPS_STATUS_SYS_AGENT_TNAME = "ALL_VIRTUAL_LONG_OPS_STATUS_SYS_AGENT"; const char *const OB_ALL_VIRTUAL_THREAD_ORA_TNAME = "ALL_VIRTUAL_THREAD"; const char *const OB_ALL_VIRTUAL_TIMESTAMP_SERVICE_ORA_TNAME = "ALL_VIRTUAL_TIMESTAMP_SERVICE"; @@ -3233,9 +3263,14 @@ const char *const OB_V_OB_ARCHIVE_DEST_STATUS_TNAME = "V$OB_ARCHIVE_DEST_STATUS" const char *const OB_DBA_OB_LS_LOG_ARCHIVE_PROGRESS_TNAME = "DBA_OB_LS_LOG_ARCHIVE_PROGRESS"; const char *const OB_CDB_OB_LS_LOG_ARCHIVE_PROGRESS_TNAME = "CDB_OB_LS_LOG_ARCHIVE_PROGRESS"; const char *const OB_DBA_OB_RSRC_IO_DIRECTIVES_TNAME = "DBA_OB_RSRC_IO_DIRECTIVES"; +const char *const OB_GV_OB_OPT_STAT_GATHER_MONITOR_TNAME = "GV$OB_OPT_STAT_GATHER_MONITOR"; +const char *const OB_V_OB_OPT_STAT_GATHER_MONITOR_TNAME = "V$OB_OPT_STAT_GATHER_MONITOR"; +const char *const OB_DBA_OB_TASK_OPT_STAT_GATHER_HISTORY_TNAME = "DBA_OB_TASK_OPT_STAT_GATHER_HISTORY"; +const char *const OB_DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY_TNAME = "DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY"; const char *const OB_GV_OB_THREAD_TNAME = "GV$OB_THREAD"; const char *const OB_V_OB_THREAD_TNAME = "V$OB_THREAD"; const char *const OB_V_OB_TIMESTAMP_SERVICE_TNAME = "V$OB_TIMESTAMP_SERVICE"; +const char *const OB_DBA_OB_TABLE_STAT_STALE_INFO_TNAME = "DBA_OB_TABLE_STAT_STALE_INFO"; const char *const OB_DBA_SYNONYMS_TNAME = "DBA_SYNONYMS"; const char *const OB_DBA_OBJECTS_ORA_TNAME = "DBA_OBJECTS"; const char *const OB_ALL_OBJECTS_TNAME = "ALL_OBJECTS"; @@ -3458,6 +3493,9 @@ const char *const OB_DBA_OB_RSRC_IO_DIRECTIVES_ORA_TNAME = "DBA_OB_RSRC_IO_DIREC const char *const OB_ALL_DB_LINKS_ORA_TNAME = "ALL_DB_LINKS"; const char *const OB_DBA_DB_LINKS_ORA_TNAME = "DBA_DB_LINKS"; const char *const OB_USER_DB_LINKS_ORA_TNAME = "USER_DB_LINKS"; +const char *const OB_DBA_OB_TASK_OPT_STAT_GATHER_HISTORY_ORA_TNAME = "DBA_OB_TASK_OPT_STAT_GATHER_HISTORY"; +const char *const OB_DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY_ORA_TNAME = "DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY"; +const char *const OB_DBA_OB_TABLE_STAT_STALE_INFO_ORA_TNAME = "DBA_OB_TABLE_STAT_STALE_INFO"; const char *const OB_GV_OB_SQL_AUDIT_ORA_TNAME = "GV$OB_SQL_AUDIT"; const char *const OB_V_OB_SQL_AUDIT_ORA_TNAME = "V$OB_SQL_AUDIT"; const char *const OB_GV_INSTANCE_TNAME = "GV$INSTANCE"; @@ -3608,6 +3646,8 @@ const char *const OB_GV_OB_SQL_PLAN_ORA_TNAME = "GV$OB_SQL_PLAN"; const char *const OB_V_OB_SQL_PLAN_ORA_TNAME = "V$OB_SQL_PLAN"; const char *const OB_V_OB_ARCHIVE_DEST_STATUS_ORA_TNAME = "V$OB_ARCHIVE_DEST_STATUS"; const char *const OB_DBA_OB_LS_LOG_ARCHIVE_PROGRESS_ORA_TNAME = "DBA_OB_LS_LOG_ARCHIVE_PROGRESS"; +const char *const OB_GV_OB_OPT_STAT_GATHER_MONITOR_ORA_TNAME = "GV$OB_OPT_STAT_GATHER_MONITOR"; +const char *const OB_V_OB_OPT_STAT_GATHER_MONITOR_ORA_TNAME = "V$OB_OPT_STAT_GATHER_MONITOR"; const char *const OB_GV_SESSION_LONGOPS_ORA_TNAME = "GV$SESSION_LONGOPS"; const char *const OB_V_SESSION_LONGOPS_ORA_TNAME = "V$SESSION_LONGOPS"; const char *const OB_GV_OB_THREAD_ORA_TNAME = "GV$OB_THREAD"; @@ -3846,6 +3886,8 @@ const char *const OB_ALL_RLS_ATTRIBUTE_HISTORY_AUX_LOB_META_TNAME = "__all_rls_a const char *const OB_ALL_TENANT_REWRITE_RULES_AUX_LOB_META_TNAME = "__all_tenant_rewrite_rules_aux_lob_meta"; const char *const OB_ALL_RESERVED_SNAPSHOT_AUX_LOB_META_TNAME = "__all_reserved_snapshot_aux_lob_meta"; const char *const OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_META_TNAME = "__all_cluster_event_history_aux_lob_meta"; +const char *const OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TNAME = "__all_task_opt_stat_gather_history_aux_lob_meta"; +const char *const OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TNAME = "__all_table_opt_stat_gather_history_aux_lob_meta"; const char *const OB_ALL_TABLE_AUX_LOB_PIECE_TNAME = "__all_table_aux_lob_piece"; const char *const OB_ALL_COLUMN_AUX_LOB_PIECE_TNAME = "__all_column_aux_lob_piece"; const char *const OB_ALL_DDL_OPERATION_AUX_LOB_PIECE_TNAME = "__all_ddl_operation_aux_lob_piece"; @@ -4079,6 +4121,8 @@ const char *const OB_ALL_RLS_ATTRIBUTE_HISTORY_AUX_LOB_PIECE_TNAME = "__all_rls_ const char *const OB_ALL_TENANT_REWRITE_RULES_AUX_LOB_PIECE_TNAME = "__all_tenant_rewrite_rules_aux_lob_piece"; const char *const OB_ALL_RESERVED_SNAPSHOT_AUX_LOB_PIECE_TNAME = "__all_reserved_snapshot_aux_lob_piece"; const char *const OB_ALL_CLUSTER_EVENT_HISTORY_AUX_LOB_PIECE_TNAME = "__all_cluster_event_history_aux_lob_piece"; +const char *const OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TNAME = "__all_task_opt_stat_gather_history_aux_lob_piece"; +const char *const OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TNAME = "__all_table_opt_stat_gather_history_aux_lob_piece"; const char *const OB_ALL_VIRTUAL_PLAN_CACHE_STAT_ALL_VIRTUAL_PLAN_CACHE_STAT_I1_TNAME = "__idx_11003_all_virtual_plan_cache_stat_i1"; const char *const OB_ALL_VIRTUAL_SESSION_EVENT_ALL_VIRTUAL_SESSION_EVENT_I1_TNAME = "__idx_11013_all_virtual_session_event_i1"; const char *const OB_ALL_VIRTUAL_SESSION_WAIT_ALL_VIRTUAL_SESSION_WAIT_I1_TNAME = "__idx_11014_all_virtual_session_wait_i1"; diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 755fe8175d..596fc16688 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -5462,8 +5462,62 @@ def_table_schema( # 449 : __all_wait_for_partition_split_tablet # 450 : __all_external_table_file -# 451 : __all_task_opt_stat_gather_history -# 452 : __all_table_opt_stat_gather_history + +def_table_schema( + owner = 'jiangxiu.wt', + table_name = '__all_task_opt_stat_gather_history', + table_id = '451', + table_type = 'SYSTEM_TABLE', + gm_columns = ['gmt_create', 'gmt_modified'], + rowkey_columns = [ + ('tenant_id', 'int'), + ('task_id', 'varchar:36') + ], + in_tenant_space = True, + is_cluster_private = True, + meta_record_in_sys = False, + normal_columns = [ + ('type', 'int', 'true'), + ('ret_code', 'int', 'true'), + ('table_count', 'int', 'true'), + ('failed_count', 'int', 'true'), + ('start_time', 'timestamp'), + ('end_time', 'timestamp'), + ('spare1', 'int', 'true'), + ('spare2', 'int', 'true'), + ('spare3', 'varchar:MAX_VALUE_LENGTH', 'true'), + ('spare4', 'varchar:MAX_VALUE_LENGTH', 'true'), + ], +) + +def_table_schema( + owner = 'jiangxiu.wt', + table_name = '__all_table_opt_stat_gather_history', + table_id = '452', + table_type = 'SYSTEM_TABLE', + gm_columns = ['gmt_create', 'gmt_modified'], + rowkey_columns = [ + ('tenant_id', 'int'), + ('task_id', 'varchar:36'), + ('table_id', 'int'), + ], + in_tenant_space = True, + is_cluster_private = True, + meta_record_in_sys = False, + normal_columns = [ + ('ret_code', 'int', 'true'), + ('start_time', 'timestamp', 'true'), + ('end_time', 'timestamp', 'true'), + ('memory_used', 'int', 'true'), + ('stat_refresh_failed_list', 'varchar:MAX_VALUE_LENGTH', 'true'), + ('properties', 'varchar:MAX_VALUE_LENGTH', 'true'), + ('spare1', 'int', 'true'), + ('spare2', 'int', 'true'), + ('spare3', 'varchar:MAX_VALUE_LENGTH', 'true'), + ('spare4', 'varchar:MAX_VALUE_LENGTH', 'true'), + ], +) + # 453 : __all_zone_storage # 454 : __all_zone_storage_operation # 455 : __wr_active_session_history @@ -11507,9 +11561,48 @@ def_table_schema( # 12378: __all_virtual_dup_ls_tablet_set # 12379: __all_virtual_dup_ls_tablets # 12380: __all_virtual_tx_data -# 12381: __all_virtual_task_opt_stat_gather_history -# 12382: __all_virtual_table_opt_stat_gather_history -# 12383: __all_virtual_opt_stat_gather_monitor + +def_table_schema(**gen_iterate_private_virtual_table_def( + table_id = '12381', + table_name = '__all_virtual_task_opt_stat_gather_history', + in_tenant_space = True, + keywords = all_def_keywords['__all_task_opt_stat_gather_history'])) + +def_table_schema(**gen_iterate_private_virtual_table_def( + table_id = '12382', + table_name = '__all_virtual_table_opt_stat_gather_history', + in_tenant_space = True, + keywords = all_def_keywords['__all_table_opt_stat_gather_history'])) + +def_table_schema( + owner = 'jiangxiu.wt', + table_name = '__all_virtual_opt_stat_gather_monitor', + table_id = '12383', + table_type = 'VIRTUAL_TABLE', + gm_columns = [], + rowkey_columns = [], + in_tenant_space = True, + normal_columns = [ + ('tenant_id', 'int'), + ('svr_ip', 'varchar:MAX_IP_ADDR_LENGTH'), + ('svr_port', 'int'), + ('session_id', 'int'), + ('trace_id', 'varchar:OB_MAX_TRACE_ID_BUFFER_SIZE'), + ('task_id', 'varchar:36'), + ('type', 'int'), + ('task_start_time', 'timestamp'), + ('task_table_count', 'int'), + ('task_duration_time', 'int'), + ('completed_table_count', 'int'), + ('running_table_owner', 'varchar:OB_MAX_DATABASE_NAME_LENGTH'), + ('running_table_name', 'varchar:OB_MAX_TABLE_NAME_LENGTH'), + ('running_table_duration_time', 'int'), + ('spare1', 'int', 'true'), + ('spare2', 'varchar:MAX_VALUE_LENGTH', 'true'), + ], + partition_columns = ['svr_ip', 'svr_port'], + vtable_route_policy = 'distributed', +) def_table_schema( owner = 'fengshuo.fs', @@ -11854,14 +11947,12 @@ def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15287' def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15288', all_def_keywords['__all_virtual_ls_arb_replica_task']))) def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15289', all_def_keywords['__all_virtual_ls_arb_replica_task_history']))) def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15290', all_def_keywords['__all_virtual_archive_dest_status']))) - # 15291: __all_virtual_backup_transferring_tablets # 15292: __all_virtual_external_table_file # 15293: __all_data_dictionary_in_log -# 15294: __all_task_opt_stat_gather_history -# 15295: __all_table_opt_stat_gather_history -# 15296: __all_virtual_opt_stat_gather_monitor - +def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15294', all_def_keywords['__all_virtual_task_opt_stat_gather_history']))) +def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15295', all_def_keywords['__all_virtual_table_opt_stat_gather_history']))) +def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15296', all_def_keywords['__all_virtual_opt_stat_gather_monitor']))) def_table_schema(**gen_sys_agent_virtual_table_def('15297', all_def_keywords['__all_virtual_long_ops_status'])) def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15298', all_def_keywords['__all_virtual_thread']))) @@ -11964,6 +12055,7 @@ def_table_schema(**no_direct_access(gen_oracle_mapping_virtual_table_def('15385' # 15387: __all_virtual_ls_log_restore_status # 15388: __all_virtual_tenant_parameter + ################################################################################ # System View (20000,30000] # MySQL System View (20000, 25000] @@ -22187,6 +22279,7 @@ def_table_schema( JOIN OCEANBASE.__ALL_TABLE T ON V.TENANT_ID = T.TENANT_ID AND V.TABLE_ID = T.TABLE_ID + AND T.TABLE_TYPE in (0, 3, 6) JOIN OCEANBASE.__ALL_DATABASE DB ON T.TENANT_ID = DB.TENANT_ID @@ -25988,10 +26081,127 @@ def_table_schema( # 21373: DBA_OB_ACCESS_POINT # 21374: CDB_OB_DATA_DICTIONARY_IN_LOG # 21375: DBA_OB_DATA_DICTIONARY_IN_LOG -# 21376: GV$OB_OPT_STAT_GATHER_MONITOR -# 21377: V$OB_OPT_STAT_GATHER_MONITOR -# 21378: DBA_OB_TASK_OPT_STAT_GATHER_HISTORY -# 21379: DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY + +def_table_schema( + owner = 'jiangxiu.wt', + table_name = 'GV$OB_OPT_STAT_GATHER_MONITOR', + table_id = '21376', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """SELECT + CAST(TENANT_ID AS SIGNED) AS TENANT_ID, + CAST(SVR_IP AS CHAR(46)) AS SVR_IP, + CAST(SVR_PORT AS SIGNED) AS SVR_PORT, + CAST(SESSION_ID AS SIGNED) AS SESSION_ID, + CAST(TRACE_ID AS CHAR(64)) AS TRACE_ID, + CAST(TASK_ID AS CHAR(36)) AS TASK_ID, + CAST((CASE WHEN TYPE = 0 THEN 'MANUAL GATHER' ELSE + (CASE WHEN TYPE = 1 THEN 'AUTO GATHER' ELSE 'UNDEFINED GATHER' END) END) AS CHAR(16)) AS TYPE, + CAST(TASK_START_TIME AS DATETIME(6)) AS TASK_START_TIME, + CAST(TASK_DURATION_TIME AS SIGNED) AS TASK_DURATION_TIME, + CAST(TASK_TABLE_COUNT AS SIGNED) AS TASK_TABLE_COUNT, + CAST(COMPLETED_TABLE_COUNT AS SIGNED) AS COMPLETED_TABLE_COUNT, + CAST(RUNNING_TABLE_OWNER AS CHAR(128)) AS RUNNING_TABLE_OWNER, + CAST(RUNNING_TABLE_NAME AS CHAR(256)) AS RUNNING_TABLE_NAME, + CAST(RUNNING_TABLE_DURATION_TIME AS SIGNED) AS RUNNING_TABLE_DURATION_TIME + FROM oceanbase.__all_virtual_opt_stat_gather_monitor +""".replace("\n", " ") +) + +def_table_schema( + owner = 'jiangxiu.wt', + table_name = 'V$OB_OPT_STAT_GATHER_MONITOR', + table_id = '21377', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT * FROM oceanbase.GV$OB_OPT_STAT_GATHER_MONITOR WHERE SVR_IP=HOST_IP() AND SVR_PORT=RPC_PORT() +""".replace("\n", " ") +) + +def_table_schema( + owner = 'jiangxiu.wt', + table_name = 'DBA_OB_TASK_OPT_STAT_GATHER_HISTORY', + table_id = '21378', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(TENANT_ID AS SIGNED) AS TENANT_ID, + CAST(TASK_ID AS CHAR(36)) AS TASK_ID, + CAST((CASE WHEN type = 0 THEN 'MANUAL GATHER' + ELSE ( CASE WHEN type = 1 THEN 'AUTO GATHER' + ELSE ( CASE WHEN type IS NULL THEN NULL + ELSE 'UNDEFINED GATHER' END )END ) END ) AS CHAR(16)) AS TYPE, + CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE 'FAILED' END) AS CHAR(8)) AS STATUS, + CAST(TABLE_COUNT AS SIGNED) AS TABLE_COUNT, + CAST(FAILED_COUNT AS SIGNED) AS FAILED_COUNT, + CAST(START_TIME AS DATETIME(6)) AS START_TIME, + CAST(END_TIME AS DATETIME(6)) AS END_TIME + FROM + oceanbase.__all_virtual_task_opt_stat_gather_history + WHERE TENANT_ID = EFFECTIVE_TENANT_ID() +""".replace("\n", " ") +) + +def_table_schema( + owner = 'jiangxiu.wt', + table_name = 'DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY', + table_id = '21379', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(DB.DATABASE_NAME AS CHAR(128)) AS OWNER, + CAST(V.TABLE_NAME AS CHAR(256)) AS TABLE_NAME, + CAST(STAT.TASK_ID AS CHAR(36)) AS TASK_ID, + CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE 'FAILED' END) AS CHAR(8)) AS STATUS, + CAST(STAT.START_TIME AS DATETIME(6)) AS START_TIME, + CAST(STAT.END_TIME AS DATETIME(6)) AS END_TIME, + CAST(STAT.MEMORY_USED AS SIGNED) AS MEMORY_USED, + CAST(STAT.STAT_REFRESH_FAILED_LIST AS CHAR(4096)) AS STAT_REFRESH_FAILED_LIST, + CAST(STAT.PROPERTIES AS CHAR(4096)) AS PROPERTIES + FROM + ( + (SELECT CAST(0 AS SIGNED) AS TENANT_ID, + DATABASE_ID, + TABLE_ID, + TABLE_NAME + FROM + OCEANBASE.__ALL_VIRTUAL_CORE_ALL_TABLE + WHERE TENANT_ID = EFFECTIVE_TENANT_ID() + UNION ALL + SELECT TENANT_ID, + DATABASE_ID, + TABLE_ID, + TABLE_NAME + FROM + oceanbase.__all_table T + WHERE T.TABLE_TYPE IN (0,2,3,6)) + ) V + JOIN + oceanbase.__all_database DB + ON DB.TENANT_ID = V.TENANT_ID + AND DB.DATABASE_ID = V.DATABASE_ID + AND V.TENANT_ID = 0 + LEFT JOIN + oceanbase.__all_virtual_table_opt_stat_gather_history STAT + ON STAT.TENANT_ID = EFFECTIVE_TENANT_ID() + AND V.TABLE_ID = STAT.TABLE_ID +""".replace("\n", " ") +) def_table_schema( owner = 'fengshuo.fs', @@ -26099,7 +26309,152 @@ def_table_schema( # 21420: V$OB_PX_P2P_DATAHUB # 21421: GV$SQL_JOIN_FILTER # 21422: V$SQL_JOIN_FILTER -# 21423: DBA_OB_TABLE_STAT_STALE_INFO + +def_table_schema( + owner = 'yibo.tyf', + table_name = 'DBA_OB_TABLE_STAT_STALE_INFO', + table_id = '21423', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ +WITH V AS +(SELECT + NVL(T.TENANT_ID, 0) AS TENANT_ID, + NVL(T.TABLE_ID, VT.TABLE_ID) AS TABLE_ID, + NVL(T.TABLET_ID, VT.TABLET_ID) AS TABLET_ID, + NVL(T.INSERTS, 0) + NVL(VT.INSERT_ROW_COUNT, 0) - NVL(T.LAST_INSERTS, 0) AS INSERTS, + NVL(T.UPDATES, 0) + NVL(VT.UPDATE_ROW_COUNT, 0) - NVL(T.LAST_UPDATES, 0) AS UPDATES, + NVL(T.DELETES, 0) + NVL(VT.DELETE_ROW_COUNT, 0) - NVL(T.LAST_DELETES, 0) AS DELETES + FROM + OCEANBASE.__ALL_MONITOR_MODIFIED T + FULL JOIN + OCEANBASE.__ALL_VIRTUAL_DML_STATS VT + ON T.TABLE_ID = VT.TABLE_ID + AND T.TABLET_ID = VT.TABLET_ID + AND VT.TENANT_ID = EFFECTIVE_TENANT_ID() +) +SELECT + CAST(TM.DATABASE_NAME AS CHAR(128)) AS DATABASE_NAME, + CAST(TM.TABLE_NAME AS CHAR(128)) AS TABLE_NAME, + CAST(TM.PART_NAME AS CHAR(128)) AS PARTITION_NAME, + CAST(TM.SUB_PART_NAME AS CHAR(128)) AS SUBPARTITION_NAME, + CAST(TS.ROW_CNT AS SIGNED) AS LAST_ANALYZED_ROWS, + TS.LAST_ANALYZED AS LAST_ANALYZED_TIME, + CAST(TM.INSERTS AS SIGNED) AS INSERTS, + CAST(TM.UPDATES AS SIGNED) AS UPDATES, + CAST(TM.DELETES AS SIGNED) AS DELETES, + CAST(NVL(CAST(UP.VALCHAR AS SIGNED), CAST(GP.SPARE4 AS SIGNED)) AS SIGNED) STALE_PERCENT, + CAST(CASE NVL((TM.INSERTS + TM.UPDATES + TM.DELETES) > TS.ROW_CNT * NVL(CAST(UP.VALCHAR AS SIGNED), CAST(GP.SPARE4 AS SIGNED)) / 100, + (TM.INSERTS + TM.UPDATES + TM.DELETES) > 0) + WHEN 0 THEN 'NO' + WHEN 1 THEN 'YES' + END AS CHAR(3)) AS IS_STALE +FROM +(SELECT + T.TENANT_ID, + T.TABLE_ID, + CASE T.PART_LEVEL WHEN 0 THEN T.TABLE_ID WHEN 1 THEN P.PART_ID WHEN 2 THEN SP.SUB_PART_ID END AS PARTITION_ID, + DB.DATABASE_NAME, + T.TABLE_NAME, + P.PART_NAME, + SP.SUB_PART_NAME, + NVL(V.INSERTS, 0) AS INSERTS, + NVL(V.UPDATES, 0) AS UPDATES, + NVL(V.DELETES, 0) AS DELETES +FROM OCEANBASE.__ALL_TABLE T +JOIN OCEANBASE.__ALL_DATABASE DB + ON T.TENANT_ID = DB.TENANT_ID AND DB.DATABASE_ID = T.DATABASE_ID +LEFT JOIN OCEANBASE.__ALL_PART P + ON T.TENANT_ID = P.TENANT_ID AND T.TABLE_ID = P.TABLE_ID +LEFT JOIN OCEANBASE.__ALL_SUB_PART SP + ON T.TENANT_ID = SP.TENANT_ID AND T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID +LEFT JOIN V +ON T.TENANT_ID = V.TENANT_ID AND T.TABLE_ID = V.TABLE_ID +AND V.TABLET_ID = CASE T.PART_LEVEL WHEN 0 THEN T.TABLET_ID WHEN 1 THEN P.TABLET_ID WHEN 2 THEN SP.TABLET_ID END +WHERE T.TABLE_TYPE IN (0, 3, 6) +UNION ALL +SELECT + MIN(T.TENANT_ID), + MIN(T.TABLE_ID), + -1 AS PARTITION_ID, + DB.DATABASE_NAME, + T.TABLE_NAME, + NULL AS PART_NAME, + NULL AS SUB_PART_NAME, + SUM(NVL(V.INSERTS, 0)) AS INSERTS, + SUM(NVL(V.UPDATES, 0)) AS UPDATES, + SUM(NVL(V.DELETES, 0)) AS DELETES +FROM OCEANBASE.__ALL_TABLE T +JOIN OCEANBASE.__ALL_DATABASE DB + ON T.TENANT_ID = DB.TENANT_ID AND DB.DATABASE_ID = T.DATABASE_ID +JOIN OCEANBASE.__ALL_PART P + ON T.TENANT_ID = P.TENANT_ID AND T.TABLE_ID = P.TABLE_ID +LEFT JOIN V +ON T.TENANT_ID = V.TENANT_ID AND T.TABLE_ID = V.TABLE_ID AND V.TABLET_ID = P.TABLET_ID +WHERE T.TABLE_TYPE IN (0, 3, 6) AND T.PART_LEVEL = 1 +GROUP BY DB.DATABASE_NAME, + T.TABLE_NAME +UNION ALL +SELECT + MIN(T.TENANT_ID), + MIN(T.TABLE_ID), + MIN(P.PART_ID) AS PARTITION_ID, + DB.DATABASE_NAME, + T.TABLE_NAME, + P.PART_NAME, + NULL AS SUB_PART_NAME, + SUM(NVL(V.INSERTS, 0)) AS INSERTS, + SUM(NVL(V.UPDATES, 0)) AS UPDATES, + SUM(NVL(V.DELETES, 0)) AS DELETES +FROM OCEANBASE.__ALL_TABLE T +JOIN OCEANBASE.__ALL_DATABASE DB + ON T.TENANT_ID = DB.TENANT_ID AND DB.DATABASE_ID = T.DATABASE_ID +JOIN OCEANBASE.__ALL_PART P + ON T.TENANT_ID = P.TENANT_ID AND T.TABLE_ID = P.TABLE_ID +JOIN OCEANBASE.__ALL_SUB_PART SP + ON T.TENANT_ID = SP.TENANT_ID AND T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID +LEFT JOIN V +ON T.TENANT_ID = V.TENANT_ID AND T.TABLE_ID = V.TABLE_ID AND V.TABLET_ID = SP.TABLET_ID +WHERE T.TABLE_TYPE IN (0, 3, 6) AND T.PART_LEVEL = 2 +GROUP BY DB.DATABASE_NAME, + T.TABLE_NAME, + P.PART_NAME +UNION ALL +SELECT + MIN(T.TENANT_ID), + MIN(T.TABLE_ID), + -1 AS PARTITION_ID, + DB.DATABASE_NAME, + T.TABLE_NAME, + NULL AS PART_NAME, + NULL AS SUB_PART_NAME, + SUM(NVL(V.INSERTS, 0)) AS INSERTS, + SUM(NVL(V.UPDATES, 0)) AS UPDATES, + SUM(NVL(V.DELETES, 0)) AS DELETES +FROM OCEANBASE.__ALL_TABLE T +JOIN OCEANBASE.__ALL_DATABASE DB + ON T.TENANT_ID = DB.TENANT_ID AND DB.DATABASE_ID = T.DATABASE_ID +JOIN OCEANBASE.__ALL_PART P + ON T.TENANT_ID = P.TENANT_ID AND T.TABLE_ID = P.TABLE_ID +JOIN OCEANBASE.__ALL_SUB_PART SP + ON T.TENANT_ID = SP.TENANT_ID AND T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID +LEFT JOIN V +ON T.TENANT_ID = V.TENANT_ID AND T.TABLE_ID = V.TABLE_ID AND V.TABLET_ID = SP.TABLET_ID +WHERE T.TABLE_TYPE IN (0, 3, 6) AND T.PART_LEVEL = 2 +GROUP BY DB.DATABASE_NAME, + T.TABLE_NAME +) TM +LEFT JOIN OCEANBASE.__ALL_TABLE_STAT TS + ON TM.TENANT_ID = TS.TENANT_ID AND TM.TABLE_ID = TS.TABLE_ID AND TM.PARTITION_ID = TS.PARTITION_ID +LEFT JOIN OCEANBASE.__ALL_OPTSTAT_USER_PREFS UP + ON TM.TENANT_ID = UP.TENANT_ID AND TM.TABLE_ID = UP.TABLE_ID AND UP.PNAME = 'STALE_PERCENT' +JOIN OCEANBASE.__ALL_OPTSTAT_GLOBAL_PREFS GP + ON GP.SNAME = 'STALE_PERCENT' +""".replace("\n", " ") +) # 21424: V$OB_LS_LOG_RESTORE_STATUS ################################################################################ @@ -41636,6 +41991,7 @@ def_table_schema( JOIN SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T ON T.TABLE_ID = V.TABLE_ID AND T.TENANT_ID = V.TENANT_ID + AND T.TABLE_TYPE IN (0, 3, 8, 9) AND T.TENANT_ID = EFFECTIVE_TENANT_ID() JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB @@ -41700,6 +42056,7 @@ def_table_schema( ON T.TABLE_ID = V.TABLE_ID AND T.TENANT_ID = V.TENANT_ID AND T.TENANT_ID = EFFECTIVE_TENANT_ID() + AND T.TABLE_TYPE IN (0, 3, 8, 9) JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB ON DB.TENANT_ID = V.TENANT_ID @@ -41759,6 +42116,7 @@ def_table_schema( JOIN SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T ON T.TABLE_ID = V.TABLE_ID AND T.TENANT_ID = V.TENANT_ID + AND T.TABLE_TYPE IN (0, 3, 8, 9) AND T.TENANT_ID = EFFECTIVE_TENANT_ID() AND T.DATABASE_ID = USERENV('SCHEMAID') JOIN @@ -43514,8 +43872,101 @@ def_table_schema( """.replace("\n", " ") ) -# 25227: DBA_OB_TASK_OPT_STAT_GATHER_HISTORY -# 25228: DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY +def_table_schema( + owner = 'jiangxiu.wt', + table_name = 'DBA_OB_TASK_OPT_STAT_GATHER_HISTORY', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25227', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(TENANT_ID AS NUMBER) AS TENANT_ID, + CAST(TASK_ID AS VARCHAR2(36)) AS TASK_ID, + CAST((CASE WHEN type = 0 THEN 'MANUAL GATHER' + ELSE ( CASE WHEN type = 1 THEN 'AUTO GATHER' + ELSE ( CASE WHEN type IS NULL THEN NULL + ELSE 'UNDEFINED GATHER' END )END ) END ) AS VARCHAR2(16)) AS TYPE, + CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE 'FAILED' END) AS VARCHAR2(8)) AS STATUS, + CAST(TABLE_COUNT AS NUMBER) AS TASK_TABLE_COUNT, + CAST(FAILED_COUNT AS NUMBER) AS FAILED_COUNT, + CAST(START_TIME AS TIMESTAMP(6)) AS TASK_START_TIME, + CAST(END_TIME AS TIMESTAMP(6)) AS TASK_END_TIME + FROM + SYS.ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY + WHERE TENANT_ID = EFFECTIVE_TENANT_ID() +""".replace("\n", " ") +) + +def_table_schema( + owner = 'jiangxiu.wt', + table_name = 'DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25228', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(DB.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, + CAST(V.TABLE_NAME AS VARCHAR2(256)) AS TABLE_NAME, + CAST(STAT.TASK_ID AS VARCHAR2(36)) AS TASK_ID, + CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE 'FAILED' END) AS VARCHAR2(8)) AS STATUS, + CAST(STAT.START_TIME AS TIMESTAMP(6)) AS START_TIME, + CAST(STAT.END_TIME AS TIMESTAMP(6)) AS END_TIME, + CAST(STAT.MEMORY_USED AS NUMBER) AS MEMORY_USED, + CAST(STAT.STAT_REFRESH_FAILED_LIST AS VARCHAR2(4096)) AS STAT_REFRESH_FAILED_LIST, + CAST(STAT.PROPERTIES AS VARCHAR2(4096)) AS PROPERTIES + FROM + ( + (SELECT TENANT_ID, + DATABASE_ID, + TABLE_ID, + TABLE_ID AS PARTITION_ID, + TABLE_NAME, + NULL AS PARTITION_NAME, + NULL AS SUBPARTITION_NAME, + NULL AS PARTITION_POSITION, + NULL AS SUBPARTITION_POSITION, + 'TABLE' AS OBJECT_TYPE + FROM + SYS.ALL_VIRTUAL_CORE_ALL_TABLE + UNION ALL + SELECT TENANT_ID, + DATABASE_ID, + TABLE_ID, + CASE WHEN PART_LEVEL = 0 THEN TABLE_ID ELSE -1 END AS PARTITION_ID, + TABLE_NAME, + NULL AS PARTITION_NAME, + NULL AS SUBPARTITION_NAME, + NULL AS PARTITION_POSITION, + NULL AS SUBPARTITION_POSITION, + 'TABLE' AS OBJECT_TYPE + FROM + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T + WHERE T.TABLE_TYPE IN (0,2,3,8,9)) + ) V + JOIN + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT db + ON db.tenant_id = V.tenant_id + AND db.database_id = V.database_id + AND V.TENANT_ID = EFFECTIVE_TENANT_ID() + AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() + LEFT JOIN + SYS.ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY STAT + ON V.TENANT_ID = STAT.TENANT_ID + AND V.TABLE_ID = STAT.TABLE_ID + AND STAT.TENANT_ID = EFFECTIVE_TENANT_ID() +""".replace("\n", " ") +) + # 25229: DBA_WR_ACTIVE_SESSION_HISTORY # 25230: DBA_WR_SNAPSHOT # 25231: DBA_WR_STAT_NAME @@ -43535,6 +43986,174 @@ def_table_schema( # 25246: V$SQL_JOIN_FILTER # 25247: DBA_OB_TABLE_STAT_STALE_INFO +def_table_schema( + owner = 'yibo.tyf', + table_name = 'DBA_OB_TABLE_STAT_STALE_INFO', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25247', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ +WITH V AS +(SELECT + NVL(T.TABLE_ID, VT.TABLE_ID) AS TABLE_ID, + NVL(T.TABLET_ID, VT.TABLET_ID) AS TABLET_ID, + NVL(T.INSERTS, 0) + NVL(VT.INSERT_ROW_COUNT, 0) - NVL(T.LAST_INSERTS, 0) AS INSERTS, + NVL(T.UPDATES, 0) + NVL(VT.UPDATE_ROW_COUNT, 0) - NVL(T.LAST_UPDATES, 0) AS UPDATES, + NVL(T.DELETES, 0) + NVL(VT.DELETE_ROW_COUNT, 0) - NVL(T.LAST_DELETES, 0) AS DELETES + FROM + SYS.ALL_VIRTUAL_MONITOR_MODIFIED_REAL_AGENT T + FULL JOIN + SYS.ALL_VIRTUAL_DML_STATS VT + ON T.TABLE_ID = VT.TABLE_ID + AND T.TABLET_ID = VT.TABLET_ID + AND T.TENANT_ID = EFFECTIVE_TENANT_ID() + AND VT.TENANT_ID = EFFECTIVE_TENANT_ID() +) +SELECT + CAST(TM.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, + CAST(TM.TABLE_NAME AS VARCHAR2(128)) AS TABLE_NAME, + CAST(TM.PART_NAME AS VARCHAR2(128)) AS PARTITION_NAME, + CAST(TM.SUB_PART_NAME AS VARCHAR2(128)) AS SUBPARTITION_NAME, + CAST(TS.ROW_CNT AS NUMBER) AS LAST_ANALYZED_ROWS, + TS.LAST_ANALYZED AS LAST_ANALYZED_TIME, + CAST(TM.INSERTS AS NUMBER) AS INSERTS, + CAST(TM.UPDATES AS NUMBER) AS UPDATES, + CAST(TM.DELETES AS NUMBER) AS DELETES, + CAST(NVL(CAST(UP.VALCHAR AS NUMBER), CAST(GP.SPARE4 AS NUMBER)) AS NUMBER) STALE_PERCENT, + CAST(CASE WHEN TS.ROW_CNT IS NOT NULL + THEN CASE WHEN (TM.INSERTS + TM.UPDATES + TM.DELETES) > TS.ROW_CNT * NVL(CAST(UP.VALCHAR AS NUMBER), CAST(GP.SPARE4 AS NUMBER)) / 100 + THEN 'YES' ELSE 'NO' END + ELSE CASE WHEN (TM.INSERTS + TM.UPDATES + TM.DELETES) > 0 + THEN 'YES' ELSE 'NO' END + END AS VARCHAR2(3)) AS IS_STALE +FROM +(SELECT + T.TENANT_ID, + T.TABLE_ID, + CASE T.PART_LEVEL WHEN 0 THEN T.TABLE_ID WHEN 1 THEN P.PART_ID WHEN 2 THEN SP.SUB_PART_ID END AS PARTITION_ID, + DB.DATABASE_NAME, + T.TABLE_NAME, + P.PART_NAME, + SP.SUB_PART_NAME, + NVL(V.INSERTS, 0) AS INSERTS, + NVL(V.UPDATES, 0) AS UPDATES, + NVL(V.DELETES, 0) AS DELETES +FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T +JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB + ON DB.DATABASE_ID = T.DATABASE_ID + AND T.TENANT_ID = EFFECTIVE_TENANT_ID() + AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() +LEFT JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P + ON T.TABLE_ID = P.TABLE_ID + AND P.TENANT_ID = EFFECTIVE_TENANT_ID() +LEFT JOIN SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SP + ON T.TABLE_ID = SP.TABLE_ID + AND P.PART_ID = SP.PART_ID + AND SP.TENANT_ID = EFFECTIVE_TENANT_ID() +LEFT JOIN V +ON T.TABLE_ID = V.TABLE_ID +AND V.TABLET_ID = CASE T.PART_LEVEL WHEN 0 THEN T.TABLET_ID WHEN 1 THEN P.TABLET_ID WHEN 2 THEN SP.TABLET_ID END +WHERE T.TABLE_TYPE IN (0, 3, 8, 9) +UNION ALL +SELECT + MIN(T.TENANT_ID), + MIN(T.TABLE_ID), + -1 AS PARTITION_ID, + DB.DATABASE_NAME, + T.TABLE_NAME, + NULL AS PART_NAME, + NULL AS SUB_PART_NAME, + SUM(NVL(V.INSERTS, 0)) AS INSERTS, + SUM(NVL(V.UPDATES, 0)) AS UPDATES, + SUM(NVL(V.DELETES, 0)) AS DELETES +FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T +JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB + ON DB.DATABASE_ID = T.DATABASE_ID + AND T.TENANT_ID = EFFECTIVE_TENANT_ID() + AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() +JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P + ON T.TABLE_ID = P.TABLE_ID + AND P.TENANT_ID = EFFECTIVE_TENANT_ID() +LEFT JOIN V +ON T.TABLE_ID = V.TABLE_ID AND V.TABLET_ID = P.TABLET_ID +WHERE T.TABLE_TYPE IN (0, 3, 8, 9) AND T.PART_LEVEL = 1 +GROUP BY DB.DATABASE_NAME, + T.TABLE_NAME +UNION ALL +SELECT + MIN(T.TENANT_ID), + MIN(T.TABLE_ID), + MIN(P.PART_ID) AS PARTITION_ID, + DB.DATABASE_NAME, + T.TABLE_NAME, + P.PART_NAME, + NULL AS SUB_PART_NAME, + SUM(NVL(V.INSERTS, 0)) AS INSERTS, + SUM(NVL(V.UPDATES, 0)) AS UPDATES, + SUM(NVL(V.DELETES, 0)) AS DELETES +FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T +JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB + ON DB.DATABASE_ID = T.DATABASE_ID + AND T.TENANT_ID = EFFECTIVE_TENANT_ID() + AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() +JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P + ON T.TENANT_ID = P.TENANT_ID AND T.TABLE_ID = P.TABLE_ID +JOIN SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SP + ON T.TENANT_ID = SP.TENANT_ID AND T.TABLE_ID = SP.TABLE_ID AND P.PART_ID = SP.PART_ID +LEFT JOIN V +ON T.TABLE_ID = V.TABLE_ID AND V.TABLET_ID = SP.TABLET_ID +WHERE T.TABLE_TYPE IN (0, 3, 8, 9) AND T.PART_LEVEL = 2 +GROUP BY DB.DATABASE_NAME, + T.TABLE_NAME, + P.PART_NAME +UNION ALL +SELECT + MIN(T.TENANT_ID), + MIN(T.TABLE_ID), + -1 AS PARTITION_ID, + DB.DATABASE_NAME, + T.TABLE_NAME, + NULL AS PART_NAME, + NULL AS SUB_PART_NAME, + SUM(NVL(V.INSERTS, 0)) AS INSERTS, + SUM(NVL(V.UPDATES, 0)) AS UPDATES, + SUM(NVL(V.DELETES, 0)) AS DELETES +FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T +JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT DB + ON DB.DATABASE_ID = T.DATABASE_ID + AND T.TENANT_ID = EFFECTIVE_TENANT_ID() + AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() +JOIN SYS.ALL_VIRTUAL_PART_REAL_AGENT P + ON T.TABLE_ID = P.TABLE_ID + AND P.TENANT_ID = EFFECTIVE_TENANT_ID() +JOIN SYS.ALL_VIRTUAL_SUB_PART_REAL_AGENT SP + ON T.TABLE_ID = SP.TABLE_ID + AND P.PART_ID = SP.PART_ID + AND SP.TENANT_ID = EFFECTIVE_TENANT_ID() +LEFT JOIN V +ON T.TABLE_ID = V.TABLE_ID AND V.TABLET_ID = SP.TABLET_ID +WHERE T.TABLE_TYPE IN (0, 3, 8, 9) AND T.PART_LEVEL = 2 +GROUP BY DB.DATABASE_NAME, + T.TABLE_NAME +) TM +LEFT JOIN SYS.ALL_VIRTUAL_TABLE_STAT_REAL_AGENT TS + ON TM.TABLE_ID = TS.TABLE_ID + AND TM.PARTITION_ID = TS.PARTITION_ID + AND TM.TENANT_ID = EFFECTIVE_TENANT_ID() +LEFT JOIN SYS.ALL_VIRTUAL_OPTSTAT_USER_PREFS_REAL_AGENT UP + ON TM.TABLE_ID = UP.TABLE_ID + AND UP.PNAME = 'STALE_PERCENT' + AND UP.TENANT_ID = EFFECTIVE_TENANT_ID() +JOIN SYS.ALL_VIRTUAL_OPTSTAT_GLOBAL_PREFS_REAL_AGENT GP + ON GP.SNAME = 'STALE_PERCENT' +""".replace("\n", " ") +) + #### End Data Dictionary View ################################################################################ @@ -49079,8 +49698,52 @@ def_table_schema( # 28180: V$OB_LOCKS # 28181: DBA_OB_ACCESS_POINT # 28182: DBA_OB_DATA_DICTIONARY_IN_LOG -# 28183: GV$OB_OPT_STAT_GATHER_MONITOR -# 28184: V$OB_OPT_STAT_GATHER_MONITOR + +def_table_schema( + owner = 'jiangxiu.wt', + table_name = 'GV$OB_OPT_STAT_GATHER_MONITOR', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '28183', + table_type = 'SYSTEM_VIEW', + gm_columns = [], + in_tenant_space = True, + rowkey_columns = [], + normal_columns = [], + view_definition = """SELECT + CAST(TENANT_ID AS NUMBER) AS TENANT_ID, + CAST(SVR_IP AS VARCHAR2(46)) AS SVR_IP, + CAST(SVR_PORT AS NUMBER) AS SVR_PORT, + CAST(SESSION_ID AS NUMBER) AS SESSION_ID, + CAST(TRACE_ID AS VARCHAR2(64)) AS TRACE_ID, + CAST(TASK_ID AS VARCHAR(36)) AS TASK_ID, + CAST(DECODE(TYPE, 0, 'MANUAL GATHER', 1, 'AUTO GATHER', 'UNDEFINED GATHER') AS VARCHAR2(16)) AS TYPE, + CAST(TASK_START_TIME AS TIMESTAMP(6)) AS TASK_START_TIME, + CAST(TASK_DURATION_TIME AS NUMBER) AS TASK_DURATION_TIME, + CAST(TASK_TABLE_COUNT AS NUMBER) AS TASK_TABLE_COUNT, + CAST(COMPLETED_TABLE_COUNT AS NUMBER) AS COMPLETED_TABLE_COUNT, + CAST(RUNNING_TABLE_OWNER AS VARCHAR2(128)) AS RUNNING_TABLE_OWNER, + CAST(RUNNING_TABLE_NAME AS VARCHAR2(256)) AS RUNNING_TABLE_NAME, + CAST(RUNNING_TABLE_DURATION_TIME AS VARCHAR2(256)) AS RUNNING_TABLE_DURATION_TIME + FROM SYS.ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR +""".replace("\n", " ") +) + +def_table_schema( + owner = 'jiangxiu.wt', + table_name = 'V$OB_OPT_STAT_GATHER_MONITOR', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '28184', + table_type = 'SYSTEM_VIEW', + gm_columns = [], + in_tenant_space = True, + rowkey_columns = [], + normal_columns = [], + view_definition = """ + SELECT * FROM SYS.GV$OB_OPT_STAT_GATHER_MONITOR WHERE SVR_IP=HOST_IP() AND SVR_PORT=RPC_PORT() +""".replace("\n", " ") +) def_table_schema( owner = 'zhenjiang.xzj', diff --git a/src/share/inner_table/ob_inner_table_schema_misc.ipp b/src/share/inner_table/ob_inner_table_schema_misc.ipp index c5b73f04a5..137d01ebe4 100644 --- a/src/share/inner_table/ob_inner_table_schema_misc.ipp +++ b/src/share/inner_table/ob_inner_table_schema_misc.ipp @@ -432,8 +432,10 @@ case OB_ALL_VIRTUAL_MERGE_INFO_TID: case OB_ALL_VIRTUAL_RESTORE_JOB_TID: case OB_ALL_VIRTUAL_RESTORE_JOB_HISTORY_TID: case OB_ALL_VIRTUAL_RESTORE_PROGRESS_TID: +case OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_TID: case OB_ALL_VIRTUAL_TABLET_META_TABLE_TID: case OB_ALL_VIRTUAL_TABLET_REPLICA_CHECKSUM_TID: +case OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_TID: case OB_ALL_VIRTUAL_TENANT_INFO_TID: case OB_ALL_VIRTUAL_TENANT_USER_FAILED_LOGIN_STAT_TID: case OB_ALL_VIRTUAL_ZONE_MERGE_INFO_TID: @@ -1168,6 +1170,22 @@ case OB_ALL_VIRTUAL_ZONE_MERGE_INFO_TID: break; } + case OB_ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY_TID: { + ObIteratePrivateVirtualTable *iter = NULL; + const bool meta_record_in_sys = false; + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIteratePrivateVirtualTable, iter))) { + SERVER_LOG(WARN, "create iterate private virtual table iterator failed", KR(ret)); + } else if (OB_FAIL(iter->init(OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_TID, meta_record_in_sys, index_schema, params))) { + SERVER_LOG(WARN, "iterate private virtual table iter init failed", KR(ret)); + iter->~ObIteratePrivateVirtualTable(); + allocator.free(iter); + iter = NULL; + } else { + vt_iter = iter; + } + break; + } + case OB_ALL_VIRTUAL_TABLET_META_TABLE_TID: { ObIteratePrivateVirtualTable *iter = NULL; const bool meta_record_in_sys = false; @@ -1200,6 +1218,22 @@ case OB_ALL_VIRTUAL_ZONE_MERGE_INFO_TID: break; } + case OB_ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY_TID: { + ObIteratePrivateVirtualTable *iter = NULL; + const bool meta_record_in_sys = false; + if (OB_FAIL(NEW_VIRTUAL_TABLE(ObIteratePrivateVirtualTable, iter))) { + SERVER_LOG(WARN, "create iterate private virtual table iterator failed", KR(ret)); + } else if (OB_FAIL(iter->init(OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_TID, meta_record_in_sys, index_schema, params))) { + SERVER_LOG(WARN, "iterate private virtual table iter init failed", KR(ret)); + iter->~ObIteratePrivateVirtualTable(); + allocator.free(iter); + iter = NULL; + } else { + vt_iter = iter; + } + break; + } + case OB_ALL_VIRTUAL_TENANT_INFO_TID: { ObIteratePrivateVirtualTable *iter = NULL; const bool meta_record_in_sys = false; @@ -3699,12 +3733,18 @@ case OB_ALL_RESTORE_PROGRESS_AUX_LOB_PIECE_TID: case OB_ALL_SERVICE_EPOCH_TID: case OB_ALL_SERVICE_EPOCH_AUX_LOB_META_TID: case OB_ALL_SERVICE_EPOCH_AUX_LOB_PIECE_TID: +case OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_TID: +case OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TID: +case OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TID: case OB_ALL_TABLET_META_TABLE_TID: case OB_ALL_TABLET_META_TABLE_AUX_LOB_META_TID: case OB_ALL_TABLET_META_TABLE_AUX_LOB_PIECE_TID: case OB_ALL_TABLET_REPLICA_CHECKSUM_TID: case OB_ALL_TABLET_REPLICA_CHECKSUM_AUX_LOB_META_TID: case OB_ALL_TABLET_REPLICA_CHECKSUM_AUX_LOB_PIECE_TID: +case OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_TID: +case OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_META_TID: +case OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_AUX_LOB_PIECE_TID: case OB_ALL_TENANT_GLOBAL_TRANSACTION_TID: case OB_ALL_TENANT_GLOBAL_TRANSACTION_IDX_XA_TRANS_ID_TID: case OB_ALL_TENANT_GLOBAL_TRANSACTION_AUX_LOB_META_TID: diff --git a/src/share/schema/ob_schema_struct.cpp b/src/share/schema/ob_schema_struct.cpp index 58cb1fac50..06849d9ef3 100644 --- a/src/share/schema/ob_schema_struct.cpp +++ b/src/share/schema/ob_schema_struct.cpp @@ -4034,7 +4034,8 @@ int ObPartitionSchema::get_tablet_and_object_id_by_index( const int64_t part_idx, const int64_t subpart_idx, ObTabletID &tablet_id, - ObObjectID &object_id) const + ObObjectID &object_id, + ObObjectID &first_level_part_id) const { int ret = OB_SUCCESS; const ObPartition *partition = NULL; @@ -4055,6 +4056,7 @@ int ObPartitionSchema::get_tablet_and_object_id_by_index( } else { tablet_id = partition->get_tablet_id(); object_id = partition->get_part_id(); + first_level_part_id = OB_INVALID_ID; ObSubPartition **subpartition_array = partition->get_subpart_array(); int64_t subpartition_num = partition->get_subpartition_num(); if (PARTITION_LEVEL_TWO != part_level || subpart_idx < 0) { @@ -4069,6 +4071,7 @@ int ObPartitionSchema::get_tablet_and_object_id_by_index( } else { const ObSubPartition *subpartition = subpartition_array[subpart_idx]; tablet_id = subpartition->get_tablet_id(); + first_level_part_id = object_id; object_id = subpartition->get_sub_part_id(); } } @@ -6205,12 +6208,13 @@ int ObPartitionUtils::fill_tablet_and_object_ids_( const uint64_t src_table_id = table_schema.get_table_id(); ObTabletID src_tablet_id; ObObjectID src_object_id; + ObObjectID src_first_level_part_id; // part_idx is valid when dealing with composited-partitioned table int64_t actual_part_idx = part_idx >= 0 ? part_idx : index.get_part_idx(); int64_t actual_subpart_idx = index.get_subpart_idx(); if (OB_FAIL(table_schema.get_tablet_and_object_id_by_index( actual_part_idx, actual_subpart_idx, - src_tablet_id, src_object_id))) { + src_tablet_id, src_object_id, src_first_level_part_id))) { LOG_WARN("fail to get tablet and object id", KR(ret), K(part_idx), K(index)); } else if (fill_tablet_id && OB_FAIL(tablet_ids.push_back(src_tablet_id))) { LOG_WARN("fail to push back tablet_id", KR(ret), K(src_tablet_id)); @@ -6223,6 +6227,7 @@ int ObPartitionUtils::fill_tablet_and_object_ids_( const uint64_t related_table_id = related_table->related_tids_->at(j); ObTabletID related_tablet_id; ObObjectID related_object_id; + ObObjectID related_first_level_part_id; const ObSimpleTableSchemaV2 *related_schema = NULL; if (OB_FAIL(guard->get_simple_table_schema(tenant_id, related_table_id, related_schema))) { LOG_WARN("fail to get simple table schema", KR(ret), K(tenant_id), K(related_table_id)); @@ -6231,10 +6236,10 @@ int ObPartitionUtils::fill_tablet_and_object_ids_( LOG_WARN("table not exist", KR(ret), K(tenant_id), K(related_table_id)); } else if (OB_FAIL(related_schema->get_tablet_and_object_id_by_index( actual_part_idx, actual_subpart_idx, - related_tablet_id, related_object_id))) { + related_tablet_id, related_object_id, related_first_level_part_id))) { LOG_WARN("fail to get tablet and object id", KR(ret), K(part_idx), K(index)); } else if (OB_FAIL(related_table->related_map_->add_related_tablet_id( - src_tablet_id, related_table_id, related_tablet_id, related_object_id))) { + src_tablet_id, related_table_id, related_tablet_id, related_object_id, related_first_level_part_id))) { LOG_WARN("fail to add related tablet info", KR(ret), K(src_table_id), K(src_tablet_id), K(src_object_id), K(related_table_id), K(related_tablet_id), K(related_object_id)); @@ -6277,7 +6282,7 @@ int ObPartitionUtils::get_tablet_and_object_id( related_tablet_id, related_object_id))) { LOG_WARN("fail to get tablet id and object id", KR(ret), K(related_table_id)); } else if (OB_FAIL(related_table->related_map_->add_related_tablet_id( - tablet_id, related_table_id, related_tablet_id, related_object_id))) { + tablet_id, related_table_id, related_tablet_id, related_object_id, OB_INVALID_ID))) { LOG_WARN("fail to add related tablet info", KR(ret), "src_table_id", table_schema.get_table_id(), "src_tablet_id", tablet_id, "src_object_id", object_id, diff --git a/src/share/schema/ob_schema_struct.h b/src/share/schema/ob_schema_struct.h index 51d9ef72bc..267c087e45 100644 --- a/src/share/schema/ob_schema_struct.h +++ b/src/share/schema/ob_schema_struct.h @@ -1837,7 +1837,8 @@ public: virtual int add_related_tablet_id(common::ObTabletID src_tablet_id, common::ObTableID related_table_id, common::ObTabletID related_tablet_id, - common::ObObjectID related_part_id) = 0; + common::ObObjectID related_part_id, + common::ObObjectID related_first_level_part_id) = 0; }; class ObSchemaGetterGuard; @@ -2294,11 +2295,26 @@ public: common::ObTabletID &tablet_id, common::ObObjectID &object_id) const; + /** + * first_level_part_id represent the first level part id of subpartition, + * otherwise its value is OB_INVALID_ID + * e.g. + * PARTITION_LEVEL_ZERO + * - object_id = table_id + * - first_level_part_id = OB_INVALID_ID + * PARTITION_LEVEL_ONE + * - object_id = part_id + * - first_level_part_id = OB_INVALID_ID + * PARTITION_LEVEL_TWO + * - object_id = sub_part_id + * - first_level_part_id = part_id + */ int get_tablet_and_object_id_by_index( const int64_t part_idx, const int64_t subpart_idx, ObTabletID &tablet_id, - ObObjectID &object_id) const; + ObObjectID &object_id, + ObObjectID &first_level_part_id) const; /** generate part name of hash partition, for resolver only * @name_type: Type of generated hash partition name: diff --git a/src/share/schema/ob_table_schema.cpp b/src/share/schema/ob_table_schema.cpp index 137b0c272c..49c8aa0ce6 100644 --- a/src/share/schema/ob_table_schema.cpp +++ b/src/share/schema/ob_table_schema.cpp @@ -1073,10 +1073,12 @@ int ObSimpleTableSchemaV2::get_part_id_by_tablet(const ObTabletID &tablet_id, in int ObSimpleTableSchemaV2::get_part_id_and_tablet_id_by_idx(const int64_t part_idx, const int64_t subpart_idx, ObObjectID &object_id, + ObObjectID &first_level_part_id, ObTabletID &tablet_id) const { int ret = OB_SUCCESS; ObBasePartition *base_part = NULL; + first_level_part_id = OB_INVALID_ID; if (PARTITION_LEVEL_ZERO == part_level_) { object_id = get_object_id(); tablet_id = get_tablet_id(); @@ -1085,6 +1087,9 @@ int ObSimpleTableSchemaV2::get_part_id_and_tablet_id_by_idx(const int64_t part_i } else { object_id = base_part->get_object_id(); tablet_id = base_part->get_tablet_id(); + if (PARTITION_LEVEL_TWO == part_level_) { + first_level_part_id = static_cast(base_part->get_part_id()); + } } return ret; } @@ -2915,12 +2920,14 @@ void ObTableSchema::reset() } int ObTableSchema::get_all_tablet_and_object_ids(ObIArray &tablet_ids, - ObIArray &partition_ids) const + ObIArray &partition_ids, + ObIArray *first_level_part_ids) const { int ret = OB_SUCCESS; if (PARTITION_LEVEL_ZERO == get_part_level()) { OZ(tablet_ids.push_back(get_tablet_id())); OZ(partition_ids.push_back(get_object_id())); + OZ(first_level_part_ids != NULL && first_level_part_ids->push_back(OB_INVALID_ID)); } else if (OB_ISNULL(partition_array_) || partition_num_ <= 0) { ret = OB_ERR_UNEXPECTED; LOG_WARN("part_array is null or is empty", K(ret), KP_(partition_array), K_(partition_num)); @@ -2932,6 +2939,7 @@ int ObTableSchema::get_all_tablet_and_object_ids(ObIArray &tablet_id } else { OZ(tablet_ids.push_back(partition_array_[i]->get_tablet_id())); OZ(partition_ids.push_back(partition_array_[i]->get_part_id())); + OZ(first_level_part_ids != NULL && first_level_part_ids->push_back(OB_INVALID_ID)); } } } else if (PARTITION_LEVEL_TWO == get_part_level()) { @@ -2956,6 +2964,7 @@ int ObTableSchema::get_all_tablet_and_object_ids(ObIArray &tablet_id partition_id = sub_part_array[j]->get_sub_part_id(); OZ(partition_ids.push_back(partition_id)); OZ(tablet_ids.push_back(sub_part_array[j]->get_tablet_id())); + OZ(first_level_part_ids != NULL && first_level_part_ids->push_back(partition_array_[i]->get_part_id())); } } } diff --git a/src/share/schema/ob_table_schema.h b/src/share/schema/ob_table_schema.h index 4fa213b1e5..ea35747cca 100644 --- a/src/share/schema/ob_table_schema.h +++ b/src/share/schema/ob_table_schema.h @@ -606,10 +606,25 @@ public: const ObTabletID &tablet_id, int64_t &part_id, int64_t &subpart_id) const; + /** + * first_level_part_id represent the first level part id of subpartition, + * otherwise its value is OB_INVALID_ID + * e.g. + * PARTITION_LEVEL_ZERO + * - object_id = table_id + * - first_level_part_id = OB_INVALID_ID + * PARTITION_LEVEL_ONE + * - object_id = part_id + * - first_level_part_id = OB_INVALID_ID + * PARTITION_LEVEL_TWO + * - object_id = sub_part_id + * - first_level_part_id = part_id + */ int get_part_id_and_tablet_id_by_idx( const int64_t part_idx, const int64_t subpart_idx, common::ObObjectID &object_id, + common::ObObjectID &first_level_part_id, common::ObTabletID &tablet_id) const; int get_part_by_idx( const int64_t part_id, @@ -1220,10 +1235,26 @@ public: virtual int deserialize_columns(const char *buf, const int64_t data_len, int64_t &pos); int serialize_constraints(char *buf, const int64_t data_len, int64_t &pos) const; int deserialize_constraints(const char *buf, const int64_t data_len, int64_t &pos); - // FIXME: move to ObPartitionSchema - // this function won't reset tablet_ids/partition_ids first, should be careful + /** + * FIXME: move to ObPartitionSchema + * this function won't reset tablet_ids/partition_ids first, should be careful!!! + * + * first_level_part_ids represent the first level part id of subpartition, + * otherwise its value is OB_INVALID_ID + * e.g. + * PARTITION_LEVEL_ZERO + * - partition_id = table_id + * - first_level_part_id = OB_INVALID_ID + * PARTITION_LEVEL_ONE + * - partition_id = part_id + * - first_level_part_id = OB_INVALID_ID + * PARTITION_LEVEL_TWO + * - partition_id = sub_part_id + * - first_level_part_id = part_id + */ int get_all_tablet_and_object_ids(common::ObIArray &tablet_ids, - common::ObIArray &partition_ids) const; + common::ObIArray &partition_ids, + ObIArray *first_level_part_ids = NULL) const; virtual int alloc_partition(const ObPartition *&partition); virtual int alloc_partition(const ObSubPartition *&subpartition); diff --git a/src/share/stat/ob_basic_stats_estimator.cpp b/src/share/stat/ob_basic_stats_estimator.cpp index 4ce31c436e..e89898a5fc 100644 --- a/src/share/stat/ob_basic_stats_estimator.cpp +++ b/src/share/stat/ob_basic_stats_estimator.cpp @@ -48,7 +48,6 @@ int ObBasicStatsEstimator::estimate(const ObTableStatParam ¶m, { int ret = OB_SUCCESS; const ObIArray &column_params = param.column_params_; - ObString hint_str("NO_REWRITE USE_PLAN_CACHE(NONE) DBMS_STATS"); ObString calc_part_id_str; ObOptTableStat tab_stat; ObOptStat src_opt_stat; @@ -69,8 +68,8 @@ int ObBasicStatsEstimator::estimate(const ObTableStatParam ¶m, column_params.count(), src_col_stats))) { LOG_WARN("failed init col stats", K(ret)); - } else if (OB_FAIL(add_hint(hint_str, allocator))) { - LOG_WARN("failed to add hint", K(ret)); + } else if (OB_FAIL(fill_hints(allocator, param.tab_name_))) { + LOG_WARN("failed to fill hints", K(ret)); } else if (OB_FAIL(add_from_table(param.db_name_, param.tab_name_))) { LOG_WARN("failed to add from table", K(ret)); } else if (OB_FAIL(fill_parallel_info(allocator, param.degree_))) { @@ -367,6 +366,7 @@ int ObBasicStatsEstimator::get_tablet_locations(ObExecContext &ctx, LOG_WARN("failed to get location", K(ret), K(loc_meta), K(tablet_ids.at(i))); } else if (OB_FAIL(candi_tablet_loc.set_part_loc_with_only_readable_replica( partition_ids.at(i), + OB_INVALID_INDEX, tablet_ids.at(i), location, session->get_retry_info().get_invalid_servers()))) { LOG_WARN("fail to set partition location with only readable replica", @@ -439,6 +439,137 @@ int ObBasicStatsEstimator::estimate_modified_count(ObExecContext &ctx, LOG_TRACE("succeed to get estimate modified count", K(table_id), K(result), K(need_inc_modified_count)); } + int tmp_ret = OB_SUCCESS; + if (NULL != client_result) { + if (OB_SUCCESS != (tmp_ret = client_result->close())) { + LOG_WARN("close result set failed", K(ret), K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + } + } + return ret; +} + +int ObBasicStatsEstimator::estimate_row_count(ObExecContext &ctx, + const uint64_t tenant_id, + const uint64_t table_id, + int64_t &row_cnt) +{ + int ret = OB_SUCCESS; + row_cnt = 0; + ObSqlString select_sql; + bool is_valid = true; + if (OB_FAIL(ObDbmsStatsUtils::check_table_read_write_valid(tenant_id, is_valid))) { + LOG_WARN("failed to check table read write valid", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(select_sql.append_fmt( + "select cast(sum(inserts) - sum(deletes) as signed) as row_cnt " \ + "from %s where tenant_id = %lu and table_id = %lu;", + share::OB_ALL_MONITOR_MODIFIED_TNAME, + share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id)))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + ObCommonSqlProxy *sql_proxy = ctx.get_sql_proxy(); + SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { + sqlclient::ObMySQLResult *client_result = NULL; + ObSQLClientRetryWeak sql_client_retry_weak(sql_proxy); + ObObj row_cnt_obj; + const int64_t obj_pos = 0; + if (OB_FAIL(sql_client_retry_weak.read(proxy_result, tenant_id, select_sql.ptr()))) { + LOG_WARN("failed to execute sql", K(ret), K(select_sql)); + } else if (OB_ISNULL(client_result = proxy_result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to execute sql", K(ret)); + } else if (OB_FAIL(client_result->next())) { + if (ret == OB_ITER_END) { + ret = OB_SUCCESS; + row_cnt = 0; + } else { + LOG_WARN("failed to get next result", K(ret)); + } + } else if (OB_FAIL(client_result->get_obj(obj_pos, row_cnt_obj))) { + LOG_WARN("failed to get object", K(ret)); + } else if (row_cnt_obj.is_null()) { + row_cnt = 0; + } else if (OB_UNLIKELY(!row_cnt_obj.is_integer_type())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected obj type", K(ret), K(row_cnt_obj.get_type())); + } else { + row_cnt = row_cnt_obj.get_int(); + } + LOG_TRACE("succeed to get table row count", K(table_id), K(row_cnt)); + int tmp_ret = OB_SUCCESS; + if (NULL != client_result) { + if (OB_SUCCESS != (tmp_ret = client_result->close())) { + LOG_WARN("close result set failed", K(ret), K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + } + } + return ret; +} + +int ObBasicStatsEstimator::get_gather_table_duration(ObExecContext &ctx, + const uint64_t tenant_id, + const uint64_t table_id, + int64_t &last_gather_duration) +{ + int ret = OB_SUCCESS; + last_gather_duration = 0; + ObSqlString select_sql; + bool is_valid = true; + if (OB_FAIL(ObDbmsStatsUtils::check_table_read_write_valid(tenant_id, is_valid))) { + LOG_WARN("failed to check table read write valid", K(ret)); + } else if (!is_valid) { + // do nothing + } else if (OB_FAIL(select_sql.append_fmt( + "select cast((time_to_usec(end_time) - time_to_usec(start_time)) as signed) as last_gather_duration" \ + " from %s where tenant_id = %lu and table_id = %lu and ret_code = 0 order by start_time desc limit 1;", + share::OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_TNAME, + share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id)))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + ObCommonSqlProxy *sql_proxy = ctx.get_sql_proxy(); + SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { + sqlclient::ObMySQLResult *client_result = NULL; + ObSQLClientRetryWeak sql_client_retry_weak(sql_proxy); + ObObj obj; + const int64_t obj_pos = 0; + if (OB_FAIL(sql_client_retry_weak.read(proxy_result, gen_meta_tenant_id(tenant_id), select_sql.ptr()))) { + LOG_WARN("failed to execute sql", K(ret), K(select_sql)); + } else if (OB_ISNULL(client_result = proxy_result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to execute sql", K(ret)); + } else if (OB_FAIL(client_result->next())) { + if (ret == OB_ITER_END) { + ret = OB_SUCCESS; + last_gather_duration = 0; + } else { + LOG_WARN("failed to get result"); + } + } else if (OB_FAIL(client_result->get_obj(obj_pos, obj))) { + LOG_WARN("failed to get object", K(ret)); + } else if (obj.is_null()) { + last_gather_duration = 0; + } else if (OB_UNLIKELY(!obj.is_integer_type())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected obj type", K(ret), K(obj.get_type())); + } else { + last_gather_duration = obj.get_int(); + } + LOG_TRACE("succeed to get last gather table duration", K(table_id), K(last_gather_duration)); + int tmp_ret = OB_SUCCESS; + if (NULL != client_result) { + if (OB_SUCCESS != (tmp_ret = client_result->close())) { + LOG_WARN("close result set failed", K(ret), K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } } } return ret; @@ -922,8 +1053,8 @@ int ObBasicStatsEstimator::check_stat_need_re_estimate(const ObTableStatParam &o opt_stat.column_stats_.at(i)->set_num_null(0); opt_stat.column_stats_.at(i)->set_num_distinct(0); opt_stat.column_stats_.at(i)->set_avg_len(0); - opt_stat.column_stats_.at(i)->set_llc_bitmap_size(ObColumnStat::NUM_LLC_BUCKET); - MEMSET(opt_stat.column_stats_.at(i)->get_llc_bitmap(), 0, ObColumnStat::NUM_LLC_BUCKET); + opt_stat.column_stats_.at(i)->set_llc_bitmap_size(ObOptColumnStat::NUM_LLC_BUCKET); + MEMSET(opt_stat.column_stats_.at(i)->get_llc_bitmap(), 0, ObOptColumnStat::NUM_LLC_BUCKET); opt_stat.column_stats_.at(i)->get_histogram().reset(); } } @@ -932,5 +1063,38 @@ int ObBasicStatsEstimator::check_stat_need_re_estimate(const ObTableStatParam &o return ret; } +int ObBasicStatsEstimator::fill_hints(common::ObIAllocator &alloc, + const ObString &table_name) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(table_name.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(table_name)); + } else { + const char *fmt_str = "NO_REWRITE USE_PLAN_CACHE(NONE) DBMS_STATS FULL(%.*s)"; + int64_t buf_len = table_name.length() + strlen(fmt_str); + char *buf = NULL; + if (OB_ISNULL(buf = static_cast(alloc.alloc(buf_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(buf), K(buf_len)); + } else { + int64_t real_len = sprintf(buf, fmt_str, table_name.length(), table_name.ptr()); + if (OB_UNLIKELY(real_len < 0 || real_len > buf_len)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(real_len)); + } else { + ObString hint_str; + hint_str.assign_ptr(buf, real_len); + if (OB_FAIL(add_hint(hint_str, alloc))) { + LOG_WARN("failed to add hint", K(ret)); + } else { + LOG_TRACE("succeed to fill index info", K(hint_str)); + } + } + } + } + return ret; +} + } // end of common } // end of oceanbase diff --git a/src/share/stat/ob_basic_stats_estimator.h b/src/share/stat/ob_basic_stats_estimator.h index 4ea7086c23..ffc95cb625 100644 --- a/src/share/stat/ob_basic_stats_estimator.h +++ b/src/share/stat/ob_basic_stats_estimator.h @@ -48,6 +48,15 @@ public: int64_t &result, const bool need_inc_modified_count = true); + static int estimate_row_count(ObExecContext &ctx, + const uint64_t tenant_id, + const uint64_t table_id, + int64_t &row_cnt); + static int get_gather_table_duration(ObExecContext &ctx, + const uint64_t tenant_id, + const uint64_t table_id, + int64_t &last_gather_duration); + static int estimate_stale_partition(ObExecContext &ctx, const uint64_t tenant_id, const uint64_t table_id, @@ -123,6 +132,7 @@ private: ObTableStatParam &new_param, ObExtraParam &new_extra); + int fill_hints(common::ObIAllocator &alloc, const ObString &table_name); }; } diff --git a/src/share/stat/ob_dbms_stats_executor.cpp b/src/share/stat/ob_dbms_stats_executor.cpp index e26655b3bf..579bad3267 100644 --- a/src/share/stat/ob_dbms_stats_executor.cpp +++ b/src/share/stat/ob_dbms_stats_executor.cpp @@ -22,6 +22,7 @@ #include "share/stat/ob_dbms_stats_lock_unlock.h" #include "share/stat/ob_index_stats_estimator.h" #include "pl/sys_package/ob_dbms_stats.h" +#include "share/stat/ob_opt_stat_gather_stat.h" namespace oceanbase { using namespace pl; namespace common { @@ -41,10 +42,13 @@ int ObDbmsStatsExecutor::gather_table_stats(ObExecContext &ctx, ObSEArray approx_opt_part_stats; ObExtraParam extra; extra.start_time_ = ObTimeUtility::current_time(); - if (OB_FAIL(extra.partition_id_block_map_.create(10000, - ObModIds::OB_HASH_BUCKET_TABLE_STATISTICS, - ObModIds::OB_HASH_BUCKET_TABLE_STATISTICS, - param.tenant_id_))) { + if (OB_ISNULL(param.allocator_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param.allocator_)); + } else if (OB_FAIL(extra.partition_id_block_map_.create(10000, + ObModIds::OB_HASH_BUCKET_TABLE_STATISTICS, + ObModIds::OB_HASH_BUCKET_TABLE_STATISTICS, + param.tenant_id_))) { LOG_WARN("failed to create hash map", K(ret)); } else if (param.need_estimate_block_ && OB_FAIL(ObBasicStatsEstimator::estimate_block_count(ctx, param, @@ -150,14 +154,15 @@ int ObDbmsStatsExecutor::do_gather_stats(ObExecContext &ctx, LOG_WARN("failed to estimate hybrid histogram", K(ret)); } else {/*do nothing*/} } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(ObIncrementalStatEstimator::try_drive_global_stat(ctx, param, - extra, approx_part_opt_stats, - opt_stats))) { - LOG_WARN("failed to try drive global stat", K(ret)); - } else if (OB_FAIL(check_all_cols_range_skew(param, opt_stats))) { - LOG_WARN("failed to check all cols range skew", K(ret)); - } else {/*do nothing*/} + if (OB_SUCC(ret)) { + if (OB_FAIL(ObIncrementalStatEstimator::try_derive_global_stat(ctx, param, + extra, approx_part_opt_stats, + opt_stats))) { + LOG_WARN("failed to try derive global stat", K(ret)); + } else if (OB_FAIL(check_all_cols_range_skew(param, opt_stats))) { + LOG_WARN("failed to check all cols range skew", K(ret)); + } else {/*do nothing*/} + } return ret; } @@ -286,9 +291,9 @@ int ObDbmsStatsExecutor::set_column_stats(ObExecContext &ctx, ObOptStatManager &mgr = ObOptStatManager::get_instance(); ObOptColumnStat::Key key; ObSEArray column_stats; - char *buf = NULL; + ObIAllocator *alloc = NULL; if (OB_UNLIKELY(param.table_param_.column_params_.count() != 1) || - OB_ISNULL(param.table_param_.allocator_)) { + OB_ISNULL(alloc = param.table_param_.allocator_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(param.table_param_.column_params_.count()), K(param.table_param_.allocator_)); @@ -306,36 +311,24 @@ int ObDbmsStatsExecutor::set_column_stats(ObExecContext &ctx, key.partition_id_ = param.table_param_.part_infos_.at(0).part_id_; stat_level = PARTITION_LEVEL; } + ObOptColumnStat *col_stat = NULL; if (OB_FAIL(mgr.get_column_stat(param.table_param_.tenant_id_, key, col_stat_handle))) { LOG_WARN("failed to get column stat", K(ret), K(key)); } else if (OB_ISNULL(col_stat_handle.stat_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(col_stat_handle.stat_), K(ret)); - } else if (OB_ISNULL(buf = static_cast(param.table_param_.allocator_->alloc(col_stat_handle.stat_->size())))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("memory is not enough", K(ret), K(buf)); - } else { - ObOptColumnStat *col_stat = new (buf) ObOptColumnStat(); - int64_t buf_len = col_stat_handle.stat_->size(); - int64_t pos = sizeof(ObOptColumnStat); - if (OB_ISNULL(col_stat)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(col_stat), K(ret)); - } else if (OB_FAIL(col_stat->deep_copy(*col_stat_handle.stat_, buf, buf_len, pos))) { - LOG_WARN("failed to deep copy", K(ret)); - } else if (OB_ISNULL(col_stat)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(col_stat), K(ret)); - } else {//reset base infos - col_stat->set_table_id(key.table_id_); - col_stat->set_partition_id(key.partition_id_); - col_stat->set_stat_level(stat_level); - col_stat->set_column_id(key.column_id_); - col_stat->set_collation_type(param.table_param_.column_params_.at(0).cs_type_); - col_stat->set_last_analyzed(0); - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(do_set_column_stats(param, col_stat))) { + } else if (OB_ISNULL(col_stat = OB_NEWx(ObOptColumnStat, alloc, (*alloc)))) { + LOG_WARN("failed to create column stat"); + } else if (OB_FAIL(col_stat->deep_copy(*col_stat_handle.stat_))) { + LOG_WARN("failed to deep copy", K(ret)); + } else {//reset base infos + col_stat->set_table_id(key.table_id_); + col_stat->set_partition_id(key.partition_id_); + col_stat->set_stat_level(stat_level); + col_stat->set_column_id(key.column_id_); + col_stat->set_collation_type(param.table_param_.column_params_.at(0).cs_type_); + col_stat->set_last_analyzed(0); + if (OB_FAIL(do_set_column_stats(param, col_stat))) { LOG_WARN("failed to do set table stats", K(ret)); } else if (OB_FAIL(column_stats.push_back(col_stat))) { LOG_WARN("failed to push back column stat", K(ret)); @@ -791,11 +784,11 @@ int ObDbmsStatsExecutor::do_gather_index_stats(ObExecContext &ctx, return ret; } -int ObDbmsStatsExecutor::update_stat_online(ObExecContext &ctx, +int ObDbmsStatsExecutor::update_online_stat(ObExecContext &ctx, ObTableStatParam ¶m, share::schema::ObSchemaGetterGuard *schema_guard, - TabStatIndMap &online_table_stats, - ColStatIndMap &online_column_stats) + const TabStatIndMap &online_table_stats, + const ColStatIndMap &online_column_stats) { int ret = OB_SUCCESS; diff --git a/src/share/stat/ob_dbms_stats_executor.h b/src/share/stat/ob_dbms_stats_executor.h index f0274c0c44..ab99673da8 100644 --- a/src/share/stat/ob_dbms_stats_executor.h +++ b/src/share/stat/ob_dbms_stats_executor.h @@ -46,11 +46,11 @@ public: const ObTableStatParam ¶m, const bool only_histogram); - static int update_stat_online(ObExecContext &ctx, + static int update_online_stat(ObExecContext &ctx, ObTableStatParam ¶m, share::schema::ObSchemaGetterGuard *schema_guard, - TabStatIndMap &online_table_stats, - ColStatIndMap &online_column_stats); + const TabStatIndMap &online_table_stats, + const ColStatIndMap &online_column_stats); private: static int do_gather_stats(ObExecContext &ctx, diff --git a/src/share/stat/ob_dbms_stats_export_import.cpp b/src/share/stat/ob_dbms_stats_export_import.cpp index f9683c4422..8414d9ca58 100644 --- a/src/share/stat/ob_dbms_stats_export_import.cpp +++ b/src/share/stat/ob_dbms_stats_export_import.cpp @@ -1152,7 +1152,7 @@ int ObDbmsStatsExportImport::get_opt_stat(ObExecContext &ctx, common::str_to_hex(hex_str.ptr(), hex_str.length(), bitmap_buf, hex_str.length()); // decompress llc bitmap; char *decomp_buf = NULL ; - int64_t decomp_size = ObColumnStat::NUM_LLC_BUCKET; + int64_t decomp_size = ObOptColumnStat::NUM_LLC_BUCKET; const int64_t bitmap_size = hex_str.length() / 2; if (OB_FAIL(ObOptStatSqlService::get_decompressed_llc_bitmap(*param.allocator_, bitmap_buf, diff --git a/src/share/stat/ob_dbms_stats_history_manager.cpp b/src/share/stat/ob_dbms_stats_history_manager.cpp index c1a4d75359..b5674d5561 100644 --- a/src/share/stat/ob_dbms_stats_history_manager.cpp +++ b/src/share/stat/ob_dbms_stats_history_manager.cpp @@ -60,11 +60,8 @@ namespace common { #define UPDATE_STATS_HISTROY_RETENTION "UPDATE %s SET sval1 = %ld, \ sval2 = CURRENT_TIMESTAMP where sname = 'STATS_RETENTION';" -#define DELETE_TAB_STAT_HISTORY_SQL "DELETE FROM %s %s;" +#define DELETE_STAT_HISTORY "DELETE FROM %s %s;" -#define DELETE_COL_STAT_HISTORY_SQL "DELETE FROM %s %s;" - -#define DELETE_HISTOGRAM_STAT_HISTORY_SQL "DELETE FROM %s %s;" int ObDbmsStatsHistoryManager::get_history_stat_handles(ObExecContext &ctx, const ObTableStatParam ¶m, @@ -109,9 +106,9 @@ int ObDbmsStatsHistoryManager::purge_stats(ObExecContext &ctx, const int64_t spe { int ret = OB_SUCCESS; ObObj retention; - int64_t retention_val; ObSQLSessionInfo *session = ctx.get_my_session(); ObSqlString time_str; + ObSqlString gather_time_str; if (OB_ISNULL(session)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(session), K(ret)); @@ -128,12 +125,20 @@ int ObDbmsStatsHistoryManager::purge_stats(ObExecContext &ctx, const int64_t spe "date_sub(CURRENT_TIMESTAMP, interval %ld day)", retention_val))) { LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(gather_time_str.append_fmt("WHERE start_time < "\ + "date_sub(CURRENT_TIMESTAMP, interval %ld day)", + retention_val))) { + LOG_WARN("failed to append fmt", K(ret)); } else {/*do nothing*/} } else if (specify_time == 0) {//delete all statistics history if (OB_FAIL(time_str.append(" "))) { LOG_WARN("failed to append", K(ret)); + } else if (OB_FAIL(gather_time_str.append(" "))) { + LOG_WARN("failed to append", K(ret)); } else {/*do nothing*/} - } else if (OB_FAIL(time_str.append_fmt("WHERE savtime < usec_to_time('%ld')", specify_time))) { + } else if (OB_FAIL(time_str.append_fmt("WHERE savtime < usec_to_time(%ld)", specify_time))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(gather_time_str.append_fmt("WHERE start_time < usec_to_time(%ld)", specify_time))) { LOG_WARN("failed to append fmt", K(ret)); } else {/*do nothing*/} if (OB_FAIL(ret)) { @@ -144,42 +149,62 @@ int ObDbmsStatsHistoryManager::purge_stats(ObExecContext &ctx, const int64_t spe ObSqlString del_tab_history; ObSqlString del_col_history; ObSqlString del_hist_history; + ObSqlString del_task_opt_stat_gather_history; + ObSqlString del_table_opt_stat_gather_history; ObMySQLTransaction trans; + ObMySQLTransaction trans1; if (OB_ISNULL(mysql_proxy)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(mysql_proxy)); - } else if (OB_FAIL(del_tab_history.append_fmt(DELETE_TAB_STAT_HISTORY_SQL, + } else if (OB_FAIL(del_tab_history.append_fmt(DELETE_STAT_HISTORY, share::OB_ALL_TABLE_STAT_HISTORY_TNAME, time_str.ptr()))) { LOG_WARN("failed to append sql stmt", K(ret), K(del_tab_history)); - } else if (OB_FAIL(del_col_history.append_fmt(DELETE_COL_STAT_HISTORY_SQL, + } else if (OB_FAIL(del_col_history.append_fmt(DELETE_STAT_HISTORY, share::OB_ALL_COLUMN_STAT_HISTORY_TNAME, time_str.ptr()))) { LOG_WARN("failed to append sql stmt", K(ret), K(del_col_history)); - } else if (OB_FAIL(del_hist_history.append_fmt(DELETE_HISTOGRAM_STAT_HISTORY_SQL, + } else if (OB_FAIL(del_hist_history.append_fmt(DELETE_STAT_HISTORY, share::OB_ALL_HISTOGRAM_STAT_HISTORY_TNAME, time_str.ptr()))) { LOG_WARN("failed to append sql stmt", K(ret), K(del_hist_history)); + } else if (OB_FAIL(del_task_opt_stat_gather_history.append_fmt(DELETE_STAT_HISTORY, + share::OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_TNAME, + gather_time_str.ptr()))) { + LOG_WARN("failed to append sql stmt", K(ret), K(del_hist_history)); + } else if (OB_FAIL(del_table_opt_stat_gather_history.append_fmt(DELETE_STAT_HISTORY, + share::OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_TNAME, + gather_time_str.ptr()))) { + LOG_WARN("failed to append sql stmt", K(ret), K(del_hist_history)); } else if (OB_FAIL(trans.start(mysql_proxy, tenant_id))) { LOG_WARN("fail to start transaction", K(ret)); + } else if (OB_FAIL(trans1.start(mysql_proxy, gen_meta_tenant_id(tenant_id)))) { + LOG_WARN("fail to start transaction", K(ret)); } else if (OB_FAIL(trans.write(tenant_id, del_tab_history.ptr(), affected_rows))) { LOG_WARN("fail to exec sql", K(del_tab_history), K(ret)); } else if (OB_FAIL(trans.write(tenant_id, del_col_history.ptr(), affected_rows))) { LOG_WARN("fail to exec sql", K(del_col_history), K(ret)); } else if (OB_FAIL(trans.write(tenant_id, del_hist_history.ptr(), affected_rows))) { LOG_WARN("fail to exec sql", K(del_hist_history), K(ret)); + } else if (OB_FAIL(trans1.write(gen_meta_tenant_id(tenant_id), del_task_opt_stat_gather_history.ptr(), affected_rows))) { + LOG_WARN("fail to exec sql", K(del_task_opt_stat_gather_history), K(ret)); + } else if (OB_FAIL(trans1.write(gen_meta_tenant_id(tenant_id), del_table_opt_stat_gather_history.ptr(), affected_rows))) { + LOG_WARN("fail to exec sql", K(del_table_opt_stat_gather_history), K(ret)); } else { LOG_TRACE("Succeed to do execute sql", K(del_tab_history), K(del_col_history), - K(del_hist_history)); + K(del_hist_history), K(del_task_opt_stat_gather_history), + K(del_table_opt_stat_gather_history)); } if (OB_SUCC(ret)) { - if (OB_FAIL(trans.end(true))) { + if (OB_FAIL(trans.end(true)) || OB_FAIL(trans1.end(true))) { LOG_WARN("fail to commit transaction", K(ret)); } } else { int tmp_ret = OB_SUCCESS; if (OB_SUCCESS != (tmp_ret = trans.end(false))) { LOG_WARN("fail to roll back transaction", K(tmp_ret)); + } else if (OB_SUCCESS != (tmp_ret = trans1.end(false))) { + LOG_WARN("fail to roll back transaction", K(tmp_ret)); } } } @@ -572,7 +597,7 @@ int ObDbmsStatsHistoryManager::fill_column_stat_history(ObIAllocator &allocator, common::str_to_hex(hex_str.ptr(), hex_str.length(), bitmap_buf, hex_str.length()); // decompress llc bitmap; char *decomp_buf = NULL ; - int64_t decomp_size = ObColumnStat::NUM_LLC_BUCKET; + int64_t decomp_size = ObOptColumnStat::NUM_LLC_BUCKET; const int64_t bitmap_size = hex_str.length() / 2; if (OB_FAIL(ObOptStatSqlService::get_decompressed_llc_bitmap(allocator, bitmap_buf, bitmap_size, decomp_buf, diff --git a/src/share/stat/ob_dbms_stats_history_manager.h b/src/share/stat/ob_dbms_stats_history_manager.h index 25e1631c57..ab09374883 100644 --- a/src/share/stat/ob_dbms_stats_history_manager.h +++ b/src/share/stat/ob_dbms_stats_history_manager.h @@ -17,7 +17,6 @@ #include "sql/engine/ob_exec_context.h" #include "share/stat/ob_stat_item.h" #include "share/stat/ob_opt_table_stat.h" -#include "share/stat/ob_column_stat.h" #include "share/stat/ob_opt_column_stat_cache.h" namespace oceanbase { diff --git a/src/share/stat/ob_dbms_stats_utils.cpp b/src/share/stat/ob_dbms_stats_utils.cpp index 7bd031f804..c02447875c 100644 --- a/src/share/stat/ob_dbms_stats_utils.cpp +++ b/src/share/stat/ob_dbms_stats_utils.cpp @@ -13,6 +13,7 @@ #define USING_LOG_PREFIX SQL_ENG #include "ob_dbms_stats_utils.h" #include "share/stat/ob_opt_column_stat.h" +#include "share/stat/ob_opt_osg_column_stat.h" #include "share/object/ob_obj_cast.h" #include "share/stat/ob_opt_stat_manager.h" #include "share/stat/ob_opt_table_stat.h" @@ -215,9 +216,9 @@ int ObDbmsStatsUtils::check_is_stat_table(share::schema::ObSchemaGetterGuard &sc } int ObDbmsStatsUtils::check_is_sys_table(share::schema::ObSchemaGetterGuard &schema_guard, - const uint64_t tenant_id, - const int64_t table_id, - bool &is_valid) + const uint64_t tenant_id, + const int64_t table_id, + bool &is_valid) { bool ret = OB_SUCCESS; const ObSimpleTenantSchema *tenant = NULL; @@ -257,6 +258,7 @@ bool ObDbmsStatsUtils::is_no_stat_virtual_table(const int64_t table_id) table_id == share::OB_TENANT_VIRTUAL_CURRENT_TENANT_TID || table_id == share::OB_TENANT_VIRTUAL_SHOW_TABLES_TID || table_id == share::OB_TENANT_VIRTUAL_SHOW_CREATE_PROCEDURE_TID || + table_id == share::OB_ALL_VIRTUAL_SESSTAT_TID || table_id == share::OB_ALL_VIRTUAL_PROXY_SCHEMA_TID || table_id == share::OB_ALL_VIRTUAL_PROXY_PARTITION_INFO_TID || table_id == share::OB_ALL_VIRTUAL_PROXY_PARTITION_TID || @@ -266,6 +268,7 @@ bool ObDbmsStatsUtils::is_no_stat_virtual_table(const int64_t table_id) table_id == share::OB_TENANT_VIRTUAL_SHOW_CREATE_TRIGGER_TID || table_id == share::OB_TENANT_VIRTUAL_ALL_TABLE_AGENT_TID || table_id == share::OB_ALL_VIRTUAL_INFORMATION_COLUMNS_TID || + table_id == share::OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_TID || table_id == share::OB_ALL_VIRTUAL_SESSTAT_ORA_TID || table_id == share::OB_TENANT_VIRTUAL_SHOW_CREATE_TABLE_ORA_TID || table_id == share::OB_TENANT_VIRTUAL_SHOW_CREATE_PROCEDURE_ORA_TID || @@ -282,7 +285,8 @@ bool ObDbmsStatsUtils::is_no_stat_virtual_table(const int64_t table_id) table_id == share::OB_ALL_VIRTUAL_SQL_AUDIT_ORA_TID || table_id == share::OB_ALL_VIRTUAL_TRACE_SPAN_INFO_ORA_TID || table_id == share::OB_ALL_VIRTUAL_LOCK_WAIT_STAT_ORA_TID || - table_id == share::OB_ALL_VIRTUAL_TRANS_STAT_ORA_TID; + table_id == share::OB_ALL_VIRTUAL_TRANS_STAT_ORA_TID || + table_id == share::OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_ORA_TID; } bool ObDbmsStatsUtils::is_virtual_index_table(const int64_t table_id) @@ -566,11 +570,7 @@ int ObDbmsStatsUtils::merge_tab_stats(const ObTableStatParam ¶m, // put all stats into array for future use. FOREACH_X(it, online_table_stats, OB_SUCC(ret)) { - bool is_valid = false; - if (OB_FAIL(check_part_id_valid(param, it->second->get_partition_id(), is_valid))) { - // if partition is locked, shouldn't gather. - LOG_WARN("fail to check part id valid", K(ret)); - } else if (is_valid) { + if (is_part_id_valid(param, it->second->get_partition_id())) { if (OB_FAIL(dst_tab_stats.push_back(it->second))) { LOG_WARN("fail to push back table stats", K(ret)); } @@ -587,77 +587,66 @@ int ObDbmsStatsUtils::merge_col_stats(const ObTableStatParam ¶m, common::ObIArray &dst_col_stats) { int ret = OB_SUCCESS; - ObOptColumnStat *tmp_col_stat; - + ObOptColumnStat *tmp_col_stat = NULL; for (int64_t i = 0; OB_SUCC(ret) && i < old_col_handles.count(); i++) { - ObOptColumnStat * old_col_stat = const_cast(old_col_handles.at(i).stat_); - ObOptColumnStat::Key key(param.tenant_id_, old_col_stat->get_table_id(), - old_col_stat->get_partition_id(), old_col_stat->get_column_id()); + ObOptColumnStat *old_col_stat = const_cast(old_col_handles.at(i).stat_); + ObOptColumnStat::Key key(param.tenant_id_, + old_col_stat->get_table_id(), + old_col_stat->get_partition_id(), + old_col_stat->get_column_id()); if (OB_FAIL(online_column_stats.get_refactored(key, tmp_col_stat))) { - if (OB_HASH_NOT_EXIST != ret) { + if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) { LOG_WARN("failed to find in hashmap", K(ret)); - } else { - if (OB_FAIL(dst_col_stats.push_back(old_col_stat))) { - LOG_WARN("fail to push back table stats", K(ret)); - } + } else if (OB_FAIL(dst_col_stats.push_back(old_col_stat))) { + LOG_WARN("fail to push back table stats", K(ret)); } } else if (OB_FAIL(tmp_col_stat->merge_column_stat(*old_col_stat))) { - //merge LOG_WARN("fail to merge new table stat with old table stat", K(ret)); } } FOREACH_X(it, online_column_stats, OB_SUCC(ret)) { - // after merge, we need to re-calc ndv from llc. + // after merge, we need to re-calc ndv from llc bool is_valid = false; - if (OB_ISNULL(it->second)) { + ObOptColumnStat *col_stat = NULL; + if (OB_ISNULL(col_stat = it->second)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null pointer", K(ret)); - } else if (OB_FAIL(check_part_id_valid(param, it->second->get_partition_id(), is_valid))) { - // if partition is locked, shouldn't gather. - LOG_WARN("fail to check part id valid", K(ret)); - } else if (is_valid) { - it->second->set_num_distinct(ObGlobalNdvEval::get_ndv_from_llc(it->second->get_llc_bitmap())); - if (OB_FAIL(dst_col_stats.push_back(it->second))) { + } else if (is_part_id_valid(param, col_stat->get_partition_id())) { + col_stat->set_num_distinct(ObGlobalNdvEval::get_ndv_from_llc(col_stat->get_llc_bitmap())); + if (OB_FAIL(dst_col_stats.push_back(col_stat))) { LOG_WARN("fail to push back table stats", K(ret)); } } } LOG_DEBUG("OSG debug", K(dst_col_stats)); - return ret; } -int ObDbmsStatsUtils::check_part_id_valid(const ObTableStatParam ¶m, - const ObObjectID part_id, - bool &is_valid) +bool ObDbmsStatsUtils::is_part_id_valid(const ObTableStatParam ¶m, + const ObObjectID part_id) { - int ret = OB_SUCCESS; - bool found = false; - LOG_DEBUG("check part_id valid", K(param), K(part_id)); - is_valid = false; + bool is_valid = false; if (param.global_stat_param_.need_modify_) { if (part_id == param.global_part_id_) { is_valid = true; } } if (!is_valid && param.part_stat_param_.need_modify_) { - for (int64_t i = 0; !found && i < param.part_infos_.count(); i++) { + for (int64_t i = 0; !is_valid && i < param.part_infos_.count(); i++) { if (part_id == param.part_infos_.at(i).part_id_) { - found = true; is_valid = true; } } } if (!is_valid && param.subpart_stat_param_.need_modify_) { - for (int64_t i = 0; !found && i < param.subpart_infos_.count(); i++) { + for (int64_t i = 0; !is_valid && i < param.subpart_infos_.count(); i++) { if (part_id == param.subpart_infos_.at(i).part_id_) { - found = true; is_valid = true; } } } - return ret; + return is_valid; } int ObDbmsStatsUtils::get_part_ids_from_param(const ObTableStatParam ¶m, @@ -717,7 +706,8 @@ int ObDbmsStatsUtils::get_part_infos(const ObTableSchema &table_schema, } } int64_t origin_cnt = subpart_infos.count(); - if (OB_FAIL(part_infos.push_back(part_info))) { + if (OB_FAIL(ret)) { + } else if (OB_FAIL(part_infos.push_back(part_info))) { LOG_WARN("failed to push back part info", K(ret)); } else if (OB_FAIL(part_ids.push_back(part_info.part_id_))) { LOG_WARN("failed to push back part id", K(ret)); @@ -768,7 +758,8 @@ int ObDbmsStatsUtils::get_subpart_infos(const ObTableSchema &table_schema, LOG_WARN("fail to add part info to hashmap", K(ret), K(part_info), K(subpart->get_sub_part_id())); } } - if (OB_FAIL(subpart_infos.push_back(subpart_info))) { + if (OB_FAIL(ret)) { + } else if (OB_FAIL(subpart_infos.push_back(subpart_info))) { LOG_WARN("failed to push back subpart_info", K(ret)); } else if (OB_FAIL(subpart_ids.push_back(subpart_info.part_id_))) { LOG_WARN("failed to push back part id", K(ret)); diff --git a/src/share/stat/ob_dbms_stats_utils.h b/src/share/stat/ob_dbms_stats_utils.h index 854e7a2693..824be55f77 100644 --- a/src/share/stat/ob_dbms_stats_utils.h +++ b/src/share/stat/ob_dbms_stats_utils.h @@ -107,7 +107,7 @@ public: common::ObIArray &history_col_handles, common::ObIArray &dst_column_stat); - static int check_part_id_valid(const ObTableStatParam ¶m, const ObObjectID part_id, bool &is_valid); + static bool is_part_id_valid(const ObTableStatParam ¶m, const ObObjectID part_id); static int get_part_ids_from_param(const ObTableStatParam ¶m, common::ObIArray &part_ids); diff --git a/src/share/stat/ob_hybrid_hist_estimator.cpp b/src/share/stat/ob_hybrid_hist_estimator.cpp index 386f50dd12..cf2dda40f9 100644 --- a/src/share/stat/ob_hybrid_hist_estimator.cpp +++ b/src/share/stat/ob_hybrid_hist_estimator.cpp @@ -15,9 +15,11 @@ #include "observer/ob_sql_client_decorator.h" #include "sql/engine/basic/ob_chunk_row_store.h" #include "share/stat/ob_dbms_stats_utils.h" +#include "sql/engine/aggregate/ob_aggregate_processor.h" namespace oceanbase { +using namespace sql; namespace common { @@ -641,6 +643,114 @@ int ObHybridHistograms::build_hybrid_hist(ObIArray &bucket_pairs, } return ret; } +int ObHybridHistograms::build_hybrid_hist(ObAggregateProcessor::HybridHistExtraResult *extra, + ObIAllocator *alloc, + int64_t bucket_num, + int64_t total_count, + int64_t num_distinct, + int64_t pop_count, + int64_t pop_freq, + const ObObjMeta &obj_meta) +{ + int ret = OB_SUCCESS; + int64_t bucket_size = -1; + bool dynamic_size = false; + int64_t dynamic_step = 0; + const ObChunkDatumStore::StoredRow *row = nullptr; + if (OB_ISNULL(extra) || OB_ISNULL(alloc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(extra), K(alloc)); + } else if (num_distinct == 0) { + // do nothing + } else { + // determine bucket size + if (OB_FAIL(extra->get_next_row_from_material(row))) { + LOG_WARN("failed to get next row from material"); + } else if (OB_ISNULL(row) || OB_UNLIKELY(row->cnt_ != 1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null stored row", K(row)); + } else if (num_distinct <= bucket_num + 2) { + bucket_size = 1; + } else if (bucket_num <= pop_count) { + bucket_size = total_count / bucket_num; + } else { + dynamic_size = true; + // first bucket always contain only one values. following code will handle first value is + // popular value or not. + BucketDesc *desc = reinterpret_cast(row->get_extra_payload()); + if (desc->is_pop_) { + bucket_size = (total_count - pop_freq) / (bucket_num - pop_count); + } else { + bucket_size = (total_count - pop_freq - desc->ep_count_) / (bucket_num - pop_count - 1); + } + } + + int64_t bucket_rows = 0; + int64_t ep_num = 0; + int64_t un_pop_count = 0; + int64_t un_pop_bucket = 0; + int64_t i = 0; + if (OB_SUCC(ret)) { + do { + if (OB_ISNULL(row) || OB_UNLIKELY(row->cnt_ != 1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null stored row", K(row)); + } else { + BucketDesc *desc = reinterpret_cast(row->get_extra_payload()); + int64_t ep_count = desc->ep_count_; + bool is_pop = desc->is_pop_; + bucket_rows += ep_count; + ep_num += ep_count; + if (!is_pop) { + un_pop_count += ep_count; + } + if (bucket_rows > bucket_size || 0 == i || extra->get_material_row_count() - 1 == i) { + bucket_rows = 0; + ObObj ep_val; + if (OB_FAIL(row->cells()[0].to_obj(ep_val, obj_meta))) { + LOG_WARN("failed to obj", K(ret)); + } else if (OB_FAIL(ob_write_obj(*alloc, ep_val, ep_val))) { + LOG_WARN("failed to write obj", K(ret), K(ep_val)); + } else { + ObHistBucket bkt(ep_val, ep_count, ep_num); + if (!is_pop) { + ++un_pop_bucket; + } + if (OB_FAIL(add_hist_bucket(bkt))) { + LOG_WARN("failed add hist bucket", K(ret)); + } + } + + if (dynamic_size && bucket_num > pop_count + un_pop_bucket) { + bucket_size = (total_count - pop_freq - un_pop_count) + / (bucket_num - pop_count - un_pop_bucket); + } + } + ++i; + } + if (OB_SUCC(ret)) { + ret = extra->get_next_row_from_material(row); + } + } while (OB_SUCC(ret)); + } + + if (OB_LIKELY(OB_ITER_END == ret)) { + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to build hybrid histogram"); + } + } + + if (OB_SUCC(ret)) { + total_count_ = total_count; + num_distinct_ = num_distinct; + pop_count_ = pop_count; + pop_freq_ = pop_freq; + LOG_TRACE("succeed to build hybrid histogram", K(bucket_num), K(bucket_size), K(total_count), + K(pop_count), K(pop_freq), K(num_distinct), K(hybrid_buckets_.count())); + } + return ret; +} OB_DEF_SERIALIZE(ObHybridHistograms) { diff --git a/src/share/stat/ob_hybrid_hist_estimator.h b/src/share/stat/ob_hybrid_hist_estimator.h index 689a5bc98a..b082e4e64a 100644 --- a/src/share/stat/ob_hybrid_hist_estimator.h +++ b/src/share/stat/ob_hybrid_hist_estimator.h @@ -16,11 +16,19 @@ #include "share/stat/ob_stats_estimator.h" #include "share/stat/ob_opt_column_stat.h" #include "share/stat/ob_opt_table_stat.h" +#include "sql/engine/aggregate/ob_aggregate_processor.h" namespace oceanbase { namespace common { +struct BucketDesc +{ + BucketDesc() : ep_count_(0), is_pop_(false) {} + int64_t ep_count_; + bool is_pop_; +}; + struct BucketNode { BucketNode() : ep_val_(), ep_count_(0), is_pop_(false) {} @@ -61,6 +69,14 @@ public: int64_t bucket_num, int64_t total_count, int64_t num_distinct); + int build_hybrid_hist(sql::ObAggregateProcessor::HybridHistExtraResult *extra, + ObIAllocator *alloc, + int64_t bucket_num, + int64_t total_count, + int64_t num_distinct, + int64_t pop_count, + int64_t pop_freq, + const ObObjMeta &obj_meta); TO_STRING_KV(K_(total_count), K_(num_distinct), K_(pop_count), diff --git a/src/share/stat/ob_incremental_stat_estimator.cpp b/src/share/stat/ob_incremental_stat_estimator.cpp index 8a835f2695..a183836dae 100644 --- a/src/share/stat/ob_incremental_stat_estimator.cpp +++ b/src/share/stat/ob_incremental_stat_estimator.cpp @@ -26,14 +26,14 @@ namespace oceanbase { using namespace pl; namespace common { -/* Now, the following scenarios will try drive global stat: +/* Now, the following scenarios will try derive global stat: * if granularity specify 'APPROX_GLOBAL AND PARTITION': * if in auto gather stats, choose gather subpartition stats and incremental gather global stats: * then we choose incremental gather global stats from subpartition stats. * else we choose incremental gather global stats from partition stats. * */ -int ObIncrementalStatEstimator::try_drive_global_stat(ObExecContext &ctx, +int ObIncrementalStatEstimator::try_derive_global_stat(ObExecContext &ctx, const ObTableStatParam ¶m, ObExtraParam &extra, ObIArray &approx_part_opt_stats, @@ -44,23 +44,23 @@ int ObIncrementalStatEstimator::try_drive_global_stat(ObExecContext &ctx, extra.type_ == TABLE_LEVEL || !(param.global_stat_param_.need_modify_ && param.global_stat_param_.gather_approx_) || param.part_level_ == share::schema::PARTITION_LEVEL_ZERO) { - LOG_TRACE("not fullfill drive global stat", K(extra.type_),K(param.global_stat_param_), + LOG_TRACE("not fullfill derive global stat", K(extra.type_),K(param.global_stat_param_), K(param.part_level_)); } else if (extra.type_ == SUBPARTITION_LEVEL) { - if (OB_FAIL(drive_part_stats_from_subpart_stats(ctx, param, opt_stats, + if (OB_FAIL(derive_part_stats_from_subpart_stats(ctx, param, opt_stats, approx_part_opt_stats))) { - LOG_WARN("failed to drive part stats from subpart stats", K(ret)); - } else {/*do nothing*/} - } else if (OB_FAIL(drive_global_stat_from_part_stats(ctx, param, + LOG_WARN("failed to derive part stats from subpart stats", K(ret)); + } + } else if (OB_FAIL(derive_global_stat_from_part_stats(ctx, param, approx_part_opt_stats, opt_stats))) { - LOG_WARN("failed to drive global stat from part stats", K(ret)); - } else {/*do nothing*/} + LOG_WARN("failed to derive global stat from part stats", K(ret)); + } return ret; } -int ObIncrementalStatEstimator::drive_global_stat_by_direct_load(ObExecContext &ctx, +int ObIncrementalStatEstimator::derive_global_stat_by_direct_load(ObExecContext &ctx, ObIArray &part_tab_stats, - ObIArray &part_column_stats) + const ObIArray &part_column_stats) { int ret = OB_SUCCESS; const share::schema::ObTableSchema *table_schema = NULL; @@ -102,11 +102,11 @@ int ObIncrementalStatEstimator::drive_global_stat_by_direct_load(ObExecContext & //derive global stat if (OB_SUCC(ret)) { ObOptStat global_opt_stat; - bool need_drive_hist = false; + bool need_derive_hist = false; bool need_gather_hybrid_hist = false; if (OB_FAIL(do_derive_global_stat(ctx, alloc, param, nee_derive_part ? all_derive_opt_stats : part_opt_stats, - need_drive_hist, + need_derive_hist, TABLE_LEVEL, param.global_part_id_, need_gather_hybrid_hist, global_opt_stat))) { LOG_WARN("Failed to derive global stat from part stat", K(ret)); @@ -133,7 +133,7 @@ int ObIncrementalStatEstimator::drive_global_stat_by_direct_load(ObExecContext & LOG_WARN("failed to write all opt stats by dircet load", K(ret)); } else {/*do nothing*/} } - LOG_TRACE("succeed to drive global stat by direct load", K(param), K(part_tab_stats), + LOG_TRACE("succeed to derive global stat by direct load", K(param), K(part_tab_stats), K(part_column_stats)); return ret; } @@ -166,7 +166,7 @@ int ObIncrementalStatEstimator::write_all_opt_stats_by_dircet_load( return ret; } -int ObIncrementalStatEstimator::drive_global_stat_from_part_stats( +int ObIncrementalStatEstimator::derive_global_stat_from_part_stats( ObExecContext &ctx, const ObTableStatParam ¶m, const ObIArray &approx_part_opt_stats, @@ -178,7 +178,7 @@ int ObIncrementalStatEstimator::drive_global_stat_from_part_stats( ObSEArray col_handles; ObSEArray tmp_opt_stats; ObSEArray no_regather_part_ids; - bool need_drive_hist = true; + bool need_derive_hist = true; bool need_gather_hybrid_hist = false; if (OB_ISNULL(param.allocator_)) { ret = OB_ERR_UNEXPECTED; @@ -190,7 +190,7 @@ int ObIncrementalStatEstimator::drive_global_stat_from_part_stats( } else if (OB_FAIL(tmp_opt_stats.assign(opt_stats))) { LOG_WARN("failed to assign", K(ret)); } else { - //if specify part/subpart name, need get other already existed part stats, but only drive base + //if specify part/subpart name, need get other already existed part stats, but only derive base //stat, not derive histogram stats. if (!param.part_name_.empty()) { if (OB_UNLIKELY(tmp_opt_stats.count() != 1)) { @@ -208,7 +208,7 @@ int ObIncrementalStatEstimator::drive_global_stat_from_part_stats( tmp_opt_stats))) { LOG_WARN("failed to generate all opt stat", K(ret)); } else { - need_drive_hist = false; + need_derive_hist = false; } //get regather partition stats(stats is locked or not stale). } else { @@ -234,21 +234,21 @@ int ObIncrementalStatEstimator::drive_global_stat_from_part_stats( } } if (OB_SUCC(ret)) { - if (OB_FAIL(do_derive_global_stat(ctx, *param.allocator_, param, tmp_opt_stats, need_drive_hist, + if (OB_FAIL(do_derive_global_stat(ctx, *param.allocator_, param, tmp_opt_stats, need_derive_hist, TABLE_LEVEL,param.global_part_id_, need_gather_hybrid_hist, global_opt_stat))) { LOG_WARN("Failed to derive global stat from part stat", K(ret)); } else if (OB_FAIL(opt_stats.push_back(global_opt_stat))) { LOG_WARN("failed to push back global stat", K(ret)); } else { - LOG_TRACE("Succeed to drive global stat from part stats", K(opt_stats.count()), K(param)); + LOG_TRACE("Succeed to derive global stat from part stats", K(opt_stats.count()), K(param)); } } } return ret; } -int ObIncrementalStatEstimator::drive_part_stats_from_subpart_stats( +int ObIncrementalStatEstimator::derive_part_stats_from_subpart_stats( ObExecContext &ctx, const ObTableStatParam ¶m, const ObIArray &gather_opt_stats, @@ -275,7 +275,7 @@ int ObIncrementalStatEstimator::drive_part_stats_from_subpart_stats( approx_part_opt_stats))) { LOG_WARN("failed to do derive part stats from subpart stats", K(ret)); } else { - LOG_TRACE("Succeed to drive part stats from subpart stats", K(approx_part_opt_stats.count()), + LOG_TRACE("Succeed to derive part stats from subpart stats", K(approx_part_opt_stats.count()), K(gather_opt_stats.count()), K(param)); } @@ -356,7 +356,8 @@ int ObIncrementalStatEstimator::do_derive_part_stats_from_subpart_stats( ObTableStatParam part_param; if (OB_FAIL(gen_part_param(param, gather_hybrid_hist_opt_stats, part_param))) { LOG_WARN("failed to gen part param", K(ret)); - } else if (OB_FAIL(hybrid_est.estimate(part_param, extra, gather_hybrid_hist_opt_stats))) { + } else if (OB_FAIL(hybrid_est.estimate(part_param, extra, + gather_hybrid_hist_opt_stats))) { LOG_WARN("failed to estimate hybrid histogram", K(ret)); } else { LOG_TRACE("succeed to gather partition hybrid hist", K(gather_hybrid_hist_opt_stats.count())); @@ -366,7 +367,7 @@ int ObIncrementalStatEstimator::do_derive_part_stats_from_subpart_stats( } int ObIncrementalStatEstimator::generate_all_opt_stat(ObIArray &table_stats, - ObIArray &col_handles, + const ObIArray &col_handles, int64_t col_cnt, ObIArray &all_opt_stats) { @@ -398,7 +399,7 @@ int ObIncrementalStatEstimator::generate_all_opt_stat(ObIArray & } int ObIncrementalStatEstimator::generate_all_opt_stat(ObIArray &table_stats, - ObIArray &column_stats, + const ObIArray &column_stats, int64_t col_cnt, ObIArray &all_opt_stats) { @@ -506,7 +507,7 @@ int ObIncrementalStatEstimator::do_derive_global_stat(ObExecContext &ctx, ObIAllocator &alloc, const ObTableStatParam ¶m, ObIArray &part_opt_stats, - bool need_drive_hist, + bool need_derive_hist, const StatLevel &approx_level, const int64_t partition_id, bool &need_gather_hybrid_hist, @@ -516,7 +517,7 @@ int ObIncrementalStatEstimator::do_derive_global_stat(ObExecContext &ctx, if (OB_FAIL(derive_global_tbl_stat(ctx, alloc, param, approx_level, partition_id, part_opt_stats, global_opt_stat))) { LOG_WARN("failed to derive global tbl stat from part tbl stat", K(ret)); - } else if (OB_FAIL(derive_global_col_stat(ctx, alloc, param, part_opt_stats, need_drive_hist, + } else if (OB_FAIL(derive_global_col_stat(ctx, alloc, param, part_opt_stats, need_derive_hist, approx_level, partition_id, need_gather_hybrid_hist, global_opt_stat))) { LOG_WARN("failed to derive global col stat from part col stat", K(ret)); @@ -586,7 +587,7 @@ int ObIncrementalStatEstimator::derive_global_col_stat(ObExecContext &ctx, ObIAllocator &alloc, const ObTableStatParam ¶m, ObIArray &part_opt_stats, - bool need_drive_hist, + bool need_derive_hist, const StatLevel &approx_level, const int64_t partition_id, bool &need_gather_hybrid_hist, @@ -642,11 +643,11 @@ int ObIncrementalStatEstimator::derive_global_col_stat(ObExecContext &ctx, K(param.column_params_.at(i))); } else if (opt_col_stat->get_num_distinct() == 0 && opt_col_stat->get_num_null() == 0) { /*do nothing*/ - } else if (need_drive_hist && opt_col_stat->get_histogram().is_valid() && + } else if (need_derive_hist && opt_col_stat->get_histogram().is_valid() && OB_FAIL(all_part_histograms.push_back(opt_col_stat->get_histogram()))) { LOG_WARN("failed to push back histogram", K(ret)); } else { - need_drive_hist &= opt_col_stat->get_histogram().is_valid(); + need_derive_hist &= opt_col_stat->get_histogram().is_valid(); null_eval.add(opt_col_stat->get_num_null()); if (opt_col_stat->get_num_distinct() != 0) { min_eval.add(opt_col_stat->get_min_value()); @@ -669,7 +670,7 @@ int ObIncrementalStatEstimator::derive_global_col_stat(ObExecContext &ctx, col_stat->set_num_distinct(ndv_eval.get()); col_stat->set_avg_len(avglen_eval.get()); ndv_eval.get_llc_bitmap(col_stat->get_llc_bitmap(), col_stat->get_llc_bitmap_size()); - col_stat->set_llc_bitmap_size(ObColumnStat::NUM_LLC_BUCKET); + col_stat->set_llc_bitmap_size(ObOptColumnStat::NUM_LLC_BUCKET); col_stat->set_collation_type(param.column_params_.at(i).cs_type_); ObObj new_min_obj, new_max_obj; //maybe the stat is from KVCACHE, need deep copy min/max obj. @@ -679,7 +680,7 @@ int ObIncrementalStatEstimator::derive_global_col_stat(ObExecContext &ctx, } else if (OB_FAIL(ob_write_obj(alloc, max_eval.get(), new_max_obj)) || FALSE_IT(col_stat->set_max_value(new_max_obj))) { LOG_WARN("failed to set max value", K(ret), K(max_eval.get()), K(new_max_obj)); - } else if (need_drive_hist && !all_part_histograms.empty() && + } else if (need_derive_hist && !all_part_histograms.empty() && OB_FAIL(derive_global_histogram(all_part_histograms, alloc, max_bucket_num, diff --git a/src/share/stat/ob_incremental_stat_estimator.h b/src/share/stat/ob_incremental_stat_estimator.h index c43a6b6e3b..241a567fd4 100644 --- a/src/share/stat/ob_incremental_stat_estimator.h +++ b/src/share/stat/ob_incremental_stat_estimator.h @@ -26,28 +26,28 @@ class ObIncrementalStatEstimator { public: - static int try_drive_global_stat(ObExecContext &ctx, - const ObTableStatParam ¶m, - ObExtraParam &extra, - ObIArray &approx_part_opt_stats, - ObIArray &opt_stats); + static int try_derive_global_stat(ObExecContext &ctx, + const ObTableStatParam ¶m, + ObExtraParam &extra, + ObIArray &approx_part_opt_stats, + ObIArray &opt_stats); static bool is_part_can_incremental_gather(const ObTableStatParam ¶m, int64_t part_id, int64_t subpart_cnt, bool is_gather_part); - static int drive_global_stat_by_direct_load(ObExecContext &ctx, + static int derive_global_stat_by_direct_load(ObExecContext &ctx, ObIArray &part_tab_stats, - ObIArray &part_column_stats); + const ObIArray &part_column_stats); private: - static int drive_global_stat_from_part_stats(ObExecContext &ctx, + static int derive_global_stat_from_part_stats(ObExecContext &ctx, const ObTableStatParam ¶m, const ObIArray &approx_part_opt_stats, ObIArray &opt_stats); - static int drive_part_stats_from_subpart_stats(ObExecContext &ctx, + static int derive_part_stats_from_subpart_stats(ObExecContext &ctx, const ObTableStatParam ¶m, const ObIArray &gather_opt_stats, ObIArray &approx_part_opt_stats); @@ -71,12 +71,12 @@ private: ObIArray &column_ids); static int generate_all_opt_stat(ObIArray &table_stats, - ObIArray &col_handles, + const ObIArray &col_handles, int64_t col_cnt, ObIArray &all_opt_stats); static int generate_all_opt_stat(ObIArray &table_stats, - ObIArray &column_stats, + const ObIArray &column_stats, int64_t col_cnt, ObIArray &all_opt_stats); @@ -84,7 +84,7 @@ private: ObIAllocator &alloc, const ObTableStatParam ¶m, ObIArray &part_opt_stats, - bool need_drive_hist, + bool need_derive_hist, const StatLevel &approx_level, const int64_t partition_id, bool &need_gather_hybrid_hist, @@ -102,7 +102,7 @@ private: ObIAllocator &alloc, const ObTableStatParam ¶m, ObIArray &part_opt_stats, - bool need_drive_hist, + bool need_derive_hist, const StatLevel &approx_level, const int64_t partition_id, bool &need_gather_hybrid_hist, diff --git a/src/share/stat/ob_index_stats_estimator.cpp b/src/share/stat/ob_index_stats_estimator.cpp index acec7534e1..0dafb774cf 100644 --- a/src/share/stat/ob_index_stats_estimator.cpp +++ b/src/share/stat/ob_index_stats_estimator.cpp @@ -31,7 +31,7 @@ int ObIndexStatsEstimator::estimate(const ObTableStatParam ¶m, { int ret = OB_SUCCESS; const ObIArray &column_params = param.column_params_; - ObString no_rewrite("NO_REWRITE"); + ObString no_rewrite("NO_REWRITE USE_PLAN_CACHE(NONE) DBMS_STATS"); ObString calc_part_id_str; ObOptTableStat tab_stat; ObOptStat src_opt_stat; diff --git a/src/share/stat/ob_opt_column_stat.cpp b/src/share/stat/ob_opt_column_stat.cpp index d56dfac6cd..40de5c5422 100644 --- a/src/share/stat/ob_opt_column_stat.cpp +++ b/src/share/stat/ob_opt_column_stat.cpp @@ -14,7 +14,6 @@ #include "share/stat/ob_opt_column_stat.h" #include "share/stat/ob_stat_define.h" -#include "share/stat/ob_column_stat.h" #include "share/stat/ob_stat_item.h" #include "sql/engine/aggregate/ob_aggregate_processor.h" #include "sql/engine/expr/ob_expr_sys_op_opnsize.h" @@ -37,6 +36,18 @@ int ObHistBucket::deep_copy(const ObHistBucket &src, return ret; } +int ObHistBucket::deep_copy(ObIAllocator &alloc, const ObHistBucket &src) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ob_write_obj(alloc, src.endpoint_value_, endpoint_value_))) { + LOG_WARN("failed to write obj"); + } else { + endpoint_repeat_count_ = src.endpoint_repeat_count_; + endpoint_num_ = src.endpoint_num_; + } + return ret; +} + void ObHistogram::reset() { type_ = ObHistType::INVALID_TYPE; @@ -94,6 +105,30 @@ int ObHistogram::deep_copy(const ObHistogram &src, char *buf, const int64_t buf_ return ret; } +int ObHistogram::deep_copy(ObIAllocator &allocator, const ObHistogram &src) +{ + int ret = OB_SUCCESS; + type_ = src.type_; + sample_size_ = src.sample_size_; + density_ = src.density_; + bucket_cnt_ = src.bucket_cnt_; + void *ptr = NULL; + if (src.buckets_.empty()) { + } else if (OB_ISNULL(ptr = allocator.alloc(sizeof(ObHistBucket) * src.buckets_.count()))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(src.buckets_.count())); + } else { + ObHistBucket *new_buckets = new (ptr) ObHistBucket[src.buckets_.count()]; + buckets_ = ObArrayWrap(new_buckets, src.buckets_.count()); + for (int64_t i = 0; OB_SUCC(ret) && i < buckets_.count(); ++i) { + if (OB_FAIL(buckets_.at(i).deep_copy(allocator, src.buckets_.at(i)))) { + LOG_WARN("deep copy bucket failed"); + } + } + } + return ret; +} + int ObHistogram::prepare_allocate_buckets(ObIAllocator &allocator, const int64_t bucket_size) { int ret = OB_SUCCESS; @@ -158,8 +193,8 @@ ObOptColumnStat::ObOptColumnStat() histogram_(), last_analyzed_(0), cs_type_(CS_TYPE_INVALID), - inner_max_allocator_("OptColStatMax"), - inner_min_allocator_("OptColStatMin") + inner_allocator_("ObOptColumnStat"), + allocator_(inner_allocator_) { min_value_.set_null(); max_value_.set_null(); @@ -182,15 +217,15 @@ ObOptColumnStat::ObOptColumnStat(ObIAllocator &allocator) histogram_(), last_analyzed_(0), cs_type_(CS_TYPE_INVALID), - inner_max_allocator_("OptColStatMax"), - inner_min_allocator_("OptColStatMin") + inner_allocator_("ObOptColumnStat"), + allocator_(allocator) { min_value_.set_null(); max_value_.set_null(); - if (NULL == (llc_bitmap_ = static_cast(allocator.alloc(ObColumnStat::NUM_LLC_BUCKET)))) { + if (NULL == (llc_bitmap_ = static_cast(allocator.alloc(ObOptColumnStat::NUM_LLC_BUCKET)))) { COMMON_LOG_RET(WARN, OB_ALLOCATE_MEMORY_FAILED, "allocate memory for llc_bitmap_ failed."); } else { - llc_bitmap_size_ = ObColumnStat::NUM_LLC_BUCKET; + llc_bitmap_size_ = ObOptColumnStat::NUM_LLC_BUCKET; MEMSET(llc_bitmap_, 0, llc_bitmap_size_); } } @@ -210,8 +245,6 @@ void ObOptColumnStat::reset() llc_bitmap_ = NULL; last_analyzed_ = 0; cs_type_ = CS_TYPE_INVALID; - inner_max_allocator_.reset(); - inner_min_allocator_.reset(); histogram_.reset(); } @@ -245,20 +278,35 @@ int ObOptColumnStat::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue return ret; } -int ObOptColumnStat::deep_copy(char *buf, const int64_t buf_len, ObOptColumnStat *&value) const +int ObOptColumnStat::deep_copy(const ObOptColumnStat &src) { int ret = OB_SUCCESS; - if (nullptr == buf || buf_len < size()) { + if (OB_UNLIKELY(!src.is_valid())) { ret = OB_INVALID_ARGUMENT; - COMMON_LOG(WARN, "invalid arguments.", - KP(buf), K(buf_len), K(size()), K(ret)); + LOG_WARN("invalid arguments.", K(src)); } else { - ObOptColumnStat *stat = new (buf) ObOptColumnStat(); - int64_t pos = sizeof(*this); - if (OB_FAIL(stat->deep_copy(*this, buf, buf_len, pos))) { - COMMON_LOG(WARN, "deep copy column stat failed.", K(ret)); + table_id_ = src.table_id_; + partition_id_ = src.partition_id_; + column_id_ = src.column_id_; + object_type_ = src.object_type_; + version_ = src.version_; + num_null_ = src.num_null_; + num_not_null_ = src.num_not_null_; + num_distinct_ = src.num_distinct_; + avg_length_ = src.avg_length_; + last_analyzed_ = src.last_analyzed_; + cs_type_ = src.cs_type_; + if (OB_FAIL(ob_write_obj(allocator_, src.min_value_, min_value_))) { + LOG_WARN("deep copy min_value_ failed.", K_(src.min_value), K(ret)); + } else if (OB_FAIL(ob_write_obj(allocator_, src.max_value_, max_value_))) { + LOG_WARN("deep copy max_value_ failed.", K_(src.max_value), K(ret)); + } else if (OB_FAIL(histogram_.deep_copy(allocator_, src.histogram_))) { + LOG_WARN("failed to deep copy histogram", K(ret)); } else { - value = stat; + void *ptr = allocator_.alloc(sizeof(char) * src.llc_bitmap_size_); + llc_bitmap_ = new (ptr) char[src.llc_bitmap_size_]; + llc_bitmap_size_ = src.llc_bitmap_size_; + MEMCPY(llc_bitmap_, src.llc_bitmap_, src.llc_bitmap_size_); } } return ret; @@ -268,7 +316,6 @@ int ObOptColumnStat::deep_copy(const ObOptColumnStat &src, char *buf, const int6 { int ret = OB_SUCCESS; - // assign base members no need to handle memory. table_id_ = src.table_id_; partition_id_ = src.partition_id_; column_id_ = src.column_id_; @@ -280,7 +327,6 @@ int ObOptColumnStat::deep_copy(const ObOptColumnStat &src, char *buf, const int6 avg_length_ = src.avg_length_; last_analyzed_ = src.last_analyzed_; cs_type_ = src.cs_type_; - if (!src.is_valid() || nullptr == buf || size <= 0) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments.", K(src), KP(buf), K(size), K(ret)); @@ -325,6 +371,38 @@ void ObHistogram::calc_density(ObHistType hist_type, } } +int ObOptColumnStat::merge_column_stat(const ObOptColumnStat &other) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(get_table_id() != other.get_table_id() || + get_partition_id() != other.get_partition_id() || + get_column_id() != other.get_column_id())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("the key not match", K(get_table_id()), K(get_partition_id()), K(get_column_id())); + } else { + const ObObj &other_min = other.get_min_value(); + const ObObj &other_max = other.get_max_value(); + if (min_value_.is_null() || (!other_min.is_null() && other_min < min_value_)) { + ret = ob_write_obj(allocator_, other_min, min_value_); + } + if (OB_SUCC(ret) && (max_value_.is_null() || (!other_max.is_null() && other_max > max_value_))) { + ret = ob_write_obj(allocator_, other_max, max_value_); + } + LOG_TRACE("succeed to merge min/max val", K(min_value_), K(max_value_), + K(other.get_min_value()), K(other.get_max_value())); + if (OB_SUCC(ret)) { + // merge avg len before update number null/not null + merge_avg_len(other.get_avg_len(), other.get_num_not_null() + other.get_num_null()); + add_num_null(other.get_num_null()); + add_num_not_null(other.get_num_not_null()); + if (get_llc_bitmap_size() == other.get_llc_bitmap_size()) { + ObGlobalNdvEval::update_llc(get_llc_bitmap(), other.get_llc_bitmap()); + } + } + } + return ret; +} + OB_DEF_SERIALIZE(ObOptColumnStat) { int ret = OB_SUCCESS; LST_DO_CODE(OB_UNIS_ENCODE, @@ -344,7 +422,6 @@ OB_DEF_SERIALIZE(ObOptColumnStat) { pos += llc_bitmap_size_; } return ret; - } OB_DEF_SERIALIZE_SIZE(ObOptColumnStat) { @@ -387,175 +464,5 @@ OB_DEF_DESERIALIZE(ObOptColumnStat) { return ret; } -// shallow copy -int ObOptColumnStat::merge_column_stat(const ObOptColumnStat &other) -{ - int ret = OB_SUCCESS; - if (table_id_ != other.get_table_id() || - partition_id_ != other.get_partition_id() || - column_id_ != other.get_column_id()) { - // do not merge stats that not match. - ret = OB_ERR_UNEXPECTED; - LOG_WARN("the key not match", K(ret)); - } else { - double avg_len = 0; - other.get_avg_len(avg_len); - merge_avg_len(avg_len, other.get_num_not_null() + other.get_num_null()); - num_null_ += other.get_num_null(); - num_not_null_ += other.get_num_not_null(); - const ObObj &min_val = other.get_min_value(); - const ObObj &max_val = other.get_max_value(); - if (!min_val.is_null() && (min_value_.is_null() || min_val < min_value_)) { - inner_min_allocator_.reuse(); - if (OB_FAIL(ob_write_obj(inner_min_allocator_, min_val, min_value_))) { - LOG_WARN("fail to deep copy obj", K(ret)); - } - } - if (OB_SUCC(ret) && (!max_val.is_null() && (max_value_.is_null() || max_val > max_value_))) { - inner_max_allocator_.reuse(); - if (OB_FAIL(ob_write_obj(inner_max_allocator_, max_val, max_value_))) { - LOG_WARN("fail to deep copy obj", K(ret)); - } - } - // llc - if (llc_bitmap_size_ == other.get_llc_bitmap_size()) { - ObGlobalNdvEval::update_llc(llc_bitmap_, other.get_llc_bitmap()); - } - // do not process histogram - } - return ret; -} - -// deep copy max/min, using inner_allocator -int ObOptColumnStat::merge_obj(const ObObj &obj) -{ - int ret = OB_SUCCESS; - //calc avg_len: avg_len should update before num_null, null_not_null -- since these are use in merge_avg_len. - // the following code is abandoned, since it is not accurate. - // set calc avg_row_len using ObExprSysOpOpnsize::calc_sys_op_opnsize by datum. - // and set avg_row_len outer side this function. - //int64_t row_len = 0; - //row_len = sizeof(ObObj) + obj.get_deep_copy_size(); - //merge_avg_len(row_len, 1); - - if (obj.is_null()) { - num_null_++; - } else { - num_not_null_++; - if (!obj.get_meta().is_enum_or_set()) {//disable online gather enum/set max/min value. TODO,jiangxiu.wt - // max/min - if (min_value_.is_null() || obj < min_value_) { - inner_min_allocator_.reuse(); - if (OB_FAIL(ob_write_obj(inner_min_allocator_, obj, min_value_))) { - LOG_WARN("fail to deep copy obj", K(ret)); - } - } - if (OB_SUCC(ret) && (max_value_.is_null() || obj > max_value_)) { - inner_max_allocator_.reuse(); - if (OB_FAIL(ob_write_obj(inner_max_allocator_, obj, max_value_))) { - LOG_WARN("fail to deep copy obj", K(ret)); - } - } - } - if (OB_SUCC(ret)) { - // calc llc. - uint64_t hash_value = 0; - if (obj.is_string_type()) { - hash_value = obj.varchar_hash(obj.get_collation_type(), hash_value); - } else { - if (OB_FAIL(obj.hash(hash_value, hash_value))) { - LOG_WARN("fail to do hash", K(ret), K(obj)); - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(ObAggregateProcessor::llc_add_value(hash_value, llc_bitmap_, llc_bitmap_size_))) { - LOG_WARN("fail to calc llc", K(ret)); - } - //don't need to get call ObGlobalNdvEval::get_ndv_from_llc here, call it later. - } - } - - return ret; -} - -// deep copy max/min -/*int ObOptColumnStat::deep_copy_max_min_obj() -{ - int ret = OB_SUCCESS; - inner_min_allocator_.reuse(); - inner_max_allocator_.reuse(); - if (OB_FAIL(ob_write_obj(inner_max_allocator_, max_value_, max_value_))) { - LOG_WARN("fail to deep copy obj", K(ret)); - } else if (OB_FAIL(ob_write_obj(inner_min_allocator_, min_value_, min_value_))) { - LOG_WARN("fail to deep copy obj", K(ret)); - } - return ret; -}*/ -/* the max_alloc and min_alloc must be res-able, only for future usage. -int ObOptColumnStat::merge_obj(ObObj &obj, - ObIAllocator &max_alloc, - ObIAllocator &min_alloc) -{ - int ret = OB_SUCCESS; - int64_t row_len = 0; - - if (obj.is_null()) { - num_null_++; - } else { - num_not_null_++; - } - // max/min - if (OB_FAIL(merge_max_val(obj, max_alloc))) { - LOG_WARN("fail to merge max value", K(ret)); - } else if (OB_FAIL(merge_min_val(obj, min_alloc))) { - LOG_WARN("fail to merge min value", K(ret)); - } - // avg_len - row_len = sizeof(ObObj) + obj.get_deep_copy_size(); - merge_avg_len(row_len, 1); - - // calc llc. - if (OB_SUCC(ret)) { - uint64_t hash_value = 0; - hash_value = obj.is_string_type() ? - obj.varchar_hash(obj.get_collation_type(), hash_value) : - obj.hash(hash_value); - if (OB_FAIL(ObAggregateProcessor::llc_add_value(hash_value, llc_bitmap_, llc_bitmap_size_))) { - LOG_WARN("fail to calc llc", K(ret)); - } else { - num_distinct_ = ObGlobalNdvEval::get_ndv_from_llc(llc_bitmap_); - } - } - return ret; -}*/ - -// this two function couldn't guarantee the alloc is reuseable. So do not use it. -// only for future usage. -/*int ObOptColumnStat::merge_min_val(const common::ObObj &min_val, - common::ObIAllocator &alloc) { - int ret = OB_SUCCESS; - LOG_TRACE("MERGE min val", K(min_val), K(min_value_)); - if (min_value_.is_null() || min_val < min_value_) { - alloc.reuse(); - if (OB_FAIL(ob_write_obj(alloc, min_val, min_value_))) { - LOG_WARN("fail to deep copy obj", K(ret)); - } - } - return ret; -} - -int ObOptColumnStat::merge_max_val(const common::ObObj &max_val, - common::ObIAllocator &alloc) { - int ret = OB_SUCCESS; - LOG_TRACE("MERGE max val", K(max_val), K(max_value_)); - if (max_value_.is_null() || max_val > max_value_) { - alloc.reuse(); - if (OB_FAIL(ob_write_obj(alloc, max_val, max_value_))) { - LOG_WARN("fail to deep copy obj", K(ret)); - } - } - return ret; -}*/ - } } diff --git a/src/share/stat/ob_opt_column_stat.h b/src/share/stat/ob_opt_column_stat.h index 4e3ab9569e..101a00ea3d 100644 --- a/src/share/stat/ob_opt_column_stat.h +++ b/src/share/stat/ob_opt_column_stat.h @@ -54,7 +54,7 @@ public: char *buf, const int64_t buf_len, int64_t &pos); - + int deep_copy(ObIAllocator &alloc, const ObHistBucket &src); int64_t deep_copy_size() const { return endpoint_value_.get_deep_copy_size(); } TO_STRING_KV(K_(endpoint_value), @@ -94,6 +94,7 @@ public: void reset(); int deep_copy(const ObHistogram &src, char *buf, const int64_t buf_len, int64_t &pos); + int deep_copy(ObIAllocator &alloc, const ObHistogram &src); int assign(const ObHistogram &other); int64_t deep_copy_size() const; @@ -157,6 +158,9 @@ class ObOptColumnStat : public common::ObIKVCacheValue OB_UNIS_VERSION_V(1); public: static const int64_t MAX_OBJECT_SERIALIZE_SIZE = 512; + static const int64_t LARGE_NDV_NUMBER = 2LL << 61; // 2 << 64 is too large for int64_t, and 2 << 61 is enough for ndv + static const int64_t BUCKET_BITS = 10; // ln2(1024) = 10; + static const int64_t NUM_LLC_BUCKET = (1 << BUCKET_BITS); struct Key : public common::ObIKVCacheKey { @@ -228,12 +232,9 @@ public: }; ObOptColumnStat(); - int merge_column_stat(const ObOptColumnStat &other); - explicit ObOptColumnStat(common::ObIAllocator &allocator); ~ObOptColumnStat() { reset(); } - void reset(); uint64_t get_table_id() const { return table_id_; } @@ -246,9 +247,11 @@ public: void set_column_id(uint64_t cid) { column_id_ = cid; } const common::ObObj &get_max_value() const { return max_value_; } + common::ObObj &get_max_value() { return max_value_; } void set_max_value(const common::ObObj &max) { max_value_ = max; } const common::ObObj &get_min_value() const { return min_value_; } + common::ObObj &get_min_value() { return min_value_; } void set_min_value(const common::ObObj &min) { min_value_ = min; } int64_t get_num_distinct() const { return num_distinct_; } @@ -261,8 +264,7 @@ public: int64_t get_num_not_null() const { return num_not_null_; } void set_avg_len(int64_t avg_len) { avg_length_ = avg_len; } - int64_t get_avg_len() const { return (int64_t)avg_length_; } - void get_avg_len(double &avg_len) const { avg_len = avg_length_; } + int64_t get_avg_len() const { return avg_length_; } int64_t get_stat_level() const { return object_type_; } void set_stat_level(int64_t object_type) { object_type_ = object_type; } @@ -274,7 +276,7 @@ public: virtual int64_t size() const override; virtual int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const override; int deep_copy(const ObOptColumnStat &src, char *buf, const int64_t size, int64_t &pos); - virtual int deep_copy(char *buf, const int64_t buf_len, ObOptColumnStat *&value) const; + int deep_copy(const ObOptColumnStat &value); int64_t get_last_analyzed() const { return last_analyzed_; } void set_last_analyzed(int64_t last) { last_analyzed_ = last; } @@ -299,53 +301,18 @@ public: && num_null_ >= 0; } - void add_num_null(int64_t num_null) { num_null += num_null; } + void add_num_null(int64_t num_null) { num_null_ += num_null; } void add_num_not_null(int64_t num_not_null) { num_not_null_ += num_not_null; } - // deep copy - //int merge_min_val(const common::ObObj &min_val, common::ObIAllocator &alloc); - - //int merge_max_val(const common::ObObj &max_val, common::ObIAllocator &alloc); - - int merge_obj(common::ObObj &obj, - ObIAllocator &max_alloc, - ObIAllocator &min_alloc); - int merge_obj(const ObObj &obj); - // normal copy - //shallow copy - void merge_max_val(const common::ObObj &max_val) { - if (max_value_.is_null() || max_val > max_value_) { - max_value_ = max_val; - } - } - - //shallow copy - void merge_min_val(const common::ObObj &min_val) { - if (min_value_.is_null() || min_val > min_value_) { - min_value_ = min_val; - } - } - - void merge_avg_len(int64_t avg_len) { - merge_avg_len(avg_len, 1); - } - - void merge_avg_len(int64_t avg_len, int64_t num_rows) { + void merge_avg_len(int64_t avg_len, int64_t num_rows) + { SQL_LOG(DEBUG, "MERGE avg len", K(column_id_), K(partition_id_), K(avg_len), K(avg_length_), K(num_not_null_), K(num_null_), K(num_rows)); if (num_not_null_ + num_null_ + num_rows != 0) { avg_length_ = (avg_length_ * (num_not_null_ + num_null_) + avg_len * num_rows) / (num_not_null_ + num_null_+ num_rows); } } - - void merge_avg_len(double avg_len, int64_t num_rows) { - SQL_LOG(DEBUG, "MERGE avg len", K(column_id_), K(partition_id_), K(avg_len), K(avg_length_), K(num_not_null_), K(num_null_), K(num_rows)); - if (num_not_null_ + num_null_ + num_rows != 0) { - avg_length_ = (avg_length_ * (num_not_null_ + num_null_) + avg_len * num_rows) / (num_not_null_ + num_null_+ num_rows); - } - } - - int deep_copy_max_min_obj(); + int merge_column_stat(const ObOptColumnStat &other); common::ObCollationType get_collation_type() const { return cs_type_; } void set_collation_type(common::ObCollationType cs_type) { cs_type_ = cs_type; } @@ -376,7 +343,7 @@ protected: int64_t num_null_; int64_t num_not_null_; int64_t num_distinct_; - double avg_length_; + int64_t avg_length_; common::ObObj min_value_; common::ObObj max_value_; int64_t llc_bitmap_size_; @@ -386,9 +353,8 @@ protected: /** last analyzed time */ int64_t last_analyzed_; common::ObCollationType cs_type_; - - ObArenaAllocator inner_max_allocator_; - ObArenaAllocator inner_min_allocator_; + common::ObArenaAllocator inner_allocator_; + common::ObIAllocator &allocator_; }; } diff --git a/src/share/stat/ob_opt_osg_column_stat.cpp b/src/share/stat/ob_opt_osg_column_stat.cpp new file mode 100644 index 0000000000..eacd3a1be6 --- /dev/null +++ b/src/share/stat/ob_opt_osg_column_stat.cpp @@ -0,0 +1,304 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_OPT + +#include "share/stat/ob_opt_osg_column_stat.h" +#include "share/stat/ob_opt_column_stat.h" +#include "share/stat/ob_stat_define.h" +#include "share/stat/ob_stat_item.h" +#include "sql/engine/aggregate/ob_aggregate_processor.h" +#include "sql/optimizer/ob_optimizer_util.h" +#include "sql/engine/expr/ob_expr_sys_op_opnsize.h" +namespace oceanbase { +namespace common { +using namespace sql; + +int ObMinMaxValEval::get_obj(ObObj &obj) const +{ + int ret = OB_SUCCESS; + if (datum_ == NULL) { + obj.set_null(); + } else if (datum_->to_obj(obj, meta_)) { + LOG_WARN("failed to to obj"); + } + return ret; +} + +int ObMinMaxValEval::deep_copy(const ObMinMaxValEval &other, ObIAllocator &alloc) +{ + int ret = OB_SUCCESS; + datum_ = OB_NEWx(ObDatum, (&alloc)); + if (OB_ISNULL(datum_)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("Failed to allocate memory"); + } else if (OB_FAIL(datum_->deep_copy(*other.datum_, alloc))) { + LOG_WARN("failed to deep copy datum"); + } else { + meta_ = other.meta_; + cmp_func_ = other.cmp_func_; + } + return ret; +} + +OB_SERIALIZE_MEMBER(ObOptOSGColumnStat, *col_stat_); + +void ObOptOSGColumnStat::reset() +{ + if (col_stat_ != NULL) { + col_stat_->reset(); + col_stat_ = NULL; + } + min_val_.reset(); + max_val_.reset(); + inner_min_allocator_.reset(); + inner_max_allocator_.reset(); +} + +ObOptOSGColumnStat* ObOptOSGColumnStat::create_new_osg_col_stat(common::ObIAllocator &allocator) +{ + ObOptOSGColumnStat *new_osg_col_stat = OB_NEWx(ObOptOSGColumnStat, (&allocator), allocator); + ObOptColumnStat *new_col_stat = OB_NEWx(ObOptColumnStat, (&allocator), allocator); + if (OB_NOT_NULL(new_osg_col_stat) && OB_NOT_NULL(new_col_stat)) { + new_osg_col_stat->col_stat_ = new_col_stat; + } else { + if (new_osg_col_stat != NULL) { + new_osg_col_stat->~ObOptOSGColumnStat(); + allocator.free(new_osg_col_stat); + new_osg_col_stat = NULL; + } + if (new_col_stat != NULL) { + new_col_stat->~ObOptColumnStat(); + allocator.free(new_col_stat); + new_col_stat = NULL; + } + } + return new_osg_col_stat; +} + +int ObOptOSGColumnStat::deep_copy(const ObOptOSGColumnStat &other) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(col_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null"); + } else if (OB_FAIL(col_stat_->deep_copy(*other.col_stat_))) { + LOG_WARN("failed to deep copy col stat"); + } else if (other.min_val_.is_valid() && OB_FAIL(min_val_.deep_copy(other.min_val_, allocator_))) { + LOG_WARN("failed to deep copy min val"); + } else if (other.max_val_.is_valid() && OB_FAIL(max_val_.deep_copy(other.max_val_, allocator_))) { + LOG_WARN("failed to deep copy max val"); + } + return ret; +} + +int ObOptOSGColumnStat::set_min_max_datum_to_obj() +{ + int ret = OB_SUCCESS; + ObObj *min_obj = NULL; + ObObj *max_obj = NULL; + if (OB_ISNULL(col_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null"); + } else if (OB_ISNULL(min_obj = OB_NEWx(ObObj, (&allocator_))) || + OB_ISNULL(max_obj = OB_NEWx(ObObj, (&allocator_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory"); + } else if (OB_FAIL(min_val_.get_obj(*min_obj))) { + LOG_WARN("failed to get min obj"); + } else if (OB_FAIL(max_val_.get_obj(*max_obj))) { + LOG_WARN("failed to get max obj"); + } else if (OB_FAIL(ObOptimizerUtil::truncate_string_for_opt_stats(min_obj, allocator_, min_obj))) { + LOG_WARN("fail to truncate string", K(ret)); + } else if (OB_FAIL(ObOptimizerUtil::truncate_string_for_opt_stats(max_obj, allocator_, max_obj))) { + LOG_WARN("fail to truncate string", K(ret)); + } else { + const ObObj &min_val = col_stat_->get_min_value(); + const ObObj &max_val = col_stat_->get_max_value(); + LOG_TRACE("set min/max val", K(min_obj), K(max_obj), K(min_val), K(max_val)); + if (min_val.is_null() || (!min_obj->is_null() && *min_obj < min_val)) { + col_stat_->set_min_value(*min_obj); + } + if (max_val.is_null() || (!max_obj->is_null() && *max_obj > max_val)) { + col_stat_->set_max_value(*max_obj); + } + } + return ret; +} + +int ObOptOSGColumnStat::merge_column_stat(const ObOptOSGColumnStat &other) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(col_stat_) || OB_ISNULL(other.col_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(col_stat_)); + } else if (OB_UNLIKELY(col_stat_->get_table_id() != other.col_stat_->get_table_id() || + col_stat_->get_partition_id() != other.col_stat_->get_partition_id() || + col_stat_->get_column_id() != other.col_stat_->get_column_id())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("the key not match", K(col_stat_->get_table_id()), + K(col_stat_->get_partition_id()), + K(col_stat_->get_column_id())); + } else if (other.min_val_.is_valid() && + OB_FAIL(inner_merge_min(*other.min_val_.datum_, + other.min_val_.meta_, + other.min_val_.cmp_func_))) { + LOG_WARN("failed to inner merge min val"); + } else if (other.max_val_.is_valid() && + OB_FAIL(inner_merge_max(*other.max_val_.datum_, + other.max_val_.meta_, + other.max_val_.cmp_func_))) { + LOG_WARN("failed to inner merge min val"); + } else { + // merge avg len before update number null/not null + col_stat_->merge_avg_len(other.col_stat_->get_avg_len(), + other.col_stat_->get_num_not_null() + other.col_stat_->get_num_null()); + col_stat_->add_num_null(other.col_stat_->get_num_null()); + col_stat_->add_num_not_null(other.col_stat_->get_num_not_null()); + if (col_stat_->get_llc_bitmap_size() == other.col_stat_->get_llc_bitmap_size()) { + ObGlobalNdvEval::update_llc(col_stat_->get_llc_bitmap(), other.col_stat_->get_llc_bitmap()); + } + } + return ret; +} + +int ObOptOSGColumnStat::update_column_stat_info(const ObDatum *datum, + const ObObjMeta &meta, + const ObDatumCmpFuncType cmp_func) +{ + int ret = OB_SUCCESS; + int64_t col_len = 0; + if (OB_ISNULL(datum) || OB_ISNULL(col_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null"); + } else if (OB_FAIL(calc_col_len(*datum, meta, col_len))) { + LOG_WARN("failed to calc col len", K(datum), K(meta)); + } else if (OB_FALSE_IT(col_stat_->merge_avg_len(col_len, 1))) { + // avg len should be set at the very begining + } else if (datum->is_null()) { + col_stat_->add_num_null(1); + } else { + col_stat_->add_num_not_null(1); + uint64_t hash_value = 0; + ObObj tmp_obj; + if (OB_FAIL(datum->to_obj(tmp_obj, meta))) { + LOG_WARN("failed to to obj"); + } else if (tmp_obj.is_string_type()) { + hash_value = tmp_obj.varchar_hash(tmp_obj.get_collation_type(), hash_value); + } else if (OB_FAIL(tmp_obj.hash(hash_value, hash_value))) { + LOG_WARN("fail to do hash", K(ret), K(tmp_obj)); + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ObAggregateProcessor::llc_add_value(hash_value, + col_stat_->get_llc_bitmap(), + col_stat_->get_llc_bitmap_size()))) { + LOG_WARN("fail to calc llc", K(ret)); + } else if (OB_FAIL(inner_merge_min(*datum, meta, cmp_func))) { + LOG_WARN("failed to inner merge min val"); + } else if (OB_FAIL(inner_merge_max(*datum, meta, cmp_func))) { + LOG_WARN("failed to inner merge max val"); + } + } + return ret; +} + +int ObOptOSGColumnStat::calc_col_len(const ObDatum &datum, const ObObjMeta &meta, int64_t &col_len) +{ + int ret = OB_SUCCESS; + if (!is_lob_storage(meta.get_type()) || datum.is_null()) { + col_len = sizeof(datum) + (datum.is_null() ? 0 : datum.len_); + } else { + ObLobLocatorV2 locator(datum.get_string(), meta.has_lob_header()); + int64_t lob_data_byte_len = 0; + if (OB_FAIL(locator.get_lob_data_byte_len(lob_data_byte_len))) { + LOG_WARN("get lob data byte length failed", K(ret), K(locator)); + } else { + col_len = sizeof(datum) + static_cast(lob_data_byte_len); + } + } + return ret; +} + +int ObOptOSGColumnStat::inner_merge_min(const ObDatum &datum, const ObObjMeta &meta, const ObDatumCmpFuncType cmp_func) +{ + int ret = OB_SUCCESS; + if (meta.is_enum_or_set()) { + //disable online gather enum/set max/min value. TODO,jiangxiu.wt + } else if (min_val_.datum_ == NULL) { + min_val_.datum_ = OB_NEWx(ObDatum, (&allocator_)); + if (OB_ISNULL(min_val_.datum_)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory"); + } else if (OB_FAIL(min_val_.datum_->deep_copy(datum, inner_min_allocator_))) { + LOG_WARN("failed to deep copy datum"); + } else { + min_val_.meta_ = meta; + min_val_.cmp_func_ = cmp_func; + } + } else if (min_val_.datum_->is_null()) { + inner_min_allocator_.reuse(); + if (OB_FAIL(min_val_.datum_->deep_copy(datum, inner_min_allocator_))) { + LOG_WARN("failed to deep copy datum"); + } + } else { + int cmp_ret = 0; + if (OB_FAIL(min_val_.cmp_func_(*min_val_.datum_, datum, cmp_ret))) { + LOG_WARN("failed to perform compare"); + } else if (cmp_ret > 0) { + inner_min_allocator_.reuse(); + if (OB_FAIL(min_val_.datum_->deep_copy(datum, inner_min_allocator_))) { + LOG_WARN("failed to deep copy datum"); + } + } + } + LOG_TRACE("succeed to merge min datum", K(*min_val_.datum_), K(datum), K(meta)); + return ret; +} + +int ObOptOSGColumnStat::inner_merge_max(const ObDatum &datum, const ObObjMeta &meta, const ObDatumCmpFuncType cmp_func) +{ + int ret = OB_SUCCESS; + if (meta.is_enum_or_set()) { + //disable online gather enum/set max/min value. TODO,jiangxiu.wt + } else if (max_val_.datum_ == NULL) { + max_val_.datum_ = OB_NEWx(ObDatum, (&allocator_)); + if (OB_ISNULL(max_val_.datum_)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory"); + } else if (OB_FAIL(max_val_.datum_->deep_copy(datum, inner_max_allocator_))) { + LOG_WARN("failed to deep copy datum"); + } else { + max_val_.meta_ = meta; + max_val_.cmp_func_ = cmp_func; + } + } else if (max_val_.datum_->is_null()) { + inner_max_allocator_.reuse(); + if (OB_FAIL(max_val_.datum_->deep_copy(datum, inner_max_allocator_))) { + LOG_WARN("failed to deep copy datum"); + } + } else { + int cmp_ret = 0; + if (OB_FAIL(max_val_.cmp_func_(*max_val_.datum_, datum, cmp_ret))) { + LOG_WARN("failed to perform compare"); + } else if (cmp_ret < 0) { + inner_max_allocator_.reuse(); + if (OB_FAIL(max_val_.datum_->deep_copy(datum, inner_max_allocator_))) { + LOG_WARN("failed to deep copy datum"); + } + } + } + LOG_TRACE("succeed to merge max datum", K(*max_val_.datum_), K(datum), K(meta)); + return ret; +} + +} +} \ No newline at end of file diff --git a/src/share/stat/ob_opt_osg_column_stat.h b/src/share/stat/ob_opt_osg_column_stat.h new file mode 100644 index 0000000000..5438379757 --- /dev/null +++ b/src/share/stat/ob_opt_osg_column_stat.h @@ -0,0 +1,111 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef _OB_OPT_OSG_COLUMN_STAT_H_ +#define _OB_OPT_OSG_COLUMN_STAT_H_ + +#include +#include "lib/allocator/ob_malloc.h" +#include "share/stat/ob_opt_column_stat.h" +#include "share/datum/ob_datum.h" +#include "share/datum/ob_datum_funcs.h" + + +namespace oceanbase { +namespace common { + +enum ValEvalType { + T_INVALID_VAL = -1, + T_MIN_VAL, + T_MAX_VAL +}; + +struct ObMinMaxValEval { + ObMinMaxValEval(ValEvalType eval_type): + val_type_(eval_type), + datum_(NULL), + meta_(), + cmp_func_(NULL) + { } + void reset() + { + val_type_ = T_INVALID_VAL; + if (datum_ != NULL) { + datum_->reset(); + } + meta_.reset(); + } + inline bool is_valid() const { return datum_ != NULL && cmp_func_ != NULL; } + int get_obj(ObObj &obj) const; + int deep_copy(const ObMinMaxValEval &other, ObIAllocator &alloc); + + ValEvalType val_type_; + ObDatum *datum_; + ObObjMeta meta_; + ObDatumCmpFuncType cmp_func_; + TO_STRING_KV(K_(val_type), + K_(datum), + K_(meta)); +}; + +class ObOptOSGColumnStat +{ + OB_UNIS_VERSION_V(1); +public: + ObOptOSGColumnStat(): + col_stat_(NULL), + min_val_(T_MIN_VAL), + max_val_(T_MAX_VAL), + inner_min_allocator_("OptOSGMin"), + inner_max_allocator_("OptOSGMax"), + inner_allocator_("OptOSG"), + allocator_(inner_allocator_) + { } + ObOptOSGColumnStat(ObIAllocator &alloc): + col_stat_(NULL), + min_val_(T_MIN_VAL), + max_val_(T_MAX_VAL), + inner_min_allocator_("OptOSGMin"), + inner_max_allocator_("OptOSGMax"), + inner_allocator_("OptOSG"), + allocator_(alloc) + { } + virtual ~ObOptOSGColumnStat() { reset(); } + void reset(); + int deep_copy(const ObOptOSGColumnStat &other); + static ObOptOSGColumnStat* create_new_osg_col_stat(common::ObIAllocator &allocator); + int get_min_obj(ObObj &obj); + int get_max_obj(ObObj &obj); + int set_min_max_datum_to_obj(); + int merge_column_stat(const ObOptOSGColumnStat &other); + int update_column_stat_info(const ObDatum *datum, const ObObjMeta &meta, const ObDatumCmpFuncType cmp_func); + + ObOptColumnStat *col_stat_; + // members below is no need to serialize + ObMinMaxValEval min_val_; + ObMinMaxValEval max_val_; + ObArenaAllocator inner_min_allocator_; + ObArenaAllocator inner_max_allocator_; + ObArenaAllocator inner_allocator_; + ObIAllocator &allocator_; + TO_STRING_KV(K_(col_stat)); +private: + int inner_merge_min(const ObDatum &datum, const ObObjMeta &meta, const ObDatumCmpFuncType cmp_func); + int inner_merge_max(const ObDatum &datum, const ObObjMeta &meta, const ObDatumCmpFuncType cmp_func); + int calc_col_len(const ObDatum &datum, const ObObjMeta &meta, int64_t &col_len); + DISALLOW_COPY_AND_ASSIGN(ObOptOSGColumnStat); +}; + +} +} + +#endif /* _OB_OPT_OSG_COLUMN_STAT_H_ */ diff --git a/src/share/stat/ob_opt_stat_gather_stat.cpp b/src/share/stat/ob_opt_stat_gather_stat.cpp new file mode 100644 index 0000000000..c7ff3052b0 --- /dev/null +++ b/src/share/stat/ob_opt_stat_gather_stat.cpp @@ -0,0 +1,328 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_OPT +#include "share/stat/ob_opt_stat_gather_stat.h" +#include "lib/string/ob_sql_string.h" + +using namespace oceanbase::common; +using namespace oceanbase::sql; + +int ObOptStatTaskInfo::init(common::ObIAllocator &allocator, + uint64_t tenant_id, + uint64_t session_id, + const common::ObCurTraceId::TraceId &trace_id, + ObString &task_id, + ObOptStatGatherType type, + uint64_t task_start_time, + int64_t task_table_cnt) +{ + int ret = OB_SUCCESS; + char *trace_id_buf = NULL; + const int32_t max_trace_id_len = 64; + if (OB_ISNULL(trace_id_buf = static_cast(allocator.alloc(max_trace_id_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed", K(ret), K(trace_id_buf)); + } else { + tenant_id_ = tenant_id; + session_id_ = session_id; + int64_t len = trace_id.to_string(trace_id_buf, max_trace_id_len); + trace_id_.assign_ptr(trace_id_buf, static_cast(len)); + task_id_ = task_id; + type_ = type; + task_start_time_ = task_start_time; + task_table_count_ = task_table_cnt; + } + return ret; +} + +int ObOptStatTaskInfo::deep_copy(ObOptStatTaskInfo &other, char *buf, int64_t buf_len, int64_t &pos) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(buf) || OB_UNLIKELY(pos + other.size() > buf_len)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret)); + } else { + session_id_ = other.session_id_; + MEMCPY(buf + pos, other.trace_id_.ptr(), other.trace_id_.length()); + trace_id_.assign_ptr(buf + pos, other.trace_id_.length()); + pos += other.trace_id_.length(); + tenant_id_ = other.tenant_id_; + MEMCPY(buf + pos, other.task_id_.ptr(), other.task_id_.length()); + task_id_.assign_ptr(buf + pos, other.task_id_.length()); + pos += other.task_id_.length(); + type_ = other.type_; + task_table_count_ = other.task_table_count_; + task_start_time_ = other.task_start_time_; + task_end_time_ = other.task_end_time_; + ret_code_ = other.ret_code_; + failed_count_ = other.failed_count_; + completed_table_count_ = other.completed_table_count_; + } + return ret; +} + +ObOptStatGatherStat::ObOptStatGatherStat() : + task_info_(), + database_name_(), + table_id_(0), + table_name_(), + ret_code_(0), + start_time_(0), + end_time_(0), + memory_used_(0), + stat_refresh_failed_list_(), + properties_() +{ +} + +ObOptStatGatherStat::ObOptStatGatherStat(ObOptStatTaskInfo &task_info) : + task_info_(task_info), + database_name_(), + table_id_(0), + table_name_(), + ret_code_(0), + start_time_(0), + end_time_(0), + memory_used_(0), + stat_refresh_failed_list_(), + properties_() +{ +} + +ObOptStatGatherStat::~ObOptStatGatherStat() +{ +} + +int ObOptStatGatherStat::assign(const ObOptStatGatherStat &other) +{ + int ret = OB_SUCCESS; + task_info_ = other.task_info_; + database_name_ = other.database_name_; + table_id_ = other.table_id_; + table_name_ = other.table_name_; + ret_code_ = other.ret_code_; + start_time_ = other.start_time_; + end_time_ = other.end_time_; + memory_used_ = other.memory_used_; + stat_refresh_failed_list_ = other.stat_refresh_failed_list_; + properties_ = other.properties_; + return ret; +} + +int64_t ObOptStatGatherStat::size() const +{ + int64_t base_size = sizeof(ObOptStatGatherStat); + base_size += task_info_.size(); + base_size += database_name_.length(); + base_size += table_name_.length(); + base_size += stat_refresh_failed_list_.length(); + base_size += properties_.length(); + return base_size; +} + +int ObOptStatGatherStat::deep_copy(common::ObIAllocator &allocator, ObOptStatGatherStat *&new_stat) +{ + int ret = OB_SUCCESS; + char *buf = NULL; + int64_t buf_len = size(); + if (OB_ISNULL(buf = static_cast(allocator.alloc(buf_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed", K(ret), K(buf_len)); + } else { + new_stat = new (buf) ObOptStatGatherStat(); + int64_t pos = sizeof(*this); + //deep copy task info + if (OB_FAIL(new_stat->task_info_.deep_copy(task_info_, buf, buf_len, pos))) { + LOG_WARN("failed to deep copy", K(ret)); + } else { + //set database_name_ + MEMCPY(buf + pos, database_name_.ptr(), database_name_.length()); + new_stat->set_database_name(buf + pos, database_name_.length()); + pos += database_name_.length(); + //set table_id_ + new_stat->set_table_id(table_id_); + //set table_name_ + MEMCPY(buf + pos, table_name_.ptr(), table_name_.length()); + new_stat->set_table_name(buf + pos, table_name_.length()); + pos += table_name_.length(); + //set ret_code_ + new_stat->set_ret_code(ret_code_); + //set start_time_ + new_stat->set_start_time(start_time_); + //set end_time_ + new_stat->set_end_time(end_time_); + //set memory_used_ + new_stat->set_memory_used(memory_used_); + ////set stat_refresh_failed_list_ + MEMCPY(buf + pos, stat_refresh_failed_list_.ptr(), stat_refresh_failed_list_.length()); + new_stat->set_stat_refresh_failed_list(buf + pos, stat_refresh_failed_list_.length()); + pos += stat_refresh_failed_list_.length(); + ////set properties_ + MEMCPY(buf + pos, properties_.ptr(), properties_.length()); + new_stat->set_properties(buf + pos, properties_.length()); + pos += properties_.length(); + if (OB_UNLIKELY(pos != buf_len)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(pos), K(buf_len)); + } + } + } + return ret; +} +//----------------------------------------------------- +int ObOptStatRunningMonitor::add_table_info(common::ObTableStatParam &table_param, + double stale_percent) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ob_write_string(allocator_, + table_param.db_name_, + opt_stat_gather_stat_.get_database_name()))) { + LOG_WARN("failed to write string", K(ret)); + } else if (OB_FAIL(ob_write_string(allocator_, + table_param.tab_name_, + opt_stat_gather_stat_.get_table_name()))) { + LOG_WARN("failed to write string", K(ret)); + } else { + opt_stat_gather_stat_.set_table_id(table_param.table_id_); + ObSqlString tmp_properties_str; + char *buf = NULL; + if (OB_FAIL(tmp_properties_str.append_fmt("GRANULARITY:%.*s;METHOD_OPT:%.*s;DEGREE:%ld;ESTIMATE_PERCENT:%lf;BLOCK_SAMPLE:%d;STALE_PERCENT:%lf;", + table_param.granularity_.length(), + table_param.granularity_.ptr(), + table_param.method_opt_.length(), + table_param.method_opt_.ptr(), + table_param.degree_, + table_param.sample_info_.is_sample_ ? table_param.sample_info_.sample_value_ : 100.0, + table_param.sample_info_.is_block_sample_, + stale_percent))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_ISNULL(buf = static_cast(allocator_.alloc(tmp_properties_str.length())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("memory is not enough", K(ret), K(tmp_properties_str)); + } else { + MEMCPY(buf, tmp_properties_str.ptr(), tmp_properties_str.length()); + opt_stat_gather_stat_.set_properties(buf, static_cast(tmp_properties_str.length())); + } + } + return ret; +} + +// int ObOptStatRunningMonitor::add_monitor_info(ObOptStatRunningPhase current_phase, +// int64_t current_memory_used) +// { +// int ret = OB_SUCCESS; +// if (current_phase > ObOptStatRunningPhase::GATHER_BEGIN && +// current_phase <= ObOptStatRunningPhase::GATHER_END) { +// int64_t current_time = ObTimeUtility::current_time(); +// ObSqlString tmp_str; +// if (current_phase != ObOptStatRunningPhase::GATHER_END && +// OB_FAIL(tmp_str.append_fmt("%s:cost_time=%ldus,cost_memory=%ldbyte;", running_phase_name[current_phase], +// current_time - last_start_time_, +// current_memory_used - last_memory_used_))) { +// LOG_WARN("failed to append fmt", K(ret)); +// } else if (current_phase == ObOptStatRunningPhase::GATHER_END && +// OB_FAIL(tmp_str.append_fmt("%s;", running_phase_name[current_phase]))) { +// LOG_WARN("failed to append fmt", K(ret)); +// } else { +// char *buf = NULL; +// ObString &running_detailed = opt_stat_gather_stat_.get_running_detailed(); +// opt_stat_gather_stat_.add_memory_used(current_memory_used - last_memory_used_); +// opt_stat_gather_stat_.add_duration_time(current_time - last_start_time_); +// int32_t buf_len = static_cast(tmp_str.length() + running_detailed.length()); +// if (OB_ISNULL(buf = static_cast(allocator_.alloc(tmp_str.length() + running_detailed.length())))) { +// ret = OB_ALLOCATE_MEMORY_FAILED; +// LOG_WARN("memory is not enough", K(ret), K(buf)); +// } else { +// MEMCPY(buf, running_detailed.ptr(), running_detailed.length()); +// MEMCPY(buf + running_detailed.length() , tmp_str.ptr(), tmp_str.length()); +// running_detailed.assign_ptr(buf, buf_len); +// running_phase_ = current_phase; +// last_start_time_ = current_time; +// last_memory_used_ = current_memory_used; +// } +// } +// } +// return ret; +// } + +void ObOptStatRunningMonitor::set_monitor_result(int ret_code, + int64_t end_time, + int64_t current_memory_used) +{ + opt_stat_gather_stat_.set_ret_code(ret_code); + opt_stat_gather_stat_.set_end_time(end_time); + opt_stat_gather_stat_.set_memory_used(current_memory_used - last_memory_used_); +} + +//------------------------------------------------------ +ObOptStatGatherStatList::ObOptStatGatherStatList() : lock_(common::ObLatchIds::OPT_STAT_GATHER_STAT_LOCK) +{ +} + +ObOptStatGatherStatList::~ObOptStatGatherStatList() +{ +} + +ObOptStatGatherStatList& ObOptStatGatherStatList::instance() +{ + static ObOptStatGatherStatList the_opt_stat_gather_stat_List; + return the_opt_stat_gather_stat_List; +} + +int ObOptStatGatherStatList::push(ObOptStatGatherStat &stat_value) +{ + int ret = OB_SUCCESS; + ObSpinLockGuard guard(lock_); + if(!stat_list_.add_last(&stat_value)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to add stat", K(ret)); + } + return ret; +} + +int ObOptStatGatherStatList::remove(ObOptStatGatherStat &stat_value) +{ + int ret = OB_SUCCESS; + ObSpinLockGuard guard(lock_); + if(NULL == stat_list_.remove(&stat_value)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to move stat", K(ret)); + } + return ret; +} + +int ObOptStatGatherStatList::list_to_array(common::ObIAllocator &allocator, + const uint64_t target_tenant_id, + ObIArray &stat_array) +{ + int ret = OB_SUCCESS; + stat_array.reset(); + ObSpinLockGuard guard(lock_); + stat_array.reserve(stat_list_.get_size()); + DLIST_FOREACH(cur, stat_list_) { + // sys tenant list all tenant stat + // non-sys tennat list self tenant stat + ObOptStatGatherStat *tmp_stat = NULL; + if (!is_sys_tenant(target_tenant_id) && cur->get_tenant_id() != target_tenant_id) { + //do nothing + } else if (cur->deep_copy(allocator, tmp_stat)) { + LOG_WARN("failed to deep copy", K(ret)); + } else if (OB_ISNULL(tmp_stat)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(tmp_stat)); + } else if (OB_FAIL(stat_array.push_back(*tmp_stat))) { + LOG_WARN("failed to push back", K(ret)); + } + } + return ret; +} diff --git a/src/share/stat/ob_opt_stat_gather_stat.h b/src/share/stat/ob_opt_stat_gather_stat.h new file mode 100644 index 0000000000..afffafd44d --- /dev/null +++ b/src/share/stat/ob_opt_stat_gather_stat.h @@ -0,0 +1,234 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef __OB_OPT_STAT_GATHER_STAT_H__ +#define __OB_OPT_STAT_GATHER_STAT_H__ + +#include "lib/list/ob_dlist.h" +#include "lib/lock/ob_spin_lock.h" +#include "lib/container/ob_array.h" +#include "lib/profile/ob_trace_id.h" +#include "share/stat/ob_stat_define.h" + +namespace oceanbase +{ +namespace common +{ +enum ObOptStatGatherType { + INVALID_GATHER_TYPE = -1, + MANUAL_GATHER, + AUTO_GATHER +}; + +// enum ObOptStatRunningPhase { +// GATHER_BEGIN = 0, +// GATHER_SUBPART_STATS, +// GATHER_PART_STATS, +// DERIVE_PART_STATS, +// GATHER_GLOBAL_STATS, +// DERIVE_GLOBAL_STATS, +// WRITE_STATS, +// GATHER_INDEX_STATS, +// GATHER_END +// }; + +// static const char *running_phase_name[ObOptStatRunningPhase::GATHER_END + 1] = +// { +// "GATHER BEGIN", +// "GATHER SUBPART STATS", +// "GATHER PART STATS", +// "DERIVE PART STATS", +// "GATHER GLOBAL STATS", +// "DERIVE GLOBAL STATS", +// "WRITE STATS", +// "GATHER INDEX STATS", +// "GATHER END" +// }; + +struct ObOptStatTaskInfo +{ + ObOptStatTaskInfo() : + tenant_id_(0), + session_id_(0), + trace_id_(), + task_id_(), + type_(ObOptStatGatherType::INVALID_GATHER_TYPE), + task_table_count_(0), + task_start_time_(0), + task_end_time_(0), + ret_code_(0), + failed_count_(0), + completed_table_count_(0) + {} + int64_t size() const { return trace_id_.length() + task_id_.length(); } + int init(common::ObIAllocator &allocator, + uint64_t tenant_id, + uint64_t session_id, + const common::ObCurTraceId::TraceId &trace_id, + ObString &task_id, + ObOptStatGatherType type, + uint64_t task_start_time, + int64_t task_table_cnt); + int deep_copy(ObOptStatTaskInfo &other, char *buf, int64_t buf_len, int64_t &pos); + TO_STRING_KV(K(tenant_id_), + K(session_id_), + K(trace_id_), + K(task_id_), + K(type_), + K(task_table_count_), + K(task_start_time_), + K(task_end_time_), + K(ret_code_), + K(failed_count_), + K(completed_table_count_)); + uint64_t tenant_id_; + uint64_t session_id_; + ObString trace_id_; + ObString task_id_; + int64_t type_; + int64_t task_table_count_; + int64_t task_start_time_; + int64_t task_end_time_; + int ret_code_; + int64_t failed_count_; + int64_t completed_table_count_; +}; + +class ObOptStatGatherStat : public common::ObDLinkBase +{ +public: + ObOptStatGatherStat(); + ObOptStatGatherStat(ObOptStatTaskInfo &task_info); + ~ObOptStatGatherStat(); + bool is_valid() const { return start_time_ > 0; } + int assign(const ObOptStatGatherStat &other); + int64_t size() const; + int deep_copy(common::ObIAllocator &allocator, ObOptStatGatherStat *&new_stat); + inline uint64_t get_session_id() const { return task_info_.session_id_; } + void set_session_id(uint64_t session_id) { task_info_.session_id_ = session_id; } + inline const ObString &get_trace_id() const { return task_info_.trace_id_; } + inline void set_trace_id(const char *ptr, int32_t len) { task_info_.trace_id_.assign_ptr(ptr, len); } + inline uint64_t get_tenant_id() const { return task_info_.tenant_id_; } + void set_tenant_id(uint64_t tenant_id) { task_info_.tenant_id_ = tenant_id; } + inline const ObString &get_task_id() const { return task_info_.task_id_; } + inline int64_t get_type() const { return task_info_.type_; } + inline void set_type(int64_t type) { task_info_.type_ = type; } + inline int64_t get_task_start_time() const { return task_info_.task_start_time_; } + inline void set_task_start_time(int64_t task_start_time) { task_info_.task_start_time_ = task_start_time; } + inline int64_t get_task_duration_time() const { return ObTimeUtility::current_time() - task_info_.task_start_time_; } + inline int64_t get_task_end_time() const { return task_info_.task_end_time_; } + inline void set_task_end_time(int64_t task_end_time) { task_info_.task_end_time_ = task_end_time; } + inline int get_task_ret_code() const { return task_info_.ret_code_; } + inline void set_task_ret_code(int ret_code) { task_info_.ret_code_ = ret_code; } + inline int64_t get_task_table_count() const { return task_info_.task_table_count_; } + inline void set_task_table_count(int64_t task_table_count) { task_info_.task_table_count_ = task_table_count; } + inline const ObString &get_database_name() const { return database_name_; } + inline ObString &get_database_name() { return database_name_; } + inline void set_database_name(const char *ptr, int32_t len) { database_name_.assign_ptr(ptr, len); } + inline uint64_t get_table_id() const { return table_id_; } + void set_table_id(uint64_t table_id) { table_id_ = table_id; } + inline const ObString &get_table_name() const { return table_name_; } + inline ObString &get_table_name() { return table_name_; } + inline void set_table_name(const char *ptr, int32_t len) { table_name_.assign_ptr(ptr, len); } + inline int get_ret_code() const { return ret_code_; } + inline void set_ret_code(int ret_code) { ret_code_ = ret_code; } + inline int64_t get_start_time() const { return start_time_; } + inline void set_start_time(int64_t start_time) { start_time_ = start_time; } + inline int64_t get_end_time() const { return end_time_; } + inline void set_end_time(int64_t end_time) { end_time_ = end_time; } + inline int64_t get_memory_used() const { return memory_used_; } + inline void set_memory_used(int64_t size) { memory_used_ = size; } + inline const ObString &get_stat_refresh_failed_list() const { return stat_refresh_failed_list_; } + inline ObString &get_stat_refresh_failed_list() { return stat_refresh_failed_list_; } + inline void set_stat_refresh_failed_list(const char *ptr, int32_t len) { stat_refresh_failed_list_.assign_ptr(ptr, len); } + inline const ObString &get_properties() const { return properties_; } + inline void set_properties(const char *ptr, int32_t len) { properties_.assign_ptr(ptr, len); } + inline int64_t get_running_table_duration_time() const { return ObTimeUtility::current_time() - start_time_; } + inline int64_t get_completed_table_count() { return task_info_.completed_table_count_; } + TO_STRING_KV(K(task_info_), + K(database_name_), + K(table_id_), + K(table_name_), + K(ret_code_), + K(start_time_), + K(end_time_), + K(memory_used_), + K(stat_refresh_failed_list_), + K(properties_)); + +private: + ObOptStatTaskInfo task_info_; + ObString database_name_; + uint64_t table_id_; + ObString table_name_; + int ret_code_; + int64_t start_time_; + int64_t end_time_; + int64_t memory_used_; + ObString stat_refresh_failed_list_; + ObString properties_; +}; + +struct ObOptStatRunningMonitor +{ + explicit ObOptStatRunningMonitor(common::ObIAllocator &allocator, + int64_t current_time, + int64_t current_memory_used, + ObOptStatGatherStat &opt_stat_gather_stat) : + allocator_(allocator), + last_start_time_(current_time), + last_memory_used_(current_memory_used), + opt_stat_gather_stat_(opt_stat_gather_stat) + { + opt_stat_gather_stat_.set_start_time(current_time); + } + ~ObOptStatRunningMonitor(){} + void init(int64_t current_time, + int64_t current_memory_used, + ObOptStatGatherStat &opt_stat_gather_stat); + int add_table_info(common::ObTableStatParam &table_param, + double stale_percent = -1.0); + void set_monitor_result(int ret_code, + int64_t current_time, + int64_t current_memory_used); + TO_STRING_KV(K(last_start_time_), + K(last_memory_used_), + K(opt_stat_gather_stat_)); + common::ObIAllocator &allocator_; + int64_t last_start_time_; + int64_t last_memory_used_; + ObOptStatGatherStat &opt_stat_gather_stat_; +}; + +class ObOptStatGatherStatList +{ +public: + static ObOptStatGatherStatList &instance(); + int push(ObOptStatGatherStat &stat_value); + int remove(ObOptStatGatherStat &stat_value); + // param[in] tenant_id if tenant is sys, list all tenant stat, else list target tenant stat + int list_to_array(common::ObIAllocator &allocator, + const uint64_t target_tenant_id, + common::ObIArray &stat_array); +private: + common::ObDList stat_list_; + mutable common::ObSpinLock lock_; +private: + ObOptStatGatherStatList(); + ~ObOptStatGatherStatList(); + DISALLOW_COPY_AND_ASSIGN(ObOptStatGatherStatList); +}; + +} // end of namespace common +} // end of namespace oceanbase + +#endif /* __OB_OPT_STAT_GATHER_STAT_H__ */ diff --git a/src/share/stat/ob_opt_stat_manager.cpp b/src/share/stat/ob_opt_stat_manager.cpp index 9b5f80b203..b77812d710 100644 --- a/src/share/stat/ob_opt_stat_manager.cpp +++ b/src/share/stat/ob_opt_stat_manager.cpp @@ -805,5 +805,15 @@ int ObOptStatManager::erase_ds_stat(const ObOptDSStat::Key &key) return ret; } +int ObOptStatManager::update_opt_stat_gather_stat(const ObOptStatGatherStat &gather_stat) +{ + return stat_service_.get_sql_service().update_opt_stat_gather_stat(gather_stat); +} + +int ObOptStatManager::update_opt_stat_task_stat(const ObOptStatTaskInfo &task_info) +{ + return stat_service_.get_sql_service().update_opt_stat_task_stat(task_info); +} + } } diff --git a/src/share/stat/ob_opt_stat_manager.h b/src/share/stat/ob_opt_stat_manager.h index 7e7c66e45d..371563f414 100644 --- a/src/share/stat/ob_opt_stat_manager.h +++ b/src/share/stat/ob_opt_stat_manager.h @@ -200,10 +200,13 @@ public: int check_stat_tables_ready(share::schema::ObSchemaGetterGuard &schema_guard, const uint64_t tenant_id, bool &are_stat_tables_ready); + int get_ds_stat(const ObOptDSStat::Key &key, ObOptDSStatHandle &ds_stat_handle); int add_ds_stat_cache(const ObOptDSStat::Key &key, const ObOptDSStat &value, ObOptDSStatHandle &ds_stat_handle); + int update_opt_stat_gather_stat(const ObOptStatGatherStat &gather_stat); + int update_opt_stat_task_stat(const ObOptStatTaskInfo &task_info); protected: static const int64_t REFRESH_STAT_TASK_NUM = 5; bool inited_; diff --git a/src/share/stat/ob_opt_stat_service.h b/src/share/stat/ob_opt_stat_service.h index 21cdc51396..496e966ddc 100644 --- a/src/share/stat/ob_opt_stat_service.h +++ b/src/share/stat/ob_opt_stat_service.h @@ -59,6 +59,7 @@ public: const ObIArray &all_tablet_ids, const ObIArray &all_ls_ids, int64_t &table_rowcnt); + private: /** * 接口load_and_put_cache(key, handle)的实现，外部不应该直接调用这个函数 diff --git a/src/share/stat/ob_opt_stat_sql_service.cpp b/src/share/stat/ob_opt_stat_sql_service.cpp index 4b7e5a5768..e3587529b3 100644 --- a/src/share/stat/ob_opt_stat_sql_service.cpp +++ b/src/share/stat/ob_opt_stat_sql_service.cpp @@ -32,7 +32,6 @@ #include "observer/ob_server_struct.h" #include "share/stat/ob_opt_column_stat.h" #include "share/stat/ob_opt_table_stat.h" -#include "share/stat/ob_column_stat.h" #include "lib/charset/ob_charset.h" #include "share/stat/ob_opt_stat_monitor_manager.h" @@ -219,6 +218,25 @@ "user_stats = VALUES(user_stats);" // TODO DAISI, add a sys_func to merge NDV by llc. +#define INSERT_TASK_OPT_STAT_GATHER_SQL "INSERT INTO %s(tenant_id," \ + "task_id," \ + "type," \ + "ret_code," \ + "failed_count,"\ + "table_count," \ + "start_time," \ + "end_time) VALUES (%s);" + +#define INSERT_TABLE_OPT_STAT_GATHER_SQL "INSERT INTO %s(tenant_id," \ + "task_id," \ + "table_id," \ + "ret_code," \ + "start_time," \ + "end_time," \ + "memory_used," \ + "stat_refresh_failed_list," \ + "properties) VALUES (%s);" + #define DEFINE_SQL_CLIENT_RETRY_WEAK_FOR_STAT(sql_client, table_name) \ const int64_t snapshot_timestamp = OB_INVALID_TIMESTAMP; \ @@ -887,7 +905,7 @@ int ObOptStatSqlService::get_column_stat_sql(const uint64_t tenant_id, int64_t llc_hex_size = 0; if (OB_FAIL(get_valid_obj_str(stat.get_min_value(), min_meta, allocator, min_str, print_params)) || OB_FAIL(get_valid_obj_str(stat.get_max_value(), max_meta, allocator, max_str, print_params))) { - LOG_WARN("failed to get valid obj str", K(ret)); + LOG_WARN("failed to get valid obj str", K(stat.get_min_value()), K(stat.get_max_value())); } else if (OB_FAIL(get_obj_binary_hex_str(stat.get_min_value(), allocator, b_min_str)) || OB_FAIL(get_obj_binary_hex_str(stat.get_max_value(), allocator, b_max_str))) { LOG_WARN("failed to convert obj to str", K(ret)); @@ -1141,11 +1159,11 @@ int ObOptStatSqlService::get_obj_str(const ObObj &obj, ObObjPrintParams copy_print_params = print_params; copy_print_params.cs_type_ = obj.get_collation_type(); if (OB_FAIL(obj.print_varchar_literal(buf, buf_len, pos, copy_print_params))) { - LOG_WARN("failed to print sql literal", K(ret)); + LOG_WARN("failed to print sql literal", K(obj)); } else { /*do nothing*/ } } else if (obj.is_valid_type()) { if (OB_FAIL(obj.print_sql_literal(buf, buf_len, pos, print_params))) { - LOG_WARN("failed to print_sql_literal", K(ret)); + LOG_WARN("failed to print_sql_literal", K(obj)); } else { /*do nothing*/ } } else { ret = OB_ERR_UNEXPECTED; @@ -1400,7 +1418,7 @@ int ObOptStatSqlService::fill_column_stat(ObIAllocator &allocator, common::str_to_hex(hex_str.ptr(), hex_str.length(), bitmap_buf, hex_str.length()); // decompress llc bitmap; char *decomp_buf = NULL ; - int64_t decomp_size = ObColumnStat::NUM_LLC_BUCKET; + int64_t decomp_size = ObOptColumnStat::NUM_LLC_BUCKET; const int64_t bitmap_size = hex_str.length() / 2; if (OB_FAIL(get_decompressed_llc_bitmap(allocator, bitmap_buf, bitmap_size, decomp_buf, decomp_size))) { @@ -1576,10 +1594,10 @@ int ObOptStatSqlService::get_decompressed_llc_bitmap(ObIAllocator &allocator, int64_t &bitmap_size) { int ret = OB_SUCCESS; - const int64_t max_bitmap_size = ObColumnStat::NUM_LLC_BUCKET; // max size of uncompressed buffer. + const int64_t max_bitmap_size = ObOptColumnStat::NUM_LLC_BUCKET; // max size of uncompressed buffer. ObCompressor* compressor = NULL; - if (comp_size >= ObColumnStat::NUM_LLC_BUCKET) { + if (comp_size >= ObOptColumnStat::NUM_LLC_BUCKET) { // not compressed bitmap, use directly; bitmap_buf = const_cast(comp_buf); bitmap_size = comp_size; @@ -2131,6 +2149,121 @@ int ObOptStatSqlService::gen_tablet_list_str(const ObIArray &all_tab return ret; } +int ObOptStatSqlService::update_opt_stat_task_stat(const ObOptStatTaskInfo &task_info) +{ + int ret = OB_SUCCESS; + ObSqlString raw_sql; + ObSqlString value_str; + int64_t affected_rows = 0; + const uint64_t tenant_id = gen_meta_tenant_id(task_info.tenant_id_); + if (OB_FAIL(get_gather_stat_task_value(task_info, value_str))) { + LOG_WARN("failed to get gather stat values list", K(ret)); + } else if (OB_FAIL(raw_sql.append_fmt(INSERT_TASK_OPT_STAT_GATHER_SQL, + share::OB_ALL_TASK_OPT_STAT_GATHER_HISTORY_TNAME, + value_str.ptr()))) { + LOG_WARN("failed to append fmt", K(ret), K(raw_sql)); + } else { + ObMySQLTransaction trans; + LOG_TRACE("sql string of update opt stat task stat", K(raw_sql)); + if (OB_FAIL(trans.start(mysql_proxy_, tenant_id))) { + LOG_WARN("fail to start transaction", K(ret), K(tenant_id)); + } else if (OB_FAIL(trans.write(tenant_id, raw_sql.ptr(), affected_rows))) { + LOG_WARN("failed to exec sql", K(ret)); + } else {/*do nothing*/} + if (OB_SUCC(ret)) { + if (OB_FAIL(trans.end(true))) { + LOG_WARN("fail to commit transaction", K(ret)); + } + } else { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(false))) { + LOG_WARN("fail to roll back transaction", K(tmp_ret)); + } + } + } + return ret; +} + +int ObOptStatSqlService::update_opt_stat_gather_stat(const ObOptStatGatherStat &gather_stat) +{ + int ret = OB_SUCCESS; + ObSqlString raw_sql; + ObSqlString value_str; + int64_t affected_rows = 0; + const uint64_t tenant_id = gen_meta_tenant_id(gather_stat.get_tenant_id()); + if (OB_FAIL(get_gather_stat_value(gather_stat, value_str))) { + LOG_WARN("failed to get gather stat value", K(ret)); + } else if (OB_FAIL(raw_sql.append_fmt(INSERT_TABLE_OPT_STAT_GATHER_SQL, + share::OB_ALL_TABLE_OPT_STAT_GATHER_HISTORY_TNAME, + value_str.ptr()))) { + LOG_WARN("failed to append fmt", K(ret), K(raw_sql)); + } else { + ObMySQLTransaction trans; + LOG_TRACE("sql string of update opt stat gather stat", K(raw_sql)); + if (OB_FAIL(trans.start(mysql_proxy_, tenant_id))) { + LOG_WARN("fail to start transaction", K(ret), K(tenant_id)); + } else if (OB_FAIL(trans.write(tenant_id, raw_sql.ptr(), affected_rows))) { + LOG_WARN("failed to exec sql", K(ret)); + } else {/*do nothing*/} + if (OB_SUCC(ret)) { + if (OB_FAIL(trans.end(true))) { + LOG_WARN("fail to commit transaction", K(ret)); + } + } else { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(false))) { + LOG_WARN("fail to roll back transaction", K(tmp_ret)); + } + } + } + return ret; +} + +int ObOptStatSqlService::get_gather_stat_task_value(const ObOptStatTaskInfo &task_info, + ObSqlString &value_str) +{ + int ret = OB_SUCCESS; + share::ObDMLSqlSplicer dml_splicer; + uint64_t tenant_id = task_info.tenant_id_; + if (OB_FAIL(dml_splicer.add_pk_column("tenant_id", tenant_id)) || + OB_FAIL(dml_splicer.add_pk_column("task_id", task_info.task_id_)) || + OB_FAIL(dml_splicer.add_column("type", task_info.type_)) || + OB_FAIL(dml_splicer.add_column("ret_code", task_info.ret_code_)) || + OB_FAIL(dml_splicer.add_column("failed_count", task_info.failed_count_)) || + OB_FAIL(dml_splicer.add_column("table_count", task_info.task_table_count_)) || + OB_FAIL(dml_splicer.add_time_column("start_time", task_info.task_start_time_)) || + OB_FAIL(dml_splicer.add_time_column("end_time", task_info.task_end_time_))) { + LOG_WARN("failed to add dml splicer column", K(ret)); + } else if (OB_FAIL(dml_splicer.splice_values(value_str))) { + LOG_WARN("failed to get sql string", K(ret)); + } else { /*do nothing*/ } + return ret; +} + +int ObOptStatSqlService::get_gather_stat_value(const ObOptStatGatherStat &gather_stat, + ObSqlString &values_ptr) +{ + int ret = OB_SUCCESS; + share::ObDMLSqlSplicer dml_splicer; + uint64_t tenant_id = gather_stat.get_tenant_id(); + uint64_t table_id = gather_stat.get_table_id(); + uint64_t pure_table_id = ObSchemaUtils::get_extract_schema_id(tenant_id, table_id); + if (OB_FAIL(dml_splicer.add_pk_column("tenant_id", tenant_id)) || + OB_FAIL(dml_splicer.add_pk_column("task_id", gather_stat.get_task_id())) || + OB_FAIL(dml_splicer.add_pk_column("table_id", pure_table_id)) || + OB_FAIL(dml_splicer.add_column("ret_code", gather_stat.get_ret_code())) || + OB_FAIL(dml_splicer.add_time_column("start_time", gather_stat.get_start_time())) || + OB_FAIL(dml_splicer.add_time_column("end_time", gather_stat.get_end_time())) || + OB_FAIL(dml_splicer.add_column("memory_used", gather_stat.get_memory_used())) || + OB_FAIL(dml_splicer.add_column("stat_refresh_failed_list", gather_stat.get_stat_refresh_failed_list())) || + OB_FAIL(dml_splicer.add_column("properties", gather_stat.get_properties()))) { + LOG_WARN("failed to add dml splicer column", K(ret)); + } else if (OB_FAIL(dml_splicer.splice_values(values_ptr))) { + LOG_WARN("failed to get sql string", K(ret)); + } + return ret; +} + } // end of namespace common } // end of namespace oceanbase diff --git a/src/share/stat/ob_opt_stat_sql_service.h b/src/share/stat/ob_opt_stat_sql_service.h index 2366ce1e69..2f5b9903c7 100644 --- a/src/share/stat/ob_opt_stat_sql_service.h +++ b/src/share/stat/ob_opt_stat_sql_service.h @@ -16,6 +16,7 @@ #include "share/stat/ob_opt_column_stat.h" #include "share/stat/ob_opt_table_stat.h" #include "share/stat/ob_stat_define.h" +#include "share/stat/ob_opt_stat_gather_stat.h" namespace oceanbase { namespace common { namespace sqlclient @@ -159,6 +160,9 @@ public: const ObIArray &all_ls_ids, ObIArray &tstats); + int update_opt_stat_gather_stat(const ObOptStatGatherStat &gather_stat); + int update_opt_stat_task_stat(const ObOptStatTaskInfo &task_info); + private: int get_table_stat_sql(const uint64_t tenant_id, const ObOptTableStat &stat, @@ -260,6 +264,12 @@ private: ObSqlString &tablet_list_str, ObSqlString &tablet_ls_list_str); + int get_gather_stat_value(const ObOptStatGatherStat &gather_stat, + ObSqlString &values_list); + + int get_gather_stat_task_value(const ObOptStatTaskInfo &task_info, + ObSqlString &values_str); + static const char *bitmap_compress_lib_name; bool inited_; diff --git a/src/share/stat/ob_stat_define.cpp b/src/share/stat/ob_stat_define.cpp index 207efec45a..b8b218ccec 100644 --- a/src/share/stat/ob_stat_define.cpp +++ b/src/share/stat/ob_stat_define.cpp @@ -74,6 +74,75 @@ int StatTable::assign(const StatTable &other) return no_regather_partition_ids_.assign(other.no_regather_partition_ids_); } +int ObGatherTableStatsHelper::get_duration_time(sql::ParamStore ¶ms) +{ + int ret = OB_SUCCESS; + duration_time_ = -1; + number::ObNumber num_duration; + if (OB_UNLIKELY(params.empty() || params.at(0).is_null())) { + // do nothing + } else if (lib::is_oracle_mode()) { + if (OB_FAIL(params.at(0).get_number(num_duration))) { + LOG_WARN("failed to get duration", K(ret), K(params.at(0))); + } else if (OB_FAIL(num_duration.extract_valid_int64_with_trunc(duration_time_))) { + LOG_WARN("extract_valid_int64_with_trunc failed", K(ret), K(num_duration)); + } + } else if (OB_FAIL(params.at(0).get_int(duration_time_))) { + LOG_WARN("failed to get duration", K(ret), K(params.at(0))); + } + return ret; +} + +/** + * @brief + * The order to gather tables + * 1. gather user tables + * if table is a 'big table', make it at last + * if table is first time to gather, then gather it at first + * else, gather them according to the last gather duration + * 2. gather sys tables + * so as to user tables + * @param other + * @return true + * @return false + */ +bool ObStatTableWrapper::operator<(const ObStatTableWrapper &other) const +{ + bool bret = true; + if (this == &other) { + bret = false; + } else if (table_type_ == ObStatTableType::ObSysTable && other.table_type_ == ObStatTableType::ObUserTable) { + bret = false; + } else if ((table_type_ == ObStatTableType::ObUserTable && other.table_type_ == ObStatTableType::ObUserTable) || + (table_type_ == ObStatTableType::ObSysTable && other.table_type_ == ObStatTableType::ObSysTable)) { + if (is_big_table_ && !other.is_big_table_) { + bret = false; + } else if ((is_big_table_ && other.is_big_table_) || + (!is_big_table_ && !other.is_big_table_)) { + if (stat_type_ == other.stat_type_) { + bret = last_gather_duration_ < other.last_gather_duration_; + } else if (stat_type_ == ObStatType::ObStale && other.stat_type_ == ObStatType::ObFirstTimeToGather) { + bret = false; + } + } + } + return bret; +} + +int ObStatTableWrapper::assign(const ObStatTableWrapper &other) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(stat_table_.assign(other.stat_table_))) { + LOG_WARN("failed to assign stat table"); + } else { + table_type_ = other.table_type_; + stat_type_ = other.stat_type_; + is_big_table_ = other.is_big_table_; + last_gather_duration_ = other.last_gather_duration_; + } + return ret; +} + int ObTableStatParam::assign(const ObTableStatParam &other) { int ret = OB_SUCCESS; diff --git a/src/share/stat/ob_stat_define.h b/src/share/stat/ob_stat_define.h index 16a3faa723..ba075b7322 100644 --- a/src/share/stat/ob_stat_define.h +++ b/src/share/stat/ob_stat_define.h @@ -20,6 +20,7 @@ #include "share/schema/ob_schema_struct.h" #include "share/stat/ob_opt_table_stat.h" #include "share/stat/ob_opt_column_stat.h" +#include "share/stat/ob_opt_osg_column_stat.h" namespace oceanbase { @@ -31,6 +32,7 @@ class ObOptColumnStat; typedef std::pair BolckNumPair; typedef hash::ObHashMap PartitionIdBlockMap; typedef common::hash::ObHashMap TabStatIndMap; +typedef common::hash::ObHashMap OSGColStatIndMap; typedef common::hash::ObHashMap ColStatIndMap; enum StatOptionFlags @@ -54,6 +56,7 @@ enum StatOptionFlags const static double OPT_DEFAULT_STALE_PERCENT = 0.1; const static int64_t OPT_DEFAULT_STATS_RETENTION = 31; const static int64_t OPT_STATS_MAX_VALUE_CHAR_LEN = 128; +const static int64_t OPT_STATS_BIG_TABLE_ROWS = 10000000; const int64_t MAGIC_SAMPLE_SIZE = 5500; const int64_t MAGIC_MAX_AUTO_SAMPLE_SIZE = 22000; const int64_t MAGIC_MIN_SAMPLE_SIZE = 2500; @@ -187,6 +190,63 @@ struct PartInfo ObSEArray no_regather_partition_ids_; }; +enum ObStatTableType { + ObUserTable = 0, + ObSysTable +}; + +enum ObStatType { + ObFirstTimeToGather = 0, + ObStale, + ObNotStale +}; + +struct ObStatTableWrapper { + ObStatTableWrapper() : + stat_table_(), + table_type_(ObUserTable), + stat_type_(ObFirstTimeToGather), + is_big_table_(false), + last_gather_duration_(0) + {} + bool operator<(const ObStatTableWrapper &other) const; + int assign(const ObStatTableWrapper &other); + StatTable stat_table_; + ObStatTableType table_type_; + ObStatType stat_type_; + bool is_big_table_; + int64_t last_gather_duration_; + TO_STRING_KV(K_(stat_table), + K_(table_type), + K_(stat_type), + K_(is_big_table), + K_(last_gather_duration)); +}; + +struct ObGatherTableStatsHelper { + ObGatherTableStatsHelper(): + stat_tables_(), + duration_time_(-1), + succeed_count_(0), + failed_count_(0) + {} + inline bool need_gather_table_stats() const + { + return !stat_tables_.empty(); + } + int get_duration_time(sql::ParamStore ¶ms); + ObArray stat_tables_; + + //duration_time to is used to mark the gather database stats job can use max time. default value + //is -1, it's meaning gather until all table have been gathered. + int64_t duration_time_; + int64_t succeed_count_; + int64_t failed_count_; + TO_STRING_KV(K_(stat_tables), + K_(duration_time), + K_(succeed_count), + K_(failed_count)); +}; struct ObGlobalStatParam { ObGlobalStatParam() @@ -641,6 +701,7 @@ enum OSG_TYPE GATHER_OSG = 1, MERGE_OSG = 2 }; + struct OSGPartInfo { OB_UNIS_VERSION(1); public: @@ -649,7 +710,6 @@ public: ObTabletID tablet_id_; TO_STRING_KV(K_(part_id), K_(tablet_id)); }; - typedef common::hash::ObHashMap OSGPartMap; } diff --git a/src/share/stat/ob_stat_item.cpp b/src/share/stat/ob_stat_item.cpp index 57186bd516..8cb3a86576 100644 --- a/src/share/stat/ob_stat_item.cpp +++ b/src/share/stat/ob_stat_item.cpp @@ -571,7 +571,7 @@ void ObGlobalNdvEval::add(int64_t ndv, const char *llc_bitmap) void ObGlobalNdvEval::update_llc(char *dst_llc_bitmap, const char *src_llc_bitmap, bool force_update) { if (dst_llc_bitmap != NULL && src_llc_bitmap != NULL) { - for (int64_t k = 0; k < ObColumnStat::NUM_LLC_BUCKET; ++k) { + for (int64_t k = 0; k < ObOptColumnStat::NUM_LLC_BUCKET; ++k) { if (force_update || static_cast(src_llc_bitmap[k]) > static_cast(dst_llc_bitmap[k])) { dst_llc_bitmap[k] = src_llc_bitmap[k]; @@ -600,27 +600,27 @@ int64_t ObGlobalNdvEval::get_ndv_from_llc(const char *llc_bitmap) LOG_WARN_RET(OB_ERR_UNEXPECTED, "get unexpected null pointer"); } else { double sum_of_pmax = 0; - double alpha = select_alpha_value(ObColumnStat::NUM_LLC_BUCKET); + double alpha = select_alpha_value(ObOptColumnStat::NUM_LLC_BUCKET); int64_t empty_bucket_num = 0; - for (int64_t i = 0; i < ObColumnStat::NUM_LLC_BUCKET; ++i) { + for (int64_t i = 0; i < ObOptColumnStat::NUM_LLC_BUCKET; ++i) { sum_of_pmax += (1 / pow(2, (llc_bitmap[i]))); if (llc_bitmap[i] == 0) { ++empty_bucket_num; } } - double estimate_ndv = (alpha * ObColumnStat::NUM_LLC_BUCKET - * ObColumnStat::NUM_LLC_BUCKET) / sum_of_pmax; + double estimate_ndv = (alpha * ObOptColumnStat::NUM_LLC_BUCKET + * ObOptColumnStat::NUM_LLC_BUCKET) / sum_of_pmax; num_distinct = static_cast(estimate_ndv); // check if estimate result too tiny or large. - if (estimate_ndv <= 5 * ObColumnStat::NUM_LLC_BUCKET / 2) { + if (estimate_ndv <= 5 * ObOptColumnStat::NUM_LLC_BUCKET / 2) { if (0 != empty_bucket_num) { // use linear count - num_distinct = static_cast(ObColumnStat::NUM_LLC_BUCKET - * log(ObColumnStat::NUM_LLC_BUCKET / double(empty_bucket_num))); + num_distinct = static_cast(ObOptColumnStat::NUM_LLC_BUCKET + * log(ObOptColumnStat::NUM_LLC_BUCKET / double(empty_bucket_num))); } } - if (estimate_ndv > (static_cast(ObColumnStat::LARGE_NDV_NUMBER) / 30)) { - num_distinct = static_cast((0-pow(2, 32)) * log(1 - estimate_ndv / ObColumnStat::LARGE_NDV_NUMBER)); + if (estimate_ndv > (static_cast(ObOptColumnStat::LARGE_NDV_NUMBER) / 30)) { + num_distinct = static_cast((0-pow(2, 32)) * log(1 - estimate_ndv / ObOptColumnStat::LARGE_NDV_NUMBER)); } } diff --git a/src/share/stat/ob_stat_item.h b/src/share/stat/ob_stat_item.h index 0110187aa8..2089ad5f70 100644 --- a/src/share/stat/ob_stat_item.h +++ b/src/share/stat/ob_stat_item.h @@ -14,7 +14,6 @@ #define OB_STAT_ITEM_H #include "share/stat/ob_stat_define.h" #include "share/stat/ob_opt_column_stat.h" -#include "share/stat/ob_column_stat.h" #include "share/stat/ob_topk_hist_estimator.h" namespace oceanbase { @@ -351,10 +350,10 @@ private: class ObGlobalNdvEval { - const int64_t NUM_LLC_BUCKET = ObColumnStat::NUM_LLC_BUCKET; + const int64_t NUM_LLC_BUCKET = ObOptColumnStat::NUM_LLC_BUCKET; public: ObGlobalNdvEval() : global_ndv_(-1), part_cnt_(0) { - MEMSET(global_llc_bitmap_, 0, ObColumnStat::NUM_LLC_BUCKET); } + MEMSET(global_llc_bitmap_, 0, ObOptColumnStat::NUM_LLC_BUCKET); } void add(int64_t ndv, const char *llc_bitmap); @@ -369,7 +368,7 @@ public: private: int64_t global_ndv_; int64_t part_cnt_; - char global_llc_bitmap_[ObColumnStat::NUM_LLC_BUCKET]; + char global_llc_bitmap_[ObOptColumnStat::NUM_LLC_BUCKET]; }; class ObGlobalMaxEval diff --git a/src/share/stat/ob_topk_hist_estimator.cpp b/src/share/stat/ob_topk_hist_estimator.cpp index 54d82f64cb..ea9fe543a1 100644 --- a/src/share/stat/ob_topk_hist_estimator.cpp +++ b/src/share/stat/ob_topk_hist_estimator.cpp @@ -57,6 +57,10 @@ struct CopyableComparer int ObTopKFrequencyHistograms::create_topk_fre_items() { int ret = OB_SUCCESS; + topk_fre_items_.set_allocator(&topk_buf_); + if (OB_FAIL(topk_fre_items_.init(used_list_.count()))) { + LOG_WARN("failed to init fixed array", K(ret)); + } for (int64_t i = 0; OB_SUCC(ret) && i < used_list_.count(); ++i) { if (OB_ISNULL(used_list_.at(i))) { ret = OB_ERR_UNEXPECTED; @@ -146,6 +150,8 @@ int ObTopKFrequencyHistograms::add_top_k_frequency_item(const ObObj &obj) const int64_t hashmap_size = 2 * (window_size_ + item_size_); if (OB_FAIL(topk_map_.create(hashmap_size, "TopkMap"))) { LOG_WARN("failed to create hash map", K(ret)); + } else { + need_deep_copy_ = obj.need_deep_copy(); } } if (OB_FAIL(ret)) { @@ -201,7 +207,7 @@ int ObTopKFrequencyHistograms::add_entry(const ObObj &obj, LOG_WARN("failed to add item", K(ret)); } else if (OB_FAIL(used_list_.push_back(item))) { LOG_WARN("failed to push back item", K(ret)); - } else if (obj.need_deep_copy()) { + } else if (need_deep_copy_) { ++ copied_count_; } } @@ -299,7 +305,7 @@ int ObTopKFrequencyHistograms::shrink_memory_usage() } else if (OB_FAIL(topk_map_.set_refactored(item->col_obj_, reinterpret_cast(item)))) { LOG_WARN("failed to add item", K(ret)); - } else if (item->col_obj_.need_deep_copy()) { + } else if (need_deep_copy_) { ++copied_count_; } } @@ -364,6 +370,10 @@ OB_DEF_DESERIALIZE(ObTopKFrequencyHistograms) int64_t items_count = 0; OB_UNIS_DECODE(N_); OB_UNIS_DECODE(items_count); + topk_fre_items_.set_allocator(&topk_buf_); + if (OB_FAIL(topk_fre_items_.init(items_count))) { + LOG_WARN("failed to init fixed array", K(ret)); + } for (int64_t i = 0; OB_SUCC(ret) && i < items_count; ++i) { ObTopkItem top_k_fre_item; OB_UNIS_DECODE(top_k_fre_item.col_obj_); diff --git a/src/share/stat/ob_topk_hist_estimator.h b/src/share/stat/ob_topk_hist_estimator.h index 4164a6d15b..6b5a23d2dd 100644 --- a/src/share/stat/ob_topk_hist_estimator.h +++ b/src/share/stat/ob_topk_hist_estimator.h @@ -57,7 +57,8 @@ class ObTopKFrequencyHistograms use_first_alloc_(true), obj_buf1_(), obj_buf2_(), - copied_count_() + copied_count_(), + need_deep_copy_(false) {} int read_result(const ObObj &result_obj); @@ -117,11 +118,11 @@ class ObTopKFrequencyHistograms int64_t window_size_; // in default, the window size is 1000 int64_t item_size_; bool is_topk_hist_need_des_row_; // for parallel topk histogram compuation - common::ObSEArray topk_fre_items_; + common::ObFixedArray topk_fre_items_; // the following are used to computation - common::ObSEArray used_list_; - common::ObSEArray free_list_; + common::ObSEArray used_list_; + common::ObSEArray free_list_; ObTopkMap topk_map_; ObArenaAllocator topk_buf_; @@ -132,6 +133,7 @@ class ObTopKFrequencyHistograms ObArenaAllocator obj_buf2_; int64_t copied_count_; + bool need_deep_copy_; }; } // end of namespace common diff --git a/src/share/table/ob_table_load_define.cpp b/src/share/table/ob_table_load_define.cpp index ffb4faff63..f77368ab98 100644 --- a/src/share/table/ob_table_load_define.cpp +++ b/src/share/table/ob_table_load_define.cpp @@ -96,31 +96,20 @@ OB_DEF_DESERIALIZE(ObTableLoadSqlStatistics) size = 0; OB_UNIS_DECODE(size) for (int64_t i = 0; OB_SUCC(ret) && i < size; ++i) { - ObArenaAllocator allocator; - ObOptColumnStat col_stat(allocator); - ObOptColumnStat *copied_col_stat = nullptr; - if (OB_FAIL(col_stat.deserialize(buf, data_len, pos))) { - LOG_WARN("deserialize datum store failed", K(ret), K(i)); - } else { - int64_t size = col_stat.size(); - char *new_buf = nullptr; - if (OB_ISNULL(new_buf = static_cast(allocator_.alloc(size)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - OB_LOG(WARN, "fail to allocate buffer", KR(ret), K(size)); - } else if (OB_FAIL(col_stat.deep_copy(new_buf, size, copied_col_stat))) { - OB_LOG(WARN, "fail to copy table stat", KR(ret)); - } else if (OB_FAIL(col_stat_array_.push_back(copied_col_stat))) { - OB_LOG(WARN, "fail to add table stat", KR(ret)); - } - if (OB_FAIL(ret)) { - if (copied_col_stat != nullptr) { - copied_col_stat->~ObOptColumnStat(); - copied_col_stat = nullptr; - } - if(new_buf != nullptr) { - allocator_.free(new_buf); - new_buf = nullptr; - } + ObOptOSGColumnStat *osg_col_stat = NULL; + if (OB_ISNULL(osg_col_stat = ObOptOSGColumnStat::create_new_osg_col_stat(allocator_)) || + OB_ISNULL(osg_col_stat->col_stat_)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "failed to create col stat"); + } else if (OB_FAIL(osg_col_stat->deserialize(buf, data_len, pos))) { + OB_LOG(WARN, "deserialize datum store failed", K(ret), K(i)); + } else if (OB_FAIL(col_stat_array_.push_back(osg_col_stat))) { + OB_LOG(WARN, "fail to add table stat", KR(ret)); + } + if (OB_FAIL(ret)) { + if (osg_col_stat != nullptr) { + osg_col_stat->~ObOptOSGColumnStat(); + osg_col_stat = nullptr; } } } diff --git a/src/share/table/ob_table_load_define.h b/src/share/table/ob_table_load_define.h index 229f1454e0..9df0e7c734 100644 --- a/src/share/table/ob_table_load_define.h +++ b/src/share/table/ob_table_load_define.h @@ -12,6 +12,7 @@ #include "lib/utility/ob_print_utils.h" #include "share/stat/ob_opt_table_stat.h" #include "share/stat/ob_opt_column_stat.h" +#include "share/stat/ob_opt_osg_column_stat.h" namespace oceanbase { @@ -413,9 +414,9 @@ public: ~ObTableLoadSqlStatistics() { reset();} void reset() { for (int64_t i = 0; i < col_stat_array_.count(); ++i) { - ObOptColumnStat *col_stat = col_stat_array_.at(i); + ObOptOSGColumnStat *col_stat = col_stat_array_.at(i); if (col_stat != nullptr) { - col_stat->~ObOptColumnStat(); + col_stat->~ObOptOSGColumnStat(); } } col_stat_array_.reset(); @@ -453,23 +454,23 @@ public: } return ret; } - int allocate_col_stat(ObOptColumnStat *&col_stat) + int allocate_col_stat(ObOptOSGColumnStat *&col_stat) { int ret = OB_SUCCESS; - ObOptColumnStat *new_col_stat = OB_NEWx(ObOptColumnStat, (&allocator_), allocator_); - if (OB_ISNULL(new_col_stat)) { + ObOptOSGColumnStat *new_osg_col_stat = ObOptOSGColumnStat::create_new_osg_col_stat(allocator_); + if (OB_ISNULL(new_osg_col_stat) || OB_ISNULL(new_osg_col_stat->col_stat_)) { ret = OB_ALLOCATE_MEMORY_FAILED; OB_LOG(WARN, "fail to allocate buffer", KR(ret)); - } else if (OB_FAIL(col_stat_array_.push_back(new_col_stat))) { + } else if (OB_FAIL(col_stat_array_.push_back(new_osg_col_stat))) { OB_LOG(WARN, "fail to push back", KR(ret)); } else { - col_stat = new_col_stat; + col_stat = new_osg_col_stat; } if (OB_FAIL(ret)) { - if (new_col_stat != nullptr) { - new_col_stat->~ObOptColumnStat(); - allocator_.free(new_col_stat); - new_col_stat = nullptr; + if (new_osg_col_stat != nullptr) { + new_osg_col_stat->~ObOptOSGColumnStat(); + allocator_.free(new_osg_col_stat); + new_osg_col_stat = nullptr; } } return ret; @@ -504,37 +505,48 @@ public: } } for (int64_t i = 0; OB_SUCC(ret)&& i < other.col_stat_array_.count(); ++i) { - ObOptColumnStat *col_stat = other.col_stat_array_.at(i); - if (col_stat != nullptr) { - ObOptColumnStat *copied_col_stat = nullptr; - int64_t size = col_stat->size(); - char *new_buf = nullptr;; - if (OB_ISNULL(new_buf = static_cast(allocator_.alloc(size)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - OB_LOG(WARN, "fail to allocate buffer", KR(ret), K(size)); - } else if (OB_FAIL(col_stat->deep_copy(new_buf, size, copied_col_stat))) { - OB_LOG(WARN, "fail to copy col stat", KR(ret)); - } else if (OB_FAIL(col_stat_array_.push_back(copied_col_stat))) { - OB_LOG(WARN, "fail to add col stat", KR(ret)); - } - if (OB_FAIL(ret)) { - if (copied_col_stat != nullptr) { - copied_col_stat->~ObOptColumnStat(); - copied_col_stat = nullptr; - } - if (new_buf != nullptr) { - allocator_.free(new_buf); - new_buf = nullptr; - } + ObOptOSGColumnStat *col_stat = other.col_stat_array_.at(i); + ObOptOSGColumnStat *copied_col_stat = nullptr; + if (OB_ISNULL(col_stat)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "get unexpected null"); + } else if (OB_ISNULL(copied_col_stat = ObOptOSGColumnStat::create_new_osg_col_stat(allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "failed to create new col stat"); + } else if (OB_FAIL(copied_col_stat->deep_copy(*col_stat))) { + OB_LOG(WARN, "fail to copy col stat", KR(ret)); + } else if (OB_FAIL(col_stat_array_.push_back(copied_col_stat))) { + OB_LOG(WARN, "fail to add col stat", KR(ret)); + } + if (OB_FAIL(ret)) { + if (copied_col_stat != nullptr) { + copied_col_stat->~ObOptOSGColumnStat(); + copied_col_stat = nullptr; } } } return ret; } + + int get_col_stat_array(ObIArray &col_stat_array) + { + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < col_stat_array_.count(); ++i) { + if (OB_ISNULL(col_stat_array_.at(i))) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "get unexpected null"); + } else if (OB_FAIL(col_stat_array_.at(i)->set_min_max_datum_to_obj())) { + OB_LOG(WARN, "failed to persistence min max"); + } else if (OB_FAIL(col_stat_array.push_back(col_stat_array_.at(i)->col_stat_))) { + OB_LOG(WARN, "failed to push back col stat"); + } + } + return ret; + } TO_STRING_KV(K_(col_stat_array), K_(table_stat_array)); public: common::ObSEArray table_stat_array_; - common::ObSEArray col_stat_array_; + common::ObSEArray col_stat_array_; common::ObArenaAllocator allocator_; }; diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index d552fefc5c..56eb9d64bf 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -152,6 +152,7 @@ ob_set_subtarget(ob_sql engine_basic engine/basic/ob_hash_partitioning_infrastructure_op.cpp engine/basic/ob_json_table_op.cpp engine/basic/ob_limit_op.cpp + engine/basic/ob_material_op_impl.cpp engine/basic/ob_material_op.cpp engine/basic/ob_monitoring_dump_op.cpp engine/basic/ob_pushdown_filter.cpp diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index ead33ceaa8..e61593ab7a 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -1209,8 +1209,8 @@ int ObStaticEngineCG::generate_spec(ObLogOptimizerStatsGathering &op, ObOptimize spec.set_target_osg_id(target_id); } } - if (OB_SUCC(ret) && (spec.part_level_ == share::schema::PARTITION_LEVEL_ONE - || spec.part_level_ == share::schema::PARTITION_LEVEL_TWO)) { + // only osg gather need calc_part_id_expr + if (OB_SUCC(ret) && spec.is_part_table() && !op.is_merge_osg()) { if (OB_ISNULL(op.get_calc_part_id_expr())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("calc_part_id_expr is null", K(ret)); @@ -3840,6 +3840,7 @@ int ObStaticEngineCG::generate_normal_tsc(ObLogTableScan &op, ObTableScanSpec &s LOG_WARN("generate rt expr failed", K(ret)); } else { spec.pdml_partition_id_ = rt_expr; + spec.partition_id_calc_type_ = op.get_tablet_id_type(); found = true; } } diff --git a/src/sql/das/ob_das_context.cpp b/src/sql/das/ob_das_context.cpp index 2c7977c117..f16ae63191 100644 --- a/src/sql/das/ob_das_context.cpp +++ b/src/sql/das/ob_das_context.cpp @@ -31,6 +31,7 @@ int ObDASCtx::init(const ObPhysicalPlan &plan, ObExecContext &ctx) int ret = OB_SUCCESS; ObPhysicalPlanCtx *plan_ctx = ctx.get_physical_plan_ctx(); ObSEArray partition_ids; + ObSEArray first_level_part_ids; ObSEArray tablet_ids; ObDataTypeCastParams dtc_params = ObBasicSessionInfo::create_dtc_params(ctx.get_my_session()); const ObIArray &normal_locations = plan.get_table_locations(); @@ -40,10 +41,12 @@ int ObDASCtx::init(const ObPhysicalPlan &plan, ObExecContext &ctx) ObDASTableLoc *table_loc = nullptr; tablet_ids.reuse(); partition_ids.reuse(); + first_level_part_ids.reuse(); if (OB_FAIL(das_location.calculate_tablet_ids(ctx, plan_ctx->get_param_store(), tablet_ids, partition_ids, + first_level_part_ids, dtc_params))) { LOG_WARN("calculate partition ids failed", K(ret)); } else if (OB_FAIL(extended_table_loc(das_location.get_loc_meta(), table_loc))) { @@ -51,7 +54,8 @@ int ObDASCtx::init(const ObPhysicalPlan &plan, ObExecContext &ctx) } for (int64_t j = 0; OB_SUCC(ret) && j < tablet_ids.count(); ++j) { ObDASTabletLoc *tablet_loc = nullptr; - if (OB_FAIL(extended_tablet_loc(*table_loc, tablet_ids.at(j), tablet_loc))) { + if (OB_FAIL(extended_tablet_loc(*table_loc, tablet_ids.at(j), tablet_loc, partition_ids.at(j), + first_level_part_ids.empty() ? OB_INVALID_ID : first_level_part_ids.at(j)))) { LOG_WARN("extended tablet location failed", K(ret)); } } @@ -136,7 +140,9 @@ ObDASTableLoc *ObDASCtx::get_table_loc_by_id(uint64_t table_loc_id, uint64_t ref int ObDASCtx::extended_tablet_loc(ObDASTableLoc &table_loc, const ObTabletID &tablet_id, - ObDASTabletLoc *&tablet_loc) + ObDASTabletLoc *&tablet_loc, + const common::ObObjectID &partition_id, + const common::ObObjectID &first_level_part_id) { int ret = OB_SUCCESS; if (OB_FAIL(table_loc.get_tablet_loc_by_id(tablet_id, tablet_loc))) { @@ -158,6 +164,8 @@ int ObDASCtx::extended_tablet_loc(ObDASTableLoc &table_loc, LOG_WARN("store tablet location info failed", K(ret)); } else { tablet_loc->loc_meta_ = table_loc.loc_meta_; + tablet_loc->partition_id_ = partition_id; + tablet_loc->first_level_part_id_ = first_level_part_id; } //build related tablet location if (OB_SUCC(ret) && OB_FAIL(build_related_tablet_loc(*tablet_loc))) { @@ -218,6 +226,8 @@ int ObDASCtx::extended_tablet_loc(ObDASTableLoc &table_loc, tablet_loc->server_ = replica_loc.get_server(); tablet_loc->tablet_id_ = opt_tablet_loc.get_tablet_id(); tablet_loc->ls_id_ = opt_tablet_loc.get_ls_id(); + tablet_loc->partition_id_ = opt_tablet_loc.get_partition_id(); + tablet_loc->first_level_part_id_ = opt_tablet_loc.get_first_level_part_id(); tablet_loc->loc_meta_ = table_loc.loc_meta_; if (OB_FAIL(table_loc.add_tablet_loc(tablet_loc))) { LOG_WARN("store tablet loc failed", K(ret), K(tablet_loc)); @@ -256,11 +266,13 @@ OB_INLINE int ObDASCtx::build_related_tablet_loc(ObDASTabletLoc &tablet_loc) LOG_WARN("get related tablet id failed", K(ret)); } else { related_tablet_loc = new(related_loc_buf) ObDASTabletLoc(); - related_tablet_loc->tablet_id_ = rv.first; + related_tablet_loc->tablet_id_ = rv.tablet_id_; related_tablet_loc->ls_id_ = tablet_loc.ls_id_; related_tablet_loc->server_ = tablet_loc.server_; related_tablet_loc->loc_meta_ = related_table_loc->loc_meta_; related_tablet_loc->next_ = tablet_loc.next_; + related_tablet_loc->partition_id_ = rv.part_id_; + related_tablet_loc->first_level_part_id_ = rv.first_level_part_id_; tablet_loc.next_ = related_tablet_loc; if (OB_FAIL(related_table_loc->add_tablet_loc(related_tablet_loc))) { LOG_WARN("add related tablet location failed", K(ret)); diff --git a/src/sql/das/ob_das_context.h b/src/sql/das/ob_das_context.h index de5f7cc19a..54e3d9a6b8 100644 --- a/src/sql/das/ob_das_context.h +++ b/src/sql/das/ob_das_context.h @@ -71,7 +71,9 @@ public: DASDelCtxList& get_das_del_ctx_list() {return del_ctx_list_;} int extended_tablet_loc(ObDASTableLoc &table_loc, const common::ObTabletID &tablet_id, - ObDASTabletLoc *&tablet_loc); + ObDASTabletLoc *&tablet_loc, + const common::ObObjectID &partition_id = OB_INVALID_ID, + const common::ObObjectID &first_level_part_id = OB_INVALID_ID); int extended_tablet_loc(ObDASTableLoc &table_loc, const ObCandiTabletLoc &candi_tablet_loc, ObDASTabletLoc *&talet_loc); diff --git a/src/sql/das/ob_das_define.cpp b/src/sql/das/ob_das_define.cpp index cdc283ba3d..98f73097bf 100644 --- a/src/sql/das/ob_das_define.cpp +++ b/src/sql/das/ob_das_define.cpp @@ -72,7 +72,9 @@ OB_SERIALIZE_MEMBER(ObDASTabletLoc, tablet_id_, ls_id_, server_, - flags_); + flags_, + partition_id_, + first_level_part_id_); OB_DEF_SERIALIZE(ObDASTableLoc) { @@ -157,6 +159,8 @@ int ObDASTableLoc::assign(const ObCandiTableLoc &candi_table_loc) tablet_loc->tablet_id_ = opt_tablet_loc.get_tablet_id(); tablet_loc->ls_id_ = opt_tablet_loc.get_ls_id(); tablet_loc->loc_meta_ = loc_meta_; + tablet_loc->partition_id_ = opt_tablet_loc.get_partition_id(); + tablet_loc->first_level_part_id_ = opt_tablet_loc.get_first_level_part_id(); if (OB_FAIL(tablet_locs_.push_back(tablet_loc))) { LOG_WARN("store tablet loc failed", K(ret), K(tablet_loc)); } diff --git a/src/sql/das/ob_das_define.h b/src/sql/das/ob_das_define.h index 40a1bc45e1..d8dc3c7dcc 100644 --- a/src/sql/das/ob_das_define.h +++ b/src/sql/das/ob_das_define.h @@ -154,14 +154,18 @@ public: server_(), loc_meta_(nullptr), next_(this), - flags_(0) + flags_(0), + partition_id_(OB_INVALID_ID), + first_level_part_id_(OB_INVALID_ID) { } ~ObDASTabletLoc() = default; TO_STRING_KV(K_(tablet_id), K_(ls_id), K_(server), - K_(need_refresh)); + K_(need_refresh), + K_(partition_id), + K_(first_level_part_id)); /** * BE CAREFUL!!! can't declare implicit allocator or * data structure holding implicit allocator here, @@ -183,6 +187,10 @@ public: uint64_t reserved_ : 63; }; }; + // partition id of this tablet + uint64_t partition_id_; + // first level part id of this tablet, only valid for subpartitioned table. + uint64_t first_level_part_id_; private: DISALLOW_COPY_AND_ASSIGN(ObDASTabletLoc); int assign(const ObDASTabletLoc &other); diff --git a/src/sql/das/ob_das_location_router.cpp b/src/sql/das/ob_das_location_router.cpp index 815ee9e3ae..92b8694d18 100644 --- a/src/sql/das/ob_das_location_router.cpp +++ b/src/sql/das/ob_das_location_router.cpp @@ -34,7 +34,8 @@ OB_SERIALIZE_MEMBER(DASRelatedTabletMap::MapEntry, src_tablet_id_, related_table_id_, related_tablet_id_, - related_part_id_); + related_part_id_, + related_first_level_part_id_); int VirtualSvrPair::init(ObIAllocator &allocator, ObTableID vt_id, @@ -115,13 +116,15 @@ void VirtualSvrPair::get_default_tablet_and_part_id(ObTabletID &tablet_id, ObObj int DASRelatedTabletMap::add_related_tablet_id(ObTabletID src_tablet_id, ObTableID related_table_id, ObTabletID related_tablet_id, - ObObjectID related_part_id) + ObObjectID related_part_id, + ObObjectID related_first_level_part_id) { MapEntry map_entry; map_entry.src_tablet_id_ = src_tablet_id; map_entry.related_table_id_ = related_table_id; map_entry.related_tablet_id_ = related_tablet_id; map_entry.related_part_id_ = related_part_id; + map_entry.related_first_level_part_id_ = related_first_level_part_id; return list_.push_back(map_entry); } @@ -138,8 +141,9 @@ int DASRelatedTabletMap::get_related_tablet_id(ObTabletID src_tablet_id, } } if (OB_LIKELY(final_entry != nullptr)) { - val.first = final_entry->related_tablet_id_; - val.second = final_entry->related_part_id_; + val.tablet_id_ = final_entry->related_tablet_id_; + val.part_id_ = final_entry->related_part_id_; + val.first_level_part_id_ = final_entry->related_first_level_part_id_; } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("get related tablet id failed", K(ret), K(src_tablet_id), K(related_table_id), K(list_)); @@ -183,6 +187,8 @@ int ObDASTabletMapper::get_tablet_and_object_id( if (OB_FAIL(ObPartitionUtils::get_tablet_and_subpart_id( *table_schema_, part_id, range, tmp_tablet_ids, tmp_part_ids, related_info_ptr))) { LOG_WARN("fail to get tablet_id and part_id", KR(ret), K(part_id), K(range), KPC_(table_schema)); + } else if (OB_FAIL(set_partition_id_map(part_id, tmp_part_ids))) { + LOG_WARN("failed to set partition id map"); } } else { ret = OB_INVALID_ARGUMENT; @@ -236,6 +242,8 @@ int ObDASTabletMapper::get_tablet_and_object_id(const ObPartitionLevel part_leve if (OB_FAIL(ObPartitionUtils::get_tablet_and_subpart_id( *table_schema_, part_id, row, tablet_id, object_id, related_info_ptr))) { LOG_WARN("fail to get tablet_id and part_id", KR(ret), K(part_id), K(row), KPC_(table_schema)); + } else if (OB_FAIL(set_partition_id_map(part_id, object_id))) { + LOG_WARN("failed to set partition id map"); } } else { ret = OB_INVALID_ARGUMENT; @@ -302,7 +310,8 @@ int ObDASTabletMapper::mock_vtable_related_tablet_id_map( if (OB_FAIL(related_info_.related_map_->add_related_tablet_id(tablet_id, related_table_id, related_tablet_id, - related_object_id))) { + related_object_id, + OB_INVALID_ID))) { LOG_WARN("add related tablet id to map failed", KR(ret), K(tablet_id), K(related_table_id), K(related_tablet_id), K(related_object_id)); } else { @@ -442,7 +451,8 @@ int ObDASTabletMapper::get_all_tablet_and_object_id(ObIArray &tablet //If the part_id calculated by the partition filter in the where clause is empty, //we will use the default part id in this query as the final part_id, //because optimizer needs at least one part_id to generate a plan -int ObDASTabletMapper::get_default_tablet_and_object_id(const ObIArray &part_hint_ids, +int ObDASTabletMapper::get_default_tablet_and_object_id(const ObPartitionLevel part_level, + const ObIArray &part_hint_ids, ObTabletID &tablet_id, ObObjectID &object_id) { @@ -476,10 +486,16 @@ int ObDASTabletMapper::get_default_tablet_and_object_id(const ObIArrayempty()) { + if (OB_FAIL(ret)) { + } else if (!tablet_id.is_valid()) { + // no nothing + } else if (PARTITION_LEVEL_TWO == part_level && + OB_NOT_NULL(info.part_) && + OB_FAIL(set_partition_id_map(info.part_->get_part_id(), object_id))) { + LOG_WARN("failed to set partition id map"); + } else if (related_info_.related_tids_ != nullptr && + !related_info_.related_tids_->empty()) { + //calculate related partition id and tablet id ObSchemaGetterGuard guard; const uint64_t tenant_id= table_schema_->get_tenant_id(); if (OB_ISNULL(GCTX.schema_service_)) { @@ -492,6 +508,7 @@ int ObDASTabletMapper::get_default_tablet_and_object_id(const ObIArrayat(i); const ObSimpleTableSchemaV2 *table_schema = nullptr; ObObjectID related_part_id = OB_INVALID_ID; + ObObjectID related_first_level_part_id = OB_INVALID_ID; ObTabletID related_tablet_id; if (OB_FAIL(guard.get_simple_table_schema(tenant_id, related_table_id, table_schema))) { LOG_WARN("get_table_schema fail", K(ret), K(tenant_id), K(related_table_id)); @@ -501,12 +518,14 @@ int ObDASTabletMapper::get_default_tablet_and_object_id(const ObIArrayget_part_id_and_tablet_id_by_idx(info.part_idx_, info.subpart_idx_, related_part_id, + related_first_level_part_id, related_tablet_id))) { LOG_WARN("get part by idx failed", K(ret), K(info), K(related_table_id)); } else if (OB_FAIL(related_info_.related_map_->add_related_tablet_id(tablet_id, related_table_id, related_tablet_id, - related_part_id))) { + related_part_id, + related_first_level_part_id))) { LOG_WARN("add related tablet id failed", K(ret), K(tablet_id), K(related_table_id), K(related_part_id), K(related_tablet_id)); } else { @@ -530,7 +549,8 @@ int ObDASTabletMapper::get_default_tablet_and_object_id(const ObIArraycount(); ++i) { ObTableID related_table_id = related_info_.related_tids_->at(i); //all related tables have the same part_id and tablet_id - if (OB_FAIL(related_info_.related_map_->add_related_tablet_id(tablet_id, related_table_id, tablet_id, object_id))) { + if (OB_FAIL(related_info_.related_map_->add_related_tablet_id(tablet_id, related_table_id, tablet_id, + object_id, OB_INVALID_ID))) { LOG_WARN("add related tablet id failed", K(ret), K(related_table_id), K(object_id)); } } @@ -576,6 +596,7 @@ int ObDASTabletMapper::get_related_partition_id(const ObTableID &src_table_id, ObSchemaGetterGuard guard; const ObSimpleTableSchemaV2 *dst_table_schema = nullptr; ObObjectID related_part_id = OB_INVALID_ID; + ObObjectID related_first_level_part_id = OB_INVALID_ID; ObTabletID related_tablet_id; if (OB_ISNULL(GCTX.schema_service_)) { ret = OB_INVALID_ARGUMENT; @@ -590,6 +611,7 @@ int ObDASTabletMapper::get_related_partition_id(const ObTableID &src_table_id, } else if (OB_FAIL(dst_table_schema->get_part_id_and_tablet_id_by_idx(info.part_idx_, info.subpart_idx_, related_part_id, + related_first_level_part_id, related_tablet_id))) { LOG_WARN("get part by idx failed", K(ret), K(info), K(dst_table_id)); } else { @@ -600,6 +622,57 @@ int ObDASTabletMapper::get_related_partition_id(const ObTableID &src_table_id, return ret; } +int ObDASTabletMapper::set_partition_id_map(ObObjectID first_level_part_id, + ObIArray &partition_ids) +{ + int ret = OB_SUCCESS; + if (OB_NOT_NULL(partition_id_map_)) { + for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); ++i) { + if (OB_FAIL(partition_id_map_->set_refactored(partition_ids.at(i), first_level_part_id))) { + if (OB_LIKELY(OB_HASH_EXIST == ret)) { + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to set partition map", K(first_level_part_id), K(partition_ids.at(i))); + } + } + } + } + return ret; +} + +int ObDASTabletMapper::set_partition_id_map(ObObjectID first_level_part_id, + ObObjectID partition_id) +{ + int ret = OB_SUCCESS; + if (OB_NOT_NULL(partition_id_map_)) { + if (OB_FAIL(partition_id_map_->set_refactored(partition_id, first_level_part_id))) { + if (OB_LIKELY(OB_HASH_EXIST == ret)) { + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to set partition map", K(first_level_part_id), K(partition_id)); + } + } + } + return ret; +} + +int ObDASTabletMapper::get_partition_id_map(ObObjectID partition_id, + ObObjectID &first_level_part_id) +{ + int ret = OB_SUCCESS; + if (OB_NOT_NULL(partition_id_map_)) { + if (OB_FAIL(partition_id_map_->get_refactored(partition_id, first_level_part_id))) { + if (OB_LIKELY(OB_HASH_NOT_EXIST == ret)) { + // do nothing + } else { + LOG_WARN("failed to set partition map", K(partition_id), K(first_level_part_id)); + } + } + } + return ret; +} + + ObDASLocationRouter::ObDASLocationRouter(ObIAllocator &allocator) : virtual_server_list_(allocator), allocator_(allocator) diff --git a/src/sql/das/ob_das_location_router.h b/src/sql/das/ob_das_location_router.h index 97aece7163..605c7173a9 100644 --- a/src/sql/das/ob_das_location_router.h +++ b/src/sql/das/ob_das_location_router.h @@ -35,6 +35,7 @@ struct ObDASTableLocMeta; struct ObDASTabletLoc; class ObDASCtx; typedef common::ObFixedArray AddrArray; +typedef common::hash::ObHashMap ObPartitionIdMap; class VirtualSvrPair { @@ -81,10 +82,16 @@ public: common::ObTableID related_table_id_; common::ObTabletID related_tablet_id_; common::ObObjectID related_part_id_; + // only valid if partition level of related table is level two + common::ObObjectID related_first_level_part_id_; + }; + struct Value { + common::ObTabletID tablet_id_; + common::ObObjectID part_id_; + // only valid if partition level of related table is level two + common::ObObjectID first_level_part_id_; }; typedef common::ObList RelatedTabletList; -public: - typedef std::pair Value; public: DASRelatedTabletMap(common::ObIAllocator &allocator) : list_(allocator), @@ -95,7 +102,8 @@ public: virtual int add_related_tablet_id(common::ObTabletID src_tablet_id, common::ObTableID related_table_id, common::ObTabletID related_tablet_id, - common::ObObjectID related_part_id) override; + common::ObObjectID related_part_id, + common::ObObjectID related_first_level_part_id) override; int get_related_tablet_id(common::ObTabletID src_tablet_id, common::ObTableID related_table_id, Value &val); @@ -122,7 +130,8 @@ public: is_non_partition_optimized_(false), tablet_id_(ObTabletID::INVALID_TABLET_ID), object_id_(OB_INVALID_ID), - related_list_(nullptr) + related_list_(nullptr), + partition_id_map_(nullptr) { } @@ -192,7 +201,8 @@ public: common::ObIArray &out_part_ids); int get_all_tablet_and_object_id(common::ObIArray &tablet_ids, common::ObIArray &out_part_ids); - int get_default_tablet_and_object_id(const common::ObIArray &part_hint_ids, + int get_default_tablet_and_object_id(const share::schema::ObPartitionLevel part_level, + const common::ObIArray &part_hint_ids, common::ObTabletID &tablet_id, common::ObObjectID &object_id); int get_related_partition_id(const common::ObTableID &src_table_id, @@ -210,11 +220,18 @@ public: related_list_ = related_list; is_non_partition_optimized_ = true; } + void set_partition_id_map(ObPartitionIdMap *partition_id_map) + { + partition_id_map_ = partition_id_map; + } + int set_partition_id_map(common::ObObjectID first_level_part_id, common::ObObjectID object_id); + int get_partition_id_map(common::ObObjectID first_level_part_id, common::ObObjectID &object_id); private: int mock_vtable_related_tablet_id_map(const common::ObIArray &tablet_ids, const common::ObIArray &out_part_ids); int mock_vtable_related_tablet_id_map(const common::ObTabletID &tablet_id, const common::ObObjectID &part_id); + int set_partition_id_map(common::ObObjectID first_level_part_id, common::ObIArray &object_ids); private: const share::schema::ObTableSchema *table_schema_; const VirtualSvrPair *vt_svr_pair_; @@ -223,6 +240,7 @@ private: ObTabletID tablet_id_; ObObjectID object_id_; const RelatedTabletList *related_list_; + ObPartitionIdMap *partition_id_map_; }; class ObDASLocationRouter diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.cpp b/src/sql/engine/aggregate/ob_aggregate_processor.cpp index e79cea1a26..a67557d41b 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.cpp +++ b/src/sql/engine/aggregate/ob_aggregate_processor.cpp @@ -27,6 +27,8 @@ #include "sql/engine/sort/ob_sort_op_impl.h" #include "sql/engine/expr/ob_expr_json_func_helper.h" #include "sql/engine/expr/ob_expr_lob_utils.h" +#include "sql/engine/basic/ob_material_op_impl.h" +#include "share/stat/ob_hybrid_hist_estimator.h" namespace oceanbase { @@ -478,6 +480,203 @@ int ObAggregateProcessor::GroupConcatExtraResult::set_bool_mark(int64_t col_inde return ret; } +ObAggregateProcessor::HybridHistExtraResult::~HybridHistExtraResult() +{ + if (sort_op_ != NULL) { + sort_op_->~ObSortOpImpl(); + alloc_.free(sort_op_); + sort_op_ = NULL; + } + if (mat_op_ != NULL) { + mat_op_->~ObMaterialOpImpl(); + alloc_.free(mat_op_); + mat_op_ = NULL; + } +} + +void ObAggregateProcessor::HybridHistExtraResult::reuse_self() +{ + if (sort_op_ != NULL) { + sort_op_->reuse(); + } + if (mat_op_ != NULL) { + mat_op_->reuse(); + } + sort_row_count_ = 0; + material_row_count_ = 0; +}; + +void ObAggregateProcessor::HybridHistExtraResult::reuse() +{ + reuse_self(); + ExtraResult::reuse(); +} + +int ObAggregateProcessor::HybridHistExtraResult::init(const uint64_t tenant_id, + const ObAggrInfo &aggr_info, ObEvalCtx &eval_ctx, const bool need_rewind, + ObIOEventObserver *io_event_observer, ObSqlWorkAreaProfile &profile, + ObMonitorNode &op_monitor_info) +{ + int ret = OB_SUCCESS; + sort_row_count_ = 0; + material_row_count_ = 0; + if (OB_UNLIKELY(OB_INVALID_ID == tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(tenant_id)); + } else { + if (OB_ISNULL(sort_op_ = static_cast(alloc_.alloc(sizeof(ObSortOpImpl))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fall to alloc buff", "size", sizeof(ObSortOpImpl)); + } else { + new (sort_op_) ObSortOpImpl(op_monitor_info_); + if (OB_FAIL(sort_op_->init(tenant_id, + &aggr_info.sort_collations_, + &aggr_info.sort_cmp_funcs_, + &eval_ctx, + &eval_ctx.exec_ctx_, + false, + false, + need_rewind))) { + LOG_WARN("init sort_op_ failed"); + } else { + sort_op_->set_io_event_observer(io_event_observer); + } + } + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(mat_op_ = static_cast(alloc_.alloc(sizeof(ObMaterialOpImpl))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fall to alloc buff", "size", sizeof(ObMaterialOpImpl)); + } else { + new (mat_op_) ObMaterialOpImpl(op_monitor_info, profile); + if (OB_FAIL(mat_op_->init(tenant_id, + &eval_ctx, + &eval_ctx.exec_ctx_, + io_event_observer))) { + LOG_WARN("init mat_op_ failed"); + } + } + } + + if (OB_FAIL(ret)) { + if (NULL != sort_op_) { + sort_op_->~ObSortOpImpl(); + alloc_.free(sort_op_); + sort_op_ = NULL; + } + if (NULL != mat_op_) { + mat_op_->~ObMaterialOpImpl(); + alloc_.free(mat_op_); + mat_op_ = NULL; + } + } + return ret; +} + +int ObAggregateProcessor::HybridHistExtraResult::add_sort_row( + const ObIArray &expr, ObEvalCtx &eval_ctx) +{ + int ret = OB_SUCCESS; + if (OB_NOT_NULL(sort_op_)) { + if (OB_FAIL(sort_op_->add_row(expr))) { + LOG_WARN("failed to add row to sort op", K(expr)); + } else { + ++sort_row_count_; + } + } + return ret; +} + +int ObAggregateProcessor::HybridHistExtraResult::add_sort_row( + const ObChunkDatumStore::StoredRow &sr) +{ + int ret = OB_SUCCESS; + if (OB_NOT_NULL(sort_op_)) { + if (OB_FAIL(sort_op_->add_stored_row(sr))) { + LOG_WARN("failed to add row to sort op", K(sr)); + } else { + ++sort_row_count_; + } + } + return ret; +} + +int ObAggregateProcessor::HybridHistExtraResult::get_next_row_from_sort( + const ObChunkDatumStore::StoredRow *&sr) +{ + int ret = OB_SUCCESS; + sr = NULL; + if (OB_NOT_NULL(sort_op_)) { + ret = sort_op_->get_next_row(sr); + } + return ret; +} + +int ObAggregateProcessor::HybridHistExtraResult::finish_add_sort_row() +{ + + int ret = OB_SUCCESS; + if (OB_NOT_NULL(sort_op_)) { + if (OB_FAIL(sort_op_->sort())) { + LOG_WARN("failed to sort rows"); + } + } + return ret; +} + +int ObAggregateProcessor::HybridHistExtraResult::add_material_row( + const ObDatum *src_datums, + const int64_t datum_cnt, + const int64_t extra_size, + const ObChunkDatumStore::StoredRow *&store_row) +{ + int ret = OB_SUCCESS; + if (OB_NOT_NULL(mat_op_)) { + if (OB_FAIL(mat_op_->add_row(src_datums, datum_cnt, extra_size, store_row))) { + LOG_WARN("failed to add row to sort op", K(src_datums), K(datum_cnt)); + } else { + ++material_row_count_; + } + } + return ret; +} +int ObAggregateProcessor::HybridHistExtraResult::get_next_row_from_material( + const ObChunkDatumStore::StoredRow *&sr) +{ + int ret = OB_SUCCESS; + sr = NULL; + if (OB_NOT_NULL(mat_op_)) { + ret = mat_op_->get_next_row(sr); + } + return ret; +} + +int ObAggregateProcessor::HybridHistExtraResult::finish_add_material_row() +{ + + int ret = OB_SUCCESS; + if (OB_NOT_NULL(mat_op_)) { + if (OB_FAIL(mat_op_->finish_add_row())) { + LOG_WARN("failed to sort rows"); + } + } + return ret; +} + +int64_t ObAggregateProcessor::HybridHistExtraResult::to_string( + char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + J_OBJ_START(); + J_KV(K_(sort_row_count), + K_(material_row_count), + KP_(sort_op), + KP_(mat_op) + ); + J_OBJ_END(); + return pos; +} + int64_t ObAggregateProcessor::ExtraResult::to_string(char *buf, const int64_t buf_len) const { @@ -536,7 +735,8 @@ ObAggregateProcessor::DllUdfExtra::~DllUdfExtra() ObAggregateProcessor::ObAggregateProcessor(ObEvalCtx &eval_ctx, ObIArray &aggr_infos, - const lib::ObLabel &label) + const lib::ObLabel &label, + ObMonitorNode &op_monitor_info) : has_distinct_(false), has_order_by_(false), has_group_concat_(false), @@ -569,7 +769,9 @@ ObAggregateProcessor::ObAggregateProcessor(ObEvalCtx &eval_ctx, io_event_observer_(nullptr), removal_info_(), support_fast_single_row_agg_(false), - op_eval_infos_(nullptr) + op_eval_infos_(nullptr), + profile_(ObSqlWorkAreaType::HASH_WORK_AREA), + op_monitor_info_(op_monitor_info) { } @@ -1578,7 +1780,6 @@ int ObAggregateProcessor::generate_group_row(GroupRow *&new_group_row, case T_FUN_KEEP_WM_CONCAT: case T_FUN_WM_CONCAT: case T_FUN_PL_AGG_UDF: - case T_FUN_HYBRID_HIST: case T_FUN_JSON_ARRAYAGG: case T_FUN_ORA_JSON_ARRAYAGG: case T_FUN_JSON_OBJECTAGG: @@ -1627,6 +1828,27 @@ int ObAggregateProcessor::generate_group_row(GroupRow *&new_group_row, } break; } + case T_FUN_HYBRID_HIST: { + void *tmp_buf = NULL; + if (OB_ISNULL(tmp_buf = aggr_alloc_.alloc(sizeof(HybridHistExtraResult)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory failed", "size", sizeof(HybridHistExtraResult)); + } else { + HybridHistExtraResult *result = new (tmp_buf) HybridHistExtraResult(aggr_alloc_); + aggr_cell.set_extra(result); + const bool need_rewind = (in_window_func_ || group_id > 0); + if (OB_FAIL(result->init(eval_ctx_.exec_ctx_.get_my_session()->get_effective_tenant_id(), + aggr_info, + eval_ctx_, + need_rewind, + io_event_observer_, + profile_, + op_monitor_info_))) { + LOG_WARN("init hybrid hist extra result failed"); + } + } + break; + } case T_FUN_TOP_FRE_HIST: { void *tmp_buf = NULL; if (OB_ISNULL(tmp_buf = aggr_alloc_.alloc(sizeof(TopKFreHistExtraResult)))) { @@ -1971,7 +2193,6 @@ int ObAggregateProcessor::rollup_aggregation(AggrCell &aggr_cell, AggrCell &roll case T_FUN_KEEP_WM_CONCAT: case T_FUN_WM_CONCAT: case T_FUN_PL_AGG_UDF: - case T_FUN_HYBRID_HIST: case T_FUN_JSON_ARRAYAGG: case T_FUN_ORA_JSON_ARRAYAGG: case T_FUN_JSON_OBJECTAGG: @@ -2016,6 +2237,12 @@ int ObAggregateProcessor::rollup_aggregation(AggrCell &aggr_cell, AggrCell &roll } break; } + case T_FUN_HYBRID_HIST: { + ret = OB_NOT_SUPPORTED; + LOG_WARN("rollup contain agg hybrid hist still not supported", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "rollup contain hybrid hist"); + break; + } case T_FUN_AGG_UDF: { ret = OB_NOT_SUPPORTED; LOG_WARN("rollup contain agg udfs still not supported", K(ret)); @@ -2164,7 +2391,6 @@ int ObAggregateProcessor::prepare_aggr_result(const ObChunkDatumStore::StoredRow case T_FUN_KEEP_WM_CONCAT: case T_FUN_WM_CONCAT: case T_FUN_PL_AGG_UDF: - case T_FUN_HYBRID_HIST: case T_FUN_JSON_ARRAYAGG: case T_FUN_ORA_JSON_ARRAYAGG: case T_FUN_JSON_OBJECTAGG: @@ -2209,6 +2435,23 @@ int ObAggregateProcessor::prepare_aggr_result(const ObChunkDatumStore::StoredRow } break; } + case T_FUN_HYBRID_HIST: { + HybridHistExtraResult *extra = NULL; + if (OB_ISNULL(extra = static_cast(aggr_cell.get_extra()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("extra is null", K(aggr_cell)); + } else { + extra->reuse_self(); + if (param_exprs == NULL && OB_FAIL(extra->add_sort_row(stored_row))) { + LOG_WARN("fail to add row"); + } else if (param_exprs != NULL && OB_FAIL(extra->add_sort_row(*param_exprs, eval_ctx_))) { + LOG_WARN("fail to add row"); + } else { + LOG_DEBUG("succ to add row", K(stored_row), KPC(extra)); + } + } + break; + } case T_FUN_TOP_FRE_HIST: { TopKFreHistExtraResult *extra = NULL; if (OB_ISNULL(extra = static_cast(aggr_cell.get_extra()))) { @@ -2386,7 +2629,6 @@ int ObAggregateProcessor::process_aggr_batch_result( case T_FUN_KEEP_WM_CONCAT: case T_FUN_WM_CONCAT: case T_FUN_PL_AGG_UDF: - case T_FUN_HYBRID_HIST: case T_FUN_JSON_ARRAYAGG: case T_FUN_ORA_JSON_ARRAYAGG: case T_FUN_JSON_OBJECTAGG: @@ -2420,6 +2662,16 @@ int ObAggregateProcessor::process_aggr_batch_result( } break; } + case T_FUN_HYBRID_HIST: { + HybridHistExtraResult *extra_info = NULL; + if (OB_ISNULL(extra_info = static_cast(aggr_cell.get_extra()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("extra is null", K(aggr_cell)); + } else if (OB_FAIL(selector.add_batch(param_exprs, extra_info, eval_ctx_))) { + LOG_WARN("add batch failed"); + } + break; + } case T_FUN_TOP_FRE_HIST: { TopKFreHistExtraResult *extra_info = NULL; if (OB_ISNULL(extra_info = static_cast(aggr_cell.get_extra()))) { @@ -2615,7 +2867,6 @@ int ObAggregateProcessor::process_aggr_result(const ObChunkDatumStore::StoredRow case T_FUN_KEEP_WM_CONCAT: case T_FUN_WM_CONCAT: case T_FUN_PL_AGG_UDF: - case T_FUN_HYBRID_HIST: case T_FUN_JSON_ARRAYAGG: case T_FUN_ORA_JSON_ARRAYAGG: case T_FUN_JSON_OBJECTAGG: @@ -2635,6 +2886,20 @@ int ObAggregateProcessor::process_aggr_result(const ObChunkDatumStore::StoredRow } break; } + case T_FUN_HYBRID_HIST: { + HybridHistExtraResult *extra = NULL; + if (OB_ISNULL(extra = static_cast(aggr_cell.get_extra()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("extra is null", K(aggr_cell)); + } else if (param_exprs == NULL && OB_FAIL(extra->add_sort_row(stored_row))) { + LOG_WARN("fail to add row"); + } else if (param_exprs != NULL && OB_FAIL(extra->add_sort_row(*param_exprs, eval_ctx_))) { + LOG_WARN("fail to add row"); + } else { + LOG_DEBUG("succ to add row", K(stored_row), KPC(extra)); + } + break; + } case T_FUN_TOP_FRE_HIST: { TopKFreHistExtraResult *extra = NULL; if (OB_ISNULL(extra = static_cast(aggr_cell.get_extra()))) { @@ -3486,7 +3751,7 @@ int ObAggregateProcessor::collect_aggr_result( break; } case T_FUN_HYBRID_HIST: { - GroupConcatExtraResult *extra = static_cast(aggr_cell.get_extra()); + HybridHistExtraResult *extra = static_cast(aggr_cell.get_extra()); if (OB_FAIL(compute_hybrid_hist_result(aggr_info, extra, result))) { LOG_WARN("failed to compute_hybrid_hist_result", K(ret)); } else { @@ -4431,6 +4696,25 @@ int ObAggregateProcessor::ObBatchRowsSlice::add_batch( return ret; } +int ObAggregateProcessor::ObBatchRowsSlice::add_batch( + const ObIArray *param_exprs, + HybridHistExtraResult *extra_info, + ObEvalCtx &eval_ctx +) const +{ + int ret = OB_SUCCESS; + if (OB_NOT_NULL(extra_info->sort_op_)) { + int64_t stored_rows_count = 0; + if (OB_FAIL(extra_info->sort_op_->add_batch( + *param_exprs, *brs_->skip_, end_pos_, begin_pos_, &stored_rows_count))) { + LOG_WARN("failed to add batch"); + } else { + extra_info->sort_row_count_ += stored_rows_count; + } + } + return ret; +} + int ObAggregateProcessor::ObSelector::add_batch( const ObIArray *param_exprs, ObSortOpImpl *unique_sort_op, @@ -4463,6 +4747,24 @@ int ObAggregateProcessor::ObSelector::add_batch( return ret; } +int ObAggregateProcessor::ObSelector::add_batch( + const ObIArray *param_exprs, + HybridHistExtraResult *extra_info, + ObEvalCtx &eval_ctx +) const +{ + int ret = OB_SUCCESS; + if (OB_NOT_NULL(extra_info->sort_op_)) { + if (OB_FAIL(extra_info->sort_op_->add_batch( + *param_exprs, *brs_->skip_, brs_->size_, selector_array_, count_))) { + LOG_WARN("failed to add batch"); + } else { + extra_info->sort_row_count_ += count_; + } + } + return ret; +} + template int ObAggregateProcessor::top_fre_hist_calc_batch( const ObAggrInfo &aggr_info, @@ -5010,15 +5312,10 @@ int ObAggregateProcessor::llc_add_value(const uint64_t value, char *llc_bitmap_b pmax = ObExprEstimateNdv::llc_leading_zeros(value << LLC_BUCKET_BITS, 64 - LLC_BUCKET_BITS) + 1; } ObString::obstr_size_t llc_num_buckets = size; - if (OB_UNLIKELY(size != get_llc_size())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("buffer size don't match", K(size), K(get_llc_size())); - } else if (OB_UNLIKELY(!ObExprEstimateNdv::llc_is_num_buckets_valid(llc_num_buckets)) - || OB_UNLIKELY(llc_num_buckets <= bucket_index)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("llc_add_value failed because number of buckets is not valid", - K(llc_num_buckets), K(bucket_index), K(ret)); - } else if (pmax > static_cast(llc_bitmap_buf[bucket_index])) { + OB_ASSERT(size == get_llc_size()); + OB_ASSERT(ObExprEstimateNdv::llc_is_num_buckets_valid(llc_num_buckets)); + OB_ASSERT(llc_num_buckets > bucket_index); + if (pmax > static_cast(llc_bitmap_buf[bucket_index])) { // 理论上pmax不会超过65. llc_bitmap_buf[bucket_index] = static_cast(pmax); } @@ -5633,19 +5930,21 @@ int ObAggregateProcessor::convert_datum_to_obj(const ObAggrInfo &aggr_info, } int ObAggregateProcessor::compute_hybrid_hist_result(const ObAggrInfo &aggr_info, - GroupConcatExtraResult *&extra, + HybridHistExtraResult *&extra, ObDatum &result) { int ret = OB_SUCCESS; ObHybridHistograms hybrid_hist; - ObSEArray bucket_nodes; ObDatum *bucket_num_result = NULL; int64_t bucket_num = 0; int64_t num_distinct = 0; int64_t null_count = 0; int64_t total_count = 0; + int64_t pop_count = 0; + int64_t pop_freq = 0; if (OB_ISNULL(extra) || OB_ISNULL(aggr_info.bucket_num_param_expr_) || - OB_UNLIKELY(extra->empty() || aggr_info.param_exprs_.count() != 1 || + OB_UNLIKELY(extra->get_sort_row_count() == 0 || + aggr_info.param_exprs_.count() != 1 || aggr_info.sort_collations_.count() != 1 || !aggr_info.bucket_num_param_expr_->obj_meta_.is_numeric_type())) { ret = OB_ERR_UNEXPECTED; @@ -5662,45 +5961,63 @@ int ObAggregateProcessor::compute_hybrid_hist_result(const ObAggrInfo &aggr_info } else if (bucket_num <= 0) { ret = OB_INVALID_ARGUMENT; LOG_WARN("get invalid argument", K(ret), K(bucket_num)); - } else if (extra->is_iterated() && OB_FAIL(extra->rewind())) { - LOG_WARN("rewind failed", KPC(extra), K(ret)); - } else if (!extra->is_iterated() && OB_FAIL(extra->finish_add_row())) { + } else if ( OB_FAIL(extra->finish_add_sort_row())) { LOG_WARN("finish_add_row failed", KPC(extra), K(ret)); } else { - ObObj pre_obj; - pre_obj.set_null(); + ObChunkDatumStore::LastStoredRow prev_row(aggr_alloc_); + const int64_t extra_size = sizeof(BucketDesc); int64_t repeat_count = 0; const ObChunkDatumStore::StoredRow *stored_row = NULL; - while (OB_SUCC(ret) && OB_SUCC(extra->get_next_row(stored_row))) { - ObObj cur_obj; + const ObChunkDatumStore::StoredRow *mat_stored_row = NULL; + // get null count + while (OB_SUCC(ret) && OB_SUCC(extra->get_next_row_from_sort(stored_row))) { if (OB_ISNULL(stored_row) || OB_UNLIKELY(stored_row->cnt_ != 1)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(stored_row)); - } else if (OB_FAIL(stored_row->cells()[0].to_obj(cur_obj, - aggr_info.param_exprs_.at(0)->obj_meta_))) { - LOG_WARN("failed to obj", K(ret)); + } else if (stored_row->cells()[0].is_null()) { + ++ null_count; + } else if (OB_FAIL(prev_row.save_store_row(*stored_row))) { + LOG_WARN("failed to deep copy limit last rows", K(ret)); } else { - ++ total_count; - if (cur_obj.is_null()) { - ++ null_count; - } else if (!pre_obj.is_null() && - pre_obj.is_equal(cur_obj, aggr_info.sort_collations_.at(0).cs_type_)) { - ++ repeat_count; - } else { - // new value is found - if (repeat_count > 0 && !pre_obj.is_null()) { - if (OB_FAIL(bucket_nodes.push_back(BucketNode(pre_obj, repeat_count)))) { - LOG_WARN("failed to push back bucket node", K(ret)); - } else {/*do nothing*/} - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(ob_write_obj(aggr_alloc_, cur_obj, pre_obj))) { - LOG_WARN("fail to clone cell", K(ret), K(cur_obj)); - } else { - repeat_count = 1; - ++ num_distinct; - } + repeat_count = 1; + ++ num_distinct; + break; + } + } + total_count = extra->get_sort_row_count() - null_count; + int64_t pop_threshold = total_count / bucket_num; + // get all bucket node and store them into chunk datum store + while (OB_SUCC(ret) && OB_SUCC(extra->get_next_row_from_sort(stored_row))) { + bool is_equal = false; + if (OB_ISNULL(stored_row) || OB_UNLIKELY(stored_row->cnt_ != 1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(stored_row)); + } else if (OB_FAIL(check_rows_equal(prev_row, *stored_row, aggr_info, is_equal))) { + LOG_WARN("failed to is order by item equal with prev row", K(ret)); + } else if (is_equal) { + ++ repeat_count; + } else if (OB_ISNULL(prev_row.store_row_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(prev_row.store_row_)); + } else if (OB_FAIL(extra->add_material_row(prev_row.store_row_->cells(), + prev_row.store_row_->cnt_, + extra_size, mat_stored_row))) { + LOG_WARN("failed to add material row"); + } else if (OB_FAIL(prev_row.save_store_row(*stored_row))) { + LOG_WARN("failed to deep copy limit last rows", K(ret)); + } else if (OB_ISNULL(mat_stored_row)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(mat_stored_row)); + } else { + BucketDesc *desc = reinterpret_cast(mat_stored_row->get_extra_payload()); + desc->ep_count_ = repeat_count; + desc->is_pop_ = repeat_count > pop_threshold; + if (desc->is_pop_) { + pop_freq += repeat_count; + ++ pop_count; } + repeat_count = 1; + ++ num_distinct; } } if (ret != OB_ITER_END && ret != OB_SUCCESS) { @@ -5708,15 +6025,35 @@ int ObAggregateProcessor::compute_hybrid_hist_result(const ObAggrInfo &aggr_info } else { ret = OB_SUCCESS; bool has_lob_header = aggr_info.expr_->obj_meta_.has_lob_header(); - if (!pre_obj.is_null() && - OB_FAIL(bucket_nodes.push_back(BucketNode(pre_obj, repeat_count)))) { - LOG_WARN("failed to add the last value into bucket", K(ret)); - } else if (OB_FAIL(hybrid_hist.build_hybrid_hist(bucket_nodes, - bucket_num, - total_count - null_count, - num_distinct))) { - LOG_WARN("failed to build hybrid hist", K(ret)); - } else if (OB_FAIL(get_hybrid_hist_result(hybrid_hist, has_lob_header, result))) { + if (prev_row.store_row_ != nullptr) { + if (OB_ISNULL(prev_row.store_row_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(prev_row.store_row_)); + } else if (OB_FAIL(extra->add_material_row(prev_row.store_row_->cells(), + prev_row.store_row_->cnt_, + extra_size, mat_stored_row))) { + LOG_WARN("failed to add material row"); + } else if (OB_ISNULL(mat_stored_row)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(mat_stored_row)); + } else { + BucketDesc *desc = reinterpret_cast(mat_stored_row->get_extra_payload()); + desc->ep_count_ = repeat_count; + desc->is_pop_ = repeat_count > pop_threshold; + if (desc->is_pop_) { + pop_freq += repeat_count; + ++ pop_count; + } + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(extra->finish_add_material_row())) { + LOG_WARN("failed to finish add material row", K(ret)); + } else if (OB_FAIL(hybrid_hist.build_hybrid_hist(extra, &aggr_alloc_, bucket_num, total_count, + num_distinct, pop_count, pop_freq, + aggr_info.param_exprs_.at(0)->obj_meta_))) { + LOG_WARN("failed to build hybrid hist", K(ret), K(&aggr_alloc_)); + } else if (OB_FAIL(get_hybrid_hist_result(&hybrid_hist, has_lob_header, result))) { LOG_WARN("failed to get hybrid hist result", K(ret)); } else {/*do nothing*/} } @@ -5724,21 +6061,24 @@ int ObAggregateProcessor::compute_hybrid_hist_result(const ObAggrInfo &aggr_info return ret; } -int ObAggregateProcessor::get_hybrid_hist_result(ObHybridHistograms &hybrid_hist, +int ObAggregateProcessor::get_hybrid_hist_result(ObHybridHistograms *hybrid_hist, bool has_lob_header, ObDatum &result_datum) { int ret = OB_SUCCESS; - if (hybrid_hist.get_buckets().count() > 0) { + if (OB_ISNULL(hybrid_hist)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null hybrid histograms", K(ret)); + } else if (hybrid_hist->get_buckets().count() > 0) { char *buf = NULL; - int64_t buf_size = hybrid_hist.get_serialize_size(); + int64_t buf_size = hybrid_hist->get_serialize_size(); int64_t buf_pos = 0; ObTextStringResult new_tmp_lob(ObLongTextType, has_lob_header, &aggr_alloc_); if (OB_FAIL(new_tmp_lob.init(buf_size))) { LOG_WARN("tmp lob init failed", K(ret), K(buf_size)); } else if (OB_FAIL(new_tmp_lob.get_reserved_buffer(buf, buf_size))) { LOG_WARN("tmp lob append failed", K(ret), K(new_tmp_lob)); - } else if (OB_FAIL(hybrid_hist.serialize(buf, buf_size, buf_pos))) { + } else if (OB_FAIL(hybrid_hist->serialize(buf, buf_size, buf_pos))) { LOG_WARN("fail serialize init task arg", KP(buf), K(buf_size), K(buf_pos), K(ret)); } else if (OB_FAIL(new_tmp_lob.lseek(buf_pos, 0))) { LOG_WARN("temp lob lseek failed", K(ret), K(new_tmp_lob), K(buf_pos)); @@ -5746,11 +6086,11 @@ int ObAggregateProcessor::get_hybrid_hist_result(ObHybridHistograms &hybrid_hist ObString lob_loc_str; new_tmp_lob.get_result_buffer(lob_loc_str); result_datum.set_string(lob_loc_str); - LOG_TRACE("succeed to get topK fre hist result", K(result_datum), K(hybrid_hist)); + LOG_TRACE("succeed to get hybrid hist result", K(result_datum), KPC(hybrid_hist)); } } else { result_datum.set_null(); - LOG_TRACE("succeed to get topK fre hist result", K(result_datum), K(hybrid_hist)); + LOG_TRACE("succeed to get hybrid hist result", K(result_datum), KPC(hybrid_hist)); } return ret; } diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.h b/src/sql/engine/aggregate/ob_aggregate_processor.h index b89ede186e..1e120e75cb 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.h +++ b/src/sql/engine/aggregate/ob_aggregate_processor.h @@ -28,13 +28,18 @@ #include "sql/engine/user_defined_function/ob_pl_user_defined_agg_function.h" #include "sql/engine/expr/ob_expr_dll_udf.h" #include "sql/engine/expr/ob_rt_datum_arith.h" -#include "share/stat/ob_hybrid_hist_estimator.h" namespace oceanbase { +namespace common +{ + class ObHybridHistograms; +} namespace sql { +class ObMaterialOpImpl; + struct RemovalInfo { RemovalInfo() @@ -375,6 +380,49 @@ public: common::ObFixedArray bool_mark_; }; + class HybridHistExtraResult : public ExtraResult + { + public: + explicit HybridHistExtraResult(common::ObIAllocator &alloc) + : ExtraResult(alloc), + sort_row_count_(0), + material_row_count_(0), + sort_op_(nullptr), + mat_op_(nullptr) + {} + virtual ~HybridHistExtraResult(); + void reuse_self(); + virtual void reuse() override; + + int init(const uint64_t tenant_id, const ObAggrInfo &aggr_info, + ObEvalCtx &eval_ctx, const bool need_rewind, + ObIOEventObserver *io_event_observer, ObSqlWorkAreaProfile &profile, + ObMonitorNode &op_monitor_info); + + int add_sort_row(const ObIArray &expr, ObEvalCtx &eval_ctx); + int add_sort_row(const ObChunkDatumStore::StoredRow &sr); + int get_next_row_from_sort(const ObChunkDatumStore::StoredRow *&sr); + int finish_add_sort_row(); + int add_material_row(const ObDatum *src_datums, + const int64_t datum_cnt, + const int64_t extra_size, + const ObChunkDatumStore::StoredRow *&store_row); + int get_next_row_from_material(const ObChunkDatumStore::StoredRow *&sr); + int finish_add_material_row(); + + // bool empty() const { return 0 == row_count_; } + int64_t get_sort_row_count() const { return sort_row_count_; } + int64_t get_material_row_count() const { return material_row_count_; } + DECLARE_VIRTUAL_TO_STRING; + + public: + int64_t sort_row_count_; + int64_t material_row_count_; + + ObSortOpImpl *sort_op_; + ObMaterialOpImpl *mat_op_; + }; + struct DllUdfExtra : public ExtraResult { explicit DllUdfExtra(common::ObIAllocator &alloc) @@ -476,6 +524,9 @@ public: uint16_t get_batch_index(uint16_t i) const { return selector_array_[i]; } int add_batch(const ObIArray *param_exprs, ObSortOpImpl *unique_sort_op, GroupConcatExtraResult *extra_info, ObEvalCtx &eval_ctx) const; + int add_batch(const ObIArray *param_exprs, + HybridHistExtraResult *extra_info, + ObEvalCtx &eval_ctx) const; TO_STRING_KV(K_(count)); const ObBatchRows *brs_; const uint16_t *selector_array_; @@ -509,6 +560,9 @@ public: uint16_t get_batch_index(uint16_t i) const { return i; } int add_batch(const ObIArray *param_exprs, ObSortOpImpl *unique_sort_op, GroupConcatExtraResult *extra_info, ObEvalCtx &eval_ctx) const; + int add_batch(const ObIArray *param_exprs, + HybridHistExtraResult *extra_info, + ObEvalCtx &eval_ctx) const; TO_STRING_KV(K_(begin_pos), K_(end_pos)); const ObBatchRows *brs_; uint16_t begin_pos_; @@ -548,7 +602,8 @@ public: public: ObAggregateProcessor(ObEvalCtx &eval_ctx, ObIArray &aggr_infos, - const lib::ObLabel &label); + const lib::ObLabel &label, + ObMonitorNode &op_monitor_info); ~ObAggregateProcessor() { destroy(); }; int init(); @@ -855,10 +910,10 @@ private: ObDatum &result_datum); int compute_hybrid_hist_result(const ObAggrInfo &aggr_info, - GroupConcatExtraResult *&extra, + HybridHistExtraResult *&extra, ObDatum &result); - int get_hybrid_hist_result(ObHybridHistograms &hybrid_hist, + int get_hybrid_hist_result(ObHybridHistograms *hybrid_hist, bool has_lob_header, ObDatum &result_datum); @@ -971,6 +1026,8 @@ private: RemovalInfo removal_info_; bool support_fast_single_row_agg_; ObIArray *op_eval_infos_; + ObSqlWorkAreaProfile profile_; + ObMonitorNode &op_monitor_info_; }; struct ObAggregateCalcFunc diff --git a/src/sql/engine/aggregate/ob_groupby_op.h b/src/sql/engine/aggregate/ob_groupby_op.h index b142166e90..1259af1f66 100644 --- a/src/sql/engine/aggregate/ob_groupby_op.h +++ b/src/sql/engine/aggregate/ob_groupby_op.h @@ -68,7 +68,8 @@ public: : ObOperator(exec_ctx, spec, input), aggr_processor_(eval_ctx_, (static_cast(const_cast(spec))).aggr_infos_, - ObModIds::OB_SQL_AGGR_FUNC_ROW) + ObModIds::OB_SQL_AGGR_FUNC_ROW, + op_monitor_info_) { } inline ObAggregateProcessor &get_aggr_processor() { return aggr_processor_; } diff --git a/src/sql/engine/basic/ob_chunk_datum_store.cpp b/src/sql/engine/basic/ob_chunk_datum_store.cpp index bc7b9a1ae8..4d65aa2553 100644 --- a/src/sql/engine/basic/ob_chunk_datum_store.cpp +++ b/src/sql/engine/basic/ob_chunk_datum_store.cpp @@ -229,7 +229,6 @@ int ObChunkDatumStore::StoredRow::build(StoredRow *&sr, return ret; } - int ObChunkDatumStore::Block::add_row(const common::ObIArray &exprs, ObEvalCtx &ctx, const int64_t row_size, uint32_t row_extend_size, StoredRow **stored_row) { @@ -342,6 +341,42 @@ int ObChunkDatumStore::Block::copy_stored_row(const StoredRow &stored_row, Store return ret; } +int ObChunkDatumStore::Block::copy_datums(const ObDatum *datums, const int64_t cnt, + const int64_t extra_size, StoredRow **dst_sr) +{ + int ret = OB_SUCCESS; + BlockBuffer *buf = get_buffer(); + int64_t head_size = sizeof(StoredRow); + int64_t datum_size = sizeof(ObDatum) * cnt; + int64_t row_size = head_size + sizeof(ObDatum) * cnt + extra_size; + if (!buf->is_inited()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(buf), K(row_size)); + } else { + StoredRow *sr = new (buf->head())StoredRow; + sr->cnt_ = cnt; + MEMCPY(sr->payload_, static_cast(datums), datum_size); + char* data_start = sr->payload_ + datum_size + extra_size; + int64_t pos = 0; + for (int64_t i = 0; i < cnt; ++i) { + MEMCPY(data_start + pos, datums[i].ptr_, datums[i].len_); + sr->cells()[i].ptr_ = data_start + pos; + pos += datums[i].len_; + row_size += datums[i].len_; + } + sr->row_size_ = row_size; + if (OB_FAIL(buf->advance(row_size))) { + LOG_WARN("fill buffer head failed", K(ret), K(buf), K(row_size)); + } else { + rows_++; + if (nullptr != dst_sr) { + *dst_sr = sr; + } + } + } + return ret; +} + //the memory of shadow stored row is not continuous, //so you cannot directly copy the memory of the entire stored row, //and you should make a deep copy of each datum in turn @@ -1157,6 +1192,35 @@ int ObChunkDatumStore::add_row( return ret; } +int ObChunkDatumStore::add_row(const ObDatum *datums, const int64_t cnt, + const int64_t extra_size, StoredRow **stored_row) +{ + int ret = OB_SUCCESS; + if (!is_inited()) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + int64_t head_size = sizeof(StoredRow); + int64_t datum_size = sizeof(ObDatum) * cnt; + int64_t data_size = 0; + for (int64_t i = 0; i < cnt; ++i) { + data_size += datums[i].len_; + } + const int64_t row_size = head_size + datum_size + extra_size + data_size; + if (OB_FAIL(ensure_write_blk(row_size))) { + LOG_WARN("ensure write block failed", K(ret)); + } else if (OB_FAIL(cur_blk_->copy_datums(datums, cnt, extra_size, stored_row))) { + LOG_WARN("add row to block failed", K(ret), K(datums), K(cnt), K(extra_size), K(row_size)); + } else { + row_cnt_++; + if (col_count_ < 0) { + col_count_ = cnt; + } + } + } + return ret; +} + int ObChunkDatumStore::add_row(const ShadowStoredRow &sr, StoredRow **stored_row) { int ret = OB_SUCCESS; diff --git a/src/sql/engine/basic/ob_chunk_datum_store.h b/src/sql/engine/basic/ob_chunk_datum_store.h index 64857416ce..6a816f5a09 100644 --- a/src/sql/engine/basic/ob_chunk_datum_store.h +++ b/src/sql/engine/basic/ob_chunk_datum_store.h @@ -443,6 +443,10 @@ public: int add_row(const common::ObIArray &exprs, ObEvalCtx &ctx, const int64_t row_size, uint32_t row_extend_size, StoredRow **stored_row = nullptr); int copy_stored_row(const StoredRow &stored_row, StoredRow **dst_sr); + int copy_datums(const ObDatum *datums, + const int64_t cnt, + const int64_t extra_size, + StoredRow **dst_sr); //the memory of shadow stored row is not continuous, //so you cannot directly copy the memory of the entire stored row, //and you should make a deep copy of each datum in turn @@ -853,6 +857,8 @@ public: int add_row(const common::ObIArray &exprs, ObEvalCtx *ctx, StoredRow **stored_row = nullptr); int add_row(const StoredRow &sr, StoredRow **stored_row = nullptr); + int add_row(const ObDatum *datums, const int64_t cnt, + const int64_t extra_size, StoredRow **stored_row); int add_row(const StoredRow &sr, ObEvalCtx *ctx, StoredRow **stored_row = nullptr); int add_row(const ShadowStoredRow &sr, StoredRow **stored_row = nullptr); diff --git a/src/sql/engine/basic/ob_material_op.cpp b/src/sql/engine/basic/ob_material_op.cpp index d8f9fb07ed..de4a54426c 100644 --- a/src/sql/engine/basic/ob_material_op.cpp +++ b/src/sql/engine/basic/ob_material_op.cpp @@ -41,45 +41,21 @@ int ObMaterialOp::inner_open() void ObMaterialOp::destroy() { - sql_mem_processor_.unregister_profile_if_necessary(); - datum_store_it_.reset(); - datum_store_.reset(); - destroy_mem_context(); + material_impl_.unregister_profile_if_necessary(); + material_impl_.~ObMaterialOpImpl(); ObOperator::destroy(); } -int ObMaterialOp::process_dump() +int ObMaterialOp::init_material_impl(int64_t tenant_id, int64_t row_count) { int ret = OB_SUCCESS; - bool updated = false; - bool dumped = false; - UNUSED(updated); - // 对于material由于需要保序，所以dump实现方式是，会dump掉所有的，同时可以保留最后的内存数据。目前选择一定是从前往后dump - // 还一种方式实现是，dump剩下到最大内存量的数据，以后写入数据则必须dump，但这种方式必须对内存进行伸缩处理 - if (OB_FAIL(sql_mem_processor_.update_max_available_mem_size_periodically( - &mem_context_->get_malloc_allocator(), - [&](int64_t cur_cnt){ return datum_store_.get_row_cnt_in_memory() > cur_cnt; }, - updated))) { - LOG_WARN("failed to update max available memory size periodically", K(ret)); - } else if (need_dump() && GCONF.is_sql_operator_dump_enabled() - && OB_FAIL(sql_mem_processor_.extend_max_memory_size( - &mem_context_->get_malloc_allocator(), - [&](int64_t max_memory_size) { - return sql_mem_processor_.get_data_size() > max_memory_size; - }, - dumped, sql_mem_processor_.get_data_size()))) { - LOG_WARN("failed to extend max memory size", K(ret)); - } else if (dumped) { - if (OB_FAIL(datum_store_.dump(false, true))) { - LOG_WARN("failed to dump row store", K(ret)); - } else { - sql_mem_processor_.reset(); - sql_mem_processor_.set_number_pass(1); - LOG_TRACE("trace material dump", - K(sql_mem_processor_.get_data_size()), - K(datum_store_.get_row_cnt_in_memory()), - K(sql_mem_processor_.get_mem_bound())); - } + if (OB_FAIL(material_impl_.init(tenant_id, &eval_ctx_, &ctx_, &io_event_observer_))) { + LOG_WARN("failed to init material impl", K(tenant_id)); + } else { + material_impl_.set_input_rows(row_count); + material_impl_.set_input_width(MY_SPEC.width_); + material_impl_.set_operator_type(MY_SPEC.type_); + material_impl_.set_operator_id(MY_SPEC.id_); } return ret; } @@ -87,51 +63,20 @@ int ObMaterialOp::process_dump() int ObMaterialOp::get_all_row_from_child(ObSQLSessionInfo &session) { int ret = OB_SUCCESS; - bool first_row = true; int64_t tenant_id = session.get_effective_tenant_id(); - if (OB_ISNULL(mem_context_)) { - lib::ContextParam param; - param.set_mem_attr(tenant_id, ObModIds::OB_SQL_SORT_ROW, ObCtxIds::WORK_AREA) - .set_properties(lib::USE_TL_PAGE_OPTIONAL); - if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(mem_context_, param))) { - LOG_WARN("create entity failed", K(ret)); - } else if (OB_ISNULL(mem_context_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("null memory entity returned", K(ret)); - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(datum_store_.init(UINT64_MAX, tenant_id, ObCtxIds::WORK_AREA))) { - LOG_WARN("init row store failed", K(ret)); - } else { - datum_store_.set_allocator(mem_context_->get_malloc_allocator()); - datum_store_.set_callback(&sql_mem_processor_); - datum_store_.set_io_event_observer(&io_event_observer_); + int64_t row_count = MY_SPEC.rows_; + if (OB_FAIL(ObPxEstimateSizeUtil::get_px_size( + &ctx_, MY_SPEC.px_est_size_factor_, row_count, row_count))) { + LOG_WARN("failed to get px size", K(ret)); + } else if (OB_FAIL(init_material_impl(tenant_id, row_count))) { + LOG_WARN("failed to init material impl"); } + while (OB_SUCCESS == ret) { clear_evaluated_flag(); if (OB_FAIL(child_->get_next_row())) { - } else if (first_row) { - int64_t row_count = MY_SPEC.rows_; - if (OB_FAIL(ObPxEstimateSizeUtil::get_px_size( - &ctx_, MY_SPEC.px_est_size_factor_, row_count, row_count))) { - LOG_WARN("failed to get px size", K(ret)); - } else if (OB_FAIL(sql_mem_processor_.init( - &mem_context_->get_malloc_allocator(), - tenant_id, - row_count * MY_SPEC.width_, MY_SPEC.type_, MY_SPEC.id_, &ctx_))) { - LOG_WARN("failed to init sql memory manager processor", K(ret)); - } else { - datum_store_.set_dir_id(sql_mem_processor_.get_dir_id()); - LOG_TRACE("trace init sql mem mgr for material", K(row_count), K(MY_SPEC.width_), - K(profile_.get_cache_size()), K(profile_.get_expect_size())); - } - first_row = false; - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(process_dump())) { - LOG_WARN("failed to process dump", K(ret)); - } else if (OB_FAIL(datum_store_.add_row(child_->get_spec().output_, &eval_ctx_))) { + // do nothing + } else if (OB_FAIL(material_impl_.add_row(child_->get_spec().output_))) { LOG_WARN("failed to add row to row store", K(ret)); } } @@ -140,10 +85,8 @@ int ObMaterialOp::get_all_row_from_child(ObSQLSessionInfo &session) } else { ret = OB_SUCCESS; // 最后一批数据retain到内存中 - if (OB_FAIL(datum_store_.finish_add_row(false))) { + if (OB_FAIL(material_impl_.finish_add_row())) { LOG_WARN("failed to finish add row to row store", K(ret)); - } else if (OB_FAIL(datum_store_.begin(datum_store_it_))) { - LOG_WARN("failed to begin iterator for chunk row store", K(ret)); } else { is_first_ = false; } @@ -154,69 +97,41 @@ int ObMaterialOp::get_all_row_from_child(ObSQLSessionInfo &session) int ObMaterialOp::get_all_batch_from_child(ObSQLSessionInfo &session) { int ret = OB_SUCCESS; - bool first_row = true; int64_t tenant_id = session.get_effective_tenant_id(); - if (OB_ISNULL(mem_context_)) { - lib::ContextParam param; - param.set_mem_attr(tenant_id, ObModIds::OB_SQL_SORT_ROW, ObCtxIds::WORK_AREA) - .set_properties(lib::USE_TL_PAGE_OPTIONAL); - if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(mem_context_, param))) { - LOG_WARN("create entity failed", K(ret)); - } else if (OB_ISNULL(mem_context_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("null memory entity returned", K(ret)); - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(datum_store_.init(UINT64_MAX, tenant_id, ObCtxIds::WORK_AREA))) { - LOG_WARN("init row store failed", K(ret)); - } else { - datum_store_.set_allocator(mem_context_->get_malloc_allocator()); - datum_store_.set_callback(&sql_mem_processor_); - datum_store_.set_io_event_observer(&io_event_observer_); + int64_t row_count = MY_SPEC.rows_; + if (OB_FAIL(ObPxEstimateSizeUtil::get_px_size( + &ctx_, MY_SPEC.px_est_size_factor_, row_count, row_count))) { + LOG_WARN("failed to get px size", K(ret)); + } else if (OB_FAIL(init_material_impl(tenant_id, row_count))) { + LOG_WARN("failed to init material impl"); } + const ObBatchRows *input_brs = nullptr; bool iter_end = false; while (OB_SUCCESS == ret && !iter_end) { clear_evaluated_flag(); if (OB_FAIL(child_->get_next_batch(MY_SPEC.max_batch_size_, input_brs))) { LOG_WARN("failed to get next batch", K(ret)); - } else if (first_row) { - int64_t row_count = MY_SPEC.rows_; - if (OB_FAIL(ObPxEstimateSizeUtil::get_px_size( - &ctx_, MY_SPEC.px_est_size_factor_, row_count, row_count))) { - LOG_WARN("failed to get px size", K(ret)); - } else if (OB_FAIL(sql_mem_processor_.init( - &mem_context_->get_malloc_allocator(), - tenant_id, - row_count * MY_SPEC.width_, MY_SPEC.type_, MY_SPEC.id_, &ctx_))) { - LOG_WARN("failed to init sql memory manager processor", K(ret)); - } else { - datum_store_.set_dir_id(sql_mem_processor_.get_dir_id()); - LOG_TRACE("trace init sql mem mgr for material", K(row_count), K(MY_SPEC.width_), - K(profile_.get_cache_size()), K(profile_.get_expect_size())); - } - first_row = false; - } - int64_t read_rows = -1; - if (OB_FAIL(ret)) { - } else if (OB_FAIL(process_dump())) { - LOG_WARN("failed to process dump", K(ret)); - } else if (OB_FAIL(datum_store_.add_batch(child_->get_spec().output_, eval_ctx_, - *input_brs->skip_, input_brs->size_, - read_rows))) { + } else if (OB_FAIL(material_impl_.add_batch(child_->get_spec().output_, + *input_brs->skip_, input_brs->size_))) { LOG_WARN("failed to add row to row store", K(ret)); } else { iter_end = input_brs->end_; } } + // normally child_ should not return OB_ITER_END. We add a defence check here to prevent + // some one return OB_ITER_END by mistake. + if (OB_UNLIKELY(OB_ITER_END == ret)) { + brs_.size_ = 0; + brs_.end_ = true; + ret = OB_SUCCESS; + } + if (OB_SUCC(ret)) { // 最后一批数据retain到内存中 - if (OB_FAIL(datum_store_.finish_add_row(false))) { + if (OB_FAIL(material_impl_.finish_add_row())) { LOG_WARN("failed to finish add row to row store", K(ret)); - } else if (OB_FAIL(datum_store_.begin(datum_store_it_))) { - LOG_WARN("failed to begin iterator for chunk row store", K(ret)); } else { is_first_ = false; } @@ -227,8 +142,7 @@ int ObMaterialOp::get_all_batch_from_child(ObSQLSessionInfo &session) int ObMaterialOp::inner_rescan() { int ret = OB_SUCCESS; - datum_store_it_.reset(); - datum_store_.reset(); + material_impl_.rescan(); // restart material op if (OB_FAIL(ObOperator::inner_rescan())) { LOG_WARN("operator rescan failed", K(ret)); @@ -243,7 +157,7 @@ int ObMaterialOp::rewind() if (OB_FAIL(ObOperator::inner_rescan())) {//do not cascade rescan LOG_WARN("failed to do inner rescan", K(ret)); } else { - datum_store_it_.reset(); + material_impl_.rewind(); } return ret; } @@ -251,9 +165,7 @@ int ObMaterialOp::rewind() int ObMaterialOp::inner_close() { int ret = OB_SUCCESS; - datum_store_it_.reset(); - datum_store_.reset(); - sql_mem_processor_.unregister_profile(); + material_impl_.reset(); return ret; } @@ -271,7 +183,7 @@ int ObMaterialOp::inner_get_next_row() clear_evaluated_flag(); if (is_first_ && OB_FAIL(get_all_row_from_child(*ctx_.get_my_session()))) { LOG_WARN("failed to get all row from child", K(child_), K(ret)); - } else if (OB_FAIL(datum_store_it_.get_next_row(child_->get_spec().output_, eval_ctx_))) { + } else if (OB_FAIL(material_impl_.get_next_row(child_->get_spec().output_))) { if (OB_ITER_END != ret) { LOG_WARN("get row from row store failed", K(ret)); } @@ -300,9 +212,9 @@ int ObMaterialOp::inner_get_next_batch(int64_t max_row_cnt) clear_evaluated_flag(); if (is_first_ && OB_FAIL(get_all_batch_from_child(*ctx_.get_my_session()))) { LOG_WARN("failed to get all batch from child", K(child_), K(ret)); - } else if (OB_FAIL(datum_store_it_.get_next_batch( - child_->get_spec().output_, eval_ctx_, - std::min(MY_SPEC.max_batch_size_, max_row_cnt), read_rows))) { + } else if (OB_FAIL(material_impl_.get_next_batch(child_->get_spec().output_, + std::min(MY_SPEC.max_batch_size_, max_row_cnt), + read_rows))) { if (OB_ITER_END != ret) { LOG_WARN("failed to get next batch from datum store", K(ret)); } diff --git a/src/sql/engine/basic/ob_material_op.h b/src/sql/engine/basic/ob_material_op.h index f981c677f5..36bf1f64bb 100644 --- a/src/sql/engine/basic/ob_material_op.h +++ b/src/sql/engine/basic/ob_material_op.h @@ -16,6 +16,7 @@ #include "sql/engine/ob_operator.h" #include "sql/engine/basic/ob_chunk_datum_store.h" #include "sql/engine/ob_sql_mem_mgr_processor.h" +#include "sql/engine/basic/ob_material_op_impl.h" namespace oceanbase { @@ -52,9 +53,9 @@ class ObMaterialOp : public ObOperator public: ObMaterialOp(ObExecContext &exec_ctx, const ObOpSpec &spec, ObOpInput *input) : ObOperator(exec_ctx, spec, input), - mem_context_(nullptr), datum_store_(), datum_store_it_(), profile_(ObSqlWorkAreaType::HASH_WORK_AREA), - sql_mem_processor_(profile_, op_monitor_info_), is_first_(false) + is_first_(false), + material_impl_(op_monitor_info_, profile_) {} virtual int inner_open() override; @@ -63,36 +64,24 @@ public: virtual int inner_get_next_batch(int64_t max_row_cnt) override; virtual int inner_close() override; virtual void destroy() override; + int init_material_impl(int64_t tenant_id, int64_t row_count); int get_material_row_count(int64_t &count) const { - count = datum_store_.get_row_cnt(); + count = material_impl_.get_material_row_count(); return common::OB_SUCCESS; } // reset material iterator, used for NLJ/NL connectby int rewind(); private: - int process_dump(); int get_all_row_from_child(ObSQLSessionInfo &session); int get_all_batch_from_child(ObSQLSessionInfo &session); - bool need_dump() - { return sql_mem_processor_.get_data_size() > sql_mem_processor_.get_mem_bound(); } - void destroy_mem_context() - { - if (nullptr != mem_context_) { - DESTROY_CONTEXT(mem_context_); - mem_context_ = nullptr; - } - } private: - lib::MemoryContext mem_context_; - ObChunkDatumStore datum_store_; - ObChunkDatumStore::Iterator datum_store_it_; friend class ObValues; ObSqlWorkAreaProfile profile_; - ObSqlMemMgrProcessor sql_mem_processor_; bool is_first_; + ObMaterialOpImpl material_impl_; }; } // end namespace sql diff --git a/src/sql/engine/basic/ob_material_op_impl.cpp b/src/sql/engine/basic/ob_material_op_impl.cpp new file mode 100644 index 0000000000..07e992b39e --- /dev/null +++ b/src/sql/engine/basic/ob_material_op_impl.cpp @@ -0,0 +1,309 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_ENG + +#include "ob_material_op_impl.h" +#include "sql/engine/ob_operator.h" +#include "sql/engine/ob_tenant_sql_memory_manager.h" +#include "storage/blocksstable/encoding/ob_encoding_query_util.h" +#include "lib/container/ob_iarray.h" + +namespace oceanbase +{ +using namespace common; +namespace sql +{ +ObMaterialOpImpl::ObMaterialOpImpl(ObMonitorNode &op_monitor_info, ObSqlWorkAreaProfile &profile) +: inited_(false), + got_first_row_(false), + tenant_id_(OB_INVALID_ID), + exec_ctx_(nullptr), + mem_context_(nullptr), + datum_store_(), + datum_store_it_(), + eval_ctx_(nullptr), + profile_(profile), + sql_mem_processor_(profile, op_monitor_info), + input_rows_(OB_INVALID_ID), + input_width_(OB_INVALID_ID), + op_type_(PHY_INVALID), + op_id_(UINT64_MAX) +{} + +ObMaterialOpImpl::~ObMaterialOpImpl() +{ + reset(); + if (nullptr != mem_context_) { + DESTROY_CONTEXT(mem_context_); + mem_context_ = nullptr; + } +} + +void ObMaterialOpImpl::reset() +{ + sql_mem_processor_.unregister_profile(); + io_event_observer_ = nullptr; + datum_store_.reset(); + datum_store_it_.reset(); + got_first_row_ = false; + inited_ = false; + // can not destroy mem_entify here, the memory may hold by %iter_ or %datum_store_ +} + +int ObMaterialOpImpl::init(const uint64_t tenant_id, + ObEvalCtx *eval_ctx, + ObExecContext *exec_ctx, + ObIOEventObserver *observer, + const int64_t default_block_size) +{ + int ret = OB_SUCCESS; + if (inited_) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice"); + } else if (OB_INVALID_ID == tenant_id) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(tenant_id)); + } else if (OB_ISNULL(eval_ctx) || OB_ISNULL(exec_ctx)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get null argument", K(eval_ctx), K(exec_ctx)); + } else { + tenant_id_ = tenant_id; + eval_ctx_ = eval_ctx; + exec_ctx_ = exec_ctx; + io_event_observer_ = observer; + if (OB_ISNULL(mem_context_)) { + lib::ContextParam param; + param.set_mem_attr(tenant_id, ObModIds::OB_SQL_SORT_ROW, ObCtxIds::WORK_AREA) + .set_properties(lib::USE_TL_PAGE_OPTIONAL); + if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(mem_context_, param))) { + LOG_WARN("create entity failed"); + } else if (OB_ISNULL(mem_context_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("null memory entity returned"); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(datum_store_.init(UINT64_MAX, tenant_id_, ObCtxIds::WORK_AREA))) { + LOG_WARN("init row store failed"); + } else { + datum_store_.set_allocator(mem_context_->get_malloc_allocator()); + datum_store_.set_callback(&sql_mem_processor_); + datum_store_.set_io_event_observer(io_event_observer_); + } + if (OB_SUCC(ret)) { + inited_ = true; + } + } + return ret; +} + +int ObMaterialOpImpl::add_row(const common::ObIArray &exprs, + const ObChunkDatumStore::StoredRow *&store_row) +{ + int ret = OB_SUCCESS; + ObChunkDatumStore::StoredRow *sr = NULL; + if (OB_FAIL(before_add_row())) { + LOG_WARN("before add row process failed"); + } else if (OB_FAIL(datum_store_.add_row(exprs, eval_ctx_, &sr))) { + LOG_WARN("add store row failed", K(mem_context_->used()), K(get_memory_limit())); + } else { + store_row = sr; + } + return ret; +} + +int ObMaterialOpImpl::add_row(const ObChunkDatumStore::StoredRow &src_sr, + const ObChunkDatumStore::StoredRow *&store_row) +{ + int ret = OB_SUCCESS; + ObChunkDatumStore::StoredRow *sr = NULL; + if (OB_FAIL(before_add_row())) { + LOG_WARN("before add row process failed"); + } else if (OB_FAIL(datum_store_.add_row(src_sr, &sr))) { + LOG_WARN("add store row failed", K(mem_context_->used()), K(get_memory_limit())); + } else { + store_row = sr; + } + return ret; +} + +int ObMaterialOpImpl::add_row(const ObDatum *src_datums, + const int64_t datum_cnt, + const int64_t extra_size, + const ObChunkDatumStore::StoredRow *&store_row) +{ + int ret = OB_SUCCESS; + ObChunkDatumStore::StoredRow *sr = NULL; + if (OB_FAIL(before_add_row())) { + LOG_WARN("before add row process failed"); + } else if (OB_FAIL(datum_store_.add_row(src_datums, datum_cnt, extra_size, &sr))) { + LOG_WARN("add store row failed", K(mem_context_->used()), K(get_memory_limit())); + } else { + store_row = sr; + } + return ret; +} + +int ObMaterialOpImpl::add_batch(const common::ObIArray &exprs, + const ObBitVector &skip, + const int64_t batch_size) +{ + int ret = OB_SUCCESS; + int64_t read_rows = -1; + if (OB_FAIL(before_add_row())) { + LOG_WARN("before add row process failed"); + } else if (OB_FAIL(datum_store_.add_batch(exprs, *eval_ctx_, skip, batch_size, read_rows))) { + LOG_WARN("add store row failed", K(mem_context_->used()), K(get_memory_limit())); + } + return ret; +} + +int ObMaterialOpImpl::finish_add_row() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(datum_store_.finish_add_row(false))) { + LOG_WARN("failed to finish add row to row store"); + } else if (OB_FAIL(datum_store_.begin(datum_store_it_))) { + LOG_WARN("failed to begin iterator for chunk row store"); + } + return ret; +} + + +int ObMaterialOpImpl::before_add_row() +{ + int ret = OB_SUCCESS; + if (!inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init"); + } else if (OB_UNLIKELY(!got_first_row_)) { + int64_t size = OB_INVALID_ID == input_rows_ ? 0 : input_rows_ * input_width_; + if (OB_FAIL(sql_mem_processor_.init(&mem_context_->get_malloc_allocator(), + tenant_id_, size, op_type_, + op_id_, exec_ctx_))) { + LOG_WARN("failed to init sql memory manager processor", K(ret)); + } else { + got_first_row_ = true; + datum_store_.set_dir_id(sql_mem_processor_.get_dir_id()); + LOG_TRACE("trace init sql mem mgr for material", K(size), K(input_width_), + K(profile_.get_cache_size()), K(profile_.get_expect_size())); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(process_dump())) { + LOG_WARN("failed to process dump", K(ret)); + } + } + return ret; +} + +int ObMaterialOpImpl::process_dump() +{ + int ret = OB_SUCCESS; + bool updated = false; + bool dumped = false; + UNUSED(updated); + // 对于material由于需要保序，所以dump实现方式是，会dump掉所有的，同时可以保留最后的内存数据。目前选择一定是从前往后dump + // 还一种方式实现是，dump剩下到最大内存量的数据，以后写入数据则必须dump，但这种方式必须对内存进行伸缩处理 + if (OB_FAIL(sql_mem_processor_.update_max_available_mem_size_periodically( + &mem_context_->get_malloc_allocator(), + [&](int64_t cur_cnt){ return datum_store_.get_row_cnt_in_memory() > cur_cnt; }, + updated))) { + LOG_WARN("failed to update max available memory size periodically", K(ret)); + } else if (need_dump() && GCONF.is_sql_operator_dump_enabled() + && OB_FAIL(sql_mem_processor_.extend_max_memory_size( + &mem_context_->get_malloc_allocator(), + [&](int64_t max_memory_size) { + return sql_mem_processor_.get_data_size() > max_memory_size; + }, + dumped, sql_mem_processor_.get_data_size()))) { + LOG_WARN("failed to extend max memory size", K(ret)); + } else if (dumped) { + if (OB_FAIL(datum_store_.dump(false, true))) { + LOG_WARN("failed to dump row store", K(ret)); + } else { + sql_mem_processor_.reset(); + sql_mem_processor_.set_number_pass(1); + LOG_TRACE("trace material dump", + K(sql_mem_processor_.get_data_size()), + K(datum_store_.get_row_cnt_in_memory()), + K(sql_mem_processor_.get_mem_bound())); + } + } + return ret; +} + +int ObMaterialOpImpl::get_next_row(const common::ObIArray &exprs) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(datum_store_it_.get_next_row(exprs, *eval_ctx_))) { + if (OB_ITER_END != ret) { + LOG_WARN("get row from row store failed"); + } + } + return ret; +} + +int ObMaterialOpImpl::get_next_row(const ObChunkDatumStore::StoredRow *&sr) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(datum_store_it_.get_next_row(sr))) { + if (OB_ITER_END != ret) { + LOG_WARN("get row from row store failed"); + } + } + return ret; +} + +int ObMaterialOpImpl::get_next_batch(const common::ObIArray &exprs, + const int64_t max_rows, + int64_t &read_rows) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(datum_store_it_.get_next_batch(exprs, *eval_ctx_, max_rows, read_rows))) { + if (OB_ITER_END != ret) { + LOG_WARN("failed to get next batch from datum store"); + } + } + return ret; +} + +int ObMaterialOpImpl::rescan() +{ + int ret = OB_SUCCESS; + got_first_row_ = false; + datum_store_it_.reset(); + datum_store_.reset(); + inited_ = false; + return ret; +} + +int ObMaterialOpImpl::reuse() +{ + int ret = OB_SUCCESS; + got_first_row_ = false; + datum_store_it_.reset(); + datum_store_.reset(); + return ret; +} + +int ObMaterialOpImpl::rewind() +{ + int ret = OB_SUCCESS; + datum_store_it_.reset(); + return ret; +} + +} +} \ No newline at end of file diff --git a/src/sql/engine/basic/ob_material_op_impl.h b/src/sql/engine/basic/ob_material_op_impl.h new file mode 100644 index 0000000000..30498c54a5 --- /dev/null +++ b/src/sql/engine/basic/ob_material_op_impl.h @@ -0,0 +1,124 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_SQL_ENGINE_BASIC_MATERIAL_OP_IMPL_H_ +#define OCEANBASE_SQL_ENGINE_BASIC_MATERIAL_OP_IMPL_H_ + +#include "lib/container/ob_array.h" +#include "lib/container/ob_heap.h" +#include "sql/engine/basic/ob_chunk_datum_store.h" +#include "sql/engine/ob_sql_mem_mgr_processor.h" + +namespace oceanbase +{ +namespace sql +{ + +class ObMaterialOpImpl { +public: + explicit ObMaterialOpImpl(ObMonitorNode &op_monitor_info, ObSqlWorkAreaProfile &profile); + virtual ~ObMaterialOpImpl(); + + void reset(); + void unregister_profile_if_necessary() + { + sql_mem_processor_.unregister_profile_if_necessary(); + } + + int init(const uint64_t tenant_id, + ObEvalCtx *eval_ctx, + ObExecContext *exec_ctx, + ObIOEventObserver *observer, + const int64_t default_block_size = ObChunkDatumStore::BLOCK_SIZE); + inline void set_input_rows(int64_t input_rows) { input_rows_ = input_rows; } + inline void set_input_width(int64_t input_width) { input_width_ = input_width; } + inline void set_operator_type(ObPhyOperatorType op_type) { op_type_ = op_type; } + inline void set_operator_id(uint64_t op_id) { op_id_ = op_id; } + int64_t get_material_row_count() const { return datum_store_.get_row_cnt(); } + + // before add row process: update date used memory, try dump ... + int before_add_row(); + + int add_row(const common::ObIArray &exprs, + const ObChunkDatumStore::StoredRow *&store_row); + int add_row(const ObChunkDatumStore::StoredRow &src_sr, + const ObChunkDatumStore::StoredRow *&store_row); + int add_row(const ObDatum *src_datums, + const int64_t datum_cnt, + const int64_t extra_size, + const ObChunkDatumStore::StoredRow *&store_row); + int add_row(const ObChunkDatumStore::StoredRow &sr) + { + const ObChunkDatumStore::StoredRow *store_row = NULL; + return add_row(sr, store_row); + } + int add_row(const common::ObIArray &exprs) + { + const ObChunkDatumStore::StoredRow *store_row = NULL; + return add_row(exprs, store_row); + } + + // add batch rows by selector + int add_batch(const common::ObIArray &exprs, + const ObBitVector &skip, + const int64_t batch_size); + int finish_add_row(); + + int get_next_row(const common::ObIArray &exprs); + int get_next_row(const ObChunkDatumStore::StoredRow *&sr); + + // get next batch rows, %max_cnt should equal or smaller than max batch size. + // return OB_ITER_END for EOF + int get_next_batch(const common::ObIArray &exprs, + const int64_t max_rows, + int64_t &read_rows); + + int rescan(); + int reuse(); + // rewind get_next_row() iterator to begin. + int rewind(); + + +private: + int process_dump(); + bool need_dump() + { return sql_mem_processor_.get_data_size() > sql_mem_processor_.get_mem_bound(); } + void destroy_mem_context() + { + if (nullptr != mem_context_) { + DESTROY_CONTEXT(mem_context_); + mem_context_ = nullptr; + } + } +private: + friend class ObValues; + bool inited_; + bool got_first_row_; + int64_t tenant_id_; + ObExecContext *exec_ctx_; + lib::MemoryContext mem_context_; + ObChunkDatumStore datum_store_; + ObChunkDatumStore::Iterator datum_store_it_; + ObEvalCtx *eval_ctx_; + ObSqlWorkAreaProfile &profile_; + ObSqlMemMgrProcessor sql_mem_processor_; + ObIOEventObserver *io_event_observer_; + int64_t input_rows_; + int64_t input_width_; + ObPhyOperatorType op_type_; + uint64_t op_id_; +}; + +} +} + +#endif \ No newline at end of file diff --git a/src/sql/engine/cmd/ob_analyze_executor.cpp b/src/sql/engine/cmd/ob_analyze_executor.cpp index b32fb96139..faac7be6d6 100644 --- a/src/sql/engine/cmd/ob_analyze_executor.cpp +++ b/src/sql/engine/cmd/ob_analyze_executor.cpp @@ -17,6 +17,8 @@ #include "sql/engine/cmd/ob_analyze_executor.h" #include "sql/engine/ob_exec_context.h" #include "share/stat/ob_dbms_stats_lock_unlock.h" +#include "share/stat/ob_dbms_stats_utils.h" +#include "share/stat/ob_opt_stat_manager.h" //#define COMPUTE_FREQUENCY_HISTOGRAM // "SELECT /*+NO_USE_PX*/ col, sum(val) over (order by col rows between unbounded preceding and current row) " @@ -59,14 +61,35 @@ int ObAnalyzeExecutor::execute(ObExecContext &ctx, ObAnalyzeStmt &stmt) } else if (OB_FAIL(stmt.fill_table_stat_param(ctx, param))) { LOG_WARN("failed to fill table stat param", K(ret)); } else if (!stmt.is_delete_histogram()) { - if (OB_FAIL(ObDbmsStatsLockUnlock::check_stat_locked(ctx, param))) { - LOG_WARN("failed check stat locked", K(ret)); - } else if (OB_FAIL(ObDbmsStatsExecutor::gather_table_stats(ctx, param))) { - LOG_WARN("failed to gather table stats", K(ret)); - } else if (OB_FAIL(pl::ObDbmsStats::update_stat_cache(session->get_rpc_tenant_id(), param))) { - LOG_WARN("failed to update stat cache", K(ret)); + int64_t task_cnt = 1; + int64_t seq_id = 1; + int64_t start_time = ObTimeUtility::current_time(); + ObOptStatTaskInfo task_info; + if (OB_FAIL(pl::ObDbmsStats::init_gather_task_info(ctx, ObOptStatGatherType::MANUAL_GATHER, + start_time, task_cnt, task_info))) { + LOG_WARN("failed to init gather task info", K(ret)); } else { - LOG_TRACE("succeed to gather table stats", K(param)); + ObOptStatGatherStat gather_stat(task_info); + ObOptStatGatherStatList::instance().push(gather_stat); + ObOptStatRunningMonitor running_monitor(ctx.get_allocator(), start_time, param.allocator_->total(), gather_stat); + if (OB_FAIL(running_monitor.add_table_info(param))) { + LOG_WARN("failed to add table info", K(ret)); + } else if (OB_FAIL(ObDbmsStatsLockUnlock::check_stat_locked(ctx, param))) { + LOG_WARN("failed check stat locked", K(ret)); + } else if (OB_FAIL(ObDbmsStatsExecutor::gather_table_stats(ctx, param))) { + LOG_WARN("failed to gather table stats", K(ret)); + } else if (OB_FAIL(pl::ObDbmsStats::update_stat_cache(session->get_rpc_tenant_id(), param))) { + LOG_WARN("failed to update stat cache", K(ret)); + } else { + LOG_TRACE("succeed to gather table stats", K(param)); + } + running_monitor.set_monitor_result(ret, ObTimeUtility::current_time(), param.allocator_->used()); + ObOptStatGatherStatList::instance().remove(gather_stat); + task_info.task_end_time_ = ObTimeUtility::current_time(); + task_info.ret_code_ = ret; + task_info.failed_count_ = ret == OB_SUCCESS ? 0 : 1; + ObOptStatManager::get_instance().update_opt_stat_task_stat(task_info); + ObOptStatManager::get_instance().update_opt_stat_gather_stat(gather_stat); } } else { if (OB_FAIL(ObDbmsStatsExecutor::delete_table_stats(ctx, param, true))) { diff --git a/src/sql/engine/expr/ob_expr_uuid.cpp b/src/sql/engine/expr/ob_expr_uuid.cpp index a645be8a80..3296b4e341 100644 --- a/src/sql/engine/expr/ob_expr_uuid.cpp +++ b/src/sql/engine/expr/ob_expr_uuid.cpp @@ -395,11 +395,6 @@ int ObExprUuid::gen_server_uuid(char *server_uuid, const int64_t uuid_len) if (OB_ISNULL(server_uuid) || OB_UNLIKELY(uuid_len != UuidCommon::LENGTH_UUID)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(server_uuid), K(uuid_len)); - } else if (OB_ISNULL(uuid_node = static_cast(calc_buf.alloc(sizeof(ObUUIDNode))))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("allocate memory failed", K(ret)); - } else if (OB_FAIL(uuid_node->init())) { - LOG_WARN("failed to init", K(ret)); } else if (OB_FAIL(calc(scratch))) { LOG_WARN("failed to calc", K(ret)); } else if (OB_FAIL(UuidCommon::bin2uuid(server_uuid, scratch))) { diff --git a/src/sql/engine/opt_statistics/ob_optimizer_stats_gathering_op.cpp b/src/sql/engine/opt_statistics/ob_optimizer_stats_gathering_op.cpp index 9dc808a579..db3aff6b40 100644 --- a/src/sql/engine/opt_statistics/ob_optimizer_stats_gathering_op.cpp +++ b/src/sql/engine/opt_statistics/ob_optimizer_stats_gathering_op.cpp @@ -22,6 +22,7 @@ #include "sql/engine/aggregate/ob_aggregate_processor.h" #include "share/stat/ob_opt_stat_manager.h" #include "share/stat/ob_dbms_stats_executor.h" +#include "pl/sys_package/ob_dbms_stats.h" namespace oceanbase { @@ -81,7 +82,7 @@ ObOptimizerStatsGatheringOp::ObOptimizerStatsGatheringOp(ObExecContext &exec_ctx : ObOperator(exec_ctx, spec, input), tenant_id_(OB_INVALID_ID), table_stats_map_(), - column_stats_map_(), + osg_col_stats_map_(), part_map_(), piece_msg_(), arena_("ObOptStatGather") @@ -96,28 +97,28 @@ void ObOptimizerStatsGatheringOp::destroy() void ObOptimizerStatsGatheringOp::reset() { - FOREACH(it, column_stats_map_) { + FOREACH(it, osg_col_stats_map_) { if (OB_NOT_NULL(it->second)) { - it->second->~ObOptColumnStat(); + it->second->~ObOptOSGColumnStat(); it->second = NULL; } } table_stats_map_.destroy(); - column_stats_map_.destroy(); + osg_col_stats_map_.destroy(); part_map_.destroy(); arena_.reset(); } void ObOptimizerStatsGatheringOp::reuse_stats() { - FOREACH(it, column_stats_map_) { + FOREACH(it, osg_col_stats_map_) { if (OB_NOT_NULL(it->second)) { it->second->reset(); it->second = NULL; } } table_stats_map_.reuse(); - column_stats_map_.reuse(); + osg_col_stats_map_.reuse(); part_map_.reuse(); arena_.reset(); } @@ -125,18 +126,25 @@ void ObOptimizerStatsGatheringOp::reuse_stats() int ObOptimizerStatsGatheringOp::inner_open() { int ret = OB_SUCCESS; - if (OB_ISNULL(child_) || OB_ISNULL(ctx_.get_my_session())) { + const ObTableSchema *tab_schema = nullptr; + share::schema::ObSchemaGetterGuard *schema_guard = ctx_.get_virtual_table_ctx().schema_guard_; + if (OB_ISNULL(schema_guard) || OB_ISNULL(ctx_.get_my_session())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected status: null pointer", K(ret)); + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_FALSE_IT(tenant_id_ = ctx_.get_my_session()->get_effective_tenant_id())) { + } else if (OB_FAIL(schema_guard->get_table_schema(tenant_id_, MY_SPEC.table_id_, tab_schema))) { + LOG_WARN("fail to get table schema", K(ret), K(tenant_id_), K(MY_SPEC.table_id_)); + } else if (OB_ISNULL(tab_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get table schema", K(ret)); } else { - tenant_id_ = ctx_.get_my_session()->get_effective_tenant_id(); arena_.set_tenant_id(tenant_id_); piece_msg_.set_tenant_id(tenant_id_); if (OB_FAIL(table_stats_map_.create(DEFAULT_HASH_MAP_BUCKETS_COUNT, "TabStatBucket", "TabStatNode"))) { LOG_WARN("fail to create table stats map", K(ret)); - } else if (OB_FAIL(column_stats_map_.create(DEFAULT_HASH_MAP_BUCKETS_COUNT, + } else if (OB_FAIL(osg_col_stats_map_.create(DEFAULT_HASH_MAP_BUCKETS_COUNT, "ColStatBucket", "ColStatNode"))) { LOG_WARN("fail to create column stats map", K(ret)); @@ -144,39 +152,14 @@ int ObOptimizerStatsGatheringOp::inner_open() "PartMapBucket", "PartMapNode"))) { LOG_WARN("fail to create part map", K(ret)); - } else if (OB_FAIL(init_part_map())) { + } else if (MY_SPEC.is_part_table() && + OB_FAIL(pl::ObDbmsStats::get_table_partition_map(*tab_schema, part_map_))) { LOG_WARN("fail to init part map", K(ret)); - } else { - // do nothing yet } } return ret; } -int ObOptimizerStatsGatheringOp::init_part_map() -{ - int ret = OB_SUCCESS; - const ObTableSchema *tab_schema = nullptr; - share::schema::ObSchemaGetterGuard *schema_guard = ctx_.get_virtual_table_ctx().schema_guard_; - if (OB_ISNULL(schema_guard)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get schema guard", K(ret)); - } else if (OB_FAIL(schema_guard->get_table_schema(tenant_id_, MY_SPEC.table_id_, tab_schema))) { - LOG_WARN("fail to get table schema", K(ret), K(tenant_id_), K(MY_SPEC.table_id_)); - } else if (OB_ISNULL(tab_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get table schema", K(ret)); - } else { - if (MY_SPEC.is_part_table()) { - if (OB_FAIL(pl::ObDbmsStats::get_table_partition_map(*tab_schema, part_map_))) { - LOG_WARN("fail to get part map", K(ret)); - } - } - } - - return ret; -} - int ObOptimizerStatsGatheringOp::inner_close() { int ret = OB_SUCCESS; @@ -303,7 +286,10 @@ int ObOptimizerStatsGatheringOp::generate_part_ids(PartIds &part_ids) if (MY_SPEC.is_part_table()) { ObObjectID partition_id = OB_INVALID_ID; ObTabletID tablet_id; - if (OB_FAIL(ObExprCalcPartitionBase::calc_part_and_tablet_id(MY_SPEC.calc_part_id_expr_, eval_ctx_, partition_id, tablet_id))) { + if (OB_FAIL(ObExprCalcPartitionBase::calc_part_and_tablet_id(MY_SPEC.calc_part_id_expr_, + eval_ctx_, + partition_id, + tablet_id))) { LOG_WARN("calc part and tablet id by expr failed", K(ret)); } else { OSGPartInfo part_info; @@ -319,6 +305,8 @@ int ObOptimizerStatsGatheringOp::generate_part_ids(PartIds &part_ids) } else { part_ids.first_part_id_ = part_info.part_id_; } + LOG_TRACE("succeed to generate part ids", K(part_ids), + K(MY_SPEC.is_two_level_part()), K(partition_id), K(tablet_id)); } } else { part_ids.part_id_ = (int64_t)MY_SPEC.table_id_; @@ -333,17 +321,20 @@ int ObOptimizerStatsGatheringOp::get_tab_stat_by_key(ObOptTableStat::Key &key, O int ret = OB_SUCCESS; void *ptr = nullptr; if (OB_FAIL(table_stats_map_.get_refactored(key, tab_stat))) { - if (OB_HASH_NOT_EXIST != ret) { + if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) { LOG_WARN("failed to find in hashmap", K(ret)); - } else if (OB_ISNULL(ptr = arena_.alloc(sizeof(ObOptTableStat)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("memory is not enough", K(ret), K(ptr)); } else { - tab_stat = new (ptr) ObOptTableStat(); - tab_stat->set_table_id(MY_SPEC.table_id_); - tab_stat->set_partition_id(key.partition_id_); - if (OB_FAIL(table_stats_map_.set_refactored(key, tab_stat))) { - LOG_WARN("fail to insert into hash map", K(key), KPC(tab_stat)); + ret = OB_SUCCESS; + if (OB_ISNULL(ptr = arena_.alloc(sizeof(ObOptTableStat)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("memory is not enough", K(ret), K(ptr)); + } else { + tab_stat = new (ptr) ObOptTableStat(); + tab_stat->set_table_id(MY_SPEC.table_id_); + tab_stat->set_partition_id(key.partition_id_); + if (OB_FAIL(table_stats_map_.set_refactored(key, tab_stat))) { + LOG_WARN("fail to insert into hash map", K(key), KPC(tab_stat)); + } } } } @@ -353,28 +344,30 @@ int ObOptimizerStatsGatheringOp::get_tab_stat_by_key(ObOptTableStat::Key &key, O return ret; } -int ObOptimizerStatsGatheringOp::get_col_stat_by_key(ObOptColumnStat::Key &key, ObOptColumnStat *&col_stat) +int ObOptimizerStatsGatheringOp::get_col_stat_by_key(ObOptColumnStat::Key &key, ObOptOSGColumnStat *&osg_col_stat) { int ret = OB_SUCCESS; - void *ptr = nullptr; - if(OB_FAIL(column_stats_map_.get_refactored(key, col_stat))) { - if (OB_HASH_NOT_EXIST != ret) { + if(OB_FAIL(osg_col_stats_map_.get_refactored(key, osg_col_stat))) { + if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) { LOG_WARN("failed to find in hashmap", K(ret)); - } else if (OB_ISNULL(ptr = arena_.alloc(sizeof(ObOptColumnStat)))){ - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("memory is not enough", K(ret), K(ptr)); } else { - col_stat = new (ptr) ObOptColumnStat(arena_); - col_stat->set_table_id(MY_SPEC.table_id_); - col_stat->set_partition_id(key.partition_id_); - col_stat->set_column_id(key.column_id_); - if (OB_FAIL(column_stats_map_.set_refactored(key, col_stat))) { - LOG_WARN("fail to insert into hash map", K(ret)); + ret = OB_SUCCESS; + if (OB_ISNULL(osg_col_stat = ObOptOSGColumnStat::create_new_osg_col_stat(arena_)) || + OB_ISNULL(osg_col_stat->col_stat_)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to create new col stat"); + } else { + osg_col_stat->col_stat_->set_table_id(MY_SPEC.table_id_); + osg_col_stat->col_stat_->set_partition_id(key.partition_id_); + osg_col_stat->col_stat_->set_column_id(key.column_id_); + if (OB_FAIL(osg_col_stats_map_.set_refactored(key, osg_col_stat))) { + LOG_WARN("fail to insert into hash map", K(ret)); + } } } } if (OB_SUCC(ret)) { - LOG_DEBUG("GET col_stat", KPC(col_stat)); + LOG_DEBUG("GET col_stat", K(key), KPC(osg_col_stat)); } return ret; } @@ -387,8 +380,8 @@ int ObOptimizerStatsGatheringOp::calc_column_stats(ObExpr *expr, { int ret = OB_SUCCESS; ObDatum *datum = NULL; - ObObj res_obj; - if (OB_ISNULL(expr)) { + int64_t col_len = 0; + if (OB_ISNULL(expr) || OB_ISNULL(expr->basic_funcs_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null pointer", K(ret)); } else if (OB_FAIL(get_col_stats_by_partinfo(part_ids, column_id, all_stats))) { @@ -396,23 +389,20 @@ int ObOptimizerStatsGatheringOp::calc_column_stats(ObExpr *expr, } else if (!ObColumnStatParam::is_valid_histogram_type(expr->obj_meta_.get_type())) { // do nothing yet, shoul use the plain stats. } else if (OB_FAIL(expr->eval(eval_ctx_, datum))) { - LOG_WARN("eval expr failed", K(ret)); - } else if (OB_FAIL(datum->to_obj(res_obj, expr->obj_meta_))) { - LOG_WARN("to obj failed", K(ret)); + LOG_WARN("failed to eval expr", K(*expr)); + } else if (OB_ISNULL(datum) ) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null"); + } else if (OB_FAIL(ObExprSysOpOpnsize::calc_sys_op_opnsize(expr, datum, col_len))) { + LOG_WARN("fail to calc sys op opnsize", K(ret)); + } else if (OB_FAIL(set_col_stats(all_stats, + datum, + expr->obj_meta_, + expr->basic_funcs_->null_first_cmp_))) { + LOG_WARN("failed to set col stats"); } else { - int64_t col_len = 0; - // we set avg_len outer size the set_col_stats function - // since derive avg_len from obj is inaccurate. - // and we need to calc avg_len before num_not_null/num_null. - if (OB_FAIL(ObExprSysOpOpnsize::calc_sys_op_opnsize(expr, datum, col_len))) { - LOG_WARN("fail to calc sys op opnsize", K(ret)); - } else if (FALSE_IT(set_col_stats_avg_len(all_stats, col_len))) { - LOG_WARN("fail to set col_stats_avg_len", K(ret), K(col_len)); - } else if (OB_FAIL(set_col_stats(all_stats, res_obj))) { - LOG_WARN("fail to set col stats", K(ret)); - } else { - row_len += col_len; - } + row_len += col_len; + LOG_TRACE("succed to calc column stat", K(*expr), K(row_len), K(*datum)); } return ret; } @@ -484,44 +474,42 @@ int ObOptimizerStatsGatheringOp::get_col_stats_by_partinfo(PartIds &part_ids, ui return ret; } -void ObOptimizerStatsGatheringOp::set_col_stats_avg_len(StatItems &all_stats, int64_t avg_len) -{ - all_stats.global_col_stat_->merge_avg_len(avg_len); - if (MY_SPEC.is_part_table()) { - all_stats.part_col_stat_->merge_avg_len(avg_len); - } - if (MY_SPEC.is_two_level_part()) { - all_stats.first_part_col_stat_->merge_avg_len(avg_len); - } -} - -int ObOptimizerStatsGatheringOp::set_col_stats(StatItems &all_stats, ObObj &obj) +int ObOptimizerStatsGatheringOp::set_col_stats(StatItems &all_stats, + ObDatum *datum, + const ObObjMeta &meta, + const ObDatumCmpFuncType cmp_func) { int ret = OB_SUCCESS; - const ObObj *tmp_obj; - if (OB_FAIL(ObOptimizerUtil::truncate_string_for_opt_stats(&obj, arena_, tmp_obj))) { - LOG_WARN("fail to truncate string", K(ret)); + if (OB_ISNULL(datum) || OB_ISNULL(all_stats.global_col_stat_) || OB_ISNULL(all_stats.global_col_stat_->col_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(all_stats.global_col_stat_), K(datum)); } else { - obj = *tmp_obj; - } - - all_stats.global_col_stat_->set_stat_level(StatLevel::TABLE_LEVEL); - if (OB_SUCC(ret)) { - if (OB_FAIL(all_stats.global_col_stat_->merge_obj(obj))) { - LOG_WARN("fail to set global column stat", K(ret), K(obj)); + all_stats.global_col_stat_->col_stat_->set_stat_level(StatLevel::TABLE_LEVEL); + if (OB_FAIL(all_stats.global_col_stat_->update_column_stat_info(datum, meta, cmp_func))) { + LOG_WARN("fail to set global column stat", K(ret)); } } if (OB_SUCC(ret) && MY_SPEC.is_part_table()) { - all_stats.part_col_stat_->set_stat_level(StatLevel::PARTITION_LEVEL); - if (OB_FAIL(all_stats.part_col_stat_->merge_obj(obj))) { - LOG_WARN("fail to set part column stat", K(ret), K(obj)); + if (OB_ISNULL(all_stats.part_col_stat_) || OB_ISNULL(all_stats.part_col_stat_->col_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(all_stats.part_col_stat_)); + } else { + all_stats.part_col_stat_->col_stat_->set_stat_level(StatLevel::PARTITION_LEVEL); + if (OB_FAIL(all_stats.part_col_stat_->update_column_stat_info(datum, meta, cmp_func))) { + LOG_WARN("fail to set part column stat", K(ret)); + } } } if (OB_SUCC(ret) && MY_SPEC.is_two_level_part()) { - all_stats.first_part_col_stat_->set_stat_level(StatLevel::PARTITION_LEVEL); - all_stats.part_col_stat_->set_stat_level(StatLevel::SUBPARTITION_LEVEL); - if (OB_FAIL(all_stats.first_part_col_stat_->merge_obj(obj))) { - LOG_WARN("fail to set first part column stat", K(ret), K(obj)); + if (OB_ISNULL(all_stats.first_part_col_stat_) || OB_ISNULL(all_stats.first_part_col_stat_->col_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(all_stats.first_part_col_stat_)); + } else { + all_stats.first_part_col_stat_->col_stat_->set_stat_level(StatLevel::PARTITION_LEVEL); + all_stats.part_col_stat_->col_stat_->set_stat_level(StatLevel::SUBPARTITION_LEVEL); + if (OB_FAIL(all_stats.first_part_col_stat_->update_column_stat_info(datum, meta, cmp_func))) { + LOG_WARN("fail to set first part column stat", K(ret)); + } } } return ret; @@ -530,19 +518,34 @@ int ObOptimizerStatsGatheringOp::set_col_stats(StatItems &all_stats, ObObj &obj) int ObOptimizerStatsGatheringOp::set_tab_stats(StatItems &all_stats, int64_t row_len) { int ret = OB_SUCCESS; - all_stats.global_tab_stat_->add_avg_row_size(row_len); - all_stats.global_tab_stat_->add_row_count(1); - all_stats.global_tab_stat_->set_object_type(StatLevel::TABLE_LEVEL); - if (MY_SPEC.is_part_table()) { - all_stats.part_tab_stat_->add_avg_row_size(row_len); - all_stats.part_tab_stat_->add_row_count(1); - all_stats.part_tab_stat_->set_object_type(StatLevel::PARTITION_LEVEL); + if (OB_ISNULL(all_stats.global_tab_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null"); + } else { + all_stats.global_tab_stat_->add_avg_row_size(row_len); + all_stats.global_tab_stat_->add_row_count(1); + all_stats.global_tab_stat_->set_object_type(StatLevel::TABLE_LEVEL); } - if (MY_SPEC.is_two_level_part()) { - all_stats.first_part_tab_stat_->add_avg_row_size(row_len); - all_stats.first_part_tab_stat_->add_row_count(1); - all_stats.first_part_tab_stat_->set_object_type(StatLevel::PARTITION_LEVEL); - all_stats.part_tab_stat_->set_object_type(StatLevel::SUBPARTITION_LEVEL); + if (OB_SUCC(ret) && MY_SPEC.is_part_table()) { + if (OB_ISNULL(all_stats.part_tab_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null"); + } else { + all_stats.part_tab_stat_->add_avg_row_size(row_len); + all_stats.part_tab_stat_->add_row_count(1); + all_stats.part_tab_stat_->set_object_type(StatLevel::PARTITION_LEVEL); + } + } + if (OB_SUCC(ret) && MY_SPEC.is_two_level_part()) { + if (OB_ISNULL(all_stats.first_part_tab_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null"); + } else { + all_stats.first_part_tab_stat_->add_avg_row_size(row_len); + all_stats.first_part_tab_stat_->add_row_count(1); + all_stats.first_part_tab_stat_->set_object_type(StatLevel::PARTITION_LEVEL); + all_stats.part_tab_stat_->set_object_type(StatLevel::SUBPARTITION_LEVEL); + } } return ret; } @@ -550,28 +553,34 @@ int ObOptimizerStatsGatheringOp::set_tab_stats(StatItems &all_stats, int64_t row int ObOptimizerStatsGatheringOp::merge_tab_stat(ObOptTableStat *src_tab_stat) { int ret = OB_SUCCESS; - void *ptr = NULL; - ObOptTableStat *tab_stat; - ObOptTableStat::Key stat_key(tenant_id_, src_tab_stat->get_table_id(), src_tab_stat->get_partition_id()); - if (OB_FAIL(table_stats_map_.get_refactored(stat_key, tab_stat))) { - if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to find in hashmap", K(ret)); - } else if (OB_ISNULL(ptr = arena_.alloc(sizeof(ObOptTableStat)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("memory is not enough", K(ret), K(ptr)); - } else { - if (OB_FAIL(src_tab_stat->deep_copy((char*)ptr, sizeof(ObOptTableStat), tab_stat))) { - LOG_WARN("fail to copy tab_stat", K(ret)); - } else if (OB_ISNULL(tab_stat)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to copy tab_stat", K(ret)); - } else if (OB_FAIL(table_stats_map_.set_refactored(stat_key, tab_stat))) { - LOG_WARN("fail to insert stats idx to map", K(ret)); - } - } + ObOptTableStat *tab_stat = NULL; + if (OB_ISNULL(src_tab_stat)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null"); } else { - if (OB_FAIL(tab_stat->merge_table_stat(*src_tab_stat))) { - LOG_WARN("fail to merge two table stats", K(ret), K(tab_stat), K(src_tab_stat)); + ObOptTableStat::Key stat_key(tenant_id_, src_tab_stat->get_table_id(), src_tab_stat->get_partition_id()); + if (OB_FAIL(table_stats_map_.get_refactored(stat_key, tab_stat))) { + void *ptr = NULL; + if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) { + LOG_WARN("failed to find in hashmap", K(ret)); + } else { + ret = OB_SUCCESS; + if (OB_ISNULL(ptr = arena_.alloc(sizeof(ObOptTableStat)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("memory is not enough", K(ret), K(ptr)); + } else if (OB_FAIL(src_tab_stat->deep_copy((char*)ptr, sizeof(ObOptTableStat), tab_stat))) { + LOG_WARN("fail to copy tab_stat", K(ret)); + } else if (OB_ISNULL(tab_stat)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to copy tab_stat", K(ret)); + } else if (OB_FAIL(table_stats_map_.set_refactored(stat_key, tab_stat))) { + LOG_WARN("fail to insert stats idx to map", K(ret)); + } + } + } else { + if (OB_FAIL(tab_stat->merge_table_stat(*src_tab_stat))) { + LOG_WARN("fail to merge two table stats", K(ret), K(tab_stat), K(src_tab_stat)); + } } } return ret; @@ -580,34 +589,41 @@ int ObOptimizerStatsGatheringOp::merge_tab_stat(ObOptTableStat *src_tab_stat) int ObOptimizerStatsGatheringOp::merge_col_stat(ObOptColumnStat *src_col_stat) { int ret = OB_SUCCESS; - void *ptr = NULL; - ObOptColumnStat *col_stat; - ObOptColumnStat::Key stat_key(tenant_id_, src_col_stat->get_table_id(), - src_col_stat->get_partition_id(), src_col_stat->get_column_id()); - if (OB_FAIL(column_stats_map_.get_refactored(stat_key, col_stat))) { - if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to find in hashmap", K(ret)); - } else if (OB_ISNULL(ptr = arena_.alloc(src_col_stat->size()))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("memory is not enough", K(ret), K(ptr)); - } else { - if (OB_FAIL(src_col_stat->deep_copy((char*)ptr, src_col_stat->size(), col_stat))) { - LOG_WARN("fail to copy tab_stat", K(ret)); - } else if (OB_ISNULL(col_stat)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to copy tab_stat", K(ret)); - } else if (OB_FAIL(column_stats_map_.set_refactored(stat_key, col_stat))) { - LOG_WARN("fail to insert stats idx to map", K(ret)); + if (OB_ISNULL(src_col_stat)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null"); + } else { + ObOptColumnStat *col_stat = NULL; + ObOptOSGColumnStat *osg_col_stat = NULL; + ObOptColumnStat::Key stat_key(tenant_id_, + src_col_stat->get_table_id(), + src_col_stat->get_partition_id(), + src_col_stat->get_column_id()); + if (OB_FAIL(osg_col_stats_map_.get_refactored(stat_key, osg_col_stat))) { + if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) { + LOG_WARN("failed to find in hashmap", K(ret)); } else { + ret = OB_SUCCESS; + if (OB_ISNULL(osg_col_stat = ObOptOSGColumnStat::create_new_osg_col_stat(arena_)) || + OB_ISNULL(osg_col_stat->col_stat_)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to create osg col stat"); + } else if (OB_FAIL(osg_col_stat->col_stat_->deep_copy(*src_col_stat))) { + LOG_WARN("fail to copy tab_stat", K(ret)); + } else if (OB_FAIL(osg_col_stats_map_.set_refactored(stat_key, osg_col_stat))) { + LOG_WARN("fail to insert stats idx to map", K(ret)); + } } + } else if (OB_ISNULL(osg_col_stat) || OB_ISNULL(osg_col_stat->col_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Get unexpected null"); + } else if (OB_FAIL(osg_col_stat->col_stat_->merge_column_stat(*src_col_stat))) { + LOG_WARN("failed to merge column stat"); } - } else if (OB_FAIL(col_stat->merge_column_stat(*src_col_stat))) { - LOG_WARN("fail to merge two table stats", K(ret), K(col_stat), K(src_col_stat)); } return ret; } - int ObOptimizerStatsGatheringOp::on_piece_msg(const ObOptStatsGatherPieceMsg &piece_msg) { int ret = OB_SUCCESS; @@ -629,6 +645,7 @@ int ObOptimizerStatsGatheringOp::on_piece_msg(const ObOptStatsGatherPieceMsg &pi } if (OB_SUCC(ret)) { LOG_DEBUG("OSG merge piece msg", K(piece_msg)); + // why reset piece_msg_? piece_msg_.reset(); } } @@ -644,20 +661,48 @@ int ObOptimizerStatsGatheringOp::msg_end() } else { share::schema::ObSchemaGetterGuard *schema_guard = nullptr; ObTableStatParam param; + ColStatIndMap col_stat_map; if (OB_ISNULL(schema_guard = ctx_.get_virtual_table_ctx().schema_guard_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null pointer", K(ret)); } else if (OB_FAIL(generate_stat_param(param))) { LOG_WARN("fail to generate param", K(ret)); - } else if (OB_FAIL(ObDbmsStatsExecutor::update_stat_online(ctx_, + } else if (OB_FAIL(get_col_stat_map(col_stat_map))) { + LOG_WARN("failed to get col stat map"); + } else if (OB_FAIL(ObDbmsStatsExecutor::update_online_stat(ctx_, param, schema_guard, get_tab_stat_map(), - get_col_stat_map()))) { + col_stat_map))) { LOG_WARN("fail to update tab/col stats", K(ret)); - } else {} + } } + return ret; +} +int ObOptimizerStatsGatheringOp::get_col_stat_map(ColStatIndMap &col_stat_map) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(col_stat_map.created())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("has been created"); + } else if (OB_FAIL(col_stat_map.create(osg_col_stats_map_.size() == 0 ? + 1 : osg_col_stats_map_.size(), + "ColStatMap"))) { + LOG_WARN("failed to create col stat map"); + } else { + FOREACH_X(it, osg_col_stats_map_, OB_SUCC(ret)) { + ObOptOSGColumnStat *osg_col_stat = NULL; + if (OB_ISNULL(osg_col_stat = it->second) || OB_ISNULL(osg_col_stat->col_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null"); + } else if (OB_FAIL(osg_col_stat->set_min_max_datum_to_obj())) { + LOG_WARN("failed to persistence min max"); + } else if (OB_FAIL(col_stat_map.set_refactored(it->first, osg_col_stat->col_stat_))) { + LOG_WARN("failed to set col stat"); + } + } + } return ret; } @@ -734,12 +779,15 @@ int ObOptimizerStatsGatheringOp::generate_stat_param(ObTableStatParam ¶m) int ObOptimizerStatsGatheringOp::get_col_stats(common::ObIArray& col_stats) { int ret = OB_SUCCESS; - FOREACH_X(it, column_stats_map_, OB_SUCC(ret)) { - if (OB_ISNULL(it->second)) { + FOREACH_X(it, osg_col_stats_map_, OB_SUCC(ret)) { + ObOptOSGColumnStat *osg_col_stat = NULL; + if (OB_ISNULL(osg_col_stat = it->second) || OB_ISNULL(osg_col_stat->col_stat_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null pointer", K(ret)); - } else if (OB_FAIL(col_stats.push_back(it->second))) { - LOG_WARN("fail to push back col stats", K(ret)); + LOG_WARN("get unexpected null"); + } else if (OB_FAIL(osg_col_stat->set_min_max_datum_to_obj())) { + LOG_WARN("failed to persistence min max"); + } else if (OB_FAIL(col_stats.push_back(osg_col_stat->col_stat_))) { + LOG_WARN("failed to push back col stat"); } } return ret; diff --git a/src/sql/engine/opt_statistics/ob_optimizer_stats_gathering_op.h b/src/sql/engine/opt_statistics/ob_optimizer_stats_gathering_op.h index 39631720ee..ed4cc5549c 100644 --- a/src/sql/engine/opt_statistics/ob_optimizer_stats_gathering_op.h +++ b/src/sql/engine/opt_statistics/ob_optimizer_stats_gathering_op.h @@ -19,6 +19,7 @@ #include "sql/engine/px/datahub/components/ob_dh_opt_stats_gather.h" #include "share/stat/ob_opt_table_stat.h" #include "share/stat/ob_opt_column_stat.h" +#include "share/stat/ob_opt_osg_column_stat.h" #include "sql/engine/px/ob_px_sqc_handler.h" #include "share/stat/ob_basic_stats_estimator.h" #include "share/stat/ob_dbms_stats_executor.h" @@ -59,7 +60,6 @@ public: class ObOptimizerStatsGatheringOp : public ObOperator { public: - // store global/part/subpart part_id. struct PartIds { PartIds() : global_part_id_(common::OB_INVALID_ID), @@ -84,9 +84,9 @@ public: ObOptTableStat *global_tab_stat_; ObOptTableStat *part_tab_stat_; ObOptTableStat *first_part_tab_stat_; - ObOptColumnStat *global_col_stat_; - ObOptColumnStat *part_col_stat_; - ObOptColumnStat *first_part_col_stat_; + ObOptOSGColumnStat *global_col_stat_; + ObOptOSGColumnStat *part_col_stat_; + ObOptOSGColumnStat *first_part_col_stat_; }; public: @@ -103,12 +103,11 @@ public: int on_piece_msg(const ObOptStatsGatherPieceMsg &piece_msg); // after dh receive all piece_msg, msg_end will be called. int msg_end(); - + inline const TabStatIndMap& get_tab_stat_map() const { return table_stats_map_;}; + int get_col_stat_map(ColStatIndMap &col_stat_map); private: static const int64_t DEFAULT_HASH_MAP_BUCKETS_COUNT = 100; void reuse_stats(); - inline TabStatIndMap& get_tab_stat_map() { return table_stats_map_;}; - inline ColStatIndMap& get_col_stat_map() { return column_stats_map_;}; // set piece_msg's basic info, copy tab/column stats from stat_map to piece_msg's array. int build_piece_msg(ObOptStatsGatherPieceMsg &piece, @@ -123,7 +122,6 @@ private: PartIds &part_ids, StatItems &all_stats, int64_t &row_len); - int init_part_map(); // generate stat_param that is used to write inner_table. int generate_stat_param(ObTableStatParam ¶m); int generate_part_ids(PartIds &part_ids); @@ -132,14 +130,13 @@ private: int get_col_stats_by_partinfo(PartIds &part_ids, uint64_t column_id, StatItems &stat_item); // get stat by part_ids - int set_col_stats(StatItems &all_stat, common::ObObj &obj); - void set_col_stats_avg_len(StatItems &all_stats, int64_t avg_len); + int set_col_stats(StatItems &all_stat, ObDatum *datum, const ObObjMeta &meta, const ObDatumCmpFuncType cmp_func); int set_tab_stats(StatItems &all_stat, int64_t row_len); // get tab stat by key(tenant_id, table_id, partition_id), if NOT_EXISTS, alloc a new one. int get_tab_stat_by_key(ObOptTableStat::Key &key, ObOptTableStat *&tab_stat); // get tab stat by key(tenant_id, table_id, partition_id, column_id), if NOT_EXISTS, alloc a new one. - int get_col_stat_by_key(ObOptColumnStat::Key &key, ObOptColumnStat *&col_stat); + int get_col_stat_by_key(ObOptColumnStat::Key &key, ObOptOSGColumnStat *&osg_col_stat); int merge_tab_stat(ObOptTableStat *src_tab_stat); int merge_col_stat(ObOptColumnStat *src_col_stat); @@ -151,14 +148,9 @@ private: uint64_t tenant_id_; TabStatIndMap table_stats_map_; - ColStatIndMap column_stats_map_; + OSGColStatIndMap osg_col_stats_map_; - // store all part information: - // 1. all part/subpart ids - // 2. the map between subpart_id and part_id. - // 3. the map tetween part_id and tablet_id. OSGPartMap part_map_; - // ------- data struct for piece msg; ObOptStatsGatherPieceMsg piece_msg_; ObArenaAllocator arena_; diff --git a/src/sql/engine/px/datahub/components/ob_dh_opt_stats_gather.cpp b/src/sql/engine/px/datahub/components/ob_dh_opt_stats_gather.cpp index 281bf7c965..31d968ff2e 100644 --- a/src/sql/engine/px/datahub/components/ob_dh_opt_stats_gather.cpp +++ b/src/sql/engine/px/datahub/components/ob_dh_opt_stats_gather.cpp @@ -42,17 +42,14 @@ OB_DEF_DESERIALIZE(ObOptStatsGatherPieceMsg) int64_t size = 0; OB_UNIS_DECODE(size) for (int64_t i = 0; OB_SUCC(ret) && i < size; ++i) { - void *tmp_buf = arena_.alloc(sizeof(ObOptTableStat)); - if (OB_ISNULL(tmp_buf)) { + ObOptTableStat *tmp_stat = OB_NEWx(ObOptTableStat, (&arena_)); + if (OB_ISNULL(tmp_stat)) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("allocate memory failed", K(ret)); - } else { - ObOptTableStat *tmp_stat = new (tmp_buf) ObOptTableStat; - if (OB_FAIL(tmp_stat->deserialize(buf, data_len, pos))) { - LOG_WARN("deserialize datum store failed", K(ret), K(i)); - } else if (OB_FAIL(table_stats_.push_back(tmp_stat))) { - LOG_WARN("push back datum store failed", K(ret), K(i)); - } + LOG_WARN("failed to allocate memory"); + } else if (OB_FAIL(tmp_stat->deserialize(buf, data_len, pos))) { + LOG_WARN("deserialize datum store failed", K(ret), K(i)); + } else if (OB_FAIL(table_stats_.push_back(tmp_stat))) { + LOG_WARN("push back datum store failed", K(ret), K(i)); } } size = 0; @@ -60,17 +57,13 @@ OB_DEF_DESERIALIZE(ObOptStatsGatherPieceMsg) for (int64_t i = 0; OB_SUCC(ret) && i < size; ++i) { int col_stat_size = 0; OB_UNIS_DECODE(col_stat_size); - void *tmp_buf = arena_.alloc(col_stat_size); - if (OB_ISNULL(tmp_buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("allocate memory failed", K(ret)); - } else { - ObOptColumnStat *tmp_stat = new (tmp_buf) ObOptColumnStat(arena_); - if (OB_FAIL(tmp_stat->deserialize(buf, data_len, pos))) { - LOG_WARN("deserialize datum store failed", K(ret), K(i)); - } else if (OB_FAIL(column_stats_.push_back(tmp_stat))) { - LOG_WARN("push back datum store failed", K(ret), K(i)); - } + ObOptColumnStat *tmp_col_stat = OB_NEWx(ObOptColumnStat, (&arena_), arena_); + if (OB_ISNULL(tmp_col_stat)) { + LOG_WARN("failed to create new col stat"); + } else if (OB_FAIL(tmp_col_stat->deserialize(buf, data_len, pos))) { + LOG_WARN("deserialize datum store failed", K(ret), K(i)); + } else if (OB_FAIL(column_stats_.push_back(tmp_col_stat))) { + LOG_WARN("push back datum store failed", K(ret), K(i)); } } OB_UNIS_DECODE(target_osg_id_); @@ -111,7 +104,7 @@ int ObOptStatsGatherPieceMsgListener::on_message( ObOptimizerStatsGatheringOp *osg_op = NULL; piece_ctx.received_++; // get the merge osg, pass piece msg to osg. - ObOperatorKit *kit = piece_ctx.osg_info_.op_kit_; + ObOperatorKit *kit = piece_ctx.op_kit_; if (OB_ISNULL(kit) || OB_ISNULL(kit->op_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get null op kit", K(ret), K(kit)); @@ -148,7 +141,7 @@ int ObOptStatsGatherPieceMsgCtx::alloc_piece_msg_ctx(const ObOptStatsGatherPiece ret = OB_ERR_UNEXPECTED; LOG_WARN("get null operator kit", K(ret), K(pkt)); } else { - static_cast(msg_ctx)->osg_info_.op_kit_ = ctx.get_operator_kit(pkt.target_osg_id_); + static_cast(msg_ctx)->op_kit_ = ctx.get_operator_kit(pkt.target_osg_id_); } } diff --git a/src/sql/engine/px/datahub/components/ob_dh_opt_stats_gather.h b/src/sql/engine/px/datahub/components/ob_dh_opt_stats_gather.h index ca1ea88d96..e415d9cb3a 100644 --- a/src/sql/engine/px/datahub/components/ob_dh_opt_stats_gather.h +++ b/src/sql/engine/px/datahub/components/ob_dh_opt_stats_gather.h @@ -23,12 +23,6 @@ class ObOptStatsGatherPieceMsgListener; class ObOptStatsGatherPieceMsgCtx; class ObPxCoordInfo; -struct OSGInfo { - OSGInfo() = default; - ~OSGInfo() = default; - ObOperatorKit *op_kit_; -}; - class ObOptStatsGatherPieceMsg : public ObDatahubPieceMsg { @@ -46,7 +40,6 @@ public: void reset() { for (int64_t i = 0; i < column_stats_.count(); i++) { if (OB_NOT_NULL(column_stats_.at(i))) { - // to clear memory allocate by inner_alloc in ObOptColumnStat column_stats_.at(i)->reset(); column_stats_.at(i) = NULL; } @@ -105,7 +98,7 @@ class ObOptStatsGatherPieceMsgCtx : public ObPieceMsgCtx { public: ObOptStatsGatherPieceMsgCtx(uint64_t op_id, int64_t task_cnt, int64_t timeout_ts) - : ObPieceMsgCtx(op_id, task_cnt, timeout_ts), received_(0), osg_info_() {} + : ObPieceMsgCtx(op_id, task_cnt, timeout_ts), received_(0) {} ~ObOptStatsGatherPieceMsgCtx() = default; virtual void reset_resource() {}; static int alloc_piece_msg_ctx(const ObOptStatsGatherPieceMsg &pkt, @@ -115,7 +108,7 @@ public: ObPieceMsgCtx *&msg_ctx); INHERIT_TO_STRING_KV("meta", ObPieceMsgCtx, K_(received)); int received_; - OSGInfo osg_info_; // store the op_kit for OSG_MERGE. So that we can call API in OSG. + ObOperatorKit *op_kit_; private: DISALLOW_COPY_AND_ASSIGN(ObOptStatsGatherPieceMsgCtx); }; diff --git a/src/sql/engine/table/ob_table_scan_op.cpp b/src/sql/engine/table/ob_table_scan_op.cpp index 7dca2a70ec..927dd9e8e5 100644 --- a/src/sql/engine/table/ob_table_scan_op.cpp +++ b/src/sql/engine/table/ob_table_scan_op.cpp @@ -378,7 +378,8 @@ ObTableScanSpec::ObTableScanSpec(ObIAllocator &alloc, const ObPhyOperatorType ty pdml_partition_id_(NULL), agent_vt_meta_(alloc), flags_(0), - tenant_id_col_idx_(0) + tenant_id_col_idx_(0), + partition_id_calc_type_(0) { } @@ -402,7 +403,8 @@ OB_SERIALIZE_MEMBER((ObTableScanSpec, ObOpSpec), pdml_partition_id_, agent_vt_meta_, ddl_output_cids_, - tenant_id_col_idx_); + tenant_id_col_idx_, + partition_id_calc_type_); DEF_TO_STRING(ObTableScanSpec) { @@ -737,14 +739,16 @@ int ObTableScanOp::prepare_das_task() const ObTableLocation &das_location = *MY_CTDEF.das_dppr_tbl_; ObSEArray tablet_ids; ObSEArray partition_ids; + ObSEArray first_level_part_ids; if (OB_FAIL(das_location.calculate_tablet_ids(ctx_, plan_ctx->get_param_store(), tablet_ids, partition_ids, + first_level_part_ids, dtc_params))) { LOG_WARN("calculate dynamic partitions failed", K(ret)); } else { - LOG_TRACE("dynamic partitions", K(tablet_ids), K(partition_ids)); + LOG_TRACE("dynamic partitions", K(tablet_ids), K(partition_ids), K(first_level_part_ids)); } for (int64_t i = 0; OB_SUCC(ret) && i < tablet_ids.count(); ++i) { ObDASTabletLoc *tablet_loc = nullptr; @@ -931,30 +935,47 @@ OB_INLINE int ObTableScanOp::init_das_scan_rtdef(const ObDASScanCtDef &das_ctdef int ObTableScanOp::update_output_tablet_id() { int ret = OB_SUCCESS; - const ObDASTabletLoc *data_tablet_loc = MY_SPEC.should_scan_index() ? - ObDASUtils::get_related_tablet_loc(*scan_result_.get_tablet_loc(), MY_SPEC.ref_table_id_) : - scan_result_.get_tablet_loc(); - if (OB_NOT_NULL(data_tablet_loc)) { - if (is_vectorized()) { - const int64_t batch_size = MY_SPEC.max_batch_size_; - if (NULL != MY_SPEC.pdml_partition_id_) { + if (NULL != MY_SPEC.pdml_partition_id_) { + const ObDASTabletLoc *data_tablet_loc = nullptr; + int64_t output_id = OB_INVALID_ID; + if (MY_SPEC.partition_id_calc_type_ > 0) { + // partition id for gather statistics, index scan should output index partition id + data_tablet_loc = scan_result_.get_tablet_loc(); + } else if (MY_SPEC.should_scan_index()) { + data_tablet_loc = ObDASUtils::get_related_tablet_loc(*scan_result_.get_tablet_loc(), MY_SPEC.ref_table_id_); + } else { + data_tablet_loc = scan_result_.get_tablet_loc(); + } + if (OB_NOT_NULL(data_tablet_loc)) { + if (MY_SPEC.partition_id_calc_type_ == 0) { + output_id = data_tablet_loc->tablet_id_.id(); + } else if (MY_SPEC.partition_id_calc_type_ == 1) { + output_id = data_tablet_loc->first_level_part_id_ != OB_INVALID_ID ? + data_tablet_loc->first_level_part_id_ : data_tablet_loc->partition_id_; + } else if (MY_SPEC.partition_id_calc_type_ == 2) { + output_id = data_tablet_loc->partition_id_; + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid partition id cacl type", K(ret)); + } + if (OB_FAIL(ret)) { + } else if (is_vectorized()) { + const int64_t batch_size = MY_SPEC.max_batch_size_; ObExpr *expr = MY_SPEC.pdml_partition_id_; ObDatum *datums = expr->locate_datums_for_update(eval_ctx_, batch_size); for (int64_t i = 0; i < batch_size; i++) { - datums[i].set_int(data_tablet_loc->tablet_id_.id()); + datums[i].set_int(output_id); } expr->set_evaluated_projected(eval_ctx_); - LOG_TRACE("find the partition id expr in pdml table scan", K(ret), KPC(expr), KPC(data_tablet_loc)); - } - } else { - // handle PDML partition id: - // if partition id expr in TSC output_exprs, - // set the TSC partition id to the corresponding expr frame - if (NULL != MY_SPEC.pdml_partition_id_) { + LOG_TRACE("find the partition id expr in pdml table scan", K(ret), KPC(expr), KPC(data_tablet_loc), K(output_id)); + } else { + // handle PDML partition id: + // if partition id expr in TSC output_exprs, + // set the TSC partition id to the corresponding expr frame ObExpr *expr = MY_SPEC.pdml_partition_id_; - expr->locate_datum_for_write(eval_ctx_).set_int(data_tablet_loc->tablet_id_.id()); + expr->locate_datum_for_write(eval_ctx_).set_int(output_id); expr->set_evaluated_projected(eval_ctx_); - LOG_TRACE("find the partition id expr in pdml table scan", K(ret), KPC(data_tablet_loc)); + LOG_TRACE("find the partition id expr in pdml table scan", K(ret), KPC(data_tablet_loc), K(output_id)); } } } diff --git a/src/sql/engine/table/ob_table_scan_op.h b/src/sql/engine/table/ob_table_scan_op.h index 0c736b6478..49467e9d92 100644 --- a/src/sql/engine/table/ob_table_scan_op.h +++ b/src/sql/engine/table/ob_table_scan_op.h @@ -354,6 +354,7 @@ public: }; }; int64_t tenant_id_col_idx_; + int64_t partition_id_calc_type_; }; class ObTableScanOp : public ObOperator diff --git a/src/sql/engine/window_function/ob_window_function_op.h b/src/sql/engine/window_function/ob_window_function_op.h index 557eb72274..79bd7e3bbd 100644 --- a/src/sql/engine/window_function/ob_window_function_op.h +++ b/src/sql/engine/window_function/ob_window_function_op.h @@ -560,7 +560,7 @@ public: AggrCell(WinFuncInfo &wf_info, ObWindowFunctionOp &op, ObIArray &aggr_infos) : WinFuncCell(wf_info, op), finish_prepared_(false), - aggr_processor_(op_.eval_ctx_, aggr_infos, "WindowAggProc"), + aggr_processor_(op_.eval_ctx_, aggr_infos, "WindowAggProc", op.get_monitor_info()), result_(), got_result_(false), remove_type_(wf_info.remove_type_) diff --git a/src/sql/ob_sql.cpp b/src/sql/ob_sql.cpp index 0fa86055ab..50320758fb 100644 --- a/src/sql/ob_sql.cpp +++ b/src/sql/ob_sql.cpp @@ -3751,6 +3751,11 @@ int ObSql::get_outline_data(ObSqlCtx &context, LOG_WARN("get null session info", K(ret)); } else if (0 != context.first_plan_hash_) { outline_content = context.first_outline_data_; + } else if (OB_INVALID_ID == context.spm_ctx_.bl_key_.db_id_ + || context.multi_stmt_item_.is_batched_multi_stmt()) { + //no outline is available when database name of session is not specified, just keep the stmt + } else if (pc_ctx.is_begin_commit_stmt()) { + /* do nothing */ } else if (OB_FAIL(get_outline_data(pc_ctx, signature_sql, outline_state, outline_content))) { LOG_WARN("failed to get outline data", K(ret)); } diff --git a/src/sql/optimizer/ob_insert_log_plan.cpp b/src/sql/optimizer/ob_insert_log_plan.cpp index f8dea00778..871f094185 100644 --- a/src/sql/optimizer/ob_insert_log_plan.cpp +++ b/src/sql/optimizer/ob_insert_log_plan.cpp @@ -39,17 +39,11 @@ using namespace oceanbase::sql::log_op_def; int ObInsertLogPlan::generate_normal_raw_plan() { int ret = OB_SUCCESS; - ObSQLSessionInfo *session_info = NULL; const ObInsertStmt *insert_stmt = get_stmt(); - if (OB_ISNULL(insert_stmt) || OB_ISNULL(session_info = get_optimizer_context().get_session_info()) || - OB_ISNULL(get_optimizer_context().get_query_ctx())) { + if (OB_ISNULL(insert_stmt)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(session_info), K(insert_stmt), K(ret)); + LOG_WARN("get unexpected error"); } else { - LOG_TRACE("start to allocate operators for ", "sql", get_optimizer_context().get_query_ctx()->get_sql_stmt()); - bool need_osg = false; - const ObInsertStmt *insert_stmt = get_stmt(); - LOG_TRACE("start to allocate operators for ", "sql", get_optimizer_context().get_query_ctx()->get_sql_stmt()); OPT_TRACE("generate plan for ", get_stmt()); if (!insert_stmt->value_from_select()) { // insert into values xxxx @@ -73,11 +67,15 @@ int ObInsertLogPlan::generate_normal_raw_plan() } // allocal optimizer stat gather operator. + bool need_osg = false; + OSGShareInfo osg_info; if (OB_SUCC(ret)) { if (OB_FAIL(check_need_online_stats_gather(need_osg))) { LOG_WARN("fail to check wether we need optimizer stats gathering operator", K(ret)); } else if (need_osg) { - if (OB_FAIL(candi_allocate_optimizer_stats_gathering())) { + if (OB_FAIL(generate_osg_share_info(osg_info))) { + LOG_WARN("fail to generated share info", K(ret)); + } else if (OB_FAIL(candi_allocate_optimizer_stats_gathering(osg_info))) { LOG_WARN("failed to allocate optimizer stats gathering", K(ret)); } } @@ -123,7 +121,7 @@ int ObInsertLogPlan::generate_normal_raw_plan() * all information collection by other OSG. */ if (OB_SUCC(ret) && need_osg) { - if (OB_FAIL(candi_allocate_root_optimizer_stats_gathering())) { + if (OB_FAIL(candi_allocate_root_optimizer_stats_gathering(osg_info))) { LOG_WARN("fail to allcate osg on top", K(ret)); } else { LOG_TRACE("succeed to allocate optimizer stat gather", @@ -1310,17 +1308,14 @@ int ObInsertLogPlan::build_column_conv_for_shadow_pk(const ObInsertTableInfo& ta return ret; } -int ObInsertLogPlan::candi_allocate_optimizer_stats_gathering() +int ObInsertLogPlan::candi_allocate_optimizer_stats_gathering(const OSGShareInfo &osg_info) { int ret = OB_SUCCESS; ObExchangeInfo exch_info; CandidatePlan best_plan; ObSEArray stats_gathering_plan; ObSEArray best_candidates; - OSGShareInfo osg_info; - if (OB_FAIL(generate_osg_share_info(osg_info))) { - LOG_WARN("fail to generated share info", K(ret)); - } else if (OB_FAIL(get_minimal_cost_candidates(candidates_.candidate_plans_, best_candidates))) { + if (OB_FAIL(get_minimal_cost_candidates(candidates_.candidate_plans_, best_candidates))) { LOG_WARN("failed to get minimal cost candidates", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < best_candidates.count(); ++i) { @@ -1346,17 +1341,14 @@ int ObInsertLogPlan::candi_allocate_optimizer_stats_gathering() return ret; } -int ObInsertLogPlan::candi_allocate_root_optimizer_stats_gathering() +int ObInsertLogPlan::candi_allocate_root_optimizer_stats_gathering(const OSGShareInfo &osg_info) { int ret = OB_SUCCESS; ObExchangeInfo exch_info; CandidatePlan candidate_plan; ObSEArray stats_gathering_plan; ObSEArray best_candidates; - OSGShareInfo osg_info; - if (OB_FAIL(generate_osg_share_info(osg_info))) { - LOG_WARN("fail to generated share info", K(ret)); - } else if (OB_FAIL(get_minimal_cost_candidates(candidates_.candidate_plans_, best_candidates))) { + if (OB_FAIL(get_minimal_cost_candidates(candidates_.candidate_plans_, best_candidates))) { LOG_WARN("failed to get minimal cost candidates", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < best_candidates.count(); i++) { @@ -1364,7 +1356,7 @@ int ObInsertLogPlan::candi_allocate_root_optimizer_stats_gathering() if (OB_ISNULL(best_candidates.at(i).plan_tree_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); - } else if (best_candidates.at(i).plan_tree_->get_allocated_osg()) { + } else if (best_candidates.at(i).plan_tree_->has_allocated_osg()) { if (best_candidates.at(i).plan_tree_->is_sharding() && best_candidates.at(i).plan_tree_->get_phy_plan_type() != ObPhyPlanType::OB_PHY_PLAN_REMOTE && OB_FAIL(allocate_exchange_as_top(best_candidates.at(i).plan_tree_, exch_info))) { @@ -1389,7 +1381,7 @@ int ObInsertLogPlan::candi_allocate_root_optimizer_stats_gathering() int ObInsertLogPlan::allocate_optimizer_stats_gathering_as_top(ObLogicalOperator *&old_top, OSG_TYPE type, - OSGShareInfo &info) + const OSGShareInfo &info) { int ret = OB_SUCCESS; ObLogOptimizerStatsGathering *osg = NULL; @@ -1397,7 +1389,7 @@ int ObInsertLogPlan::allocate_optimizer_stats_gathering_as_top(ObLogicalOperator ret = OB_INVALID_ARGUMENT; LOG_WARN("Get unexpected null", K(ret), K(old_top)); } else if (OB_ISNULL(osg = static_cast(get_log_op_factory(). - allocate(*this, LOG_OPTIMIZER_STATS_GATHERING)))) { + allocate(*this, log_op_def::LOG_OPTIMIZER_STATS_GATHERING)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to allocate sequence operator", K(ret)); } else { @@ -1405,13 +1397,15 @@ int ObInsertLogPlan::allocate_optimizer_stats_gathering_as_top(ObLogicalOperator osg->set_osg_type(type); osg->set_table_id(info.table_id_); osg->set_part_level(info.part_level_); - osg->set_calc_part_id_expr(info.calc_part_id_expr_); osg->set_generated_column_exprs(info.generated_column_exprs_); osg->set_col_conv_exprs(info.col_conv_exprs_); osg->set_column_ids(info.column_ids_); if (type == OSG_TYPE::GATHER_OSG) { osg->set_allocated_osg(true); } + if (type != OSG_TYPE::MERGE_OSG) { + osg->set_calc_part_id_expr(info.calc_part_id_expr_); + } if (OB_FAIL(osg->compute_property())) { LOG_WARN("failed to compute property", K(ret)); } else { @@ -1444,15 +1438,13 @@ int ObInsertLogPlan::generate_osg_share_info(OSGShareInfo &info) const ObInsertTableInfo& table_info = stmt->get_insert_table_info(); ObSEArray generated_column_ids; info.table_id_ = ref_table_id; - // generated calc_part_id; if (tab_schema->is_partitioned_table()) { info.part_level_ = tab_schema->get_part_level(); - // should init calc_part_id here. - if (OB_FAIL(ObOptimizerUtil::init_calc_part_id_expr(this, - table_id, - ref_table_id, - info.calc_part_id_expr_))) { - LOG_WARN("failed to init calc part id", K(ret)); + if (OB_FAIL(gen_calc_part_id_expr(table_id, + ref_table_id, + CALC_PARTITION_TABLET_ID, + info.calc_part_id_expr_))) { + LOG_WARN("failed to gen calc part id expr"); } else if (OB_FAIL(ObOptimizerUtil::replace_column_with_select_for_partid(stmt, get_optimizer_context(), info.calc_part_id_expr_))) { diff --git a/src/sql/optimizer/ob_insert_log_plan.h b/src/sql/optimizer/ob_insert_log_plan.h index 44393e1f34..61945321df 100644 --- a/src/sql/optimizer/ob_insert_log_plan.h +++ b/src/sql/optimizer/ob_insert_log_plan.h @@ -65,12 +65,12 @@ protected: ObShardingInfo *insert_sharding, bool is_multi_part); int candi_allocate_pdml_insert(); - int candi_allocate_optimizer_stats_gathering(); - int candi_allocate_root_optimizer_stats_gathering(); + int candi_allocate_optimizer_stats_gathering(const OSGShareInfo &osg_info); + int candi_allocate_root_optimizer_stats_gathering(const OSGShareInfo &osg_info); int allocate_optimizer_stats_gathering_as_top(ObLogicalOperator *&old_top, OSG_TYPE type, - OSGShareInfo &info); + const OSGShareInfo &osg_info); int generate_osg_share_info(OSGShareInfo &info); virtual int check_insert_need_multi_partition_dml(ObLogicalOperator &top, diff --git a/src/sql/optimizer/ob_log_del_upd.cpp b/src/sql/optimizer/ob_log_del_upd.cpp index f75fc31070..b2af5b6a02 100644 --- a/src/sql/optimizer/ob_log_del_upd.cpp +++ b/src/sql/optimizer/ob_log_del_upd.cpp @@ -503,7 +503,6 @@ int ObLogDelUpd::find_pdml_part_id_producer(ObLogicalOperator &op, producer = &op; } } else if (op.get_type() == log_op_def::LOG_TABLE_SCAN) { - // PDML partition id expr在table scan分配的逻辑 // pdml table scan分配partition id expr的producer // table scan中分配partition id expr的producer的逻辑比较特殊： @@ -521,14 +520,12 @@ int ObLogDelUpd::find_pdml_part_id_producer(ObLogicalOperator &op, // 其会被裁剪掉，因此目前insert与subplan之间会添加一个EX算子. // 后期会进行优化，如果insert与subplan是一个full partition wise // join，那么就在insert算子上分配一个GI算子，目前先使用在subplan上分配EX算子的方式实现 - ObLogTableScan &tsc = static_cast(op); if (tid == (tsc.get_is_index_global() ? tsc.get_index_table_id() : tsc.get_ref_table_id())) { producer = &op; } } - for (int64_t i = 0; OB_SUCC(ret) && NULL == producer && i < op.get_num_of_child(); i++) { if (OB_ISNULL(op.get_child(i))) { ret = OB_ERR_UNEXPECTED; diff --git a/src/sql/optimizer/ob_log_optimizer_stats_gathering.cpp b/src/sql/optimizer/ob_log_optimizer_stats_gathering.cpp index 60110566a6..d8702d01d8 100644 --- a/src/sql/optimizer/ob_log_optimizer_stats_gathering.cpp +++ b/src/sql/optimizer/ob_log_optimizer_stats_gathering.cpp @@ -37,11 +37,7 @@ const char *ObLogOptimizerStatsGathering::get_name() const "OPTIMIZER STATS MERGE", "OPTIMIZER STATS GATHER", }; - int name_idx = 1; - if (osg_type_ == OSG_TYPE::MERGE_OSG) { - name_idx = 0; - } - return stats_gathering_name_[name_idx]; + return stats_gathering_name_[osg_type_ == OSG_TYPE::MERGE_OSG ? 0 : 1]; } int ObLogOptimizerStatsGathering::get_op_exprs(ObIArray &all_exprs) { diff --git a/src/sql/optimizer/ob_log_optimizer_stats_gathering.h b/src/sql/optimizer/ob_log_optimizer_stats_gathering.h index 74e7e56879..785dec34b8 100644 --- a/src/sql/optimizer/ob_log_optimizer_stats_gathering.h +++ b/src/sql/optimizer/ob_log_optimizer_stats_gathering.h @@ -71,14 +71,7 @@ public: inline share::schema::ObPartitionLevel get_part_level() { return part_level_; }; inline void set_part_level(share::schema::ObPartitionLevel level) { part_level_ = level; }; inline ObRawExpr *&get_calc_part_id_expr() { return calc_part_id_expr_; }; - int init_calc_part_id_expr(); int get_target_osg_id(uint64_t &target_id); - int add_col_conv_expr(ObRawExpr *expr) { - return col_conv_exprs_.push_back(expr); - } - int add_generated_column_expr(ObRawExpr *expr) { - return generated_column_exprs_.push_back(expr); - } int add_column_id(uint64_t column_id) { return column_ids_.push_back(column_id); } diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index 4f631a9f15..a89be8fa8c 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -101,6 +101,7 @@ ObLogPlan::ObLogPlan(ObOptimizerContext &ctx, const ObDMLStmt *stmt) candidates_(), group_replaced_exprs_(), window_function_replaced_exprs_(), + stat_partition_id_expr_(nullptr), query_ref_(NULL), root_(NULL), sql_text_(), @@ -11531,6 +11532,10 @@ int ObLogPlan::adjust_final_plan_info(ObLogicalOperator *&op) LOG_WARN("failed to perform window function push down", K(ret)); } else if (OB_FAIL(op->get_plan()->perform_adjust_onetime_expr(op))) { LOG_WARN("failed to perform adjust onetime expr", K(ret)); + } else if (GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_4_2_0_0 && + get_optimizer_context().get_query_ctx()->get_global_hint().has_dbms_stats_hint() && + OB_FAIL(op->get_plan()->perform_gather_stat_replace(op))) { + LOG_WARN("failed to perform gather stat replace"); } else if (OB_FAIL(update_re_est_cost(op))) { LOG_WARN("failed to re est cost", K(ret)); } else if (OB_FAIL(op->reorder_filter_exprs())) { @@ -12769,6 +12774,8 @@ int ObLogPlan::get_cache_calc_part_id_expr(int64_t table_id, int64_t ref_table_i calc_type == part_id_expr.calc_type_) { expr = part_id_expr.calc_part_id_expr_; find = true; + LOG_TRACE("succeed to get chache calc_part_id_expr", + K(table_id), K(ref_table_id), K(calc_type), K(*expr)); } } return ret; @@ -13531,3 +13538,44 @@ int ObLogPlan::fill_join_filter_info(JoinFilterInfo &join_filter_info) } return ret; } + +int ObLogPlan::perform_gather_stat_replace(ObLogicalOperator *op) +{ + int ret = OB_SUCCESS; + ObLogTableScan *table_scan = NULL; + ObLogGroupBy *group_by = NULL; + if (NULL != (table_scan = dynamic_cast(op))) { + ObOpPseudoColumnRawExpr *partition_id_expr = nullptr; + if (OB_FAIL(table_scan->generate_pseudo_partition_id_expr(partition_id_expr))) { + LOG_WARN("fail allocate part id expr", K(table_id), K(ret)); + } else { + stat_partition_id_expr_ = partition_id_expr; + stat_table_scan_ = table_scan; + table_scan->set_tablet_id_expr(partition_id_expr); + } + } else { + if (NULL != (group_by = dynamic_cast(op))) { + if (group_by->get_rollup_exprs().empty() && group_by->get_group_by_exprs().count() == 1) { + ObRawExpr* group_by_expr = group_by->get_group_by_exprs().at(0); + if (OB_ISNULL(group_by_expr) || OB_ISNULL(stat_partition_id_expr_) || OB_ISNULL(stat_table_scan_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(group_by_expr), K(stat_partition_id_expr_), K(stat_table_scan_)); + } else if (T_FUN_SYS_CALC_PARTITION_ID != group_by_expr->get_expr_type()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected group by expr", KPC(group_by_expr)); + } else if (OB_FAIL(stat_gather_replaced_exprs_.push_back( + std::pair(group_by_expr, stat_partition_id_expr_)))) { + LOG_WARN("failed to push back replaced expr", K(ret)); + } else if (group_by_expr->get_partition_id_calc_type() == CALC_IGNORE_SUB_PART) { + stat_table_scan_->set_tablet_id_type(1); + } else { + stat_table_scan_->set_tablet_id_type(2); + } + } + } + if (OB_SUCC(ret) && OB_FAIL(op->replace_op_exprs(stat_gather_replaced_exprs_))) { + LOG_WARN("failed to replace generated aggr expr", K(ret)); + } + } + return ret; +} \ No newline at end of file diff --git a/src/sql/optimizer/ob_log_plan.h b/src/sql/optimizer/ob_log_plan.h index b3104480f1..bebf0a4094 100644 --- a/src/sql/optimizer/ob_log_plan.h +++ b/src/sql/optimizer/ob_log_plan.h @@ -1324,6 +1324,8 @@ public: int fill_join_filter_info(JoinFilterInfo &join_filter_info); + int perform_gather_stat_replace(ObLogicalOperator *op); + protected: virtual int generate_normal_raw_plan() = 0; virtual int generate_dblink_raw_plan(); @@ -1733,6 +1735,11 @@ protected: // member variable common::ObSEArray, 4, common::ModulePageAllocator, true > window_function_replaced_exprs_; common::ObSEArray, 4, common::ModulePageAllocator, true > gen_col_replaced_exprs_; + common::ObSEArray, 1, common::ModulePageAllocator, true > stat_gather_replaced_exprs_; + // used for gather statistic begin + ObRawExpr* stat_partition_id_expr_; + ObLogTableScan* stat_table_scan_; + // used for gather statistics end //上层stmt条件下推下来的谓词，已经抽出？ common::ObSEArray pushdown_filters_; common::ObSEArray startup_filters_; diff --git a/src/sql/optimizer/ob_log_table_scan.h b/src/sql/optimizer/ob_log_table_scan.h index 9e89014c35..c30b5af6b3 100644 --- a/src/sql/optimizer/ob_log_table_scan.h +++ b/src/sql/optimizer/ob_log_table_scan.h @@ -75,6 +75,7 @@ public: use_batch_(false), access_path_(NULL), tablet_id_expr_(NULL), + tablet_id_type_(0), calc_part_id_expr_(NULL), global_index_back_table_partition_info_(NULL), has_index_scan_filter_(false), @@ -414,6 +415,8 @@ public: inline const AccessPath* get_access_path() const { return access_path_; } void set_tablet_id_expr(ObOpPseudoColumnRawExpr *expr) { tablet_id_expr_ = expr; } ObOpPseudoColumnRawExpr *get_tablet_id_expr() const { return tablet_id_expr_; } + void set_tablet_id_type(int64_t type) { tablet_id_type_ = type; } + int64_t get_tablet_id_type() const { return tablet_id_type_; } const common::ObIArray &get_rowkey_exprs() const { return rowkey_exprs_; } const common::ObIArray &get_part_exprs() const { return part_exprs_; } inline const ObRawExpr *get_calc_part_id_expr() const { return calc_part_id_expr_; } @@ -547,6 +550,9 @@ protected: // memeber variables bool use_batch_; AccessPath *access_path_; ObOpPseudoColumnRawExpr *tablet_id_expr_; + // decide tablet_id_expr should reture which id + // 0 for tablet id, 1 for logical part id, 2 for logical subpart id + int64_t tablet_id_type_; ObRawExpr *calc_part_id_expr_; // begin for global index lookup diff --git a/src/sql/optimizer/ob_logical_operator.cpp b/src/sql/optimizer/ob_logical_operator.cpp index 24e55373a2..ae802121fb 100644 --- a/src/sql/optimizer/ob_logical_operator.cpp +++ b/src/sql/optimizer/ob_logical_operator.cpp @@ -882,7 +882,7 @@ int ObLogicalOperator::compute_op_other_info() ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); } else { - allocated_osg_ |= get_child(i)->get_allocated_osg(); + allocated_osg_ |= get_child(i)->has_allocated_osg(); } } } diff --git a/src/sql/optimizer/ob_logical_operator.h b/src/sql/optimizer/ob_logical_operator.h index 71dbfc9cb2..50cdedede8 100644 --- a/src/sql/optimizer/ob_logical_operator.h +++ b/src/sql/optimizer/ob_logical_operator.h @@ -1480,7 +1480,7 @@ public: inherit_sharding_index_ = inherit_sharding_index; } - inline bool get_allocated_osg() const { return allocated_osg_; } + inline bool has_allocated_osg() const { return allocated_osg_; } inline void set_allocated_osg(bool allocated_osg) { allocated_osg_ = allocated_osg; @@ -1635,6 +1635,8 @@ public: const ObIArray &exec_params, ObIArray &left_above_params, ObIArray &right_above_params); + // 生成 partition id 表达式 + int generate_pseudo_partition_id_expr(ObOpPseudoColumnRawExpr *&expr); public: ObSEArray child_; ObSEArray equal_param_constraints_; @@ -1675,8 +1677,6 @@ protected: const bool two_level, char *buf, int64_t &buf_len, int64_t &pos); protected: - // 生成 partition id 表达式 - int generate_pseudo_partition_id_expr(ObOpPseudoColumnRawExpr *&expr); void add_dist_flag(uint64_t &flags, DistAlgo method) const { flags |= method; diff --git a/src/sql/optimizer/ob_opt_selectivity.cpp b/src/sql/optimizer/ob_opt_selectivity.cpp index cba8631ab8..519491202c 100644 --- a/src/sql/optimizer/ob_opt_selectivity.cpp +++ b/src/sql/optimizer/ob_opt_selectivity.cpp @@ -2343,11 +2343,11 @@ int ObOptSelectivity::calc_column_range_selectivity(const OptTableMetas &table_m ObObj maxscalar; ObObj startscalar; ObObj endscalar; - const ObObj *new_start_obj = NULL; - const ObObj *new_end_obj = NULL; + ObObj *new_start_obj = NULL; + ObObj *new_end_obj = NULL; ObArenaAllocator tmp_alloc("ObOptSel"); - if (OB_FAIL(convert_valid_obj_for_opt_stats(&start_obj, tmp_alloc, new_start_obj)) || - OB_FAIL(convert_valid_obj_for_opt_stats(&end_obj, tmp_alloc, new_end_obj))) { + if (OB_FAIL(ObOptimizerUtil::truncate_string_for_opt_stats(&start_obj, tmp_alloc, new_start_obj)) || + OB_FAIL(ObOptimizerUtil::truncate_string_for_opt_stats(&end_obj, tmp_alloc, new_end_obj))) { LOG_WARN("failed to convert valid obj for opt stats", K(ret), K(start_obj), K(end_obj), KPC(new_start_obj), KPC(new_end_obj)); } else if (OB_ISNULL(new_start_obj) || OB_ISNULL(new_end_obj)) { @@ -3586,9 +3586,9 @@ int ObOptSelectivity::get_equal_pred_sel(const ObHistogram &histogram, int ret = OB_SUCCESS; int64_t idx = -1; bool is_equal = false; - const ObObj *new_value = NULL; + ObObj *new_value = NULL; ObArenaAllocator tmp_alloc("ObOptSel"); - if (OB_FAIL(convert_valid_obj_for_opt_stats(&value, tmp_alloc, new_value))) { + if (OB_FAIL(ObOptimizerUtil::truncate_string_for_opt_stats(&value, tmp_alloc, new_value))) { LOG_WARN("failed to convert valid obj for opt stats", K(ret), K(value), KPC(new_value)); } else if (OB_ISNULL(new_value)) { ret = OB_ERR_UNEXPECTED; @@ -3632,11 +3632,11 @@ int ObOptSelectivity::get_range_sel_by_histogram(const ObHistogram &histogram, 1 == startkey.get_obj_cnt()) { const ObObj *startobj = &startkey.get_obj_ptr()[0]; const ObObj *endobj = &endkey.get_obj_ptr()[0]; - const ObObj *new_startobj = NULL; - const ObObj *new_endobj = NULL; + ObObj *new_startobj = NULL; + ObObj *new_endobj = NULL; ObArenaAllocator tmp_alloc("ObOptSel"); - if (OB_FAIL(convert_valid_obj_for_opt_stats(startobj, tmp_alloc, new_startobj)) || - OB_FAIL(convert_valid_obj_for_opt_stats(endobj, tmp_alloc, new_endobj))) { + if (OB_FAIL(ObOptimizerUtil::truncate_string_for_opt_stats(startobj, tmp_alloc, new_startobj)) || + OB_FAIL(ObOptimizerUtil::truncate_string_for_opt_stats(endobj, tmp_alloc, new_endobj))) { LOG_WARN("failed to convert valid obj for opt stats", K(ret), KPC(startobj), KPC(endobj), KPC(new_startobj), KPC(new_endobj)); } else if (OB_ISNULL(new_startobj) || OB_ISNULL(new_endobj)) { @@ -4367,20 +4367,6 @@ int ObOptSelectivity::get_join_pred_rows(const ObHistogram &left_hist, return ret; } -int ObOptSelectivity::convert_valid_obj_for_opt_stats(const ObObj *old_obj, - ObIAllocator &alloc, - const ObObj *&new_obj) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(ObOptimizerUtil::truncate_string_for_opt_stats(old_obj, - alloc, - new_obj))) { - LOG_WARN("fail to truncated string", K(ret)); - } - LOG_TRACE("Succeed to convert valid obj for opt stats", KPC(old_obj), KPC(new_obj)); - return ret; -} - //will useful in dynamic sampling join in the future // int ObOptSelectivity::calculate_join_selectivity_by_dynamic_sampling(const OptTableMetas &table_metas, // const OptSelectivityCtx &ctx, diff --git a/src/sql/optimizer/ob_opt_selectivity.h b/src/sql/optimizer/ob_opt_selectivity.h index 5a8ed675ea..d8cff6474a 100644 --- a/src/sql/optimizer/ob_opt_selectivity.h +++ b/src/sql/optimizer/ob_opt_selectivity.h @@ -883,10 +883,6 @@ private: const bool is_semi, double &rows); - static int convert_valid_obj_for_opt_stats(const ObObj *old_obj, - ObIAllocator &alloc, - const ObObj *&new_obj); - static int calc_complex_predicates_selectivity_by_ds(const OptTableMetas &table_metas, const OptSelectivityCtx &ctx, const ObIArray &predicates, diff --git a/src/sql/optimizer/ob_optimizer.cpp b/src/sql/optimizer/ob_optimizer.cpp index 5ab343586f..ee261edc38 100644 --- a/src/sql/optimizer/ob_optimizer.cpp +++ b/src/sql/optimizer/ob_optimizer.cpp @@ -655,7 +655,7 @@ int ObOptimizer::init_env_info(ObDMLStmt &stmt) } else if (OB_FAIL(session->is_serial_set_order_forced(force_serial_set_order, lib::is_oracle_mode()))) { LOG_WARN("fail to get force_serial_set_order", K(ret)); } else if (OB_FAIL(check_force_default_stat())) { - LOG_WARN("failed to check force default stat", K(ret)); + LOG_WARN("failed to check force default stat"); } else if (OB_FAIL(calc_link_stmt_count(*target_stmt, link_stmt_count))) { LOG_WARN("calc link stmt count failed", K(ret)); } else { @@ -1066,6 +1066,10 @@ int ObOptimizer::check_force_default_stat() is_restore))) { LOG_WARN("fail to check if tenant is restore", K(session->get_effective_tenant_id())); } else if (is_restore) { + // 为避免物理恢复阶段系统表恢复过程中，SQL依赖需要恢复的统计信息表， + // 对恢复中租户，仅需获取缺省统计信息即可 + ctx_.set_use_default_stat(); + } else if (query_ctx->get_global_hint().has_dbms_stats_hint()) { ctx_.set_use_default_stat(); } else if (OB_FAIL(query_ctx->get_global_hint().opt_params_.get_bool_opt_param(ObOptParamHint::USE_DEFAULT_OPT_STAT, use_default_opt_stat, diff --git a/src/sql/optimizer/ob_optimizer_util.cpp b/src/sql/optimizer/ob_optimizer_util.cpp index ca02d3d215..da5ce7d40e 100644 --- a/src/sql/optimizer/ob_optimizer_util.cpp +++ b/src/sql/optimizer/ob_optimizer_util.cpp @@ -8524,74 +8524,6 @@ int ObOptimizerUtil::check_pushdown_join_filter_for_set(const ObSelectStmt &pare return ret; } - -int ObOptimizerUtil::init_calc_part_id_expr(ObLogPlan * log_plan, - const uint64_t table_id, - const uint64_t ref_table_id, - ObRawExpr *&calc_part_id_expr) -{ - int ret = OB_SUCCESS; - calc_part_id_expr = NULL; - share::schema::ObPartitionLevel part_level = share::schema::PARTITION_LEVEL_MAX; - ObSQLSessionInfo *session = NULL; - ObRawExpr *part_expr = NULL; - ObRawExpr *subpart_expr = NULL; - ObRawExpr *new_part_expr = NULL; - ObRawExpr *new_subpart_expr = NULL; - if (OB_ISNULL(log_plan) || OB_UNLIKELY(OB_INVALID_ID == ref_table_id)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get invalid argument", K(ret), K(ref_table_id)); - } else if (OB_ISNULL(session = log_plan->get_optimizer_context().get_session_info())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("session info is null", K(ret)); - } else { - share::schema::ObSchemaGetterGuard *schema_guard = NULL; - const share::schema::ObTableSchema *table_schema = NULL; - if (OB_ISNULL(schema_guard = log_plan->get_optimizer_context().get_schema_guard())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("NULL ptr", K(ret)); - } else if (OB_FAIL(schema_guard->get_table_schema( - session->get_effective_tenant_id(), - ref_table_id, table_schema))) { - LOG_WARN("get table schema failed", K(ref_table_id), K(ret)); - } else if (OB_ISNULL(table_schema)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table schema is null", K(ret), K(table_schema)); - } else if (OB_FAIL(log_plan->gen_calc_part_id_expr(table_id, - ref_table_id, - CALC_PARTITION_TABLET_ID, - calc_part_id_expr))) { - LOG_WARN("failed to build calc part id expr", K(ret)); - } else if (!table_schema->is_heap_table() && - OB_NOT_NULL(calc_part_id_expr) && - OB_FAIL(replace_gen_column(log_plan, calc_part_id_expr, calc_part_id_expr))) { - LOG_WARN("failed to replace gen column", K(ret)); - } else { - // For no-pk table partitioned by generated column, it is no need to replace generated - // column as dependent exprs, because index table scan will add dependant columns - // into access_exprs_ - // eg: - // create table t1(c1 int, c2 int, c3 int generated always as (c1 + 1)) partition by hash(c3); - // create index idx on t1(c2) global; - // select /*+ index(t1 idx) */ * from t1\G - // TLU - // TSC // output([pk_inc],[calc_part_id_expr(c3)], [c1]), access([pk_inc], [c1]) - - // For pk table partitioned by generated column, we can replace generated column by - // dependant exprs, because pk must be a superset of partition columns - // eg: - // create table t1(c1 int primary key, c2 int, c3 int generated always as (c1 + 1)) partition by hash(c3); - // create index idx on t1(c2) global; - // select /*+ index(t1 idx) */ * from t1\G - // TLU - // TSC // output([c1],[calc_part_id_expr(c1 + 1)]), access([c1]) - LOG_TRACE("log table scan init calc part id expr", KPC(calc_part_id_expr)); - } - } - - return ret; -} - int ObOptimizerUtil::replace_column_with_select_for_partid(const ObInsertStmt *stmt, ObOptimizerContext &opt_ctx, ObRawExpr *&calc_part_id_expr) @@ -8664,11 +8596,14 @@ int ObOptimizerUtil::replace_gen_column(ObLogPlan *log_plan, ObRawExpr *part_exp int ObOptimizerUtil::truncate_string_for_opt_stats(const ObObj *old_obj, ObIAllocator &alloc, - const ObObj *&new_obj) + ObObj *&new_obj) { int ret = OB_SUCCESS; bool is_truncated = false; - if (old_obj->is_string_type()) { + if (OB_ISNULL(old_obj)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null"); + } else if (old_obj->is_string_type()) { ObString str; if (OB_FAIL(old_obj->get_string(str))) { LOG_WARN("failed to get string", K(ret), K(str)); @@ -8698,7 +8633,8 @@ int ObOptimizerUtil::truncate_string_for_opt_stats(const ObObj *old_obj, } } if (OB_SUCC(ret) && !is_truncated) { - new_obj = old_obj; + new_obj = const_cast(old_obj); } + LOG_TRACE("Succeed to truncate string obj for opt stats", KPC(old_obj), KPC(new_obj), K(is_truncated)); return ret; } diff --git a/src/sql/optimizer/ob_optimizer_util.h b/src/sql/optimizer/ob_optimizer_util.h index 98a1c80ec6..8ba88fa89d 100644 --- a/src/sql/optimizer/ob_optimizer_util.h +++ b/src/sql/optimizer/ob_optimizer_util.h @@ -1426,11 +1426,6 @@ public: ObIArray &candi_left_quals, ObIArray &candi_right_quals); - static int init_calc_part_id_expr(ObLogPlan * log_plan, - const uint64_t table_id, - const uint64_t ref_table_id, - ObRawExpr *&calc_part_id_expr); - static int replace_gen_column(ObLogPlan *log_plan, ObRawExpr *part_expr, ObRawExpr *&new_part_expr); @@ -1443,7 +1438,7 @@ public: static int truncate_string_for_opt_stats(const ObObj *old_obj, ObIAllocator &alloc, - const ObObj *&new_obj); + ObObj *&new_obj); private: //disallow construct diff --git a/src/sql/optimizer/ob_phy_table_location_info.cpp b/src/sql/optimizer/ob_phy_table_location_info.cpp index b87c79c26a..2c2d757b2f 100644 --- a/src/sql/optimizer/ob_phy_table_location_info.cpp +++ b/src/sql/optimizer/ob_phy_table_location_info.cpp @@ -24,6 +24,7 @@ namespace sql ObOptTabletLoc::ObOptTabletLoc() : partition_id_(OB_INVALID_INDEX), + first_level_part_id_(OB_INVALID_INDEX), replica_locations_(ObModIds::OB_SQL_OPTIMIZER_LOCATION_CACHE, OB_MALLOC_NORMAL_BLOCK_SIZE), renew_time_(0) { @@ -36,6 +37,7 @@ ObOptTabletLoc::~ObOptTabletLoc() void ObOptTabletLoc::reset() { partition_id_ = OB_INVALID_INDEX; + first_level_part_id_ = OB_INVALID_INDEX; tablet_id_.reset(); ls_id_.reset(); replica_locations_.reset(); @@ -48,6 +50,7 @@ int ObOptTabletLoc::assign(const ObOptTabletLoc &other) tablet_id_ = other.tablet_id_; ls_id_ = other.ls_id_; partition_id_ = other.partition_id_; + first_level_part_id_ = other.first_level_part_id_; renew_time_ = other.renew_time_; if (OB_FAIL(replica_locations_.assign(other.replica_locations_))) { LOG_WARN("Failed to assign replica locations", K(ret)); @@ -55,8 +58,8 @@ int ObOptTabletLoc::assign(const ObOptTabletLoc &other) return ret; } -int ObOptTabletLoc::assign_with_only_readable_replica( - const ObObjectID &partition_id, +int ObOptTabletLoc::assign_with_only_readable_replica(const ObObjectID &partition_id, + const ObObjectID &first_level_part_id, const common::ObTabletID &tablet_id, const ObLSLocation &ls_location, const ObIArray &invalid_servers) @@ -64,6 +67,7 @@ int ObOptTabletLoc::assign_with_only_readable_replica( int ret = OB_SUCCESS; reset(); partition_id_ = partition_id; + first_level_part_id_ = first_level_part_id; tablet_id_ = tablet_id; ls_id_ = ls_location.get_ls_id(); renew_time_ = ls_location.get_renew_time(); @@ -330,6 +334,7 @@ int ObCandiTabletLoc::get_selected_replica(ObRoutePolicy::CandidateReplica &repl int ObCandiTabletLoc::set_part_loc_with_only_readable_replica( const ObObjectID &partition_id, + const ObObjectID &first_level_part_id, const common::ObTabletID &tablet_id, const ObLSLocation &partition_location, const ObIArray &invalid_servers) @@ -340,7 +345,7 @@ int ObCandiTabletLoc::set_part_loc_with_only_readable_replica( LOG_ERROR("partition location has not been set yet, but replica idx has been selected", K(ret), K(*this), K(partition_location)); } else if (OB_FAIL(opt_tablet_loc_.assign_with_only_readable_replica( - partition_id, tablet_id, + partition_id, first_level_part_id, tablet_id, partition_location, invalid_servers))) { LOG_WARN("fail to assign partition location with only readable replica", K(ret), K(partition_location), K(invalid_servers)); @@ -593,7 +598,7 @@ int ObCandiTableLoc::replace_local_index_loc(DASRelatedTabletMap &map, ObTableID if (OB_FAIL(map.get_related_tablet_id(tablet_loc.get_tablet_id(), ref_table_id, rv))) { LOG_WARN("related tablet info is invalid", K(ret)); } else { - tablet_loc.set_tablet_info(rv.first, rv.second); + tablet_loc.set_tablet_info(rv.tablet_id_, rv.part_id_, rv.first_level_part_id_); } } return ret; diff --git a/src/sql/optimizer/ob_phy_table_location_info.h b/src/sql/optimizer/ob_phy_table_location_info.h index 4f40bba412..ca208cc04d 100644 --- a/src/sql/optimizer/ob_phy_table_location_info.h +++ b/src/sql/optimizer/ob_phy_table_location_info.h @@ -35,8 +35,8 @@ public: void reset(); int assign(const ObOptTabletLoc &partition_location); - int assign_with_only_readable_replica( - const ObObjectID &partition_id, + int assign_with_only_readable_replica(const ObObjectID &partition_id, + const ObObjectID &first_level_part_id, const common::ObTabletID &tablet_id, const share::ObLSLocation &partition_location, const common::ObIArray &invalid_servers); @@ -49,13 +49,17 @@ public: int get_strong_leader(share::ObLSReplicaLocation &replica_location) const; void set_tablet_info(common::ObTabletID tablet_id, - common::ObPartID part_id) + common::ObPartID part_id, + common::ObPartID first_level_part_id) { tablet_id_ = tablet_id; partition_id_ = part_id; + first_level_part_id_ = first_level_part_id; } inline int64_t get_partition_id() const { return partition_id_; } + inline int64_t get_first_level_part_id() const { return first_level_part_id_; } + inline common::ObTabletID get_tablet_id() const { return tablet_id_; } inline const share::ObLSID &get_ls_id() const { return ls_id_; } @@ -70,6 +74,8 @@ public: private: int64_t partition_id_; + // first level part id, only valid for subpartitioned table + int64_t first_level_part_id_; common::ObTabletID tablet_id_; share::ObLSID ls_id_; ObSmartReplicaLocationArray replica_locations_; @@ -99,6 +105,7 @@ public: int get_priority_replica_base(int64_t selected_replica_idx, T &replica_loc) const; int set_part_loc_with_only_readable_replica( const ObObjectID &partition_id, + const ObObjectID &first_level_part_id, const common::ObTabletID &tablet_id, const share::ObLSLocation &partition_location, const common::ObIArray &invalid_servers); diff --git a/src/sql/optimizer/ob_table_location.cpp b/src/sql/optimizer/ob_table_location.cpp index a519fe6997..2a09a42498 100644 --- a/src/sql/optimizer/ob_table_location.cpp +++ b/src/sql/optimizer/ob_table_location.cpp @@ -1366,6 +1366,7 @@ int ObTableLocation::calculate_candi_tablet_locations( { int ret = OB_SUCCESS; ObSEArray partition_ids; + ObSEArray first_level_part_ids; ObSEArray tablet_ids; if (!inited_) { ret = OB_NOT_INIT; @@ -1374,6 +1375,7 @@ int ObTableLocation::calculate_candi_tablet_locations( params, tablet_ids, partition_ids, + first_level_part_ids, dtc_params))) { LOG_WARN("Failed to calculate partition ids", K(ret)); } else if (OB_FAIL(get_tablet_locations(exec_ctx.get_das_ctx(), @@ -1381,6 +1383,7 @@ int ObTableLocation::calculate_candi_tablet_locations( loc_meta_.ref_table_id_, tablet_ids, partition_ids, + first_level_part_ids, candi_tablet_locs, nonblock))) { LOG_WARN("Failed to set partition locations", K(ret), K(partition_ids)); @@ -1602,10 +1605,12 @@ int ObTableLocation::calculate_tablet_ids(ObExecContext &exec_ctx, const ParamStore ¶ms, ObIArray &tablet_ids, ObIArray &partition_ids, + ObIArray &first_level_part_ids, const ObDataTypeCastParams &dtc_params) const { int ret = OB_SUCCESS; ObDASTabletMapper tablet_mapper; + ObPartitionIdMap partition_id_map; if (!inited_) { ret = OB_NOT_INIT; LOG_WARN("ObTableLocation not inited", K(ret)); @@ -1634,11 +1639,14 @@ int ObTableLocation::calculate_tablet_ids(ObExecContext &exec_ctx, } else { ObSEArray tmp_tablet_ids; ObSEArray tmp_part_ids; - if (OB_FAIL(calc_partition_ids_by_calc_node(exec_ctx, tablet_mapper, params, - calc_node_, gen_col_node_, - se_gen_col_expr_, - part_get_all_, tmp_tablet_ids, - tmp_part_ids, dtc_params))) { + if (OB_FAIL(partition_id_map.create(128, "PartitionIdMap", "PartitionIdMap"))) { + LOG_WARN("failed to create partition id map"); + } else if (OB_FALSE_IT(tablet_mapper.set_partition_id_map(&partition_id_map))) { + } else if (OB_FAIL(calc_partition_ids_by_calc_node(exec_ctx, tablet_mapper, params, + calc_node_, gen_col_node_, + se_gen_col_expr_, + part_get_all_, tmp_tablet_ids, + tmp_part_ids, dtc_params))) { LOG_WARN("Calc partition ids by calc node error", K_(stmt_type), K(ret)); } else if (OB_FAIL(calc_partition_ids_by_calc_node(exec_ctx, tablet_mapper, params, subcalc_node_, sub_gen_col_node_, @@ -1665,7 +1673,8 @@ int ObTableLocation::calculate_tablet_ids(ObExecContext &exec_ctx, if (OB_SUCC(ret) && partition_ids.empty()) { ObObjectID default_partition_id = OB_INVALID_ID; ObTabletID default_tablet_id; - if (OB_FAIL(tablet_mapper.get_default_tablet_and_object_id(part_hint_ids_, + if (OB_FAIL(tablet_mapper.get_default_tablet_and_object_id(part_level_, + part_hint_ids_, default_tablet_id, default_partition_id))) { LOG_WARN("get default tablet and object id failed", K(ret), K(part_hint_ids_)); @@ -1675,6 +1684,16 @@ int ObTableLocation::calculate_tablet_ids(ObExecContext &exec_ctx, LOG_WARN("store default tablet id failed", K(ret)); } } + if (OB_SUCC(ret) && PARTITION_LEVEL_TWO == part_level_) { + for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); ++i) { + ObObjectID first_level_part_id; + if (OB_FAIL(tablet_mapper.get_partition_id_map(partition_ids.at(i), first_level_part_id))) { + LOG_WARN("failed to get partition id map", K(i), K(partition_ids.at(i))); + } else if (OB_FAIL(first_level_part_ids.push_back(first_level_part_id))) { + LOG_WARN("faield to push back log part id"); + } + } + } } LOG_TRACE("calculate tablet ids end", K(loc_meta_), K(partition_ids), K(tablet_ids)); @@ -1687,6 +1706,7 @@ int ObTableLocation::get_tablet_locations(ObDASCtx &das_ctx, const uint64_t ref_table_id, const ObIArray &tablet_ids, const ObIArray &partition_ids, + const ObIArray &first_level_part_ids, ObCandiTabletLocIArray &candi_tablet_locs, bool nonblock /*false*/) const { @@ -1694,6 +1714,9 @@ int ObTableLocation::get_tablet_locations(ObDASCtx &das_ctx, if (OB_INVALID_ID == ref_table_id) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(ref_table_id), K(tablet_ids.empty())); + } else if (PARTITION_LEVEL_TWO == part_level_ && partition_ids.count() != first_level_part_ids.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected log part id count", K(partition_ids.count()), K(first_level_part_ids.count())); } else { NG_TRACE(get_location_cache_begin); candi_tablet_locs.reset(); @@ -1722,6 +1745,7 @@ int ObTableLocation::get_tablet_locations(ObDASCtx &das_ctx, LOG_WARN("invalid argument", K(session), K(ret)); } else if (OB_FAIL(candi_tablet_loc.set_part_loc_with_only_readable_replica( partition_ids.at(i), + PARTITION_LEVEL_TWO == part_level_ ? first_level_part_ids.at(i) : OB_INVALID_ID, tablet_ids.at(i), location, session->get_retry_info().get_invalid_servers()))) { LOG_WARN("fail to set partition location with only readable replica", @@ -5310,6 +5334,7 @@ int ObTableLocation::get_full_leader_table_loc(ObIAllocator &allocator, const int64_t expire_renew_time = 2 * 1000000; // 2s ObSEArray tablet_ids; ObSEArray partition_ids; + ObSEArray first_level_part_ids; ObSchemaGetterGuard schema_guard; OZ(GCTX.schema_service_->get_tenant_schema_guard(tenant_id, schema_guard)); OZ(schema_guard.get_table_schema(tenant_id, ref_table_id, table_schema)); @@ -5317,8 +5342,9 @@ int ObTableLocation::get_full_leader_table_loc(ObIAllocator &allocator, ret = OB_SCHEMA_ERROR; LOG_WARN("table schema is null", K(ret), K(table_id), K(tenant_id), K(ref_table_id)); } else { - OZ(table_schema->get_all_tablet_and_object_ids(tablet_ids, partition_ids)); + OZ(table_schema->get_all_tablet_and_object_ids(tablet_ids, partition_ids, &first_level_part_ids)); CK(table_schema->has_tablet()); + CK(tablet_ids.count() == partition_ids.count() && tablet_ids.count() == first_level_part_ids.count()); } if (OB_SUCC(ret)) { ObDASTableLocMeta *loc_meta = NULL; @@ -5339,6 +5365,8 @@ int ObTableLocation::get_full_leader_table_loc(ObIAllocator &allocator, CK(OB_NOT_NULL(tablet_buf)); OX(tablet_loc = new(tablet_buf) ObDASTabletLoc()); OX(tablet_loc->loc_meta_ = loc_meta); + OX(tablet_loc->partition_id_ = partition_ids.at(i)); + OX(tablet_loc->first_level_part_id_ = first_level_part_ids.at(i)); OZ(ObDASLocationRouter::get_leader(tenant_id, tablet_ids.at(i), *tablet_loc, expire_renew_time)); OZ(table_loc->add_tablet_loc(tablet_loc)); } @@ -5370,7 +5398,9 @@ int ObTableLocation::get_tablet_and_object_id_with_phy_rowid(ObNewRange &range, ObSEArray all_tablet_ids; ObSEArray all_index_tablet_ids; ObSEArray all_partition_ids; + ObSEArray all_first_level_part_ids; ObSEArray all_index_partition_ids; + ObSEArray all_index_first_level_part_ids; bool is_index_table = table_schema->is_index_table(); ObURowIDData urowid_data = range.start_key_.get_obj_ptr()[0].get_urowid(); if (OB_FAIL(urowid_data.get_tablet_id_for_heap_organized_table(tablet_id))) { @@ -5389,19 +5419,24 @@ int ObTableLocation::get_tablet_and_object_id_with_phy_rowid(ObNewRange &range, ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(table_schema->get_data_table_id())); } else if (OB_FAIL(data_table_schema->get_all_tablet_and_object_ids(all_tablet_ids, - all_partition_ids))) { + all_partition_ids, + &all_first_level_part_ids))) { LOG_WARN("failed to get all tablet and object ids", K(ret)); } else if (OB_FAIL(table_schema->get_all_tablet_and_object_ids(all_index_tablet_ids, - all_index_partition_ids))) { + all_index_partition_ids, + &all_index_first_level_part_ids))) { LOG_WARN("failed to get all tablet and object ids", K(ret)); } else {/*do nothing*/} } else if (OB_FAIL(table_schema->get_all_tablet_and_object_ids(all_tablet_ids, - all_partition_ids))) { + all_partition_ids, + &all_first_level_part_ids))) { LOG_WARN("failed to get all tablet and object ids", K(ret)); } else {/*do nothing*/} if (OB_FAIL(ret)) { } else if (OB_UNLIKELY(all_tablet_ids.count() != all_partition_ids.count() || + all_tablet_ids.count() != all_first_level_part_ids.count() || all_index_tablet_ids.count() != all_index_partition_ids.count() || + all_index_tablet_ids.count() != all_index_first_level_part_ids.count() || (is_index_table && all_tablet_ids.count() != all_index_tablet_ids.count()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(all_tablet_ids), K(all_partition_ids), @@ -5410,10 +5445,15 @@ int ObTableLocation::get_tablet_and_object_id_with_phy_rowid(ObNewRange &range, bool find_it = false; for (int64_t i = 0; OB_SUCC(ret) && !find_it && i < all_tablet_ids.count(); ++i) { if (tablet_id == all_tablet_ids.at(i)) { + ObObjectID partition_id = !is_index_table ? all_partition_ids.at(i) : all_index_partition_ids.at(i); + ObObjectID first_level_part_id = !is_index_table ? all_first_level_part_ids.at(i) : all_index_first_level_part_ids.at(i); if (OB_FAIL(tablet_ids.push_back(!is_index_table ? all_tablet_ids.at(i) : all_index_tablet_ids.at(i)))) { LOG_WARN("failed to add tablet id", K(ret), K(all_tablet_ids.at(i))); - } else if (OB_FAIL(partition_ids.push_back(!is_index_table ? all_partition_ids.at(i) : all_index_partition_ids.at(i)))) { + } else if (OB_FAIL(partition_ids.push_back(partition_id))) { LOG_WARN("failed to push back", K(ret), K(all_partition_ids.at(i))); + } else if (PARTITION_LEVEL_TWO == part_level_ && + OB_FAIL(tablet_mapper.set_partition_id_map(first_level_part_id, partition_id))) { + LOG_WARN("failed to set partition id map", K(first_level_part_id), K(partition_id)); } else if (OB_FAIL(fill_related_tablet_and_object_ids(tablet_mapper, table_schema->get_tenant_id(), !is_index_table ? all_tablet_ids.at(i) : all_index_tablet_ids.at(i), @@ -5448,7 +5488,8 @@ int ObTableLocation::fill_related_tablet_and_object_ids(ObDASTabletMapper &table for (int64_t i = 0; OB_SUCC(ret) && i < related_table.related_tids_->count(); i++) { const uint64_t related_table_id = related_table.related_tids_->at(i); ObSEArray all_related_tablet_ids; - ObSEArray all_related_object_ids; + ObSEArray all_partition_ids; + ObSEArray all_first_level_part_ids; const ObTableSchema *related_schema = NULL; if (OB_FAIL(guard->get_table_schema(tenant_id, related_table_id, related_schema))) { LOG_WARN("fail to get simple table schema", KR(ret), K(tenant_id), K(related_table_id)); @@ -5456,22 +5497,25 @@ int ObTableLocation::fill_related_tablet_and_object_ids(ObDASTabletMapper &table ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(tenant_id), K(related_table_id)); } else if (OB_FAIL(related_schema->get_all_tablet_and_object_ids(all_related_tablet_ids, - all_related_object_ids))) { + all_partition_ids, + &all_first_level_part_ids))) { LOG_WARN("failed to get all tablet and object ids", K(ret)); } else if (OB_UNLIKELY(idx < 0 || idx >= all_related_tablet_ids.count() || - all_related_tablet_ids.count() != all_related_object_ids.count())) { + all_related_tablet_ids.count() != all_partition_ids.count() || + all_related_tablet_ids.count() != all_first_level_part_ids.count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(idx), K(all_related_tablet_ids), - K(all_related_object_ids)); + K(all_partition_ids), K(all_first_level_part_ids)); } else if (OB_FAIL(related_table.related_map_->add_related_tablet_id(src_tablet_id, related_table_id, all_related_tablet_ids.at(idx), - all_related_object_ids.at(idx)))) { + all_partition_ids.at(idx), + all_first_level_part_ids.at(idx)))) { LOG_WARN("fail to add related tablet info", K(ret), K(src_tablet_id), K(related_table_id), - K(all_related_tablet_ids.at(idx)), K(all_related_object_ids.at(idx))); + K(all_related_tablet_ids.at(idx)), K(all_partition_ids.at(idx)), K(all_first_level_part_ids.at(idx))); } else { LOG_TRACE("succ to add related tablet info", K(ret), K(src_tablet_id), K(related_table_id), - K(all_related_tablet_ids.at(idx)), K(all_related_object_ids.at(idx))); + K(all_related_tablet_ids.at(idx)), K(all_partition_ids.at(idx)), K(all_first_level_part_ids.at(idx))); } } } diff --git a/src/sql/optimizer/ob_table_location.h b/src/sql/optimizer/ob_table_location.h index 7dcc52b0f8..c27d8e84bd 100644 --- a/src/sql/optimizer/ob_table_location.h +++ b/src/sql/optimizer/ob_table_location.h @@ -631,6 +631,7 @@ public: const ParamStore ¶ms, ObIArray &tablet_ids, ObIArray &partition_ids, + ObIArray &first_level_part_ids, const ObDataTypeCastParams &dtc_params) const; int init_partition_ids_by_rowkey2(ObExecContext &exec_ctx, @@ -664,6 +665,7 @@ public: const uint64_t ref_table_id, const ObIArray &tablet_ids, const ObIArray &partition_ids, + const ObIArray &first_level_part_ids, ObCandiTabletLocIArray &candi_tablet_locs, bool nonblock = false) const; diff --git a/src/sql/optimizer/ob_table_partition_info.cpp b/src/sql/optimizer/ob_table_partition_info.cpp index 12d8b04706..410e605f54 100644 --- a/src/sql/optimizer/ob_table_partition_info.cpp +++ b/src/sql/optimizer/ob_table_partition_info.cpp @@ -101,30 +101,6 @@ int ObTablePartitionInfo::calculate_phy_table_location_info( return ret; } -int ObTablePartitionInfo::calculate_phy_table_location_info( - ObExecContext &exec_ctx, - const ParamStore ¶ms, - const common::ObDataTypeCastParams &dtc_params, - const common::ObIArray &tablet_ids, - const common::ObIArray &partition_ids) -{ - int ret = OB_SUCCESS; - ObCandiTableLoc candi_table_loc; - if (OB_FAIL(table_location_.get_tablet_locations( - exec_ctx.get_das_ctx(), - exec_ctx.get_my_session(), - table_location_.get_ref_table_id(), - tablet_ids, - partition_ids, - candi_table_loc_.get_phy_part_loc_info_list_for_update()))) { - LOG_WARN("failed to set partition locations", K(ret)); - } else { - candi_table_loc_.set_table_location_key( - table_location_.get_table_id(), table_location_.get_ref_table_id()); - } - return ret; -} - // 全部选择主，并且将direction设上 int ObTablePartitionInfo::calc_phy_table_loc_and_select_leader(ObExecContext &exec_ctx, const ParamStore ¶ms, diff --git a/src/sql/optimizer/ob_table_partition_info.h b/src/sql/optimizer/ob_table_partition_info.h index 2732e7300b..3eb483ce05 100644 --- a/src/sql/optimizer/ob_table_partition_info.h +++ b/src/sql/optimizer/ob_table_partition_info.h @@ -56,12 +56,6 @@ public: const ParamStore ¶ms, const common::ObDataTypeCastParams &dtc_params); - int calculate_phy_table_location_info(ObExecContext &exec_ctx, - const ParamStore ¶ms, - const common::ObDataTypeCastParams &dtc_params, - const common::ObIArray &tablet_ids, - const common::ObIArray &partition_ids); - int calc_phy_table_loc_and_select_leader(ObExecContext &exec_ctx, const ParamStore ¶ms, const common::ObDataTypeCastParams &dtc_params); diff --git a/src/sql/plan_cache/ob_plan_cache_util.cpp b/src/sql/plan_cache/ob_plan_cache_util.cpp index 5c55bd5caf..fd1a83713b 100644 --- a/src/sql/plan_cache/ob_plan_cache_util.cpp +++ b/src/sql/plan_cache/ob_plan_cache_util.cpp @@ -364,12 +364,14 @@ int ObPhyLocationGetter::build_related_tablet_info(const ObTableLocation &table_ ObDataTypeCastParams dtc_params = ObBasicSessionInfo::create_dtc_params(exec_ctx.get_my_session()); ObPhysicalPlanCtx *plan_ctx = exec_ctx.get_physical_plan_ctx(); ObArray partition_ids; + ObArray first_level_part_ids; ObArray tablet_ids; if (OB_FAIL(table_location.calculate_tablet_ids(exec_ctx, plan_ctx->get_param_store(), tablet_ids, partition_ids, + first_level_part_ids, dtc_params))) { LOG_WARN("calculate tablet ids failed", K(ret)); } else { diff --git a/src/sql/resolver/dml/ob_select_resolver.cpp b/src/sql/resolver/dml/ob_select_resolver.cpp index eac0bb6358..921dcd0b06 100644 --- a/src/sql/resolver/dml/ob_select_resolver.cpp +++ b/src/sql/resolver/dml/ob_select_resolver.cpp @@ -5653,7 +5653,7 @@ int ObSelectResolver::check_need_use_sys_tenant(bool &use_sys_tenant) const int ObSelectResolver::check_in_sysview(bool &in_sysview) const { int ret = OB_SUCCESS; - in_sysview = params_.is_from_show_resolver_; + in_sysview = params_.is_from_show_resolver_ || params_.is_in_sys_view_; return ret; } diff --git a/src/sql/resolver/dml/ob_view_table_resolver.h b/src/sql/resolver/dml/ob_view_table_resolver.h index d15bbab7c6..f477ba0e96 100644 --- a/src/sql/resolver/dml/ob_view_table_resolver.h +++ b/src/sql/resolver/dml/ob_view_table_resolver.h @@ -28,16 +28,25 @@ public: materialized_(false), auto_name_id_(1), view_db_name_(view_db_name), - view_name_(view_name) + view_name_(view_name), + ori_is_in_sys_view_(false) { is_resolving_view_ = true; params_.is_from_create_view_ = params.is_from_create_view_; params_.is_from_create_table_ = params.is_from_create_table_; params_.is_specified_col_name_ = params.is_specified_col_name_; } - virtual ~ObViewTableResolver() {} + virtual ~ObViewTableResolver() + { + params_.is_in_sys_view_ = ori_is_in_sys_view_; + } - void set_current_view_item(const TableItem &view_item) { current_view_item = view_item; } + void set_current_view_item(const TableItem &view_item) + { + current_view_item = view_item; + ori_is_in_sys_view_ = params_.is_in_sys_view_; + params_.is_in_sys_view_ = params_.is_in_sys_view_ || is_sys_view(current_view_item.ref_id_); + } void set_parent_view_resolver(ObViewTableResolver *parent_view_resolver) { parent_view_resolver_ = parent_view_resolver; } int check_need_use_sys_tenant(bool &use_sys_tenant) const; @@ -72,6 +81,7 @@ protected: uint64_t auto_name_id_; const ObString view_db_name_; const ObString view_name_; + bool ori_is_in_sys_view_; }; } // namespace sql } // namespace oceanbase diff --git a/src/sql/resolver/ob_resolver_define.h b/src/sql/resolver/ob_resolver_define.h index 5210fd3a2b..1d5c1558c6 100644 --- a/src/sql/resolver/ob_resolver_define.h +++ b/src/sql/resolver/ob_resolver_define.h @@ -411,6 +411,7 @@ public: bool enable_res_map_; bool need_check_col_dup_; bool is_specified_col_name_;//mark if specify the column name in create view or create table as.. + bool is_in_sys_view_; }; } // end namespace sql } // end namespace oceanbase diff --git a/src/sql/rewrite/ob_expand_aggregate_utils.cpp b/src/sql/rewrite/ob_expand_aggregate_utils.cpp index 7111621cee..7769696b5a 100644 --- a/src/sql/rewrite/ob_expand_aggregate_utils.cpp +++ b/src/sql/rewrite/ob_expand_aggregate_utils.cpp @@ -2165,7 +2165,7 @@ int ObExpandAggregateUtils::expand_approx_count_distinct_expr(ObTransformerCtx * } else if (OB_FAIL(synopsis->get_real_param_exprs_for_update().assign( aggr_expr->get_real_param_exprs()))) { LOG_WARN("failed to assign real param expr for synopsis", K(ret)); - } else if (OB_FAIL(new_aggr_items.push_back(synopsis))) { + } else if (OB_FAIL(add_aggr_item(new_aggr_items, synopsis))) { LOG_WARN("failed to push back synopsis", K(ret)); } else if (OB_FAIL(ctx->expr_factory_->create_raw_expr(T_FUN_SYS_ESTIMATE_NDV, sys_func_expr))) { diff --git a/src/storage/blocksstable/ob_datum_row.h b/src/storage/blocksstable/ob_datum_row.h index 7efae1e337..0d956878ea 100644 --- a/src/storage/blocksstable/ob_datum_row.h +++ b/src/storage/blocksstable/ob_datum_row.h @@ -437,6 +437,7 @@ public: ObStorageDatumCmpFunc() = default; ~ObStorageDatumCmpFunc() = default; int compare(const ObStorageDatum &left, const ObStorageDatum &right, int &cmp_ret) const; + OB_INLINE const common::ObCmpFunc &get_cmp_func() const { return cmp_func_; } TO_STRING_KV(K_(cmp_func)); private: common::ObCmpFunc cmp_func_; diff --git a/src/storage/direct_load/ob_direct_load_fast_heap_table.cpp b/src/storage/direct_load/ob_direct_load_fast_heap_table.cpp index bc6f3ba1a4..fdd482ce54 100644 --- a/src/storage/direct_load/ob_direct_load_fast_heap_table.cpp +++ b/src/storage/direct_load/ob_direct_load_fast_heap_table.cpp @@ -43,8 +43,8 @@ ObDirectLoadFastHeapTable::ObDirectLoadFastHeapTable() ObDirectLoadFastHeapTable::~ObDirectLoadFastHeapTable() { for (int64_t i = 0; i < column_stat_array_.count(); ++i) { - ObOptColumnStat *col_stat = column_stat_array_.at(i); - col_stat->~ObOptColumnStat(); + ObOptOSGColumnStat *col_stat = column_stat_array_.at(i); + col_stat->~ObOptOSGColumnStat(); col_stat = nullptr; } } @@ -53,29 +53,22 @@ int ObDirectLoadFastHeapTable::copy_col_stat(const ObDirectLoadFastHeapTableCrea { int ret = OB_SUCCESS; for (int64_t i = 0; OB_SUCC(ret)&& i < param.column_stat_array_->count(); ++i) { - ObOptColumnStat *col_stat = param.column_stat_array_->at(i); - if (col_stat != nullptr) { - ObOptColumnStat *copied_col_stat = nullptr; - int64_t size = col_stat->size(); - char *new_buf = nullptr; - if (OB_ISNULL(new_buf = static_cast(allocator_.alloc(size)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate buffer", KR(ret), K(size)); - } else if (OB_FAIL(col_stat->deep_copy(new_buf, size, copied_col_stat))) { - LOG_WARN("fail to copy colstat", KR(ret)); - } else if (OB_FAIL(column_stat_array_.push_back(copied_col_stat))) { - LOG_WARN("fail to add table", KR(ret)); - } - if (OB_FAIL(ret)) { - if (copied_col_stat != nullptr) { - copied_col_stat->~ObOptColumnStat(); - copied_col_stat = nullptr; - } - if(new_buf != nullptr) { - allocator_.free(new_buf); - new_buf = nullptr; - } - } + ObOptOSGColumnStat *col_stat = param.column_stat_array_->at(i); + ObOptOSGColumnStat *copied_col_stat = NULL; + if (OB_ISNULL(col_stat)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null"); + } else if (OB_ISNULL(copied_col_stat = ObOptOSGColumnStat::create_new_osg_col_stat(allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory"); + } else if (OB_FAIL(copied_col_stat->deep_copy(*col_stat))) { + LOG_WARN("fail to copy colstat", KR(ret)); + } else if (OB_FAIL(column_stat_array_.push_back(copied_col_stat))) { + LOG_WARN("fail to add table", KR(ret)); + } + if (OB_FAIL(ret) && OB_NOT_NULL(copied_col_stat)) { + copied_col_stat->~ObOptOSGColumnStat(); + copied_col_stat = nullptr; } } return ret; diff --git a/src/storage/direct_load/ob_direct_load_fast_heap_table.h b/src/storage/direct_load/ob_direct_load_fast_heap_table.h index eee61ccdac..ba11e3f3ca 100644 --- a/src/storage/direct_load/ob_direct_load_fast_heap_table.h +++ b/src/storage/direct_load/ob_direct_load_fast_heap_table.h @@ -7,6 +7,7 @@ #include "common/ob_tablet_id.h" #include "storage/direct_load/ob_direct_load_i_table.h" #include "share/stat/ob_opt_column_stat.h" +#include "share/stat/ob_opt_osg_column_stat.h" namespace oceanbase { @@ -23,7 +24,7 @@ public: public: common::ObTabletID tablet_id_; int64_t row_count_; - common::ObArray *column_stat_array_; + common::ObArray *column_stat_array_; }; struct ObDirectLoadFastHeapTableMeta @@ -39,7 +40,7 @@ public: ObDirectLoadFastHeapTable(); virtual ~ObDirectLoadFastHeapTable(); int init(const ObDirectLoadFastHeapTableCreateParam ¶m); - const common::ObIArray &get_column_stat_array() const + const common::ObIArray &get_column_stat_array() const { return column_stat_array_; } @@ -53,7 +54,7 @@ private: private: ObDirectLoadFastHeapTableMeta meta_; common::ObArenaAllocator allocator_; - common::ObArray column_stat_array_; + common::ObArray column_stat_array_; bool is_inited_; DISABLE_COPY_ASSIGN(ObDirectLoadFastHeapTable); }; diff --git a/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp b/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp index ddc49c85e8..5b4d34f9f2 100644 --- a/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp +++ b/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.cpp @@ -26,6 +26,7 @@ using namespace share; ObDirectLoadFastHeapTableBuildParam::ObDirectLoadFastHeapTableBuildParam() : snapshot_version_(0), + datum_utils_(nullptr), col_descs_(nullptr), insert_table_ctx_(nullptr), fast_heap_table_ctx_(nullptr), @@ -42,7 +43,7 @@ bool ObDirectLoadFastHeapTableBuildParam::is_valid() const { return tablet_id_.is_valid() && snapshot_version_ > 0 && table_data_desc_.is_valid() && nullptr != col_descs_ && nullptr != insert_table_ctx_ && nullptr != fast_heap_table_ctx_ && - nullptr != result_info_; + nullptr != result_info_ && nullptr != datum_utils_; } /** @@ -67,8 +68,8 @@ ObDirectLoadFastHeapTableBuilder::~ObDirectLoadFastHeapTableBuilder() slice_writer_ = nullptr; } for (int64_t i = 0; i < column_stat_array_.count(); ++i) { - ObOptColumnStat *col_stat = column_stat_array_.at(i); - col_stat->~ObOptColumnStat(); + ObOptOSGColumnStat *col_stat = column_stat_array_.at(i); + col_stat->~ObOptOSGColumnStat(); col_stat = nullptr; } } @@ -77,18 +78,18 @@ int ObDirectLoadFastHeapTableBuilder::init_sql_statistics() { int ret = OB_SUCCESS; for (int64_t i = 0; OB_SUCC(ret) && i < param_.table_data_desc_.column_count_; ++i) { - ObOptColumnStat *col_stat = OB_NEWx(ObOptColumnStat, (&allocator_), allocator_); - if (OB_ISNULL(col_stat)) { + ObOptOSGColumnStat *new_osg_col_stat = ObOptOSGColumnStat::create_new_osg_col_stat(allocator_); + if (OB_ISNULL(new_osg_col_stat)) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate buffer", KR(ret)); - } else if (OB_FAIL(column_stat_array_.push_back(col_stat))) { + LOG_WARN("failed to allocate col stat"); + } else if (OB_FAIL(column_stat_array_.push_back(new_osg_col_stat))) { LOG_WARN("fail to push back", KR(ret)); } if (OB_FAIL(ret)) { - if (col_stat != nullptr) { - col_stat->~ObOptColumnStat(); - allocator_.free(col_stat); - col_stat = nullptr; + if (new_osg_col_stat != nullptr) { + new_osg_col_stat->~ObOptOSGColumnStat(); + allocator_.free(new_osg_col_stat); + new_osg_col_stat = nullptr; } } } @@ -98,18 +99,20 @@ int ObDirectLoadFastHeapTableBuilder::init_sql_statistics() int ObDirectLoadFastHeapTableBuilder::collect_obj(const ObDatumRow &datum_row) { int ret = OB_SUCCESS; + const int64_t extra_rowkey_cnt = ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); for (int64_t i = 0; OB_SUCC(ret) && i < param_.table_data_desc_.column_count_; i++) { const ObStorageDatum &datum = - datum_row.storage_datums_[i + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt() + 1]; + datum_row.storage_datums_[i + extra_rowkey_cnt + 1]; + const common::ObCmpFunc &cmp_func = param_.datum_utils_->get_cmp_funcs().at( + i + extra_rowkey_cnt + 1).get_cmp_func(); const ObColDesc &col_desc = param_.col_descs_->at(i + 1); - ObOptColumnStat *col_stat = column_stat_array_.at(i); + ObOptOSGColumnStat *col_stat = column_stat_array_.at(i); bool is_valid = ObColumnStatParam::is_valid_histogram_type(col_desc.col_type_.get_type()); if (col_stat != nullptr && is_valid) { - ObObj obj; - if (OB_FAIL(datum.to_obj_enhance(obj, col_desc.col_type_))) { - LOG_WARN("Failed to transform datum to obj", K(ret), K(i), K(datum_row.storage_datums_[i])); - } else if (OB_FAIL(col_stat->merge_obj(obj))) { - LOG_WARN("Failed to merge obj", K(ret), K(obj), KP(col_stat)); + if (OB_FAIL(col_stat->update_column_stat_info(&datum, + col_desc.col_type_, + cmp_func.cmp_func_))) { + LOG_WARN("failed to update column stat info"); } } } diff --git a/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.h b/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.h index 1099e7ff89..b486eb8ffa 100644 --- a/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.h +++ b/src/storage/direct_load/ob_direct_load_fast_heap_table_builder.h @@ -8,6 +8,7 @@ #include "storage/direct_load/ob_direct_load_fast_heap_table_ctx.h" #include "storage/direct_load/ob_direct_load_i_table.h" #include "storage/direct_load/ob_direct_load_table_data_desc.h" +#include "sql/engine/expr/ob_expr_sys_op_opnsize.h" namespace oceanbase { @@ -17,7 +18,7 @@ class ObTableLoadResultInfo; } // namespace table namespace common { -class ObOptColumnStat; +class ObOptOSGColumnStat; } // namespace common namespace storage { @@ -36,6 +37,7 @@ public: common::ObTabletID tablet_id_; int64_t snapshot_version_; ObDirectLoadTableDataDesc table_data_desc_; + const blocksstable::ObStorageDatumUtils *datum_utils_; const common::ObIArray *col_descs_; ObDirectLoadInsertTableContext *insert_table_ctx_; ObDirectLoadFastHeapTableContext *fast_heap_table_ctx_; @@ -68,7 +70,7 @@ private: ObSSTableInsertSliceWriter *slice_writer_; ObDirectLoadFastHeapTableTabletWriteCtx write_ctx_; blocksstable::ObDatumRow datum_row_; - common::ObArray column_stat_array_; + common::ObArray column_stat_array_; int64_t row_count_; bool is_closed_; bool is_inited_; diff --git a/src/storage/direct_load/ob_direct_load_merge_ctx.cpp b/src/storage/direct_load/ob_direct_load_merge_ctx.cpp index 4db2dd3a68..c0040cb9bf 100644 --- a/src/storage/direct_load/ob_direct_load_merge_ctx.cpp +++ b/src/storage/direct_load/ob_direct_load_merge_ctx.cpp @@ -230,58 +230,58 @@ int ObDirectLoadTabletMergeCtx::collect_sql_statistics( int64_t col_id = param_.is_heap_table_ ? i + 1 : i; int64_t row_count = 0; int64_t avg_len = 0; - ObOptColumnStat *col_stat = nullptr; - if (OB_FAIL(sql_statistics.allocate_col_stat(col_stat))) { + ObOptOSGColumnStat *osg_col_stat = nullptr; + if (OB_FAIL(sql_statistics.allocate_col_stat(osg_col_stat))) { LOG_WARN("fail to allocate table stat", KR(ret)); } // scan task_array for (int64_t j = 0; OB_SUCC(ret) && j < task_array_.count(); ++j) { - ObOptColumnStat *tmp_col_stat = task_array_.at(j)->get_column_stat_array().at(i); - if (tmp_col_stat != nullptr) { - ObOptColumnStat *copied_col_stat = nullptr; - int64_t size = tmp_col_stat->size(); - char *new_buf = nullptr; - if (OB_ISNULL(new_buf = static_cast(sql_statistics.allocator_.alloc(size)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate buffer", KR(ret), K(size)); - } else if (OB_FAIL(tmp_col_stat->deep_copy(new_buf, size, copied_col_stat))) { - LOG_WARN("fail to copy colstat", KR(ret)); - } else if (OB_FAIL(col_stat->merge_column_stat(*copied_col_stat))) { - LOG_WARN("fail to merge column stat", KR(ret)); - } else { - row_count += task_array_.at(j)->get_row_count(); - avg_len += col_stat->get_avg_len(); - } + ObOptOSGColumnStat *tmp_col_stat = task_array_.at(j)->get_column_stat_array().at(i); + ObOptOSGColumnStat *copied_col_stat = NULL; + if (OB_ISNULL(tmp_col_stat)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null"); + } else if (OB_ISNULL(copied_col_stat = + ObOptOSGColumnStat::create_new_osg_col_stat(sql_statistics.allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory"); + } else if (OB_FAIL(copied_col_stat->deep_copy(*tmp_col_stat))) { + LOG_WARN("fail to copy colstat", KR(ret)); + } else if (OB_FAIL(osg_col_stat->merge_column_stat(*copied_col_stat))) { + LOG_WARN("fail to merge column stat", KR(ret)); + } else { + row_count += task_array_.at(j)->get_row_count(); + avg_len += osg_col_stat->col_stat_->get_avg_len(); } } // scan fast heap table for (int64_t j = 0; OB_SUCC(ret) && j < fast_heap_table_array.count(); ++j) { - ObOptColumnStat *tmp_col_stat = fast_heap_table_array.at(j)->get_column_stat_array().at(i); - if (tmp_col_stat != nullptr) { - ObOptColumnStat *copied_col_stat = nullptr; - int64_t size = tmp_col_stat->size(); - char *new_buf = nullptr; - if (OB_ISNULL(new_buf = static_cast(sql_statistics.allocator_.alloc(size)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate buffer", KR(ret), K(size)); - } else if (OB_FAIL(tmp_col_stat->deep_copy(new_buf, size, copied_col_stat))) { - LOG_WARN("fail to copy colstat", KR(ret)); - } else if (OB_FAIL(col_stat->merge_column_stat(*copied_col_stat))) { - LOG_WARN("fail to merge column stat", KR(ret)); - } else { - row_count += fast_heap_table_array.at(j)->get_row_count(); - avg_len += col_stat->get_avg_len(); - } + ObOptOSGColumnStat *tmp_col_stat = fast_heap_table_array.at(j)->get_column_stat_array().at(i); + ObOptOSGColumnStat *copied_col_stat = NULL; + if (OB_ISNULL(tmp_col_stat)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null"); + } else if (OB_ISNULL(copied_col_stat = + ObOptOSGColumnStat::create_new_osg_col_stat(sql_statistics.allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory"); + } else if (OB_FAIL(copied_col_stat->deep_copy(*tmp_col_stat))) { + LOG_WARN("fail to copy colstat", KR(ret)); + } else if (OB_FAIL(osg_col_stat->merge_column_stat(*copied_col_stat))) { + LOG_WARN("fail to merge column stat", KR(ret)); + } else { + row_count += fast_heap_table_array.at(j)->get_row_count(); + avg_len += osg_col_stat->col_stat_->get_avg_len(); } } if (OB_SUCC(ret)) { table_row_cnt = row_count; table_avg_len += avg_len; - col_stat->set_table_id(param_.target_table_id_); - col_stat->set_partition_id(target_partition_id_); - col_stat->set_stat_level(stat_level); - col_stat->set_column_id(param_.col_descs_->at(col_id).col_id_); - col_stat->set_num_distinct(ObGlobalNdvEval::get_ndv_from_llc(col_stat->get_llc_bitmap())); + osg_col_stat->col_stat_->set_table_id(param_.target_table_id_); + osg_col_stat->col_stat_->set_partition_id(target_partition_id_); + osg_col_stat->col_stat_->set_stat_level(stat_level); + osg_col_stat->col_stat_->set_column_id(param_.col_descs_->at(col_id).col_id_); + osg_col_stat->col_stat_->set_num_distinct(ObGlobalNdvEval::get_ndv_from_llc(osg_col_stat->col_stat_->get_llc_bitmap())); } } if (OB_SUCC(ret)) { diff --git a/src/storage/direct_load/ob_direct_load_merge_ctx.h b/src/storage/direct_load/ob_direct_load_merge_ctx.h index 4a2bbcdfe2..cd6e7f1cb4 100644 --- a/src/storage/direct_load/ob_direct_load_merge_ctx.h +++ b/src/storage/direct_load/ob_direct_load_merge_ctx.h @@ -9,6 +9,7 @@ #include "share/schema/ob_table_param.h" #include "share/stat/ob_stat_define.h" #include "share/stat/ob_opt_column_stat.h" +#include "share/stat/ob_opt_osg_column_stat.h" #include "share/stat/ob_opt_table_stat.h" #include "share/table/ob_table_load_define.h" #include "storage/direct_load/ob_direct_load_origin_table.h" @@ -23,7 +24,7 @@ class ObTableLoadErrorRowHandler; class ObTableLoadSchema; } // namespace observer namespace common { -class ObOptColumnStat; +class ObOptOSGColumnStat; class ObOptTableStat; } // namespace common namespace storage diff --git a/src/storage/direct_load/ob_direct_load_partition_merge_task.cpp b/src/storage/direct_load/ob_direct_load_partition_merge_task.cpp index 159c934c62..4b17275420 100644 --- a/src/storage/direct_load/ob_direct_load_partition_merge_task.cpp +++ b/src/storage/direct_load/ob_direct_load_partition_merge_task.cpp @@ -14,6 +14,7 @@ #include "storage/direct_load/ob_direct_load_multiple_heap_table.h" #include "storage/direct_load/ob_direct_load_origin_table.h" + namespace oceanbase { namespace storage @@ -41,9 +42,9 @@ ObDirectLoadPartitionMergeTask::ObDirectLoadPartitionMergeTask() ObDirectLoadPartitionMergeTask::~ObDirectLoadPartitionMergeTask() { for (int64_t i = 0; i < column_stat_array_.count(); ++i) { - ObOptColumnStat *col_stat = column_stat_array_.at(i); + ObOptOSGColumnStat *col_stat = column_stat_array_.at(i); if (col_stat != nullptr) { - col_stat->~ObOptColumnStat(); + col_stat->~ObOptOSGColumnStat(); col_stat = nullptr; } } @@ -58,18 +59,16 @@ int ObDirectLoadPartitionMergeTask::process() } else { const ObTabletID &tablet_id = merge_ctx_->get_tablet_id(); const ObTabletID &target_tablet_id = merge_ctx_->get_target_tablet_id(); - ObArenaAllocator allocator("TLD_MergeTask"); ObIStoreRowIterator *row_iter = nullptr; ObSSTableInsertSliceWriter *writer = nullptr; ObMacroDataSeq block_start_seq; - allocator.set_tenant_id(MTL_ID()); block_start_seq.set_parallel_degree(parallel_idx_); if (merge_param_->online_opt_stat_gather_ && OB_FAIL(init_sql_statistics())) { LOG_WARN("fail to init sql statistics", KR(ret)); - } else if (OB_FAIL(construct_row_iter(allocator, row_iter))) { + } else if (OB_FAIL(construct_row_iter(allocator_, row_iter))) { LOG_WARN("fail to construct row iter", KR(ret)); } else if (OB_FAIL(merge_param_->insert_table_ctx_->construct_sstable_slice_writer( - target_tablet_id, block_start_seq, writer, allocator))) { + target_tablet_id, block_start_seq, writer, allocator_))) { LOG_WARN("fail to construct sstable slice writer", KR(ret), K(target_tablet_id), K(block_start_seq)); } else { @@ -101,12 +100,12 @@ int ObDirectLoadPartitionMergeTask::process() } if (OB_NOT_NULL(row_iter)) { row_iter->~ObIStoreRowIterator(); - allocator.free(row_iter); + allocator_.free(row_iter); row_iter = nullptr; } if (OB_NOT_NULL(writer)) { writer->~ObSSTableInsertSliceWriter(); - allocator.free(writer); + allocator_.free(writer); writer = nullptr; } if (OB_SUCC(ret)) { @@ -126,7 +125,7 @@ int ObDirectLoadPartitionMergeTask::init_sql_statistics() { int ret = OB_SUCCESS; for (int64_t i = 0; OB_SUCC(ret)&& i < merge_param_->table_data_desc_.column_count_; ++i) { - ObOptColumnStat *col_stat = OB_NEWx(ObOptColumnStat, (&allocator_), allocator_); + ObOptOSGColumnStat *col_stat = ObOptOSGColumnStat::create_new_osg_col_stat(allocator_); if (OB_ISNULL(col_stat)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to allocate buffer", KR(ret)); @@ -135,7 +134,7 @@ int ObDirectLoadPartitionMergeTask::init_sql_statistics() } if (OB_FAIL(ret)) { if (col_stat != nullptr) { - col_stat->~ObOptColumnStat(); + col_stat->~ObOptOSGColumnStat(); allocator_.free(col_stat); col_stat = nullptr; } @@ -151,44 +150,39 @@ int ObDirectLoadPartitionMergeTask::collect_obj(const ObDatumRow &datum_row) if (merge_param_->is_heap_table_ ) { for (int64_t i = 0; OB_SUCC(ret) && i < merge_param_->table_data_desc_.column_count_; i++) { const ObStorageDatum &datum = datum_row.storage_datums_[i + extra_rowkey_cnt + 1]; + const common::ObCmpFunc &cmp_func = merge_param_->datum_utils_->get_cmp_funcs().at( + i + extra_rowkey_cnt + 1).get_cmp_func(); const ObColDesc &col_desc = merge_param_->col_descs_->at(i + 1); - ObOptColumnStat *col_stat = column_stat_array_.at(i); + ObOptOSGColumnStat *col_stat = column_stat_array_.at(i); bool is_valid = ObColumnStatParam::is_valid_histogram_type(col_desc.col_type_.get_type()); if (col_stat != nullptr && is_valid) { - ObObj obj; - if (OB_FAIL(datum.to_obj_enhance(obj, col_desc.col_type_))) { - LOG_WARN("Failed to transform datum to obj", K(ret), K(i), K(datum_row.storage_datums_[i])); - } else if (OB_FAIL(col_stat->merge_obj(obj))) { - LOG_WARN("Failed to merge obj", K(ret), K(obj), KP(col_stat)); + if (OB_FAIL(col_stat->update_column_stat_info(&datum, col_desc.col_type_, cmp_func.cmp_func_))) { + LOG_WARN("Failed to merge obj", K(ret), KP(col_stat)); } } } } else { for (int64_t i = 0; OB_SUCC(ret) && i < merge_param_->rowkey_column_num_; i++) { const ObStorageDatum &datum = datum_row.storage_datums_[i]; + const common::ObCmpFunc &cmp_func = merge_param_->datum_utils_->get_cmp_funcs().at(i).get_cmp_func(); const ObColDesc &col_desc = merge_param_->col_descs_->at(i); - ObOptColumnStat *col_stat = column_stat_array_.at(i); + ObOptOSGColumnStat *col_stat = column_stat_array_.at(i); bool is_valid = ObColumnStatParam::is_valid_histogram_type(col_desc.col_type_.get_type()); if (col_stat != nullptr && is_valid) { - ObObj obj; - if (OB_FAIL(datum.to_obj_enhance(obj, col_desc.col_type_))) { - LOG_WARN("Failed to transform datum to obj", K(ret), K(i), K(datum_row.storage_datums_[i])); - } else if (OB_FAIL(col_stat->merge_obj(obj))) { - LOG_WARN("Failed to merge obj", K(ret), K(obj), KP(col_stat)); + if (OB_FAIL(col_stat->update_column_stat_info(&datum, col_desc.col_type_, cmp_func.cmp_func_))) { + LOG_WARN("Failed to merge obj", K(ret), KP(col_stat)); } } } for (int64_t i = merge_param_->rowkey_column_num_; OB_SUCC(ret) && i < merge_param_->table_data_desc_.column_count_; i++) { const ObStorageDatum &datum = datum_row.storage_datums_[i + extra_rowkey_cnt]; + const common::ObCmpFunc &cmp_func = merge_param_->datum_utils_->get_cmp_funcs().at(i + extra_rowkey_cnt).get_cmp_func(); const ObColDesc &col_desc = merge_param_->col_descs_->at(i); - ObOptColumnStat *col_stat = column_stat_array_.at(i); + ObOptOSGColumnStat *col_stat = column_stat_array_.at(i); bool is_valid = ObColumnStatParam::is_valid_histogram_type(col_desc.col_type_.get_type()); if (col_stat != nullptr && is_valid) { - ObObj obj; - if (OB_FAIL(datum.to_obj_enhance(obj, col_desc.col_type_))) { - LOG_WARN("Failed to transform datum to obj", K(ret), K(i), K(datum_row.storage_datums_[i])); - } else if (OB_FAIL(col_stat->merge_obj(obj))) { - LOG_WARN("Failed to merge obj", K(ret), K(obj), KP(col_stat)); + if (OB_FAIL(col_stat->update_column_stat_info(&datum, col_desc.col_type_, cmp_func.cmp_func_))) { + LOG_WARN("Failed to merge obj", K(ret), KP(col_stat)); } } } diff --git a/src/storage/direct_load/ob_direct_load_partition_merge_task.h b/src/storage/direct_load/ob_direct_load_partition_merge_task.h index ec3c2d818a..a3633ab280 100644 --- a/src/storage/direct_load/ob_direct_load_partition_merge_task.h +++ b/src/storage/direct_load/ob_direct_load_partition_merge_task.h @@ -10,6 +10,7 @@ #include "storage/direct_load/ob_direct_load_external_scanner.h" #include "storage/direct_load/ob_direct_load_merge_ctx.h" #include "storage/direct_load/ob_direct_load_multiple_heap_table_scanner.h" +#include "sql/engine/expr/ob_expr_sys_op_opnsize.h" namespace oceanbase { @@ -19,7 +20,7 @@ class ObDatumRange; } // namespace blocksstable namespace common { -class ObOptColumnStat; +class ObOptOSGColumnStat; } // namespace common namespace storage { @@ -34,7 +35,7 @@ public: ObDirectLoadPartitionMergeTask(); virtual ~ObDirectLoadPartitionMergeTask(); int process(); - const common::ObIArray &get_column_stat_array() const + const common::ObIArray &get_column_stat_array() const { return column_stat_array_; } @@ -52,7 +53,7 @@ protected: ObDirectLoadTabletMergeCtx *merge_ctx_; int64_t parallel_idx_; int64_t affected_rows_; - common::ObArray column_stat_array_; + common::ObArray column_stat_array_; common::ObArenaAllocator allocator_; volatile bool is_stop_; bool is_inited_; diff --git a/src/storage/direct_load/ob_direct_load_table_store.cpp b/src/storage/direct_load/ob_direct_load_table_store.cpp index 5c93e0f260..093e4b6977 100644 --- a/src/storage/direct_load/ob_direct_load_table_store.cpp +++ b/src/storage/direct_load/ob_direct_load_table_store.cpp @@ -101,6 +101,7 @@ int ObDirectLoadTableStoreBucket::init(const ObDirectLoadTableStoreParam ¶m, fast_heap_table_build_param.tablet_id_ = tablet_id; fast_heap_table_build_param.snapshot_version_ = param.snapshot_version_; fast_heap_table_build_param.table_data_desc_ = param.table_data_desc_; + fast_heap_table_build_param.datum_utils_ = param.datum_utils_; fast_heap_table_build_param.col_descs_ = param.col_descs_; fast_heap_table_build_param.insert_table_ctx_ = param.insert_table_ctx_; fast_heap_table_build_param.fast_heap_table_ctx_ = param.fast_heap_table_ctx_; diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result index 72dde9c172..27e79406d1 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result @@ -233,6 +233,8 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 443 __all_tenant_rewrite_rules 0 201001 1 444 __all_reserved_snapshot 0 201001 1 445 __all_cluster_event_history 0 201001 1 +451 __all_task_opt_stat_gather_history 0 201001 1 +452 __all_table_opt_stat_gather_history 0 201001 1 10001 __tenant_virtual_all_table 2 201001 1 10002 __tenant_virtual_table_column 2 201001 1 10003 __tenant_virtual_table_index 2 201001 1 @@ -585,6 +587,9 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 12365 __all_virtual_ls_arb_replica_task_history 2 201001 1 12366 __all_virtual_archive_dest_status 2 201001 1 12369 __all_virtual_io_scheduler 2 201001 1 +12381 __all_virtual_task_opt_stat_gather_history 2 201001 1 +12382 __all_virtual_table_opt_stat_gather_history 2 201001 1 +12383 __all_virtual_opt_stat_gather_monitor 2 201001 1 12384 __all_virtual_thread 2 201001 1 12393 __all_virtual_virtual_long_ops_status_mysql_sys_agent 2 201001 1 12395 __all_virtual_timestamp_service 2 201001 1 @@ -877,9 +882,14 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 21363 DBA_OB_LS_LOG_ARCHIVE_PROGRESS 1 201001 1 21364 CDB_OB_LS_LOG_ARCHIVE_PROGRESS 1 201001 1 21369 DBA_OB_RSRC_IO_DIRECTIVES 1 201001 1 +21376 GV$OB_OPT_STAT_GATHER_MONITOR 1 201001 1 +21377 V$OB_OPT_STAT_GATHER_MONITOR 1 201001 1 +21378 DBA_OB_TASK_OPT_STAT_GATHER_HISTORY 1 201001 1 +21379 DBA_OB_TABLE_OPT_STAT_GATHER_HISTORY 1 201001 1 21380 GV$OB_THREAD 1 201001 1 21381 V$OB_THREAD 1 201001 1 21404 V$OB_TIMESTAMP_SERVICE 1 201001 1 +21423 DBA_OB_TABLE_STAT_STALE_INFO 1 201001 1 check sys table count and table_id range success check count and table_id range for virtual table success select * from information_schema.CHARACTER_SETS limit 1; diff --git a/unittest/share/test_ob_column_stat.cpp b/unittest/share/test_ob_column_stat.cpp index 40d014d4fd..84c5d43950 100644 --- a/unittest/share/test_ob_column_stat.cpp +++ b/unittest/share/test_ob_column_stat.cpp @@ -16,7 +16,6 @@ #include "lib/allocator/ob_malloc.h" #include "lib/allocator/page_arena.h" -#include "share/stat/ob_column_stat.h" using namespace oceanbase; using namespace common; diff --git a/unittest/sql/optimizer/ob_mock_stat_manager.h b/unittest/sql/optimizer/ob_mock_stat_manager.h index 007518e8ba..ad60924621 100644 --- a/unittest/sql/optimizer/ob_mock_stat_manager.h +++ b/unittest/sql/optimizer/ob_mock_stat_manager.h @@ -205,9 +205,9 @@ public: ObObj min; ObObj max; ObColumnStat *column_stat = (ObColumnStat *)allocator_.alloc(sizeof(ObColumnStat)); - char *llc_bitmap = (char *)allocator_.alloc(oceanbase::common::ObColumnStat::NUM_LLC_BUCKET); + char *llc_bitmap = (char *)allocator_.alloc(oceanbase::common::ObOptColumnStat::NUM_LLC_BUCKET); MEMSET(llc_bitmap, 1, 96); - MEMSET(llc_bitmap + 96, 0, oceanbase::common::ObColumnStat::NUM_LLC_BUCKET - 96); + MEMSET(llc_bitmap + 96, 0, oceanbase::common::ObOptColumnStat::NUM_LLC_BUCKET - 96); if (NULL == column_stat) { ret = OB_ERR_UNEXPECTED; COMMON_LOG(WARN, "failed to allocate column stat", K(ret)); @@ -221,7 +221,7 @@ public: column_stat->set_column_id(column_id); column_stat->set_num_distinct(int_info_.distinct_num_); column_stat->set_num_null(int_info_.null_num_); - column_stat->set_llc_bitmap(llc_bitmap, oceanbase::common::ObColumnStat::NUM_LLC_BUCKET); + column_stat->set_llc_bitmap(llc_bitmap, oceanbase::common::ObOptColumnStat::NUM_LLC_BUCKET); column_stat->set_version(1); } if (OB_SUCC(ret) && NULL != schema_checker_) { @@ -236,7 +236,7 @@ public: column_stat->set_num_null(0); min.set_min_value(); max.set_max_value(); - MEMSET(llc_bitmap, 0, oceanbase::common::ObColumnStat::NUM_LLC_BUCKET); + MEMSET(llc_bitmap, 0, oceanbase::common::ObOptColumnStat::NUM_LLC_BUCKET); } else if (oceanbase::common::ObStringTC == col_schema->get_meta_type().get_type_class()) { column_stat->set_num_distinct(str_info_.distinct_num_); column_stat->set_num_null(str_info_.null_num_);