From 5c5e6da6ce9949f2fe0c8ecb95bbd7ac2bf721eb Mon Sep 17 00:00:00 2001 From: pe-99y <315053752@qq.com> Date: Mon, 24 Jun 2024 13:57:14 +0000 Subject: [PATCH] [FEAT MERGE] DAS iterator refactor and keep order optimization Co-authored-by: saltonz Co-authored-by: zhenhan.gong@gmail.com Co-authored-by: Tyshawn --- deps/oblib/src/common/ob_range.cpp | 1 + deps/oblib/src/common/ob_range.h | 27 +- deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h | 2 +- src/observer/ob_srv_xlator_rootserver.cpp | 1 + src/rootserver/CMakeLists.txt | 1 + src/rootserver/ddl_task/ob_ddl_scheduler.cpp | 134 +- src/rootserver/ddl_task/ob_ddl_scheduler.h | 12 + src/rootserver/ddl_task/ob_ddl_task.cpp | 2 + .../ddl_task/ob_drop_fts_index_task.cpp | 16 +- .../ddl_task/ob_fts_index_build_task.cpp | 1667 +++++++++++++++++ .../ddl_task/ob_fts_index_build_task.h | 155 ++ .../ddl_task/ob_index_build_task.cpp | 124 +- src/rootserver/ob_ddl_service.cpp | 155 +- src/rootserver/ob_ddl_service.h | 9 + src/rootserver/ob_index_builder.cpp | 42 +- src/rootserver/ob_root_service.cpp | 19 +- src/rootserver/ob_root_service.h | 3 + src/rootserver/ob_rs_rpc_processor.h | 1 + src/share/ob_common_rpc_proxy.h | 1 + src/share/ob_ddl_common.h | 20 +- src/share/ob_fts_index_builder_util.cpp | 89 +- src/share/ob_fts_index_builder_util.h | 6 + src/share/ob_rpc_struct.cpp | 20 + src/share/ob_rpc_struct.h | 58 + src/share/parameter/ob_parameter_seed.ipp | 3 + src/sql/CMakeLists.txt | 8 +- src/sql/code_generator/ob_tsc_cg_service.cpp | 96 +- src/sql/code_generator/ob_tsc_cg_service.h | 6 +- .../das/iter/ob_das_global_lookup_iter.cpp | 241 +++ src/sql/das/iter/ob_das_global_lookup_iter.h | 72 + src/sql/das/iter/ob_das_group_fold_iter.cpp | 10 +- src/sql/das/iter/ob_das_group_fold_iter.h | 7 +- src/sql/das/iter/ob_das_iter.cpp | 29 +- src/sql/das/iter/ob_das_iter.h | 71 +- src/sql/das/iter/ob_das_iter_define.h | 89 + src/sql/das/iter/ob_das_iter_utils.cpp | 956 +++++++--- src/sql/das/iter/ob_das_iter_utils.h | 171 +- src/sql/das/iter/ob_das_local_lookup_iter.cpp | 366 ++++ src/sql/das/iter/ob_das_local_lookup_iter.h | 94 + src/sql/das/iter/ob_das_lookup_iter.cpp | 241 +-- src/sql/das/iter/ob_das_lookup_iter.h | 71 +- src/sql/das/iter/ob_das_merge_iter.cpp | 61 +- src/sql/das/iter/ob_das_merge_iter.h | 22 +- src/sql/das/iter/ob_das_scan_iter.cpp | 141 ++ src/sql/das/iter/ob_das_scan_iter.h | 77 + src/sql/das/iter/ob_das_sort_iter.cpp | 272 +++ src/sql/das/iter/ob_das_sort_iter.h | 95 + .../das/iter/ob_das_text_retrieval_iter.cpp} | 506 +++-- src/sql/das/iter/ob_das_text_retrieval_iter.h | 151 ++ .../iter/ob_das_text_retrieval_merge_iter.cpp | 708 +++++++ .../iter/ob_das_text_retrieval_merge_iter.h | 165 ++ src/sql/das/ob_das_factory.cpp | 2 +- src/sql/das/ob_das_ir_define.cpp | 39 + ...text_retrieval_op.h => ob_das_ir_define.h} | 190 +- src/sql/das/ob_das_scan_op.cpp | 448 ++--- src/sql/das/ob_das_scan_op.h | 11 +- .../das/ob_das_spatial_index_lookup_op.cpp | 2 +- src/sql/das/ob_domain_index_lookup_op.cpp | 293 +-- src/sql/das/ob_domain_index_lookup_op.h | 38 - src/sql/engine/table/ob_table_scan_op.cpp | 153 +- src/sql/engine/table/ob_table_scan_op.h | 14 +- src/sql/ob_optimizer_trace_impl.cpp | 1 + src/sql/optimizer/ob_join_order.cpp | 26 +- src/sql/optimizer/ob_log_plan.cpp | 38 +- src/sql/optimizer/ob_log_plan.h | 1 + src/sql/optimizer/ob_log_table_scan.cpp | 51 +- src/sql/optimizer/ob_log_table_scan.h | 10 +- src/sql/optimizer/ob_logical_operator.cpp | 56 +- src/sql/optimizer/ob_optimizer.cpp | 5 + src/sql/optimizer/ob_optimizer_context.h | 4 + src/sql/plan_cache/ob_plan_cache_util.cpp | 12 +- src/sql/plan_cache/ob_plan_cache_util.h | 2 + .../resolver/ddl/ob_create_index_resolver.cpp | 16 +- src/sql/resolver/ddl/ob_ddl_resolver.cpp | 15 +- src/sql/resolver/dml/ob_hint.cpp | 5 + src/sql/resolver/dml/ob_hint.h | 1 + src/storage/CMakeLists.txt | 1 - src/storage/access/ob_table_scan_range.cpp | 2 +- src/storage/blocksstable/ob_datum_range.h | 2 +- src/storage/blocksstable/ob_datum_rowkey.h | 4 +- src/storage/fts/ob_text_retrieval_iterator.h | 177 -- .../all_virtual_sys_parameter_stat.result | 1 + .../r/mysql/skyline_business_mysql.result | 2 +- .../subquery/r/mysql/subquery.result | 6 +- .../test_suite/update/r/mysql/update2.result | 8 +- unittest/sql/rewrite/test_query_range.cpp | 12 +- .../rewrite/test_query_range_collation.result | 414 ++-- .../rewrite/test_query_range_filter.result | 138 +- 88 files changed, 7062 insertions(+), 2432 deletions(-) create mode 100644 src/rootserver/ddl_task/ob_fts_index_build_task.cpp create mode 100644 src/rootserver/ddl_task/ob_fts_index_build_task.h create mode 100644 src/sql/das/iter/ob_das_global_lookup_iter.cpp create mode 100644 src/sql/das/iter/ob_das_global_lookup_iter.h create mode 100644 src/sql/das/iter/ob_das_iter_define.h create mode 100644 src/sql/das/iter/ob_das_local_lookup_iter.cpp create mode 100644 src/sql/das/iter/ob_das_local_lookup_iter.h create mode 100644 src/sql/das/iter/ob_das_scan_iter.cpp create mode 100644 src/sql/das/iter/ob_das_scan_iter.h create mode 100644 src/sql/das/iter/ob_das_sort_iter.cpp create mode 100644 src/sql/das/iter/ob_das_sort_iter.h rename src/{storage/fts/ob_text_retrieval_iterator.cpp => sql/das/iter/ob_das_text_retrieval_iter.cpp} (54%) create mode 100644 src/sql/das/iter/ob_das_text_retrieval_iter.h create mode 100644 src/sql/das/iter/ob_das_text_retrieval_merge_iter.cpp create mode 100644 src/sql/das/iter/ob_das_text_retrieval_merge_iter.h create mode 100644 src/sql/das/ob_das_ir_define.cpp rename src/sql/das/{ob_text_retrieval_op.h => ob_das_ir_define.h} (54%) delete mode 100644 src/storage/fts/ob_text_retrieval_iterator.h diff --git a/deps/oblib/src/common/ob_range.cpp b/deps/oblib/src/common/ob_range.cpp index 83cf2165c..d6de4ead9 100644 --- a/deps/oblib/src/common/ob_range.cpp +++ b/deps/oblib/src/common/ob_range.cpp @@ -96,6 +96,7 @@ int64_t ObNewRange::to_simple_string(char *buffer, const int64_t length) const databuff_printf(buffer, length, pos, "table_id:null,"); } databuff_printf(buffer, length, pos, "group_idx:%d,", group_idx_); + databuff_printf(buffer, length, pos, "index_ordered_idx:%d,", index_ordered_idx_); if (border_flag_.inclusive_start()) { databuff_printf(buffer, length, pos, "["); } else { diff --git a/deps/oblib/src/common/ob_range.h b/deps/oblib/src/common/ob_range.h index d247c69b7..d198082ec 100644 --- a/deps/oblib/src/common/ob_range.h +++ b/deps/oblib/src/common/ob_range.h @@ -17,6 +17,7 @@ #include "lib/utility/utility.h" #include "common/rowkey/ob_rowkey.h" #include "common/ob_string_buf.h" +#include "share/ob_cluster_version.h" namespace oceanbase @@ -335,7 +336,8 @@ public: struct { int64_t group_idx_: 32; int64_t is_physical_rowid_range_: 1; - int64_t reserved_: 31; + int64_t index_ordered_idx_ : 16; // used for keep order of global index lookup + int64_t reserved_: 15; }; }; @@ -382,6 +384,29 @@ public: { return group_idx_; } + inline int32_t get_index_ordered_idx() const + { + return index_ordered_idx_; + } + + // pseudo-column [GROUP_ID], with high 32 bits as group_idx_ and low 32 bits as index_ordered_idx_ + // when cluster version < 4.3.2, the das keep order optimization is disabled, we should only fill + // group_idx to [GROUP_ID] for compatibility. + inline int64_t get_group_id() const + { + return GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_3_2_0 ? group_idx_ : + (static_cast(group_idx_) << 32) | (index_ordered_idx_ & 0xffffffff); + } + // get group_idx from [GROUP_ID] + static int64_t get_group_idx(int64_t group_id) + { + return GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_3_2_0 ? group_id : (group_id >> 32); + } + // get index_order_idx from [GROUP_ID] + static int64_t get_index_ordered_idx(int64_t group_id) + { + return group_id & 0xffffffff; + } int build_range(uint64_t table_id, ObRowkey rowkey) { diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index 182f199b8..065f05f1c 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -1161,7 +1161,7 @@ PCODE_DEF(OB_CAL_STANDBY_TENANT_PHY_RESOURCE, 0x1623) //PCODE_DEF(OB_UPDATE_MVIEW_REFERENCE_TABLE_STATUS, 0x1624) //PCODE_DEF(OB_DO_EVENT_DDL, 0x1625) -//PCODE_DEF(OB_GENERATE_AUX_INDEX_SCHEMA, 0x1626) +PCODE_DEF(OB_GENERATE_AUX_INDEX_SCHEMA, 0x1626) //PCODE_DEF(OB_DUMP_SS_PHY_BLOCK, 0x1627) //PCODE_DEF(OB_DUMP_SS_MACRO_BLOCK, 0x1628) diff --git a/src/observer/ob_srv_xlator_rootserver.cpp b/src/observer/ob_srv_xlator_rootserver.cpp index 7a532d6a2..027e9efef 100644 --- a/src/observer/ob_srv_xlator_rootserver.cpp +++ b/src/observer/ob_srv_xlator_rootserver.cpp @@ -111,6 +111,7 @@ void oceanbase::observer::init_srv_xlator_for_rootserver(ObSrvRpcXlator *xlator) RPC_PROCESSOR(rootserver::ObRpcRenameTableP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcTruncateTableP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcTruncateTableV2P, *gctx_.root_service_); + RPC_PROCESSOR(rootserver::ObRpcGenerateAuxIndexSchemaP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcCreateIndexP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcDropIndexP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcCreateMLogP, *gctx_.root_service_); diff --git a/src/rootserver/CMakeLists.txt b/src/rootserver/CMakeLists.txt index acfc6147c..60e0cac88 100644 --- a/src/rootserver/CMakeLists.txt +++ b/src/rootserver/CMakeLists.txt @@ -125,6 +125,7 @@ ob_set_subtarget(ob_rootserver ddl_task ddl_task/ob_drop_primary_key_task.cpp ddl_task/ob_index_build_task.cpp ddl_task/ob_build_mview_task.cpp + ddl_task/ob_fts_index_build_task.cpp ddl_task/ob_modify_autoinc_task.cpp ddl_task/ob_table_redefinition_task.cpp ddl_task/ob_recover_restore_table_task.cpp diff --git a/src/rootserver/ddl_task/ob_ddl_scheduler.cpp b/src/rootserver/ddl_task/ob_ddl_scheduler.cpp index 52eeed999..916b9d3c4 100755 --- a/src/rootserver/ddl_task/ob_ddl_scheduler.cpp +++ b/src/rootserver/ddl_task/ob_ddl_scheduler.cpp @@ -26,6 +26,7 @@ #include "rootserver/ddl_task/ob_drop_primary_key_task.h" #include "rootserver/ddl_task/ob_index_build_task.h" #include "rootserver/ddl_task/ob_build_mview_task.h" +#include "rootserver/ddl_task/ob_fts_index_build_task.h" #include "rootserver/ddl_task/ob_modify_autoinc_task.h" #include "rootserver/ddl_task/ob_table_redefinition_task.h" #include "rootserver/ddl_task/ob_recover_restore_table_task.h" @@ -1002,6 +1003,20 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam ¶m, LOG_WARN("fail to create build index task", K(ret)); } break; + case DDL_CREATE_FTS_INDEX: + create_index_arg = static_cast(param.ddl_arg_); + if (OB_FAIL(create_build_fts_index_task(proxy, + param.src_table_schema_, + param.dest_table_schema_, + param.parallelism_, + param.parent_task_id_, + param.consumer_group_id_, + create_index_arg, + *param.allocator_, + task_record))) { + LOG_WARN("fail to create build fts index task", K(ret)); + } + break; case DDL_DROP_INDEX: case DDL_DROP_MLOG: // in this case, src_table_schema is data table, dest_table_schema is index table @@ -1519,6 +1534,50 @@ int ObDDLScheduler::start_redef_table(const obrpc::ObStartRedefTableArg &arg, ob return ret; } +int ObDDLScheduler::create_build_fts_index_task( + common::ObISQLClient &proxy, + const ObTableSchema *data_table_schema, + const ObTableSchema *index_schema, + const int64_t parallelism, + const int64_t parent_task_id, + const int64_t consumer_group_id, + const obrpc::ObCreateIndexArg *create_index_arg, + ObIAllocator &allocator, + ObDDLTaskRecord &task_record) +{ + int ret = OB_SUCCESS; + int64_t task_id = 0; + SMART_VAR(ObFtsIndexBuildTask, index_task) { + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_ISNULL(create_index_arg) || OB_ISNULL(data_table_schema) || OB_ISNULL(index_schema)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), KPC(create_index_arg), + KPC(data_table_schema), KPC(index_schema)); + } else if (OB_FAIL(ObDDLTask::fetch_new_task_id(root_service_->get_sql_proxy(), data_table_schema->get_tenant_id(), task_id))) { + LOG_WARN("fetch new task id failed", K(ret)); + } else if (OB_FAIL(index_task.init(data_table_schema->get_tenant_id(), + task_id, + data_table_schema, + index_schema, + data_table_schema->get_schema_version(), + parallelism, + consumer_group_id, + *create_index_arg, + parent_task_id))) { + LOG_WARN("init fts index task failed", K(ret), K(data_table_schema), K(index_schema)); + } else if (OB_FAIL(index_task.set_trace_id(*ObCurTraceId::get_trace_id()))) { + LOG_WARN("set trace id failed", K(ret)); + } else if (OB_FAIL(insert_task_record(proxy, index_task, allocator, task_record))) { + LOG_WARN("fail to insert task record", K(ret)); + } + + LOG_INFO("ddl_scheduler create build index task finished", K(ret), K(index_task)); + } + return ret; +} + int ObDDLScheduler::create_build_index_task( common::ObISQLClient &proxy, const share::ObDDLType &ddl_type, @@ -1543,7 +1602,7 @@ int ObDDLScheduler::create_build_index_task( } else if (OB_ISNULL(create_index_arg) || OB_ISNULL(data_table_schema) || OB_ISNULL(index_schema) || OB_UNLIKELY(tenant_data_version <= 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(create_index_arg), K(data_table_schema), K(index_schema), K(tenant_data_version)); + LOG_WARN("invalid argument", K(ret), KPC(create_index_arg), KPC(data_table_schema), KPC(index_schema), K(tenant_data_version)); } else if (OB_FAIL(ObDDLTask::fetch_new_task_id(root_service_->get_sql_proxy(), data_table_schema->get_tenant_id(), task_id))) { LOG_WARN("fetch new task id failed", K(ret)); } else if (OB_FAIL(index_task.init(data_table_schema->get_tenant_id(), @@ -1643,33 +1702,44 @@ int ObDDLScheduler::create_drop_fts_index_task( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), KP(index_schema)); } else if (FALSE_IT(is_fts_index = index_schema->is_fts_index_aux())) { - } else if (OB_ISNULL(rowkey_doc_schema) - || OB_ISNULL(doc_rowkey_schema) - || (is_fts_index && OB_ISNULL(doc_word_schema)) - || OB_UNLIKELY(schema_version <= 0)) { + } else if (OB_UNLIKELY(schema_version <= 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), KP(index_schema), KP(rowkey_doc_schema), K(doc_rowkey_schema), - K(doc_word_schema), K(schema_version)); + LOG_WARN("invalid argument", K(ret), KP(index_schema), K(schema_version)); } else if (OB_FAIL(ObDDLTask::fetch_new_task_id(root_service_->get_sql_proxy(), index_schema->get_tenant_id(), task_id))) { LOG_WARN("fetch new task id failed", K(ret)); } else if (OB_FAIL(index_schema->get_index_name(domain_index_name))) { LOG_WARN("fail to get domain index name", K(ret), KPC(index_schema)); - } else if (is_fts_index && OB_FAIL(doc_word_schema->get_index_name(fts_doc_word_name))) { - LOG_WARN("fail to get fts doc word name", K(ret), KPC(doc_word_schema)); - } else if (OB_FAIL(rowkey_doc_schema->get_index_name(rowkey_doc_name))) { - LOG_WARN("fail to get rowkey doc name", K(ret), KPC(rowkey_doc_schema)); - } else if (OB_FAIL(doc_rowkey_schema->get_index_name(doc_rowkey_name))) { - LOG_WARN("fail to get doc rowkey name", K(ret), KPC(doc_rowkey_schema)); } else { + if (is_fts_index) { + if (OB_FAIL(ret) || OB_ISNULL(doc_word_schema)) { + } else if (OB_FAIL(doc_word_schema->get_index_name(fts_doc_word_name))) { + LOG_WARN("fail to get fts doc word name", K(ret), KPC(doc_word_schema)); + } + } + if (OB_FAIL(ret) || OB_ISNULL(rowkey_doc_schema)) { + } else if (OB_FAIL(rowkey_doc_schema->get_index_name(rowkey_doc_name))) { + LOG_WARN("fail to get rowkey doc name", K(ret), KPC(rowkey_doc_schema)); + } + if (OB_FAIL(ret) || OB_ISNULL(doc_rowkey_schema)) { + } else if (OB_FAIL(doc_rowkey_schema->get_index_name(doc_rowkey_name))) { + LOG_WARN("fail to get doc rowkey name", K(ret), KPC(doc_rowkey_schema)); + } const uint64_t data_table_id = index_schema->get_data_table_id(); const ObFTSDDLChildTaskInfo domain_index(domain_index_name, index_schema->get_table_id(), 0/*task_id*/); + uint64_t rowkey_doc_table_id = OB_ISNULL(rowkey_doc_schema) ? OB_INVALID_ID : + rowkey_doc_schema->get_table_id(); + uint64_t doc_rowkey_table_id = OB_ISNULL(doc_rowkey_schema) ? OB_INVALID_ID : + doc_rowkey_schema->get_table_id(); + uint64_t doc_word_table_id = OB_ISNULL(doc_word_schema) ? OB_INVALID_ID : + doc_word_schema->get_table_id(); const ObFTSDDLChildTaskInfo fts_doc_word(fts_doc_word_name, - is_fts_index ? doc_word_schema->get_table_id() : OB_INVALID_ID, 0/*task_id*/); - const ObFTSDDLChildTaskInfo rowkey_doc(rowkey_doc_name, rowkey_doc_schema->get_table_id(), 0/*task_id*/); - const ObFTSDDLChildTaskInfo doc_rowkey(doc_rowkey_name, doc_rowkey_schema->get_table_id(), 0/*task_id*/); + is_fts_index ? doc_word_table_id : OB_INVALID_ID, 0/*task_id*/); + const ObFTSDDLChildTaskInfo rowkey_doc(rowkey_doc_name, rowkey_doc_table_id, 0/*task_id*/); + const ObFTSDDLChildTaskInfo doc_rowkey(doc_rowkey_name, doc_rowkey_table_id, 0/*task_id*/); const ObDDLType ddl_type = is_fts_index ? DDL_DROP_FTS_INDEX : DDL_DROP_MULVALUE_INDEX; - if (OB_FAIL(index_task.init(index_schema->get_tenant_id(), + if (OB_FAIL(ret)) { + } else if (OB_FAIL(index_task.init(index_schema->get_tenant_id(), task_id, data_table_id, ddl_type, @@ -2158,6 +2228,9 @@ int ObDDLScheduler::schedule_ddl_task(const ObDDLTaskRecord &record) case ObDDLType::DDL_DROP_MLOG: ret = schedule_drop_index_task(record); break; + case ObDDLType::DDL_CREATE_FTS_INDEX: + ret = schedule_build_fts_index_task(record); + break; case ObDDLType::DDL_DROP_FTS_INDEX: case ObDDLType::DDL_DROP_MULVALUE_INDEX: ret = schedule_drop_fts_index_task(record); @@ -2222,6 +2295,33 @@ int ObDDLScheduler::schedule_ddl_task(const ObDDLTaskRecord &record) return ret; } +int ObDDLScheduler::schedule_build_fts_index_task( + const ObDDLTaskRecord &task_record) +{ + int ret = OB_SUCCESS; + ObFtsIndexBuildTask *build_index_task = nullptr; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_FAIL(alloc_ddl_task(build_index_task))) { + LOG_WARN("alloc ddl task failed", K(ret)); + } else if (OB_FAIL(build_index_task->init(task_record))) { + LOG_WARN("init global_index_task failed", K(ret), K(task_record)); + } else if (OB_FAIL(build_index_task->set_trace_id(task_record.trace_id_))) { + LOG_WARN("init build index task failed", K(ret)); + } else if (OB_FAIL(inner_schedule_ddl_task(build_index_task, task_record))) { + if (OB_ENTRY_EXIST != ret) { + LOG_WARN("inner schedule task failed", K(ret), K(*build_index_task)); + } + } + if (OB_FAIL(ret) && nullptr != build_index_task) { + build_index_task->~ObFtsIndexBuildTask(); + allocator_.free(build_index_task); + build_index_task = nullptr; + } + return ret; +} + int ObDDLScheduler::schedule_build_index_task( const ObDDLTaskRecord &task_record) { diff --git a/src/rootserver/ddl_task/ob_ddl_scheduler.h b/src/rootserver/ddl_task/ob_ddl_scheduler.h index f5ec751fb..8442639e1 100755 --- a/src/rootserver/ddl_task/ob_ddl_scheduler.h +++ b/src/rootserver/ddl_task/ob_ddl_scheduler.h @@ -367,6 +367,16 @@ private: const uint64_t tenant_data_version, ObIAllocator &allocator, ObDDLTaskRecord &task_record); + int create_build_fts_index_task( + common::ObISQLClient &proxy, + const share::schema::ObTableSchema *data_table_schema, + const share::schema::ObTableSchema *index_schema, + const int64_t parallelism, + const int64_t parent_task_id, + const int64_t consumer_group_id, + const obrpc::ObCreateIndexArg *create_index_arg, + ObIAllocator &allocator, + ObDDLTaskRecord &task_record); int create_constraint_task( common::ObISQLClient &proxy, const share::schema::ObTableSchema *table_schema, @@ -492,6 +502,8 @@ private: ObIAllocator &allocator, ObDDLTaskRecord &task_record); + int schedule_build_fts_index_task( + const ObDDLTaskRecord &task_record); int schedule_build_index_task( const ObDDLTaskRecord &task_record); int schedule_build_mview_task(const ObDDLTaskRecord &task_record); diff --git a/src/rootserver/ddl_task/ob_ddl_task.cpp b/src/rootserver/ddl_task/ob_ddl_task.cpp index b2355d452..0e47ddb6f 100644 --- a/src/rootserver/ddl_task/ob_ddl_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_task.cpp @@ -736,6 +736,8 @@ int ObDDLTask::get_ddl_type_str(const int64_t ddl_type, const char *&ddl_type_st case DDL_CREATE_INDEX: ddl_type_str = "create index"; break; + case DDL_CREATE_FTS_INDEX: + ddl_type_str = "create fts index"; case DDL_CREATE_PARTITIONED_LOCAL_INDEX: ddl_type_str = "create partitioned local index"; break; diff --git a/src/rootserver/ddl_task/ob_drop_fts_index_task.cpp b/src/rootserver/ddl_task/ob_drop_fts_index_task.cpp index 4bbf4e1e8..e101decde 100644 --- a/src/rootserver/ddl_task/ob_drop_fts_index_task.cpp +++ b/src/rootserver/ddl_task/ob_drop_fts_index_task.cpp @@ -57,14 +57,11 @@ int ObDropFTSIndexTask::init( if (OB_UNLIKELY(OB_INVALID_ID == tenant_id || task_id <= 0 || OB_INVALID_ID == data_table_id - || !rowkey_doc.is_valid() - || !doc_rowkey.is_valid() || !domain_index.is_valid() - || (is_fts_task && !fts_doc_word.is_valid()) || schema_version <= 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(tenant_id), K(task_id), K(data_table_id), K(rowkey_doc), - K(doc_rowkey), K(domain_index), K(fts_doc_word), K(schema_version)); + LOG_WARN("invalid arguments", K(ret), K(tenant_id), K(task_id), K(data_table_id), + K(domain_index), K(schema_version)); } else if (OB_ISNULL(root_service_ = GCTX.root_service_)) { ret = OB_ERR_SYS; LOG_WARN("error sys, root service is null", K(ret)); @@ -377,7 +374,7 @@ int ObDropFTSIndexTask::wait_child_task_finish( for (int64_t i = 0; OB_SUCC(ret) && finished && i < child_task_ids.count(); ++i) { const ObFTSDDLChildTaskInfo &task_info = child_task_ids.at(i); finished = false; - if (-1 == task_info.task_id_) { + if (-1 == task_info.task_id_ || task_info.table_id_ == OB_INVALID_ID) { finished = true; } else if (OB_FAIL(check_drop_index_finish(tenant_id_, task_info.task_id_, task_info.table_id_, finished))) { LOG_WARN("fail to check fts index child task finish", K(ret)); @@ -434,9 +431,12 @@ int ObDropFTSIndexTask::create_drop_index_task( if (OB_ISNULL(root_service_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error, root service is nullptr", K(ret), KP(root_service_)); - } else if (OB_UNLIKELY(OB_INVALID_ID == index_tid || index_name.empty())) { + } else if (OB_INVALID_ID == index_tid) { + // nothing to do, just by pass. + task_id = -1; + } else if (OB_UNLIKELY(index_name.empty())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), K(index_tid), K(index_name)); + LOG_WARN("invalid arguments", K(ret), K(index_name)); } else if (OB_FAIL(guard.check_table_exist(tenant_id_, index_tid, is_index_exist))) { LOG_WARN("fail to check table exist", K(ret), K(tenant_id_), K(index_tid)); } else if (!is_index_exist) { diff --git a/src/rootserver/ddl_task/ob_fts_index_build_task.cpp b/src/rootserver/ddl_task/ob_fts_index_build_task.cpp new file mode 100644 index 000000000..88728fd21 --- /dev/null +++ b/src/rootserver/ddl_task/ob_fts_index_build_task.cpp @@ -0,0 +1,1667 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX RS + +#include "rootserver/ddl_task/ob_fts_index_build_task.h" +#include "share/ob_ddl_common.h" +#include "share/ob_fts_index_builder_util.h" +#include "share/ob_ddl_error_message_table_operator.h" +#include "rootserver/ob_root_service.h" +#include "rootserver/ob_index_builder.h" +#include "storage/ddl/ob_ddl_lock.h" + +using namespace oceanbase::share; + +namespace oceanbase +{ +namespace rootserver +{ +/*************** ObFtsIndexBuildTask *************/ + +ObFtsIndexBuildTask::ObFtsIndexBuildTask() + : ObDDLTask(ObDDLType::DDL_CREATE_FTS_INDEX), + index_table_id_(target_object_id_), + rowkey_doc_aux_table_id_(OB_INVALID_ID), + doc_rowkey_aux_table_id_(OB_INVALID_ID), + fts_index_aux_table_id_(OB_INVALID_ID), + fts_doc_word_aux_table_id_(OB_INVALID_ID), + rowkey_doc_schema_generated_(false), + doc_rowkey_schema_generated_(false), + fts_index_aux_schema_generated_(false), + fts_doc_word_schema_generated_(false), + rowkey_doc_task_submitted_(false), + doc_rowkey_task_submitted_(false), + fts_index_aux_task_submitted_(false), + fts_doc_word_task_submitted_(false), + drop_index_task_id_(0), + drop_index_task_submitted_(false), + root_service_(nullptr), + create_index_arg_(), + dependent_task_result_map_() +{ +} + +ObFtsIndexBuildTask::~ObFtsIndexBuildTask() +{ +} + +int ObFtsIndexBuildTask::init( + const uint64_t tenant_id, + const int64_t task_id, + const ObTableSchema *data_table_schema, + const ObTableSchema *index_schema, + const int64_t schema_version, + const int64_t parallelism, + const int64_t consumer_group_id, + const obrpc::ObCreateIndexArg &create_index_arg, + const int64_t parent_task_id /* = 0 */, + const int64_t task_status /* PREPARE */, + const int64_t snapshot_version) +{ + int ret = OB_SUCCESS; + uint64_t tenant_data_format_version = 0; + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret)); + } else if (OB_ISNULL(root_service_ = GCTX.root_service_)) { + ret = OB_ERR_SYS; + LOG_WARN("root_service is null", K(ret), KP(root_service_)); + } else if (!root_service_->in_service()) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("root service not in service", K(ret)); + } else if (OB_UNLIKELY(tenant_id == OB_INVALID_TENANT_ID || + task_id <= 0 || + OB_ISNULL(data_table_schema) || + OB_ISNULL(index_schema) || + schema_version <= 0 || + parallelism <= 0 || + consumer_group_id < 0 || + !create_index_arg.is_valid() || + task_status < ObDDLTaskStatus::PREPARE || + task_status > ObDDLTaskStatus::SUCCESS || + snapshot_version < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(tenant_id), K(task_id), + KPC(data_table_schema), KPC(index_schema), K(schema_version), K(parallelism), + K(consumer_group_id), K(create_index_arg.is_valid()), K(create_index_arg), + K(task_status), K(snapshot_version)); + } else if (OB_FAIL(deep_copy_index_arg(allocator_, + create_index_arg, + create_index_arg_))) { + LOG_WARN("fail to copy create index arg", K(ret), K(create_index_arg)); + } else if (OB_FAIL(ObShareUtil::fetch_current_data_version(*GCTX.sql_proxy_, + tenant_id, + tenant_data_format_version))) { + LOG_WARN("get min data version failed", K(ret), K(tenant_id)); + } else { + set_gmt_create(ObTimeUtility::current_time()); + tenant_id_ = tenant_id; + task_id_ = task_id; + schema_version_ = schema_version; + parallelism_ = parallelism; + consumer_group_id_ = consumer_group_id; + parent_task_id_ = parent_task_id; + if (snapshot_version > 0) { + snapshot_version_ = snapshot_version; + } + object_id_ = data_table_schema->get_table_id(); + target_object_id_ = index_schema->get_table_id(); + index_table_id_ = index_schema->get_table_id(); + create_index_arg_.exec_tenant_id_ = tenant_id; + fts_index_aux_table_id_ = index_table_id_; + // fts_index aux schema already generated before ddl task begin + fts_index_aux_schema_generated_ = true; + task_version_ = OB_FTS_INDEX_BUILD_TASK_VERSION; + start_time_ = ObTimeUtility::current_time(); + data_format_version_ = tenant_data_format_version; + if (OB_FAIL(ret)) { + } else if (FALSE_IT(task_status_ = static_cast(task_status))) { + } else if (OB_FAIL(init_ddl_task_monitor_info(index_schema->get_table_id()))) { + LOG_WARN("init ddl task monitor info failed", K(ret)); + } else { + dst_tenant_id_ = tenant_id_; + dst_schema_version_ = schema_version_; + is_inited_ = true; + } + } + return ret; +} + +int ObFtsIndexBuildTask::init(const ObDDLTaskRecord &task_record) +{ + int ret = OB_SUCCESS; + const uint64_t data_table_id = task_record.object_id_; + const uint64_t index_table_id = task_record.target_object_id_; + const int64_t schema_version = task_record.schema_version_; + int64_t pos = 0; + const ObTableSchema *data_schema = nullptr; + const char *ddl_type_str = nullptr; + const char *target_name = nullptr; + ObSchemaGetterGuard schema_guard; + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret)); + } else if (OB_ISNULL(root_service_ = GCTX.root_service_)) { + ret = OB_ERR_SYS; + LOG_WARN("root_service is null", K(ret), KP(root_service_)); + } else if (!root_service_->in_service()) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("root service not in service", K(ret)); + } else if (!task_record.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), K(task_record)); + } else if (OB_FAIL(deserialize_params_from_message(task_record.tenant_id_, + task_record.message_.ptr(), + task_record.message_.length(), + pos))) { + LOG_WARN("deserialize params from message failed", K(ret)); + } else if (OB_FAIL(ObMultiVersionSchemaService::get_instance(). + get_tenant_schema_guard(task_record.tenant_id_, + schema_guard, + schema_version))) { + LOG_WARN("fail to get schema guard", K(ret), K(index_table_id), + K(schema_version)); + } else if (OB_FAIL(schema_guard.check_formal_guard())) { + LOG_WARN("schema_guard is not formal", K(ret), K(index_table_id)); + } else if (OB_FAIL(schema_guard.get_table_schema(task_record.tenant_id_, + data_table_id, + data_schema))) { + LOG_WARN("fail to get table schema", K(ret), K(data_table_id)); + } else if (OB_ISNULL(data_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("fail to get table schema", K(ret), K(data_schema)); + } else { + tenant_id_ = task_record.tenant_id_; + task_id_ = task_record.task_id_; + schema_version_ = schema_version; + parent_task_id_ = task_record.parent_task_id_; + task_status_ = static_cast(task_record.task_status_); + snapshot_version_ = task_record.snapshot_version_; + object_id_ = data_table_id; + target_object_id_ = index_table_id; + index_table_id_ = index_table_id; + fts_index_aux_table_id_ = index_table_id_; + fts_index_aux_schema_generated_ = true; + execution_id_ = task_record.execution_id_; + ret_code_ = task_record.ret_code_; + start_time_ = ObTimeUtility::current_time(); + dst_tenant_id_ = tenant_id_; + dst_schema_version_ = schema_version_; + if (OB_FAIL(ret)) { + } else if (OB_FAIL(init_ddl_task_monitor_info(index_table_id))) { + LOG_WARN("init ddl task monitor info failed", K(ret)); + } else { + is_inited_ = true; + // set up span during recover task + ddl_tracing_.open_for_recovery(); + } + } + return ret; +} + +int ObFtsIndexBuildTask::process() +{ + int ret = OB_SUCCESS; + ObIndexType index_type = create_index_arg_.index_type_; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_FAIL(check_health())) { + LOG_WARN("check health failed", K(ret)); + } else if (!share::schema::is_fts_index(index_type)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expect index type is of fts index", K(ret), K(index_type)); + } else if (!need_retry()) { + // by pass + } else { + // switch case for diff create_index_arg, since there are 4 aux fts tables + ddl_tracing_.restore_span_hierarchy(); + const ObDDLTaskStatus status = static_cast(task_status_); + switch (status) { + case ObDDLTaskStatus::PREPARE: { + if (OB_FAIL(prepare())) { + LOG_WARN("prepare failed", K(ret), K(*this)); + } + break; + } + case ObDDLTaskStatus::GENERATE_ROWKEY_DOC_SCHEMA: { + if (OB_FAIL(prepare_rowkey_doc_table())) { + LOG_WARN("generate schema failed", K(ret), K(*this)); + } + break; + } + case ObDDLTaskStatus::WAIT_ROWKEY_DOC_TABLE_COMPLEMENT: { + if (OB_FAIL(wait_aux_table_complement())) { + LOG_WARN("wait rowkey_doc table complement failed", K(ret), K(*this)); + } + break; + } + case ObDDLTaskStatus::GENERATE_DOC_AUX_SCHEMA: { + if (OB_FAIL(prepare_aux_index_tables())) { + LOG_WARN("generate schema failed", K(ret), K(*this)); + } + break; + } + case ObDDLTaskStatus::WAIT_AUX_TABLE_COMPLEMENT: { + if (OB_FAIL(wait_aux_table_complement())) { + LOG_WARN("wait aux fts table complement failed", K(ret), K(*this)); + } + break; + } + case ObDDLTaskStatus::VALIDATE_CHECKSUM: { + if (OB_FAIL(validate_checksum())) { + LOG_WARN("validate checksum failed", K(ret), K(*this)); + } + break; + } + case ObDDLTaskStatus::FAIL: { + if (OB_FAIL(clean_on_failed())) { + LOG_WARN("clean failed_task failed", K(ret), K(*this)); + } + break; + } + case ObDDLTaskStatus::SUCCESS: { + if (OB_FAIL(succ())) { + LOG_WARN("clean task on finish failed", K(ret), K(*this)); + } + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("not expected status", K(ret), K(status), K(*this)); + } + } // end switch + ddl_tracing_.release_span_hierarchy(); + } + return ret; +} + +bool ObFtsIndexBuildTask::is_valid() const +{ + return is_inited_ && !trace_id_.is_invalid(); +} + +int ObFtsIndexBuildTask::deep_copy_index_arg( + common::ObIAllocator &allocator, + const obrpc::ObCreateIndexArg &source_arg, + obrpc::ObCreateIndexArg &dest_arg) +{ + int ret = OB_SUCCESS; + const int64_t serialize_size = source_arg.get_serialize_size(); + char *buf = nullptr; + int64_t pos = 0; + if (OB_ISNULL(buf = static_cast(allocator.alloc(serialize_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory failed", K(ret), K(serialize_size)); + } else if (OB_FAIL(source_arg.serialize(buf, serialize_size, pos))) { + LOG_WARN("serialize alter table arg", K(ret)); + } else if (FALSE_IT(pos = 0)) { + } else if (OB_FAIL(dest_arg.deserialize(buf, serialize_size, pos))) { + LOG_WARN("deserialize alter table arg failed", K(ret)); + } + if (OB_FAIL(ret) && nullptr != buf) { + allocator.free(buf); + } + return ret; +} + +int ObFtsIndexBuildTask::check_health() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (!root_service_->in_service()) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("root service not in service, not need retry", K(ret)); + need_retry_ = false; // only stop run the task, need not clean up task context + } else if (OB_FAIL(refresh_status())) { // refresh task status + LOG_WARN("refresh status failed", K(ret)); + } else if (OB_FAIL(refresh_schema_version())) { + LOG_WARN("refresh schema version failed", K(ret)); + } else { + ObMultiVersionSchemaService &schema_service = root_service_->get_schema_service(); + ObSchemaGetterGuard schema_guard; + const ObTableSchema *index_schema = nullptr; + bool is_data_table_exist = false; + bool is_all_indexes_exist = false; + if (OB_FAIL(schema_service.get_tenant_schema_guard(tenant_id_, + schema_guard))) { + LOG_WARN("get tenant schema guard failed", K(ret), K(tenant_id_)); + } else if (OB_FAIL(schema_guard.check_table_exist(tenant_id_, + object_id_, + is_data_table_exist))) { + LOG_WARN("check data table exist failed", K(ret), K(tenant_id_), K(object_id_)); + } else if (OB_FAIL(check_aux_table_schemas_exist(is_all_indexes_exist))) { + LOG_WARN("check aux index table exist failed", K(ret), K(tenant_id_)); + } else if (!is_data_table_exist || !is_all_indexes_exist) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("data table or index table not exist", K(ret), K(is_data_table_exist), + K(is_all_indexes_exist)); + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id_, + index_table_id_, + index_schema))) { + LOG_WARN("get table schema failed", K(ret), K(tenant_id_), K(index_table_id_)); + } else if (OB_ISNULL(index_schema)) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("index schema is null, but index table exist", K(ret), + K(index_table_id_)); + } else if (ObIndexStatus::INDEX_STATUS_INDEX_ERROR == index_schema->get_index_status()) { + ret = OB_SUCCESS == ret_code_ ? OB_ERR_ADD_INDEX : ret_code_; + LOG_WARN("index status error", K(ret), K(index_table_id_), + K(index_schema->get_index_status())); + } + #ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = check_errsim_error(); + } + #endif + if (OB_FAIL(ret) && !ObIDDLTask::in_ddl_retry_white_list(ret)) { + const ObDDLTaskStatus old_status = static_cast(task_status_); + const ObDDLTaskStatus new_status = ObDDLTaskStatus::FAIL; + switch_status(new_status, false, ret); + LOG_WARN("switch status to build_failed", K(ret), K(old_status), K(new_status)); + } + if (ObDDLTaskStatus::FAIL == static_cast(task_status_) || + ObDDLTaskStatus::SUCCESS == static_cast(task_status_)) { + ret = OB_SUCCESS; // allow clean up + } + } + check_ddl_task_execute_too_long(); + return ret; +} + +int ObFtsIndexBuildTask::check_aux_table_schemas_exist(bool &is_all_exist) +{ + int ret = OB_SUCCESS; + is_all_exist = false; + const ObDDLTaskStatus status = static_cast(task_status_); + ObMultiVersionSchemaService &schema_service = root_service_->get_schema_service(); + ObSchemaGetterGuard schema_guard; + const ObTableSchema *index_schema = nullptr; + if (OB_FAIL(schema_service.get_tenant_schema_guard(tenant_id_, schema_guard))) { + LOG_WARN("get tenant schema guard failed", K(ret), K(tenant_id_)); + } else { + bool rowkey_doc_exist = true; + bool doc_rowkey_exist = true; + bool fts_index_aux_exist = true; + bool fts_doc_word_exist = true; + if (status <= ObDDLTaskStatus::GENERATE_ROWKEY_DOC_SCHEMA) { + if (OB_FAIL(schema_guard.check_table_exist(tenant_id_, + fts_index_aux_table_id_, + fts_index_aux_exist))) { + LOG_WARN("check data table exist failed", K(ret), K(tenant_id_), + K(fts_index_aux_table_id_)); + } else { + is_all_exist = fts_index_aux_exist; + } + } else if (status <= ObDDLTaskStatus::GENERATE_DOC_AUX_SCHEMA) { + if (OB_FAIL(schema_guard.check_table_exist(tenant_id_, + rowkey_doc_aux_table_id_, + rowkey_doc_exist))) { + LOG_WARN("check data table exist failed", K(ret), K(tenant_id_), + K(rowkey_doc_aux_table_id_)); + } else if (OB_FAIL(schema_guard.check_table_exist(tenant_id_, + fts_index_aux_table_id_, + fts_index_aux_exist))) { + LOG_WARN("check data table exist failed", K(ret), K(tenant_id_), + K(fts_index_aux_table_id_)); + } else { + is_all_exist = (rowkey_doc_exist && fts_index_aux_exist); + } + } else { + if (OB_FAIL(schema_guard.check_table_exist(tenant_id_, + rowkey_doc_aux_table_id_, + rowkey_doc_exist))) { + LOG_WARN("check data table exist failed", K(ret), K(tenant_id_), + K(rowkey_doc_aux_table_id_)); + } else if (OB_FAIL(schema_guard.check_table_exist(tenant_id_, + doc_rowkey_aux_table_id_, + doc_rowkey_exist))) { + LOG_WARN("check data table exist failed", K(ret), K(tenant_id_), + K(doc_rowkey_aux_table_id_)); + } else if (OB_FAIL(schema_guard.check_table_exist(tenant_id_, + fts_index_aux_table_id_, + fts_index_aux_exist))) { + LOG_WARN("check data table exist failed", K(ret), K(tenant_id_), + K(fts_index_aux_table_id_)); + } else if (OB_FAIL(schema_guard.check_table_exist(tenant_id_, + fts_doc_word_aux_table_id_, + fts_doc_word_exist))) { + LOG_WARN("check data table exist failed", K(ret), K(tenant_id_), + K(fts_doc_word_aux_table_id_)); + } else { + is_all_exist = (rowkey_doc_exist && doc_rowkey_exist && + fts_index_aux_exist && fts_doc_word_exist); + if (!is_all_exist) { + LOG_WARN("fts aux table not exist", K(rowkey_doc_exist), + K(doc_rowkey_exist), K(fts_index_aux_exist), K(fts_doc_word_exist)); + } + } + } + } + return ret; +} + +int ObFtsIndexBuildTask::get_next_status(share::ObDDLTaskStatus &next_status) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + ObIndexType index_type = create_index_arg_.index_type_; + const ObDDLTaskStatus status = static_cast(task_status_); + switch (status) { + case ObDDLTaskStatus::PREPARE: { + next_status = ObDDLTaskStatus::GENERATE_ROWKEY_DOC_SCHEMA; + break; + } + case ObDDLTaskStatus::GENERATE_ROWKEY_DOC_SCHEMA: { + next_status = ObDDLTaskStatus::WAIT_ROWKEY_DOC_TABLE_COMPLEMENT; + break; + } + case ObDDLTaskStatus::WAIT_ROWKEY_DOC_TABLE_COMPLEMENT: { + next_status = ObDDLTaskStatus::GENERATE_DOC_AUX_SCHEMA; + break; + } + case ObDDLTaskStatus::GENERATE_DOC_AUX_SCHEMA: { + next_status = ObDDLTaskStatus::WAIT_AUX_TABLE_COMPLEMENT; + break; + } + case ObDDLTaskStatus::WAIT_AUX_TABLE_COMPLEMENT: { + next_status = ObDDLTaskStatus::VALIDATE_CHECKSUM; + break; + } + case ObDDLTaskStatus::VALIDATE_CHECKSUM: { + next_status = ObDDLTaskStatus::SUCCESS; + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("not expected status", K(ret), K(status), K(*this)); + } + } // end switch + } + return ret; +} + +int ObFtsIndexBuildTask::prepare() +{ + int ret = OB_SUCCESS; + bool state_finished = false; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (ObDDLTaskStatus::PREPARE != task_status_) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("task status not match", K(ret), K(task_status_)); + } else { + state_finished = true; + } + + if (state_finished) { + ObDDLTaskStatus next_status; + if (OB_FAIL(get_next_status(next_status))) { + LOG_WARN("failed to get next status", K(ret)); + } else { + (void)switch_status(next_status, true, ret); + LOG_INFO("prepare finished", K(ret), K(parent_task_id_), K(task_id_), K(*this)); + } + } + return ret; +} + +int ObFtsIndexBuildTask::prepare_rowkey_doc_table() +{ + int ret = OB_SUCCESS; + bool state_finished = false; + const int64_t num_fts_child_task = 4; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (ObDDLTaskStatus::GENERATE_ROWKEY_DOC_SCHEMA != task_status_) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("task status not match", K(ret), K(task_status_)); + } else if (!dependent_task_result_map_.created() && + OB_FAIL(dependent_task_result_map_.create(num_fts_child_task, + lib::ObLabel("DepTasMap")))) { + LOG_WARN("create dependent task map failed", K(ret)); + } else { + const uint64_t data_table_id = object_id_; + int64_t ddl_rpc_timeout = 0; + SMART_VARS_4((obrpc::ObCreateIndexArg, rowkey_doc_arg), + (ObDDLTaskRecord, rowkey_doc_task_record), + (obrpc::ObGenerateAuxIndexSchemaArg, arg), + (obrpc::ObGenerateAuxIndexSchemaRes, res)) { + ObDDLService &ddl_service = root_service_->get_ddl_service(); + arg.tenant_id_ = tenant_id_; + arg.exec_tenant_id_ = tenant_id_; + arg.data_table_id_ = data_table_id; + arg.task_id_ = task_id_; + obrpc::ObCommonRpcProxy *common_rpc = nullptr; + if (OB_FAIL(construct_rowkey_doc_arg(rowkey_doc_arg))) { + LOG_WARN("failed to construct rowkey doc id arg", K(ret)); + } else if (!rowkey_doc_schema_generated_) { + if (OB_FAIL(ObDDLUtil::get_ddl_rpc_timeout(tenant_id_, + data_table_id, + ddl_rpc_timeout))) { + LOG_WARN("get ddl rpc timeout fail", K(ret)); + } else if (OB_ISNULL(root_service_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("root_service is nullptr", K(ret)); + } else if (OB_FAIL(arg.create_index_arg_.assign(rowkey_doc_arg))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to assign create index arg", K(ret)); + } else if (OB_FALSE_IT(common_rpc = root_service_->get_ddl_service().get_common_rpc())) { + } else if (OB_ISNULL(common_rpc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("common rpc is nullptr", K(ret)); + } else if (OB_FAIL(common_rpc-> to(obrpc::ObRpcProxy::myaddr_). + timeout(ddl_rpc_timeout).generate_aux_index_schema(arg, + res))) { + LOG_WARN("generate fts aux index schema failed", K(ret), K(arg)); + } else if (res.schema_generated_) { + rowkey_doc_schema_generated_ = true; + rowkey_doc_aux_table_id_ = res.aux_table_id_; + } + } + if (OB_FAIL(ret)) { + } else if (!rowkey_doc_schema_generated_ ) { + } else if (rowkey_doc_task_submitted_) { + } else if (OB_FAIL(submit_build_aux_index_task(rowkey_doc_arg, + rowkey_doc_task_record, + rowkey_doc_task_submitted_))) { + LOG_WARN("fail to submit build rowkey doc id index task", K(ret)); + } else { + state_finished = true; + } + } + } + if (state_finished) { + ObDDLTaskStatus next_status; + if (OB_FAIL(get_next_status(next_status))) { + LOG_WARN("failed to get next status", K(ret)); + } else { + (void)switch_status(next_status, true, ret); + LOG_INFO("generate schema finished", K(ret), K(parent_task_id_), K(task_id_), + K(*this)); + } + } + return ret; +} + +int ObFtsIndexBuildTask::prepare_aux_index_tables() +{ + int ret = OB_SUCCESS; + bool state_finished = false; + const int64_t num_fts_child_task = 4; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (ObDDLTaskStatus::GENERATE_DOC_AUX_SCHEMA != task_status_) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("task status not match", K(ret), K(task_status_)); + } else if (!dependent_task_result_map_.created() && + OB_FAIL(dependent_task_result_map_.create(num_fts_child_task, + lib::ObLabel("DepTasMap")))) { + LOG_WARN("create dependent task map failed", K(ret)); + } else { + const uint64_t data_table_id = object_id_; + int64_t ddl_rpc_timeout = 0; + SMART_VARS_3((obrpc::ObCreateIndexArg, doc_rowkey_arg), + (obrpc::ObCreateIndexArg, fts_index_aux_arg), + (obrpc::ObCreateIndexArg, fts_doc_word_arg)) { + SMART_VARS_3((ObDDLTaskRecord, doc_rowkey_task_record), + (ObDDLTaskRecord, fts_index_aux_task_record), + (ObDDLTaskRecord, fts_doc_word_task_record)) { + SMART_VARS_4((obrpc::ObGenerateAuxIndexSchemaArg, doc_rowkey_schema_arg), + (obrpc::ObGenerateAuxIndexSchemaRes, doc_rowkey_schema_res), + (obrpc::ObGenerateAuxIndexSchemaArg, fts_doc_word_schema_arg), + (obrpc::ObGenerateAuxIndexSchemaRes, fts_doc_word_schema_res)) { + ObDDLService &ddl_service = root_service_->get_ddl_service(); + doc_rowkey_schema_arg.tenant_id_ = tenant_id_; + doc_rowkey_schema_arg.data_table_id_ = data_table_id; + doc_rowkey_schema_arg.exec_tenant_id_ = tenant_id_; + doc_rowkey_schema_arg.task_id_ = task_id_; + fts_doc_word_schema_arg.tenant_id_ = tenant_id_; + fts_doc_word_schema_arg.data_table_id_ = data_table_id; + fts_doc_word_schema_arg.exec_tenant_id_ = tenant_id_; + fts_doc_word_schema_arg.task_id_ = task_id_; + obrpc::ObCommonRpcProxy *common_rpc = nullptr; + if (OB_ISNULL(root_service_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("root_service is nullptr", K(ret)); + } else if (OB_FAIL(ObDDLUtil::get_ddl_rpc_timeout(tenant_id_, + data_table_id, + ddl_rpc_timeout))) { + LOG_WARN("get ddl rpc timeout fail", K(ret)); + } else if (OB_FAIL(construct_doc_rowkey_arg(doc_rowkey_arg))) { + LOG_WARN("fail to construct doc id rowkey arg", K(ret)); + } else if (!doc_rowkey_schema_generated_) { + if (OB_FAIL(doc_rowkey_schema_arg.create_index_arg_.assign(doc_rowkey_arg))) { + LOG_WARN("fail to assign create index arg", K(ret)); + } else if (OB_FALSE_IT(common_rpc = root_service_->get_ddl_service().get_common_rpc())) { + } else if (OB_ISNULL(common_rpc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("common rpc is nullptr", K(ret)); + } else if (OB_FAIL(common_rpc->to(obrpc::ObRpcProxy::myaddr_). + timeout(ddl_rpc_timeout). + generate_aux_index_schema(doc_rowkey_schema_arg, + doc_rowkey_schema_res))) { + LOG_WARN("generate fts doc rowkey schema failed", K(ret), K(doc_rowkey_schema_arg)); + } else if (doc_rowkey_schema_res.schema_generated_) { + doc_rowkey_schema_generated_ = true; + doc_rowkey_aux_table_id_ = doc_rowkey_schema_res.aux_table_id_; + } + } + if (OB_FAIL(ret)) { + } else if (!doc_rowkey_schema_generated_) { + } else if (doc_rowkey_task_submitted_) { + } else if (OB_FAIL(submit_build_aux_index_task(doc_rowkey_arg, + doc_rowkey_task_record, + doc_rowkey_task_submitted_))) { + LOG_WARN("fail to submit build doc id rowkey index task", K(ret)); + } + // NOTE unlike other 3 aux index schemas which require rpc to generate schema, + // fts index schema is generated before this ddl task start + if (OB_FAIL(ret)) { + } else if (OB_FAIL(construct_fts_index_aux_arg(fts_index_aux_arg))) { + LOG_WARN("fail to construct fts index aux arg", K(ret)); + } else if (fts_index_aux_task_submitted_) { + } else if (OB_FAIL(submit_build_aux_index_task(fts_index_aux_arg, + fts_index_aux_task_record, + fts_index_aux_task_submitted_))) { + LOG_WARN("fail to submit build fts index aux task", K(ret)); + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(construct_fts_doc_word_arg(fts_doc_word_arg))) { + LOG_WARN("fail to construct fts doc word arg", K(ret)); + } else if (!fts_doc_word_schema_generated_) { + if (OB_FAIL(fts_doc_word_schema_arg.create_index_arg_.assign(fts_doc_word_arg))) { + LOG_WARN("fail to assign create index arg", K(ret)); + } else if (OB_FAIL(common_rpc-> to(obrpc::ObRpcProxy::myaddr_). + timeout(ddl_rpc_timeout). + generate_aux_index_schema(fts_doc_word_schema_arg, + fts_doc_word_schema_res))) { + LOG_WARN("generate fts doc word schema failed", K(ret), K(fts_doc_word_schema_arg)); + } else if (fts_doc_word_schema_res.schema_generated_) { + fts_doc_word_schema_generated_ = true; + fts_doc_word_aux_table_id_ = fts_doc_word_schema_res.aux_table_id_; + } + } + if (OB_FAIL(ret)) { + } else if (!fts_doc_word_schema_generated_) { + } else if (fts_doc_word_task_submitted_) { + } else if (OB_FAIL(submit_build_aux_index_task(fts_doc_word_arg, + fts_doc_word_task_record, + fts_doc_word_task_submitted_))) { + LOG_WARN("fail to submit build fts doc word index task", K(ret)); + } + if (doc_rowkey_task_submitted_ && fts_index_aux_task_submitted_ && + fts_doc_word_task_submitted_) { + state_finished = true; + } + } + } // SMART_VARS + } // SMART_VARS + } // SMART_VARS + if (state_finished) { + ObDDLTaskStatus next_status; + if (OB_FAIL(get_next_status(next_status))) { + LOG_WARN("failed to get next status", K(ret)); + } else { + (void)switch_status(next_status, true, ret); + LOG_INFO("generate schema finished", K(ret), K(parent_task_id_), K(task_id_), + K(*this)); + } + } + return ret; +} + +int ObFtsIndexBuildTask::construct_rowkey_doc_arg(obrpc::ObCreateIndexArg &arg) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(deep_copy_index_arg(allocator_, create_index_arg_, arg))) { + LOG_WARN("failed to deep copy index arg", K(ret)); + } else if (FALSE_IT(arg.index_type_ = INDEX_TYPE_ROWKEY_DOC_ID_LOCAL)) { + } else if (OB_FAIL(ObFtsIndexBuilderUtil::generate_fts_aux_index_name(arg, + &allocator_))) { + LOG_WARN("failed to generate index name", K(ret)); + } + return ret; +} + +int ObFtsIndexBuildTask::construct_doc_rowkey_arg(obrpc::ObCreateIndexArg &arg) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(deep_copy_index_arg(allocator_, create_index_arg_, arg))) { + LOG_WARN("failed to deep copy index arg", K(ret)); + } else if (FALSE_IT(arg.index_type_ = INDEX_TYPE_DOC_ID_ROWKEY_LOCAL)) { + } else if (OB_FAIL(ObFtsIndexBuilderUtil::generate_fts_aux_index_name(arg, + &allocator_))) { + LOG_WARN("failed to generate index name", K(ret)); + } + return ret; +} + +int ObFtsIndexBuildTask::construct_fts_index_aux_arg(obrpc::ObCreateIndexArg &arg) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(deep_copy_index_arg(allocator_, create_index_arg_, arg))) { + LOG_WARN("failed to deep copy index arg", K(ret)); + } else if (FALSE_IT(arg.index_type_ = INDEX_TYPE_FTS_INDEX_LOCAL)) { + } else if (OB_FAIL(ObFtsIndexBuilderUtil::generate_fts_aux_index_name(arg, + &allocator_))) { + LOG_WARN("failed to generate index name", K(ret)); + } + return ret; +} + +int ObFtsIndexBuildTask::construct_fts_doc_word_arg(obrpc::ObCreateIndexArg &arg) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(deep_copy_index_arg(allocator_, create_index_arg_, arg))) { + LOG_WARN("failed to deep copy index arg", K(ret)); + } else if (FALSE_IT(arg.index_type_ = INDEX_TYPE_FTS_DOC_WORD_LOCAL)) { + } else if (OB_FAIL(ObFtsIndexBuilderUtil::generate_fts_aux_index_name(arg, + &allocator_))) { + LOG_WARN("failed to generate index name", K(ret)); + } + return ret; +} + +int ObFtsIndexBuildTask::record_index_table_id( + const obrpc::ObCreateIndexArg *create_index_arg, + uint64_t &aux_table_id) +{ + int ret = OB_SUCCESS; + ObIndexType index_type = create_index_arg->index_type_; + if (share::schema::is_rowkey_doc_aux(index_type)) { + rowkey_doc_aux_table_id_ = aux_table_id; + } else if (share::schema::is_doc_rowkey_aux(index_type)) { + doc_rowkey_aux_table_id_ = aux_table_id; + } else if (share::schema::is_fts_index_aux(index_type)) { + fts_index_aux_table_id_ = aux_table_id; + } else if (share::schema::is_fts_doc_word_aux(index_type)) { + fts_doc_word_aux_table_id_ = aux_table_id; + } + return ret; +} + +int ObFtsIndexBuildTask::get_index_table_id( + const obrpc::ObCreateIndexArg *create_index_arg, + uint64_t &index_table_id) +{ + int ret = OB_SUCCESS; + ObIndexType index_type = create_index_arg->index_type_; + if (share::schema::is_rowkey_doc_aux(index_type)) { + index_table_id = rowkey_doc_aux_table_id_; + } else if (share::schema::is_doc_rowkey_aux(index_type)) { + index_table_id = doc_rowkey_aux_table_id_; + } else if (share::schema::is_fts_index_aux(index_type)) { + index_table_id = fts_index_aux_table_id_; + } else if (share::schema::is_fts_doc_word_aux(index_type)) { + index_table_id = fts_doc_word_aux_table_id_; + } + return ret; +} + +// wait data complement of aux index tables +int ObFtsIndexBuildTask::wait_aux_table_complement() +{ + using task_iter = common::hash::ObHashMap::const_iterator; + int ret = OB_SUCCESS; + bool child_task_failed = false; + bool state_finished = false; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (ObDDLTaskStatus::WAIT_ROWKEY_DOC_TABLE_COMPLEMENT != task_status_ && + ObDDLTaskStatus::WAIT_AUX_TABLE_COMPLEMENT != task_status_) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("task status not match", K(ret), K(task_status_)); + } else { + int64_t finished_task_cnt = 0; + for (task_iter iter = dependent_task_result_map_.begin(); + OB_SUCC(ret) && iter != dependent_task_result_map_.end(); ++iter) { + const uint64_t task_key = iter->first; + const int64_t target_object_id = -1; + const int64_t child_task_id = iter->second.task_id_; + if (iter->second.ret_code_ == INT64_MAX) { + // maybe ddl already finish when switching rs + HEAP_VAR(ObDDLErrorMessageTableOperator::ObBuildDDLErrorMessage, error_message) { + int64_t unused_user_msg_len = 0; + ObAddr unused_addr; + if (OB_FAIL(ObDDLErrorMessageTableOperator::get_ddl_error_message( + dst_tenant_id_, + child_task_id, + target_object_id, + unused_addr, + false /* is_ddl_retry_task */, + *GCTX.sql_proxy_, + error_message, + unused_user_msg_len))) { + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_SUCCESS; + LOG_INFO("ddl task not finish", K(dst_tenant_id_), K(task_key), + K(child_task_id), K(target_object_id)); + } else { + LOG_WARN("fail to get ddl error message", K(ret), K(task_key), + K(child_task_id), K(target_object_id)); + } + } else { + finished_task_cnt++; + if (error_message.ret_code_ != OB_SUCCESS) { + ret = error_message.ret_code_; + child_task_failed = true; + state_finished = true; + break; + } + } + } + } else { + finished_task_cnt++; + if (iter->second.ret_code_ != OB_SUCCESS) { + ret = iter->second.ret_code_; + } + } + } + if (finished_task_cnt == dependent_task_result_map_.size() || OB_FAIL(ret)) { + // 1. all child tasks finish. + // 2. the parent task exits if any child task fails. + state_finished = true; + } + } + + if (state_finished) { + ObDDLTaskStatus next_status; + // 1. get next_status + if (child_task_failed) { + next_status = ObDDLTaskStatus::FAIL; + } else { + if (OB_FAIL(get_next_status(next_status))) { + LOG_WARN("failed to get next status", K(ret)); + } + } + // 2. switch to next_status + if (OB_FAIL(ret)) { + } else { + (void)switch_status(next_status, true, ret); + LOG_INFO("wait aux table complement finished", K(ret), K(parent_task_id_), + K(task_id_), K(*this)); + } + } + return ret; +} + +// submit child task of build aux index table +int ObFtsIndexBuildTask::submit_build_aux_index_task( + const obrpc::ObCreateIndexArg &create_index_arg, + ObDDLTaskRecord &task_record, + bool &task_submitted) +{ + int ret = OB_SUCCESS; + const ObTableSchema *data_schema = nullptr; + const ObTableSchema *index_schema = nullptr; + ObSchemaGetterGuard schema_guard; + uint64_t index_table_id = 0; + if (OB_FAIL(ObMultiVersionSchemaService::get_instance(). + get_tenant_schema_guard(tenant_id_, + schema_guard))) { + LOG_WARN("fail to get schema guard", K(ret)); + } else if (OB_FAIL(schema_guard.check_formal_guard())) { + LOG_WARN("schema_guard is not formal", K(ret)); + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id_, + object_id_, + data_schema))) { + LOG_WARN("fail to get table schema", K(ret), K(object_id_)); + } else if (OB_FAIL(get_index_table_id(&create_index_arg, index_table_id))) { + LOG_WARN("fail to get index table id", K(ret)); + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id_, + index_table_id, + index_schema))) { + LOG_WARN("fail to get table schema", K(ret), K(index_table_id)); + } else if (OB_ISNULL(data_schema) || OB_ISNULL(index_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("fail to get table schema", K(ret), K(data_schema), K(index_schema), + K(object_id_), K(index_table_id)); + } else { + ObCreateDDLTaskParam param(tenant_id_, + ObDDLType::DDL_CREATE_INDEX, + data_schema, + index_schema, + 0/*object_id*/, + index_schema->get_schema_version(), + parallelism_, + consumer_group_id_, + &allocator_, + &create_index_arg, + task_id_); + param.tenant_data_version_ = data_format_version_; + if (OB_FAIL(GCTX.root_service_->get_ddl_task_scheduler(). + create_ddl_task(param, *GCTX.sql_proxy_, task_record))) { + if (OB_ENTRY_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("submit create index ddl task failed", K(ret)); + } + } else if (OB_FAIL(GCTX.root_service_->get_ddl_task_scheduler(). + schedule_ddl_task(task_record))) { + LOG_WARN("fail to schedule ddl task", K(ret), K(task_record)); + } else { + TCWLockGuard guard(lock_); + DependTaskStatus status; + // check if child task is already added + if (OB_FAIL(dependent_task_result_map_.get_refactored(index_table_id, + status))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + status.task_id_ = task_record.task_id_; + if (OB_FAIL(dependent_task_result_map_.set_refactored(index_table_id, + status))) { + LOG_WARN("set dependent task map failed", K(ret), K(index_table_id)); + } + } else { + LOG_WARN("get from dependent task map failed", K(ret)); + } + } + if (OB_SUCC(ret)) { + task_submitted = true; + LOG_INFO("add build fts index task", K(ret), K(index_table_id), + K(create_index_arg.index_name_), K(index_schema->get_index_type()), + K(status), K(index_schema->get_schema_version()), + K(data_schema->get_schema_version()), K(param.schema_version_)); + } + } + } + return ret; +} + +int ObFtsIndexBuildTask::on_child_task_finish( + const uint64_t child_task_key, + const int ret_code) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("ObFtsIndexBuildTask has not been inited", K(ret)); + } else if (OB_UNLIKELY(common::OB_INVALID_ID == child_task_key)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), K(child_task_key)); + } else { + TCWLockGuard guard(lock_); + int64_t org_ret = INT64_MAX; + DependTaskStatus status; + if (OB_FAIL(dependent_task_result_map_.get_refactored(child_task_key, + status))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_ENTRY_NOT_EXIST; + } + LOG_WARN("get from dependent_task_result_map failed", K(ret), + K(child_task_key)); + } else if (org_ret != INT64_MAX && org_ret != ret_code) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("error unexpected, ddl result triggers twice", K(ret), + K(child_task_key)); + } else if (FALSE_IT(status.ret_code_ = ret_code)) { + } else if (OB_FAIL(dependent_task_result_map_.set_refactored(child_task_key, + status, + true/*overwrite*/))) { + LOG_WARN("set dependent_task_result_map failed", K(ret), K(child_task_key)); + } else { + LOG_INFO("child task finish successfully", K(child_task_key)); + } + } + return ret; +} + +int ObFtsIndexBuildTask::update_index_status_in_schema( + const ObTableSchema &index_schema, + const ObIndexStatus new_status) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + obrpc::ObUpdateIndexStatusArg arg; + arg.index_table_id_ = index_schema.get_table_id(); + arg.status_ = new_status; + arg.exec_tenant_id_ = tenant_id_; + arg.in_offline_ddl_white_list_ = true; + arg.task_id_ = task_id_; + int64_t ddl_rpc_timeout = 0; + int64_t tmp_timeout = 0; + if (INDEX_STATUS_AVAILABLE == new_status) { + const bool is_create_index_syntax = create_index_arg_.ddl_stmt_str_.trim().prefix_match_ci("create"); + if (create_index_arg_.ddl_stmt_str_.empty()) { + // alter table syntax. + } else if (OB_UNLIKELY(!is_create_index_syntax)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected err", K(ret), "ddl_stmt_str", + create_index_arg_.ddl_stmt_str_, K(create_index_arg_)); + } else { + // For create index syntax, create_index_arg_ will record the user sql, + // and generate the ddl_stmt_str when anabling index. + // For alter table add index syntax, create_index_arg_ will not record + // the user sql, and generate the ddl_stmt_str when generating index schema. + arg.ddl_stmt_str_ = create_index_arg_.ddl_stmt_str_; + } + } + + DEBUG_SYNC(BEFORE_UPDATE_GLOBAL_INDEX_STATUS); + obrpc::ObCommonRpcProxy *common_rpc = nullptr; + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ObDDLUtil::get_ddl_rpc_timeout(index_schema.get_all_part_num(), + ddl_rpc_timeout))) { + LOG_WARN("get ddl rpc timeout fail", K(ret)); + } else if (OB_FAIL(ObDDLUtil::get_ddl_rpc_timeout(tenant_id_, + index_schema.get_data_table_id(), + tmp_timeout))) { + LOG_WARN("get ddl rpc timeout fail", K(ret)); + } else if (OB_FALSE_IT(ddl_rpc_timeout += tmp_timeout)) { + } else if (OB_FALSE_IT(common_rpc = root_service_->get_ddl_service().get_common_rpc())) { + } else if (OB_ISNULL(common_rpc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("common rpc is nullptr", K(ret)); + } else if (OB_FAIL(common_rpc->to(GCTX.self_addr()).timeout(ddl_rpc_timeout). + update_index_status(arg))) { + LOG_WARN("update index status failed", K(ret), K(arg)); + } else { + LOG_INFO("notify index status changed finish", K(new_status), + K(index_table_id_), K(ddl_rpc_timeout), "ddl_stmt_str", arg.ddl_stmt_str_); + } + } + return ret; +} + +int ObFtsIndexBuildTask::serialize_params_to_message( + char *buf, + const int64_t buf_len, + int64_t &pos) const +{ + int ret = OB_SUCCESS; + int8_t rowkey_doc_generated = static_cast(rowkey_doc_schema_generated_); + int8_t doc_rowkey_generated = static_cast(doc_rowkey_schema_generated_); + int8_t fts_index_aux_generated = static_cast(fts_index_aux_schema_generated_); + int8_t fts_doc_word_generated = static_cast(fts_doc_word_schema_generated_); + int8_t rowkey_doc_submitted = static_cast(rowkey_doc_task_submitted_); + int8_t doc_rowkey_submitted = static_cast(doc_rowkey_task_submitted_); + int8_t fts_index_aux_submitted = static_cast(fts_index_aux_task_submitted_); + int8_t fts_doc_word_submitted = static_cast(fts_doc_word_task_submitted_); + int8_t drop_index_submitted = static_cast(drop_index_task_submitted_); + if (OB_UNLIKELY(nullptr == buf || buf_len <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), KP(buf), K(buf_len)); + } else if (OB_FAIL(ObDDLTask::serialize_params_to_message(buf, buf_len, pos))) { + LOG_WARN("ObDDLTask serialize failed", K(ret)); + } else if (OB_FAIL(create_index_arg_.serialize(buf, buf_len, pos))) { + LOG_WARN("serialize create index arg failed", K(ret)); + } else if (OB_FAIL(serialization::encode(buf, + buf_len, + pos, + rowkey_doc_aux_table_id_))) { + LOG_WARN("serialize rowkey doc table id failed", K(ret)); + } else if (OB_FAIL(serialization::encode(buf, + buf_len, + pos, + doc_rowkey_aux_table_id_))) { + LOG_WARN("serialize doc rowkey table id failed", K(ret)); + } else if (OB_FAIL(serialization::encode(buf, + buf_len, + pos, + fts_index_aux_table_id_))) { + LOG_WARN("serialize fts index table id failed", K(ret)); + } else if (OB_FAIL(serialization::encode(buf, + buf_len, + pos, + fts_doc_word_aux_table_id_))) { + LOG_WARN("serialize fts doc word table id failed", K(ret)); + } else if (OB_FAIL(serialization::encode_i8(buf, + buf_len, + pos, + rowkey_doc_generated))) { + LOG_WARN("serialize rowkey doc schema generated failed", K(ret)); + } else if (OB_FAIL(serialization::encode_i8(buf, + buf_len, + pos, + doc_rowkey_generated))) { + LOG_WARN("serialize doc rowkey schema generated failed", K(ret)); + } else if (OB_FAIL(serialization::encode_i8(buf, + buf_len, + pos, + fts_index_aux_generated))) { + LOG_WARN("serialize fts index aux schema generated failed", K(ret)); + } else if (OB_FAIL(serialization::encode_i8(buf, + buf_len, + pos, + fts_doc_word_generated))) { + LOG_WARN("serialize fts doc word schema generated failed", K(ret)); + } else if (OB_FAIL(serialization::encode_i8(buf, + buf_len, + pos, + rowkey_doc_submitted))) { + LOG_WARN("serialize rowkey doc task submitted failed", K(ret)); + } else if (OB_FAIL(serialization::encode_i8(buf, + buf_len, + pos, + doc_rowkey_submitted))) { + LOG_WARN("serialize doc rowkey task submitted failed", K(ret)); + } else if (OB_FAIL(serialization::encode_i8(buf, + buf_len, + pos, + fts_index_aux_submitted))) { + LOG_WARN("serialize fts index aux task submitted failed", K(ret)); + } else if (OB_FAIL(serialization::encode_i8(buf, + buf_len, + pos, + fts_doc_word_submitted))) { + LOG_WARN("serialize fts doc word task submitted failed", K(ret)); + } else if (OB_FAIL(serialization::encode_i8(buf, + buf_len, + pos, + drop_index_submitted))) { + LOG_WARN("serialize drop fts index task submitted failed", K(ret)); + } else if (OB_FAIL(serialization::encode_i64(buf, + buf_len, + pos, + drop_index_task_id_))) { + LOG_WARN("serialize drop index task id failed", K(ret)); + } + return ret; +} + +int ObFtsIndexBuildTask::deserialize_params_from_message( + const uint64_t tenant_id, + const char *buf, + const int64_t data_len, + int64_t &pos) +{ + int ret = OB_SUCCESS; + int8_t rowkey_doc_generated = 0; + int8_t doc_rowkey_generated = 0; + int8_t fts_index_aux_generated = 0; + int8_t fts_doc_word_generated = 0; + int8_t rowkey_doc_submitted = 0; + int8_t doc_rowkey_submitted = 0; + int8_t fts_index_aux_submitted = 0; + int8_t fts_doc_word_submitted = 0; + int8_t drop_index_submitted = 0; + obrpc::ObCreateIndexArg tmp_arg; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || + nullptr == buf || + data_len <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), K(tenant_id), KP(buf), K(data_len)); + } else if (OB_FAIL(ObDDLTask::deserialize_params_from_message(tenant_id, + buf, + data_len, + pos))) { + LOG_WARN("ObDDLTask deserlize failed", K(ret)); + } else if (OB_FAIL(tmp_arg.deserialize(buf, data_len, pos))) { + LOG_WARN("deserialize table failed", K(ret)); + } else if (OB_FAIL(ObDDLUtil::replace_user_tenant_id(tenant_id, tmp_arg))) { + LOG_WARN("replace user tenant id failed", K(ret), K(tenant_id), K(tmp_arg)); + } else if (OB_FAIL(deep_copy_table_arg(allocator_, tmp_arg, create_index_arg_))) { + LOG_WARN("deep copy create index arg failed", K(ret)); + } else if (OB_FAIL(serialization::decode(buf, + data_len, pos, rowkey_doc_aux_table_id_))) { + LOG_WARN("fail to deserialize rowkey doc table id", K(ret)); + } else if (OB_FAIL(serialization::decode(buf, + data_len, + pos, + doc_rowkey_aux_table_id_))) { + LOG_WARN("fail to deserialize doc rowkey table id", K(ret)); + } else if (OB_FAIL(serialization::decode(buf, + data_len, + pos, + fts_index_aux_table_id_))) { + LOG_WARN("fail to deserialize fts index aux table id", K(ret)); + } else if (OB_FAIL(serialization::decode(buf, + data_len, + pos, + fts_doc_word_aux_table_id_))) { + LOG_WARN("fail to deserialize fts doc word table id", K(ret)); + } else if (OB_FAIL(serialization::decode_i8(buf, + data_len, + pos, + &rowkey_doc_generated))) { + LOG_WARN("fail to deserialize rowkey doc schema generated", K(ret)); + } else if (OB_FAIL(serialization::decode_i8(buf, + data_len, + pos, + &doc_rowkey_generated))) { + LOG_WARN("fail to deserialize doc rowkey schema generated", K(ret)); + } else if (OB_FAIL(serialization::decode_i8(buf, + data_len, + pos, + &fts_index_aux_generated))) { + LOG_WARN("fail to deserialize fts index aux schema generated", K(ret)); + } else if (OB_FAIL(serialization::decode_i8(buf, + data_len, + pos, + &fts_doc_word_generated))) { + LOG_WARN("fail to deserialize fts doc word schema generated", K(ret)); + } else if (OB_FAIL(serialization::decode_i8(buf, + data_len, + pos, + &rowkey_doc_submitted))) { + LOG_WARN("fail to deserialize rowkey doc task submmitted", K(ret)); + } else if (OB_FAIL(serialization::decode_i8(buf, + data_len, + pos, + &doc_rowkey_submitted))) { + LOG_WARN("fail to deserialize doc rowkey task submmitted", K(ret)); + } else if (OB_FAIL(serialization::decode_i8(buf, + data_len, + pos, + &fts_index_aux_submitted))) { + LOG_WARN("fail to deserialize fts index aux task submmitted", K(ret)); + } else if (OB_FAIL(serialization::decode_i8(buf, + data_len, + pos, + &fts_doc_word_submitted))) { + LOG_WARN("fail to deserialize fts doc word task submmitted", K(ret)); + } else if (OB_FAIL(serialization::decode_i8(buf, + data_len, + pos, + &drop_index_submitted))) { + LOG_WARN("fail to deserialize drop fts index task submmitted", K(ret)); + } else if (OB_FAIL(serialization::decode_i64(buf, + data_len, + pos, + &drop_index_task_id_))) { + LOG_WARN("fail to deserialize drop fts index task id", K(ret)); + } else { + rowkey_doc_schema_generated_ = rowkey_doc_generated; + doc_rowkey_schema_generated_ = doc_rowkey_generated; + fts_index_aux_schema_generated_ = fts_index_aux_generated; + fts_doc_word_schema_generated_ = fts_doc_word_generated; + rowkey_doc_task_submitted_ = rowkey_doc_submitted; + doc_rowkey_task_submitted_ = doc_rowkey_submitted; + fts_index_aux_task_submitted_ = fts_index_aux_submitted; + fts_doc_word_task_submitted_ = fts_doc_word_submitted; + drop_index_task_submitted_ = drop_index_submitted; + } + return ret; +} + +int64_t ObFtsIndexBuildTask::get_serialize_param_size() const +{ + int8_t rowkey_doc_generated = static_cast(rowkey_doc_schema_generated_); + int8_t doc_rowkey_generated = static_cast(doc_rowkey_schema_generated_); + int8_t fts_index_aux_generated = static_cast(fts_index_aux_schema_generated_); + int8_t fts_doc_word_generated = static_cast(fts_doc_word_schema_generated_); + int8_t rowkey_doc_submitted = static_cast(rowkey_doc_task_submitted_); + int8_t doc_rowkey_submitted = static_cast(doc_rowkey_task_submitted_); + int8_t fts_index_aux_submitted = static_cast(fts_index_aux_task_submitted_); + int8_t fts_doc_word_submitted = static_cast(fts_doc_word_task_submitted_); + int8_t drop_index_submitted = static_cast(drop_index_task_submitted_); + return create_index_arg_.get_serialize_size() + + ObDDLTask::get_serialize_param_size() + + serialization::encoded_length(rowkey_doc_aux_table_id_) + + serialization::encoded_length(doc_rowkey_aux_table_id_) + + serialization::encoded_length(fts_index_aux_table_id_) + + serialization::encoded_length(fts_doc_word_aux_table_id_) + + serialization::encoded_length_i8(rowkey_doc_generated) + + serialization::encoded_length_i8(doc_rowkey_generated) + + serialization::encoded_length_i8(fts_index_aux_generated) + + serialization::encoded_length_i8(fts_doc_word_generated) + + serialization::encoded_length_i8(rowkey_doc_submitted) + + serialization::encoded_length_i8(doc_rowkey_submitted) + + serialization::encoded_length_i8(fts_index_aux_submitted) + + serialization::encoded_length_i8(fts_doc_word_submitted) + + serialization::encoded_length_i8(drop_index_submitted) + + serialization::encoded_length_i64(drop_index_task_id_); +} + +int ObFtsIndexBuildTask::clean_on_failed() +{ + using task_iter = common::hash::ObHashMap::const_iterator; + int ret = OB_SUCCESS; + bool state_finished = false; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (ObDDLTaskStatus::FAIL != task_status_) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("task status not match", K(ret), K(task_status_)); + } else { + // 1. cancel ongoing build index task + for (task_iter iter = dependent_task_result_map_.begin(); + OB_SUCC(ret) && iter != dependent_task_result_map_.end(); ++iter) { + const uint64_t task_key = iter->first; + const int64_t target_object_id = -1; + const int64_t child_task_id = iter->second.task_id_; + if (iter->second.ret_code_ == INT64_MAX) { + // maybe ddl already finish when switching rs + HEAP_VAR(ObDDLErrorMessageTableOperator::ObBuildDDLErrorMessage, error_message) { + int64_t unused_user_msg_len = 0; + ObAddr unused_addr; + if (OB_FAIL(ObDDLErrorMessageTableOperator::get_ddl_error_message( + dst_tenant_id_, + child_task_id, + target_object_id, + unused_addr, + false /* is_ddl_retry_task */, + *GCTX.sql_proxy_, + error_message, + unused_user_msg_len))) { + if (OB_ENTRY_NOT_EXIST == ret) { + // ongoing child task + ret = OB_SUCCESS; + ObMySQLTransaction trans; + if (OB_FAIL(trans.start(&root_service_->get_sql_proxy(), + dst_tenant_id_))) { + LOG_WARN("start transaction failed", K(ret)); + } else if (OB_FAIL(ObDDLTaskRecordOperator::update_task_status( + trans, dst_tenant_id_, child_task_id, ObDDLTaskStatus::FAIL))) { + LOG_WARN("update child task status failed", K(ret), K(child_task_id)); + } else { + int tmp_ret = trans.end(true/*commit*/); + if (OB_SUCCESS != tmp_ret) { + ret = (OB_SUCCESS == ret) ? tmp_ret : ret; + } + LOG_INFO("cancel not finished ddl task", K(dst_tenant_id_), + K(task_key), K(child_task_id), K(target_object_id)); + } + } else { + LOG_WARN("fail to get ddl error message", K(ret), K(task_key), + K(child_task_id), K(target_object_id)); + } + } + } + } + } + // 2. drop already built index + if (OB_FAIL(ret)) { + } else if (!drop_index_task_submitted_) { + if (OB_FAIL(submit_drop_fts_index_task())) { + LOG_WARN("failed to drop fts index", K(ret)); + } + } else { + bool drop_index_finished = false; + if (OB_FAIL(wait_drop_index_finish(drop_index_finished))) { + LOG_WARN("failed to wait drop index task finish", K(ret)); + } else if (drop_index_finished) { + state_finished = true; + } + } + } + // judge index status to choose clean_on_failed() and drop index + if (OB_SUCC(ret) && state_finished) { + if (OB_FAIL(cleanup())) { + LOG_WARN("cleanup failed", K(ret)); + } + } + return ret; +} + +int ObFtsIndexBuildTask::submit_drop_fts_index_task() +{ + int ret = OB_SUCCESS; + ObSchemaGetterGuard schema_guard; + const ObTableSchema *fts_index_aux_schema = nullptr; + const ObTableSchema *rowkey_doc_schema = nullptr; + const ObTableSchema *doc_rowkey_schema = nullptr; + const ObTableSchema *fts_doc_word_schema = nullptr; + if (OB_FAIL(ObMultiVersionSchemaService::get_instance(). + get_tenant_schema_guard(tenant_id_, + schema_guard))) { + LOG_WARN("fail to get schema guard", K(ret)); + } else if (OB_FAIL(schema_guard.check_formal_guard())) { + LOG_WARN("schema_guard is not formal", K(ret)); + } else { + if (OB_FAIL(schema_guard.get_table_schema(tenant_id_, + fts_index_aux_table_id_, + fts_index_aux_schema))) { + if (OB_TABLE_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get table schema", K(ret), K(fts_index_aux_table_id_)); + } + } else if (OB_INVALID_ID != rowkey_doc_aux_table_id_ && + OB_FAIL(schema_guard.get_table_schema(tenant_id_, + rowkey_doc_aux_table_id_, + rowkey_doc_schema))) { + if (OB_TABLE_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get table schema", K(ret), K(rowkey_doc_aux_table_id_)); + } + } else if (OB_INVALID_ID != doc_rowkey_aux_table_id_ && + OB_FAIL(schema_guard.get_table_schema(tenant_id_, + doc_rowkey_aux_table_id_, + doc_rowkey_schema))) { + if (OB_TABLE_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get table schema", K(ret), K(doc_rowkey_aux_table_id_)); + } + } else if (OB_INVALID_ID != fts_doc_word_aux_table_id_ && + OB_FAIL(schema_guard.get_table_schema(tenant_id_, + fts_doc_word_aux_table_id_, + fts_doc_word_schema))) { + if (OB_TABLE_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get table schema", K(ret), K(fts_doc_word_aux_table_id_)); + } + } + ObDDLTaskRecord task_record; + ObCreateDDLTaskParam param(tenant_id_, + ObDDLType::DDL_DROP_FTS_INDEX, + fts_index_aux_schema, + nullptr, + 0/*object_id*/, + fts_index_aux_schema->get_schema_version(), + parallelism_, + consumer_group_id_, + &allocator_); + param.tenant_data_version_ = data_format_version_; + param.aux_rowkey_doc_schema_ = rowkey_doc_schema; + param.aux_doc_rowkey_schema_ = doc_rowkey_schema; + param.aux_doc_word_schema_ = fts_doc_word_schema; + if (OB_FAIL(GCTX.root_service_->get_ddl_task_scheduler(). + create_ddl_task(param, *GCTX.sql_proxy_, task_record))) { + if (OB_ENTRY_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("submit drop fts index ddl task failed", K(ret)); + } + } else if (OB_FAIL(GCTX.root_service_->get_ddl_task_scheduler(). + schedule_ddl_task(task_record))) { + LOG_WARN("fail to schedule ddl task", K(ret), K(task_record)); + } else { + if (OB_SUCC(ret)) { + drop_index_task_submitted_ = true; + drop_index_task_id_ = task_record.task_id_; + LOG_INFO("add drop fts index task", K(ret), K(fts_index_aux_table_id_), + K(rowkey_doc_aux_table_id_), K(doc_rowkey_aux_table_id_), + K(fts_doc_word_aux_table_id_), K(create_index_arg_.index_name_), + K(fts_index_aux_schema->get_schema_version()), K(param.schema_version_)); + } + } + } + return ret; +} + +int ObFtsIndexBuildTask::wait_drop_index_finish(bool &is_finish) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (ObDDLTaskStatus::FAIL != task_status_) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("task status not match", K(ret), K(task_status_)); + } else if (OB_INVALID_ID == drop_index_task_id_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("drop index task id is invalid", K(ret)); + } else { + HEAP_VAR(ObDDLErrorMessageTableOperator::ObBuildDDLErrorMessage, error_message) { + const int64_t target_object_id = -1; + int64_t unused_user_msg_len = 0; + ObAddr unused_addr; + if (OB_FAIL(ObDDLErrorMessageTableOperator::get_ddl_error_message( + dst_tenant_id_, + drop_index_task_id_, + target_object_id, + unused_addr, + false /* is_ddl_retry_task */, + *GCTX.sql_proxy_, + error_message, + unused_user_msg_len))) { + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_SUCCESS; + LOG_INFO("ddl task not finish", K(dst_tenant_id_), K(drop_index_task_id_)); + } else { + LOG_WARN("fail to get ddl error message", K(ret), K(drop_index_task_id_)); + } + } else { + if (error_message.ret_code_ != OB_SUCCESS) { + ret = error_message.ret_code_; + drop_index_task_submitted_ = false; // retry + } else { + is_finish = true; + } + } + } + } + return ret; +} + +int ObFtsIndexBuildTask::succ() +{ + return cleanup(); +} + +int ObFtsIndexBuildTask::validate_checksum() +{ + int ret = OB_SUCCESS; + bool state_finished = false; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (ObDDLTaskStatus::VALIDATE_CHECKSUM != task_status_) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("task status not match", K(ret), K(task_status_)); + } else { + // TODO hanxuan validate checksum, set next status to FAIL if validation failed + if (OB_SUCC(ret)) { + state_finished = true; + } + } + if (state_finished) { + ObDDLTaskStatus next_status; + if (OB_FAIL(get_next_status(next_status))) { + LOG_WARN("failed to get next status", K(ret)); + } else { + (void)switch_status(next_status, true, ret); + LOG_INFO("validate checksum finished", K(ret), K(parent_task_id_), + K(task_id_), K(*this)); + } + } + return ret; +} + +int ObFtsIndexBuildTask::collect_longops_stat(ObLongopsValue &value) +{ + return OB_SUCCESS; +} + +int ObFtsIndexBuildTask::cleanup_impl() +{ + int ret = OB_SUCCESS; + ObString unused_str; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_FAIL(report_error_code(unused_str))) { + LOG_WARN("report error code failed", K(ret)); + } else { + const uint64_t data_table_id = object_id_; + const uint64_t index_table_id = fts_index_aux_table_id_; + ObMultiVersionSchemaService &schema_service = root_service_->get_schema_service(); + ObSchemaGetterGuard schema_guard; + const ObTableSchema *data_schema = nullptr; + const ObTableSchema *index_schema = nullptr; + int64_t refreshed_schema_version = 0; + ObTableLockOwnerID owner_id; + ObMySQLTransaction trans; + if (OB_FAIL(schema_service.get_tenant_schema_guard(tenant_id_, + schema_guard))) { + LOG_WARN("get tenant schema guard failed", K(ret), K(tenant_id_)); + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id_, + data_table_id, + data_schema))) { + LOG_WARN("fail to get table schema", K(ret), K(data_table_id)); + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id_, + index_table_id, + index_schema))) { + LOG_WARN("fail to get table schema", K(ret), K(index_table_id)); + } else if (OB_ISNULL(data_schema) || OB_ISNULL(index_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("fail to get table schema", K(ret), KPC(data_schema), KPC(index_schema)); + } else if (OB_FAIL(trans.start(&root_service_->get_sql_proxy(), dst_tenant_id_))) { + LOG_WARN("start transaction failed", K(ret)); + } else if (OB_FAIL(owner_id.convert_from_value(ObLockOwnerType::DEFAULT_OWNER_TYPE, + task_id_))) { + LOG_WARN("failed to get owner id", K(ret), K(task_id_)); + } else if (OB_FAIL(ObDDLLock::unlock_for_add_drop_index(*data_schema, + *index_schema, + owner_id, + trans))) { + LOG_WARN("failed to unlock online ddl lock", K(ret)); + } + if (trans.is_started()) { + int tmp_ret = trans.end(true/*commit*/); + if (OB_SUCCESS != tmp_ret) { + LOG_WARN("trans end failed", "is_commit", OB_SUCCESS == ret, K(tmp_ret)); + ret = (OB_SUCCESS == ret) ? tmp_ret : ret; + } + } + } + + DEBUG_SYNC(CREATE_INDEX_SUCCESS); + + if(OB_FAIL(ret)) { + } else if (OB_FAIL(ObDDLTaskRecordOperator::delete_record(root_service_->get_sql_proxy(), + tenant_id_, + task_id_))) { + LOG_WARN("delete task record failed", K(ret), K(task_id_), K(schema_version_)); + } else { + need_retry_ = false; // clean succ, stop the task + } + + if (OB_SUCC(ret) && parent_task_id_ > 0) { + const ObDDLTaskID parent_task_id(tenant_id_, parent_task_id_); + root_service_->get_ddl_task_scheduler().on_ddl_task_finish(parent_task_id, + get_task_key(), + ret_code_, trace_id_); + } + LOG_INFO("clean task finished", K(ret), K(*this)); + return ret; +} + +void ObFtsIndexBuildTask::flt_set_task_span_tag() const +{ +} + +void ObFtsIndexBuildTask::flt_set_status_span_tag() const +{ +} + +} // end namespace rootserver +} // end namespace oceanbase diff --git a/src/rootserver/ddl_task/ob_fts_index_build_task.h b/src/rootserver/ddl_task/ob_fts_index_build_task.h new file mode 100644 index 000000000..fac415189 --- /dev/null +++ b/src/rootserver/ddl_task/ob_fts_index_build_task.h @@ -0,0 +1,155 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_ROOTSERVER_OB_FTS_INDEX_BUILD_TASK_H_ +#define OCEANBASE_ROOTSERVER_OB_FTS_INDEX_BUILD_TASK_H_ + +#include "rootserver/ddl_task/ob_ddl_task.h" + +namespace oceanbase +{ +namespace rootserver +{ + +class ObFtsIndexBuildTask : public ObDDLTask +{ +public: + ObFtsIndexBuildTask(); + virtual ~ObFtsIndexBuildTask(); + int init( + const uint64_t tenant_id, + const int64_t task_id, + const ObTableSchema *data_table_schema, + const ObTableSchema *index_schema, + const int64_t schema_version, + const int64_t parallelism, + const int64_t consumer_group_id, + const obrpc::ObCreateIndexArg &create_index_arg, + const int64_t parent_task_id = 0, + const int64_t task_status = share::ObDDLTaskStatus::PREPARE, + const int64_t snapshot_version = 0); + int init(const ObDDLTaskRecord &task_record); + virtual int process() override; + virtual void flt_set_task_span_tag() const override; + virtual void flt_set_status_span_tag() const override; + virtual int cleanup_impl() override; + virtual bool is_valid() const override; + virtual int collect_longops_stat(share::ObLongopsValue &value) override; + virtual int serialize_params_to_message( + char *buf, + const int64_t buf_size, + int64_t &pos) const override; + virtual int deserialize_params_from_message( + const uint64_t tenant_id, + const char *buf, + const int64_t buf_size, + int64_t &pos) override; + virtual int64_t get_serialize_param_size() const override; + virtual bool support_longops_monitoring() const override { return false; } + virtual int on_child_task_finish( + const uint64_t child_task_key, + const int ret_code) override; + TO_STRING_KV(K(index_table_id_), K(rowkey_doc_aux_table_id_), + K(doc_rowkey_aux_table_id_), K(fts_index_aux_table_id_), + K(fts_doc_word_aux_table_id_), K(rowkey_doc_schema_generated_), + K(doc_rowkey_schema_generated_), K(fts_index_aux_schema_generated_), + K(fts_doc_word_schema_generated_), K(rowkey_doc_task_submitted_), + K(doc_rowkey_task_submitted_), K(fts_index_aux_task_submitted_), + K(fts_doc_word_task_submitted_), K(drop_index_task_id_), + K(drop_index_task_submitted_), K(schema_version_), K(execution_id_), + K(consumer_group_id_), K(trace_id_), K(parallelism_), K(create_index_arg_)); + +private: + int get_next_status(share::ObDDLTaskStatus &next_status); + int prepare_rowkey_doc_table(); + int prepare_aux_index_tables(); + int construct_rowkey_doc_arg(obrpc::ObCreateIndexArg &arg); + int construct_doc_rowkey_arg(obrpc::ObCreateIndexArg &arg); + int construct_fts_index_aux_arg(obrpc::ObCreateIndexArg &arg); + int construct_fts_doc_word_arg(obrpc::ObCreateIndexArg &arg); + int record_index_table_id( + const obrpc::ObCreateIndexArg *create_index_arg_, + uint64_t &aux_table_id); + int get_index_table_id( + const obrpc::ObCreateIndexArg *create_index_arg, + uint64_t &index_table_id); + int prepare(); + int wait_aux_table_complement(); + int submit_build_aux_index_task( + const obrpc::ObCreateIndexArg &create_index_arg, + ObDDLTaskRecord &task_record, + bool &task_submitted); + int validate_checksum(); + int clean_on_failed(); + int submit_drop_fts_index_task(); + int wait_drop_index_finish(bool &is_finish); + int succ(); + int update_index_status_in_schema( + const ObTableSchema &index_schema, + const ObIndexStatus new_status); + int check_health(); + int check_aux_table_schemas_exist(bool &is_all_exist); + int deep_copy_index_arg( + common::ObIAllocator &allocator, + const obrpc::ObCreateIndexArg &source_arg, + obrpc::ObCreateIndexArg &dest_arg); + +private: + struct DependTaskStatus final + { + public: + DependTaskStatus() + : ret_code_(INT64_MAX), task_id_(0) + {} + ~DependTaskStatus() = default; + TO_STRING_KV(K_(task_id), K_(ret_code)); + public: + int64_t ret_code_; + int64_t task_id_; + }; + static const int64_t OB_FTS_INDEX_BUILD_TASK_VERSION = 1; + using ObDDLTask::tenant_id_; + using ObDDLTask::task_id_; + using ObDDLTask::schema_version_; + using ObDDLTask::parallelism_; + using ObDDLTask::consumer_group_id_; + using ObDDLTask::parent_task_id_; + using ObDDLTask::task_status_; + using ObDDLTask::snapshot_version_; + using ObDDLTask::object_id_; + using ObDDLTask::target_object_id_; + using ObDDLTask::is_inited_; + uint64_t &index_table_id_; + uint64_t rowkey_doc_aux_table_id_; + uint64_t doc_rowkey_aux_table_id_; + uint64_t fts_index_aux_table_id_; + uint64_t fts_doc_word_aux_table_id_; + bool rowkey_doc_schema_generated_; + bool doc_rowkey_schema_generated_; + bool fts_index_aux_schema_generated_; + bool fts_doc_word_schema_generated_; + bool rowkey_doc_task_submitted_; + bool doc_rowkey_task_submitted_; + bool fts_index_aux_task_submitted_; + bool fts_doc_word_task_submitted_; + int64_t drop_index_task_id_; + bool drop_index_task_submitted_; + ObRootService *root_service_; + ObDDLWaitTransEndCtx wait_trans_ctx_; + obrpc::ObCreateIndexArg create_index_arg_; + common::hash::ObHashMap dependent_task_result_map_; +}; + +} // end namespace rootserver +} // end namespace oceanbase + +#endif /* OCEANBASE_ROOTSERVER_OB_FTS_INDEX_BUILD_TASK_H_*/ diff --git a/src/rootserver/ddl_task/ob_index_build_task.cpp b/src/rootserver/ddl_task/ob_index_build_task.cpp index 3e6e1cd61..eb2e78552 100755 --- a/src/rootserver/ddl_task/ob_index_build_task.cpp +++ b/src/rootserver/ddl_task/ob_index_build_task.cpp @@ -1051,67 +1051,73 @@ bool ObIndexBuildTask::is_create_partitioned_local_index() int ObIndexBuildTask::wait_data_complement() { int ret = OB_SUCCESS; - bool state_finished = false; - bool is_request_end = false; - share::ObLSID ls_id; - common::ObAddr leader_addr; - ObArray index_partition_ids; - if (OB_UNLIKELY(!is_inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else if (ObDDLTaskStatus::REDEFINITION != task_status_) { - LOG_WARN("task status not match", K(ret), K(task_status_)); - } else if (OB_UNLIKELY(snapshot_version_ <= 0)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected snapshot", K(ret), KPC(this)); - } - // submit a job to complete sstable for the index table on snapshot_version - if (OB_SUCC(ret) && !state_finished && !is_sstable_complete_task_submitted()) { - bool need_exec_new_inner_sql = false; - if (OB_FAIL(reap_old_replica_build_task(need_exec_new_inner_sql))) { - if (OB_EAGAIN == ret) { - ret = OB_SUCCESS; // retry - } else { - LOG_WARN("failed to reap old task", K(ret)); - } - } else if (!need_exec_new_inner_sql) { - state_finished = true; - } else if (OB_FAIL(send_build_single_replica_request(false, parallelism_, 0, ls_id, leader_addr, index_partition_ids))) { - LOG_WARN("fail to send build single replica request", K(ret), K(parallelism_), K(index_partition_ids)); - } - } - - DEBUG_SYNC(CREATE_INDEX_REPLICA_BUILD); - - if (OB_SUCC(ret) && !state_finished && is_sstable_complete_task_submitted()) { - if (OB_FAIL(check_build_single_replica(is_request_end))) { - LOG_WARN("fail to check build single replica", K(ret)); - } else if (OB_UNLIKELY(is_request_end)) { - ret = complete_sstable_job_ret_code_; - state_finished = true; - } - } - if (OB_SUCC(ret) && state_finished && !create_index_arg_.is_spatial_index()) { - bool dummy_equal = false; - bool need_verify_checksum = true; -#ifdef ERRSIM - // when the major compaction is delayed, skip verify column checksum - need_verify_checksum = 0 == GCONF.errsim_ddl_major_delay_time; -#endif - if (need_verify_checksum && OB_FAIL(ObDDLChecksumOperator::check_column_checksum( - tenant_id_, get_execution_id(), object_id_, index_table_id_, task_id_, false/*index build*/, dummy_equal, root_service_->get_sql_proxy()))) { - if (OB_ITER_END != ret) { - LOG_WARN("fail to check column checksum", K(ret), K(index_table_id_), K(object_id_), K(task_id_)); - state_finished = true; - } else if (REACH_TIME_INTERVAL(1000L * 1000L)) { - LOG_INFO("index checksum has not been reported", K(ret), K(index_table_id_), K(object_id_), K(task_id_)); - } - } - } - - if (state_finished || OB_FAIL(ret)) { + // temporary bypass data complement for fts index + if (share::schema::is_fts_index(create_index_arg_.index_type_)) { (void)switch_status(ObDDLTaskStatus::VALIDATE_CHECKSUM, true, ret); LOG_INFO("wait data complement finished", K(ret), K(*this)); + } else { + bool state_finished = false; + bool is_request_end = false; + share::ObLSID ls_id; + common::ObAddr leader_addr; + ObArray index_partition_ids; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (ObDDLTaskStatus::REDEFINITION != task_status_) { + LOG_WARN("task status not match", K(ret), K(task_status_)); + } else if (OB_UNLIKELY(snapshot_version_ <= 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected snapshot", K(ret), KPC(this)); + } + // submit a job to complete sstable for the index table on snapshot_version + if (OB_SUCC(ret) && !state_finished && !is_sstable_complete_task_submitted()) { + bool need_exec_new_inner_sql = false; + if (OB_FAIL(reap_old_replica_build_task(need_exec_new_inner_sql))) { + if (OB_EAGAIN == ret) { + ret = OB_SUCCESS; // retry + } else { + LOG_WARN("failed to reap old task", K(ret)); + } + } else if (!need_exec_new_inner_sql) { + state_finished = true; + } else if (OB_FAIL(send_build_single_replica_request(false, parallelism_, 0, ls_id, leader_addr, index_partition_ids))) { + LOG_WARN("fail to send build single replica request", K(ret), K(parallelism_), K(index_partition_ids)); + } + } + + DEBUG_SYNC(CREATE_INDEX_REPLICA_BUILD); + + if (OB_SUCC(ret) && !state_finished && is_sstable_complete_task_submitted()) { + if (OB_FAIL(check_build_single_replica(is_request_end))) { + LOG_WARN("fail to check build single replica", K(ret)); + } else if (is_request_end) { + ret = complete_sstable_job_ret_code_; + state_finished = true; + } + } + if (OB_SUCC(ret) && state_finished && !create_index_arg_.is_spatial_index()) { + bool dummy_equal = false; + bool need_verify_checksum = true; + #ifdef ERRSIM + // when the major compaction is delayed, skip verify column checksum + need_verify_checksum = 0 == GCONF.errsim_ddl_major_delay_time; + #endif + if (need_verify_checksum && OB_FAIL(ObDDLChecksumOperator::check_column_checksum( + tenant_id_, get_execution_id(), object_id_, index_table_id_, task_id_, false/*index build*/, dummy_equal, root_service_->get_sql_proxy()))) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to check column checksum", K(ret), K(index_table_id_), K(object_id_), K(task_id_)); + state_finished = true; + } else if (REACH_TIME_INTERVAL(1000L * 1000L)) { + LOG_INFO("index checksum has not been reported", K(ret), K(index_table_id_), K(object_id_), K(task_id_)); + } + } + } + + if (state_finished || OB_FAIL(ret)) { + (void)switch_status(ObDDLTaskStatus::VALIDATE_CHECKSUM, true, ret); + LOG_INFO("wait data complement finished", K(ret), K(*this)); + } } return ret; } diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index d5a317df6..93a64f71b 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -6207,6 +6207,160 @@ int ObDDLService::lock_tables_of_database(const ObDatabaseSchema &database_schem return ret; } +int ObDDLService::check_schema_generated_for_aux_index_schema_( + const obrpc::ObGenerateAuxIndexSchemaArg &arg, + ObSchemaGetterGuard &schema_guard, + const ObTableSchema *data_schema, + bool &schema_generated, + uint64_t &aux_index_table_id) +{ + int ret = OB_SUCCESS; + schema_generated = false; + aux_index_table_id = OB_INVALID_ID; + ObArenaAllocator allocator(ObModIds::OB_SCHEMA); + const uint64_t tenant_id = arg.tenant_id_; + ObIndexType index_type = arg.create_index_arg_.index_type_; + ObString index_table_name; + if (OB_ISNULL(data_schema)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), KPC(data_schema)); + } else if (OB_FAIL(ObTableSchema::build_index_table_name(allocator, + data_schema->get_table_id(), + arg.create_index_arg_.index_name_, + index_table_name))) { + LOG_WARN("failed to construct index table name", K(ret), + K(arg.create_index_arg_.index_name_)); + } else if (share::schema::is_fts_index(index_type)) { + const ObTableSchema *index_schema = nullptr; + if (OB_FAIL(schema_guard.get_table_schema(tenant_id, + data_schema->get_database_id(), + index_table_name, + true/*is_index*/, + index_schema, + false/*with_hidden_flag*/, + share::schema::is_built_in_fts_index(index_type)))) { + LOG_WARN("failed to get index schema", K(ret), K(tenant_id), K(index_table_name)); + } else if (OB_NOT_NULL(index_schema)) { + schema_generated = true; + aux_index_table_id = index_schema->get_table_id(); + LOG_INFO("fts index aux table already exist, no need to generate", + K(index_table_name)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected index type", K(ret), K(index_type)); + } + return ret; +} + +int ObDDLService::generate_aux_index_schema( + const obrpc::ObGenerateAuxIndexSchemaArg &arg, + obrpc::ObGenerateAuxIndexSchemaRes &result) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = arg.tenant_id_; + const uint64_t data_table_id = arg.data_table_id_; + SMART_VARS_2((obrpc::ObCreateIndexArg, create_index_arg), + (ObTableSchema, nonconst_data_schema)) { + ObSchemaGetterGuard schema_guard; + const ObTableSchema *data_schema = nullptr; + ObSEArray gen_columns; + ObArenaAllocator allocator(lib::ObLabel("DdlTaskTmp")); + if (!arg.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(arg)); + } else if (OB_FAIL(get_tenant_schema_guard_with_version_in_inner_table(tenant_id, + schema_guard))) { + LOG_WARN("get schema guard failed", K(ret)); + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, + data_table_id, + data_schema))) { + LOG_WARN("get table schema failed", K(ret), K(tenant_id), K(data_table_id)); + } else if (OB_ISNULL(data_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("error unexpected, table schema is nullptr", K(ret), K(data_table_id)); + } else if (OB_FAIL(nonconst_data_schema.assign(*data_schema))) { + LOG_WARN("failed to assign to nonconst data schema", K(ret)); + } else if (OB_FAIL(create_index_arg.assign(arg.create_index_arg_))) { + LOG_WARN("fail to assign create index arg", K(ret)); + } else if (OB_FAIL(ObFtsIndexBuilderUtil::adjust_fts_args(create_index_arg, + nonconst_data_schema, + allocator, + gen_columns))) { + LOG_WARN("fail to adjust expr index args", K(ret)); + } else { + if (OB_FAIL(check_schema_generated_for_aux_index_schema_(arg, + schema_guard, + data_schema, + result.schema_generated_, + result.aux_table_id_) )) { + LOG_WARN("failed to check if schema is generated for aux index table", K(ret), K(arg)); + } else if (result.schema_generated_) { + // do nothing + } else { + ObIndexBuilder index_builder(*this); + ObDDLSQLTransaction trans(&get_schema_service()); + const bool global_index_without_column_info = true; + int64_t refreshed_schema_version = 0; + uint64_t tenant_data_version = 0; + ObTableSchema index_schema; + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) { + LOG_WARN("get min data version failed", K(ret), K(tenant_id)); + } else if (OB_FAIL(schema_guard.get_schema_version(tenant_id, + refreshed_schema_version))) { + LOG_WARN("failed to get tenant schema version", KR(ret), K(tenant_id)); + } else if (OB_FAIL(trans.start(&get_sql_proxy(), + tenant_id, + refreshed_schema_version))) { + LOG_WARN("start transaction failed", KR(ret), K(tenant_id), + K(refreshed_schema_version)); + } else if (OB_FAIL(index_builder.generate_schema(create_index_arg, + nonconst_data_schema, + global_index_without_column_info, + true/*generate_id*/, + index_schema))) { + LOG_WARN("fail to generate schema", K(ret), K(create_index_arg)); + } else if (OB_FAIL(nonconst_data_schema.check_create_index_on_hidden_primary_key(index_schema))) { + LOG_WARN("failed to check create index on table", K(ret), K(index_schema)); + } else if (gen_columns.empty()) { + if (OB_FAIL(create_index_table(create_index_arg, + tenant_data_version, + index_schema, + trans))) { + LOG_WARN("fail to create index", K(ret), K(index_schema)); + } + } else { + if (OB_FAIL(create_inner_expr_index(trans, + *data_schema, + tenant_data_version, + nonconst_data_schema, + gen_columns, + index_schema))) { + LOG_WARN("fail to create inner expr index", K(ret)); + } + } + if (trans.is_started()) { + int temp_ret = OB_SUCCESS; + if (OB_SUCCESS != (temp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("trans end failed", "is_commit", OB_SUCCESS == ret, K(temp_ret)); + ret = (OB_SUCC(ret)) ? temp_ret : ret; + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(publish_schema(tenant_id))) { + LOG_WARN("fail to publish schema", K(ret), K(tenant_id)); + } else { + result.schema_generated_ = true; + result.aux_table_id_ = index_schema.get_table_id(); + } + } + } + } + } + LOG_INFO("finish generate aux index schema", K(ret), K(arg), K(result), "ddl_event_info", ObDDLEventInfo()); + return ret; +} + int ObDDLService::lock_tables_in_recyclebin(const ObDatabaseSchema &database_schema, ObMySQLTransaction &trans) { @@ -7642,7 +7796,6 @@ int ObDDLService::get_dropping_domain_index_invisiable_aux_table_schema( const share::schema::ObTableSchema *doc_word_schema = nullptr; const share::schema::ObTableSchema *rowkey_doc_schema = nullptr; const share::schema::ObTableSchema *doc_rowkey_schema = nullptr; - for (int64_t i = 0; OB_SUCC(ret) && i < indexs.count(); ++i) { const share::schema::ObAuxTableMetaInfo &info = indexs.at(i); if (share::schema::is_rowkey_doc_aux(info.index_type_)) { diff --git a/src/rootserver/ob_ddl_service.h b/src/rootserver/ob_ddl_service.h index 902af285e..e0b976653 100644 --- a/src/rootserver/ob_ddl_service.h +++ b/src/rootserver/ob_ddl_service.h @@ -2127,6 +2127,9 @@ private: ObDDLSQLTransaction &trans); public: + int generate_aux_index_schema( + const obrpc::ObGenerateAuxIndexSchemaArg &arg, + obrpc::ObGenerateAuxIndexSchemaRes &result); int check_parallel_ddl_conflict( share::schema::ObSchemaGetterGuard &schema_guard, const obrpc::ObDDLArg &arg); @@ -2186,6 +2189,12 @@ public: common::ObIAllocator *allocator = NULL); #endif private: + int check_schema_generated_for_aux_index_schema_( + const obrpc::ObGenerateAuxIndexSchemaArg &arg, + ObSchemaGetterGuard &schema_guard, + const ObTableSchema *data_schema, + bool &schema_generated, + uint64_t &index_table_id); int adjust_cg_for_offline(ObTableSchema &new_table_schema); int add_column_group(const obrpc::ObAlterTableArg &alter_table_arg, const share::schema::ObTableSchema &ori_table_schema, diff --git a/src/rootserver/ob_index_builder.cpp b/src/rootserver/ob_index_builder.cpp index d33f2ed7e..e96ac96c9 100644 --- a/src/rootserver/ob_index_builder.cpp +++ b/src/rootserver/ob_index_builder.cpp @@ -26,9 +26,11 @@ #include "share/schema/ob_multi_version_schema_service.h" #include "share/schema/ob_schema_struct.h" #include "share/schema/ob_part_mgr_util.h" +#include "share/schema/ob_table_schema.h" #include "share/ob_common_rpc_proxy.h" #include "share/config/ob_server_config.h" #include "share/ob_index_builder_util.h" +#include "share/ob_fts_index_builder_util.h" #include "observer/ob_server_struct.h" #include "sql/resolver/ddl/ob_ddl_resolver.h" #include "ob_zone_manager.h" @@ -455,14 +457,20 @@ int ObIndexBuilder::submit_build_index_task( if (OB_UNLIKELY(nullptr == data_schema || nullptr == index_schema || tenant_data_version <= 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("schema is invalid", K(ret), KP(data_schema), KP(index_schema), K(tenant_data_version)); - } else if (OB_FAIL(GCTX.root_service_->get_ddl_task_scheduler().create_ddl_task(param, trans, task_record))) { - LOG_WARN("submit create index ddl task failed", K(ret)); - } else if (OB_FAIL(owner_id.convert_from_value(ObLockOwnerType::DEFAULT_OWNER_TYPE, - task_record.task_id_))) { - LOG_WARN("failed to get owner id", K(ret), K(task_record.task_id_)); - } else if (OB_FAIL(ObDDLLock::lock_for_add_drop_index( - *data_schema, inc_data_tablet_ids, del_data_tablet_ids, *index_schema, owner_id, trans))) { - LOG_WARN("failed to lock online ddl lock", K(ret)); + } else { + bool is_create_fts_index = share::schema::is_fts_index(create_index_arg.index_type_); + if (is_create_fts_index) { + param.type_ = ObDDLType::DDL_CREATE_FTS_INDEX; + } + if (OB_FAIL(GCTX.root_service_->get_ddl_task_scheduler().create_ddl_task(param, trans, task_record))) { + LOG_WARN("submit create index ddl task failed", K(ret)); + } else if (OB_FAIL(owner_id.convert_from_value(ObLockOwnerType::DEFAULT_OWNER_TYPE, + task_record.task_id_))) { + LOG_WARN("failed to get owner id", K(ret), K(task_record.task_id_)); + } else if (OB_FAIL(ObDDLLock::lock_for_add_drop_index( + *data_schema, inc_data_tablet_ids, del_data_tablet_ids, *index_schema, owner_id, trans))) { + LOG_WARN("failed to lock online ddl lock", K(ret)); + } } return ret; } @@ -637,11 +645,7 @@ int ObIndexBuilder::do_create_local_index( int64_t refreshed_schema_version = 0; const uint64_t tenant_id = table_schema.get_tenant_id(); uint64_t tenant_data_version = 0; - // TODO hanxuan support fulltext index after table created - if (share::schema::is_fts_index(create_index_arg.index_type_)) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("not supported", K(ret)); - } else if (OB_FAIL(schema_guard.get_schema_version(tenant_id, refreshed_schema_version))) { + if (OB_FAIL(schema_guard.get_schema_version(tenant_id, refreshed_schema_version))) { LOG_WARN("failed to get tenant schema version", KR(ret), K(tenant_id)); } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) { LOG_WARN("get tenant data version failed", K(ret)); @@ -676,7 +680,11 @@ int ObIndexBuilder::do_create_local_index( my_arg.index_schema_.set_index_type(INDEX_TYPE_SPATIAL_GLOBAL_LOCAL_STORAGE); } } - if (FAILEDx(ObIndexBuilderUtil::adjust_expr_index_args( + if (OB_FAIL(ret)) { + } else if (share::schema::is_fts_index(my_arg.index_type_) && + OB_FAIL(ObFtsIndexBuilderUtil::generate_fts_aux_index_name(my_arg, &allocator))) { + LOG_WARN("failed to adjust fts index name", K(ret)); + } else if (OB_FAIL(ObIndexBuilderUtil::adjust_expr_index_args( my_arg, new_table_schema, allocator, gen_columns))) { LOG_WARN("fail to adjust expr index args", K(ret)); } else if (OB_FAIL(generate_schema( @@ -887,10 +895,10 @@ int ObIndexBuilder::generate_schema( if (OB_FAIL(GET_MIN_DATA_VERSION(data_schema.get_tenant_id(), tenant_data_version))) { LOG_WARN("failed to get tenant data version", K(ret)); - } else if (tenant_data_version < DATA_VERSION_4_3_1_0) { + } else if (tenant_data_version < DATA_VERSION_4_3_2_0) { ret = OB_NOT_SUPPORTED; - LOG_WARN("tenant data version is less than 4.3.1, fulltext index is not supported", K(ret), K(tenant_data_version)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "tenant data version is less than 4.3.1, fulltext index"); + LOG_WARN("tenant data version is less than 4.3.2, create fulltext index on existing table is not supported", K(ret), K(tenant_data_version)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "tenant data version is less than 4.3.2, fulltext index"); } } else if (is_multivalue_index(arg.index_type_)) { uint64_t tenant_data_version = 0; diff --git a/src/rootserver/ob_root_service.cpp b/src/rootserver/ob_root_service.cpp index be2e5ebb7..68c1c1ae7 100755 --- a/src/rootserver/ob_root_service.cpp +++ b/src/rootserver/ob_root_service.cpp @@ -51,6 +51,7 @@ #include "share/ob_time_zone_info_manager.h" #include "share/ob_server_status.h" #include "share/ob_index_builder_util.h" +#include "share/ob_fts_index_builder_util.h" #include "observer/ob_server_struct.h" #include "observer/omt/ob_tenant_config_mgr.h" #include "observer/ob_server_event_history_table_operator.h" @@ -4647,6 +4648,21 @@ int ObRootService::exchange_partition(const obrpc::ObExchangePartitionArg &arg, return ret; } +int ObRootService::generate_aux_index_schema( + const ObGenerateAuxIndexSchemaArg &arg, + ObGenerateAuxIndexSchemaRes &result) +{ + int ret = OB_SUCCESS; + if (!arg.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(arg)); + } else if (OB_FAIL(ddl_service_.generate_aux_index_schema(arg, result))) { + LOG_WARN("failed to generate aux index schema", K(ret), K(arg), K(result)); + } + LOG_INFO("finish generate aux index schema", K(ret), K(arg), K(result), "ddl_event_info", ObDDLEventInfo()); + return ret; +} + int ObRootService::create_index(const ObCreateIndexArg &arg, obrpc::ObAlterTableRes &res) { int ret = OB_SUCCESS; @@ -4658,8 +4674,7 @@ int ObRootService::create_index(const ObCreateIndexArg &arg, obrpc::ObAlterTable } else if (!arg.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(arg), K(ret)); - } else if (is_fts_index(arg.index_type_) || is_multivalue_index(arg.index_type_)) { - // TODO hanxuan support create fulltext index + } else if (is_multivalue_index(arg.index_type_)) { // todo yunyi not dynamic create multivlaue index ret = OB_NOT_SUPPORTED; LOG_WARN("not supported", K(ret)); diff --git a/src/rootserver/ob_root_service.h b/src/rootserver/ob_root_service.h index b4f114e82..a8a154405 100644 --- a/src/rootserver/ob_root_service.h +++ b/src/rootserver/ob_root_service.h @@ -502,6 +502,9 @@ public: int truncate_table(const obrpc::ObTruncateTableArg &arg, obrpc::ObDDLRes &res); int truncate_table_v2(const obrpc::ObTruncateTableArg &arg, obrpc::ObDDLRes &res); int exchange_partition(const obrpc::ObExchangePartitionArg &arg, obrpc::ObAlterTableRes &res); + int generate_aux_index_schema( + const obrpc::ObGenerateAuxIndexSchemaArg &arg, + obrpc::ObGenerateAuxIndexSchemaRes &result); int create_index(const obrpc::ObCreateIndexArg &arg, obrpc::ObAlterTableRes &res); int drop_table(const obrpc::ObDropTableArg &arg, obrpc::ObDDLRes &res); int drop_database(const obrpc::ObDropDatabaseArg &arg, obrpc::ObDropDatabaseRes &drop_database_res); diff --git a/src/rootserver/ob_rs_rpc_processor.h b/src/rootserver/ob_rs_rpc_processor.h index b712ee95b..c39d9505f 100644 --- a/src/rootserver/ob_rs_rpc_processor.h +++ b/src/rootserver/ob_rs_rpc_processor.h @@ -348,6 +348,7 @@ DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_DROP_TABLE, ObRpcDropTableP, drop_table(ar DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_RENAME_TABLE, ObRpcRenameTableP, rename_table(arg_)); DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_TRUNCATE_TABLE, ObRpcTruncateTableP, truncate_table(arg_, result_)); DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_TRUNCATE_TABLE_V2, ObRpcTruncateTableV2P, truncate_table_v2(arg_, result_)); +DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_GENERATE_AUX_INDEX_SCHEMA, ObRpcGenerateAuxIndexSchemaP, generate_aux_index_schema(arg_, result_)); DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_CREATE_INDEX, ObRpcCreateIndexP, create_index(arg_, result_)); DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_DROP_INDEX, ObRpcDropIndexP, drop_index(arg_, result_)); DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_CREATE_MLOG, ObRpcCreateMLogP, create_mlog(arg_, result_)); diff --git a/src/share/ob_common_rpc_proxy.h b/src/share/ob_common_rpc_proxy.h index 86af4aa25..3891de036 100644 --- a/src/share/ob_common_rpc_proxy.h +++ b/src/share/ob_common_rpc_proxy.h @@ -76,6 +76,7 @@ public: RPC_S(PRD rename_table, obrpc::OB_RENAME_TABLE, (ObRenameTableArg)); RPC_S(PRD truncate_table, obrpc::OB_TRUNCATE_TABLE, (ObTruncateTableArg), ObDDLRes); RPC_S(PRD truncate_table_v2, obrpc::OB_TRUNCATE_TABLE_V2, (ObTruncateTableArg), ObDDLRes); + RPC_S(PRD generate_aux_index_schema, obrpc::OB_GENERATE_AUX_INDEX_SCHEMA, (obrpc::ObGenerateAuxIndexSchemaArg), obrpc::ObGenerateAuxIndexSchemaRes); RPC_S(PRD create_index, obrpc::OB_CREATE_INDEX, (ObCreateIndexArg), ObAlterTableRes); RPC_S(PRD drop_index, obrpc::OB_DROP_INDEX, (ObDropIndexArg), ObDropIndexRes); RPC_S(PRD create_mlog, obrpc::OB_CREATE_MLOG, (ObCreateMLogArg), ObCreateMLogRes); diff --git a/src/share/ob_ddl_common.h b/src/share/ob_ddl_common.h index 4f4d4a325..01921c5f0 100644 --- a/src/share/ob_ddl_common.h +++ b/src/share/ob_ddl_common.h @@ -164,10 +164,10 @@ enum ObDDLTaskStatus { WAIT_LOCAL_INDEX_SPLIT_END = 22, WAIT_LOB_TABLE_SPLIT_END = 23, WAIT_PARTITION_SPLIT_RECOVERY_TASK_FINISH = 24, - //GENERATE_ROWKEY_DOC_SCHEMA = 25, - //WAIT_ROWKEY_DOC_TABLE_COMPLEMENT = 26, - //GENERATE_DOC_AUX_SCHEMA = 27, - //WAIT_AUX_TABLE_COMPLEMENT = 28, + GENERATE_ROWKEY_DOC_SCHEMA = 25, + WAIT_ROWKEY_DOC_TABLE_COMPLEMENT = 26, + GENERATE_DOC_AUX_SCHEMA = 27, + WAIT_AUX_TABLE_COMPLEMENT = 28, FAIL = 99, SUCCESS = 100 }; @@ -275,6 +275,18 @@ static const char* ddl_task_status_to_str(const ObDDLTaskStatus &task_status) { case ObDDLTaskStatus::WAIT_PARTITION_SPLIT_RECOVERY_TASK_FINISH: str = "WAIT_PARTITION_SPLIT_RECOVERY_TASK_FINISH"; break; + case ObDDLTaskStatus::GENERATE_ROWKEY_DOC_SCHEMA: + str = "GENERATE_ROWKEY_DOC_SCHEMA"; + break; + case ObDDLTaskStatus::GENERATE_DOC_AUX_SCHEMA: + str = "GENERATE_DOC_AUX_SCHEMA"; + break; + case ObDDLTaskStatus::WAIT_ROWKEY_DOC_TABLE_COMPLEMENT: + str = "WAIT_ROWKEY_DOC_TABLE_COMPLEMENT"; + break; + case ObDDLTaskStatus::WAIT_AUX_TABLE_COMPLEMENT: + str = "WAIT_AUX_TABLE_COMPLEMENT"; + break; case ObDDLTaskStatus::FAIL: str = "FAIL"; break; diff --git a/src/share/ob_fts_index_builder_util.cpp b/src/share/ob_fts_index_builder_util.cpp index a8d2a7032..01d71aa57 100644 --- a/src/share/ob_fts_index_builder_util.cpp +++ b/src/share/ob_fts_index_builder_util.cpp @@ -155,6 +155,42 @@ int ObFtsIndexBuilderUtil::append_fts_doc_word_arg( return ret; } +int ObFtsIndexBuilderUtil::fts_doc_word_schema_exist( + uint64_t tenant_id, + uint64_t database_id, + ObSchemaGetterGuard &schema_guard, + const ObString &index_name, + bool &is_exist) +{ + int ret = OB_SUCCESS; + is_exist = false; + const int64_t buf_size = OB_MAX_TABLE_NAME_BUF_LENGTH; + char buf[buf_size] = {0}; + int64_t pos = 0; + ObString doc_word_index_name; + const ObTableSchema *fts_doc_word_schema = nullptr; + if (OB_FAIL(databuff_printf(buf, + buf_size, + pos, + "%.*s_fts_doc_word", + index_name.length(), + index_name.ptr()))) { + LOG_WARN("fail to printf fts doc word name str", K(ret), K(index_name)); + } else if (OB_FALSE_IT(doc_word_index_name.assign_ptr(buf, pos))) { + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, + database_id, + doc_word_index_name, + true/*is_index*/, + fts_doc_word_schema, + false/*with_hidden_flag*/, + true/*is_built_in_index*/))) { + LOG_WARN("failed to get index schema", K(ret), K(tenant_id)); + } else if (OB_NOT_NULL(fts_doc_word_schema)) { + is_exist = true; + } + return ret; +} + int ObFtsIndexBuilderUtil::generate_fts_aux_index_name( obrpc::ObCreateIndexArg &arg, ObIAllocator *allocator) @@ -273,32 +309,22 @@ int ObFtsIndexBuilderUtil::adjust_fts_args( ObColumnSchemaV2 *generated_word_col = nullptr; ObColumnSchemaV2 *generated_doc_len_col = nullptr; ObColumnSchemaV2 *generated_word_count_col = nullptr; - if (is_rowkey_doc || is_doc_rowkey) { - if (OB_ISNULL(existing_doc_id_col)) { // need to generate doc id col - doc_id_col_id = available_col_id++; - if (OB_FAIL(ret)) { - } else if (OB_FAIL(generate_doc_id_column(&index_arg, - doc_id_col_id, - data_schema, - generated_doc_id_col))) { - LOG_WARN("failed to generate doc id column", K(ret)); - } else if (OB_FAIL(gen_columns.push_back(generated_doc_id_col))) { - LOG_WARN("failed to push back doc id col", K(ret)); - } - } + if (OB_ISNULL(existing_doc_id_col)) { // need to generate doc id col + doc_id_col_id = available_col_id++; if (OB_FAIL(ret)) { - } else if (OB_FAIL(push_back_gen_col(tmp_cols, - existing_doc_id_col, - generated_doc_id_col))) { + } else if (OB_FAIL(generate_doc_id_column(&index_arg, + doc_id_col_id, + data_schema, + generated_doc_id_col))) { + LOG_WARN("failed to generate doc id column", K(ret)); + } else if (OB_FAIL(gen_columns.push_back(generated_doc_id_col))) { LOG_WARN("failed to push back doc id col", K(ret)); - } else if (OB_FAIL(adjust_fts_arg(&index_arg, - data_schema, - allocator, - tmp_cols))) { - LOG_WARN("failed to append fts_index arg", K(ret)); } + } + if (is_rowkey_doc || is_doc_rowkey) { } else if (is_fts_index || is_doc_word) { - if (OB_ISNULL(existing_word_col)) { + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(existing_word_col)) { word_col_id = available_col_id++; if (OB_FAIL(ret)) { } else if (OB_FAIL(generate_word_segment_column(&index_arg, @@ -334,6 +360,20 @@ int ObFtsIndexBuilderUtil::adjust_fts_args( LOG_WARN("fail to push back generated document length", K(ret)); } } + } + if (is_rowkey_doc || is_doc_rowkey) { + if (OB_FAIL(ret)) { + } else if (OB_FAIL(push_back_gen_col(tmp_cols, + existing_doc_id_col, + generated_doc_id_col))) { + LOG_WARN("failed to push back doc id col", K(ret)); + } else if (OB_FAIL(adjust_fts_arg(&index_arg, + data_schema, + allocator, + tmp_cols))) { + LOG_WARN("failed to append fts_index arg", K(ret)); + } + } else if (is_fts_index || is_doc_word) { if (OB_FAIL(ret)) { } else if (is_fts_index) { if (OB_FAIL(push_back_gen_col(tmp_cols, @@ -384,6 +424,7 @@ int ObFtsIndexBuilderUtil::adjust_fts_args( } } } + FLOG_INFO("adjust fts arg finished", K(index_arg)); return ret; } @@ -802,8 +843,8 @@ int ObFtsIndexBuilderUtil::inner_adjust_fts_arg( !share::schema::is_fts_doc_word_aux(fts_arg->index_type_)) || fts_cols.count() != index_column_cnt + 2) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid argument", K(ret), KPC(fts_arg), - K(fts_cols.count()), K(index_column_cnt)); + LOG_WARN("invalid argument", K(ret), KPC(fts_arg), K(fts_cols.count()), + K(index_column_cnt)); } else { // 1. add doc id column, word column to arg->index_columns for (int64_t i = 0; OB_SUCC(ret) && i < index_column_cnt; ++i) { diff --git a/src/share/ob_fts_index_builder_util.h b/src/share/ob_fts_index_builder_util.h index 56e778041..5cc3d2bdc 100644 --- a/src/share/ob_fts_index_builder_util.h +++ b/src/share/ob_fts_index_builder_util.h @@ -46,6 +46,12 @@ public: const obrpc::ObCreateIndexArg &index_arg, ObIAllocator *allocator, ObIArray &index_arg_list); + static int fts_doc_word_schema_exist( + uint64_t tenant_id, + uint64_t database_id, + ObSchemaGetterGuard &schema_guard, + const ObString &index_name, + bool &is_exist); static int generate_fts_aux_index_name( obrpc::ObCreateIndexArg &arg, ObIAllocator *allocator); diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index d17266908..2389cf8b8 100755 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -3175,6 +3175,26 @@ OB_SERIALIZE_MEMBER((ObCreateIndexArg, ObIndexArg), exist_all_column_group_, index_cgs_); +int ObGenerateAuxIndexSchemaArg::assign(const ObGenerateAuxIndexSchemaArg &other) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(create_index_arg_.assign(other.create_index_arg_))) { + SHARE_LOG(WARN, "fail to assign arg", K(ret)); + } else { + tenant_id_ = other.tenant_id_; + data_table_id_ = other.data_table_id_; + } + return ret; +} + +OB_SERIALIZE_MEMBER((ObGenerateAuxIndexSchemaArg, ObDDLArg), + tenant_id_, + data_table_id_, + create_index_arg_); +OB_SERIALIZE_MEMBER(ObGenerateAuxIndexSchemaRes, + aux_table_id_, + schema_generated_); + bool ObAlterIndexArg::is_valid() const { // store_columns_ can be empty diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index 9213d930a..fed952fe5 100755 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -2700,6 +2700,64 @@ public: common::ObSEArray index_cgs_; }; +struct ObGenerateAuxIndexSchemaArg : public ObDDLArg +{ + OB_UNIS_VERSION_V(1); +public: + ObGenerateAuxIndexSchemaArg() + : tenant_id_(OB_INVALID_TENANT_ID), + data_table_id_(OB_INVALID_ID) + {} + ~ObGenerateAuxIndexSchemaArg() {} + bool is_valid() const + { + return tenant_id_ != OB_INVALID_TENANT_ID && + data_table_id_ != OB_INVALID_ID&& + create_index_arg_.is_valid(); + } + int assign(const ObGenerateAuxIndexSchemaArg &other); + void reset() + { + tenant_id_ = OB_INVALID_TENANT_ID; + data_table_id_ = OB_INVALID_ID; + create_index_arg_.reset(); + } + TO_STRING_KV(K(tenant_id_), K(data_table_id_), K(create_index_arg_)); + +public: + uint64_t tenant_id_; + uint64_t data_table_id_; + ObCreateIndexArg create_index_arg_; +}; + +struct ObGenerateAuxIndexSchemaRes final +{ + OB_UNIS_VERSION_V(1); +public: + ObGenerateAuxIndexSchemaRes() + : aux_table_id_(OB_INVALID_ID), + schema_generated_(false) + {} + ~ObGenerateAuxIndexSchemaRes() {} + int assign(const ObGenerateAuxIndexSchemaRes &other) + { + int ret = OB_SUCCESS; + aux_table_id_ = other.aux_table_id_; + schema_generated_ = other.schema_generated_; + return ret; + } + void reset() + { + aux_table_id_ = OB_INVALID_ID; + schema_generated_ = false; + } + TO_STRING_KV(K(aux_table_id_), K(schema_generated_)); + +public: + uint64_t aux_table_id_; + bool schema_generated_; +}; + typedef ObCreateIndexArg ObAlterPrimaryArg; struct ObCreateMLogArg : public ObDDLArg diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index 54d8a9b0a..bb42f9574 100644 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -1399,6 +1399,9 @@ DEF_BOOL(_enable_px_batch_rescan, OB_TENANT_PARAMETER, "True", DEF_BOOL(_enable_spf_batch_rescan, OB_TENANT_PARAMETER, "False", "enable das batch rescan for subplan filter", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_BOOL(_enable_das_keep_order, OB_TENANT_PARAMETER, "True", + "enable das keep order optimization", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_INT(_parallel_max_active_sessions, OB_TENANT_PARAMETER, "0", "[0,]", "max active parallel sessions allowed for tenant. Range: [0,+∞)", diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index 4ea0cf8a3..1eabd7239 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -57,18 +57,24 @@ ob_set_subtarget(ob_sql das das/ob_das_id_service.cpp das/ob_das_id_rpc.cpp das/ob_das_id_cache.cpp + das/ob_das_ir_define.cpp das/ob_das_task_result.cpp das/ob_das_spatial_index_lookup_op.cpp das/ob_das_retry_ctrl.cpp das/ob_das_simple_op.cpp das/ob_das_domain_utils.cpp - das/ob_text_retrieval_op.cpp das/ob_das_attach_define.cpp das/ob_group_scan_iter.cpp das/iter/ob_das_iter.cpp das/iter/ob_das_merge_iter.cpp das/iter/ob_das_lookup_iter.cpp das/iter/ob_das_group_fold_iter.cpp + das/iter/ob_das_scan_iter.cpp + das/iter/ob_das_local_lookup_iter.cpp + das/iter/ob_das_global_lookup_iter.cpp + das/iter/ob_das_sort_iter.cpp + das/iter/ob_das_text_retrieval_iter.cpp + das/iter/ob_das_text_retrieval_merge_iter.cpp das/iter/ob_das_iter_utils.cpp ) diff --git a/src/sql/code_generator/ob_tsc_cg_service.cpp b/src/sql/code_generator/ob_tsc_cg_service.cpp index 5ba73aa0d..6935fe72d 100644 --- a/src/sql/code_generator/ob_tsc_cg_service.cpp +++ b/src/sql/code_generator/ob_tsc_cg_service.cpp @@ -105,6 +105,9 @@ int ObTscCgService::generate_tsc_ctdef(ObLogTableScan &op, ObTableScanCtDef &tsc root_ctdef = &scan_ctdef; } } + if (OB_SUCC(ret) && op.das_need_keep_ordering()) { + tsc_ctdef.is_das_keep_order_ = true; + } //to generate dynamic tsc partition pruning info if (OB_SUCC(ret)) { @@ -166,7 +169,8 @@ int ObTscCgService::generate_tsc_ctdef(ObLogTableScan &op, ObTableScanCtDef &tsc if (OB_SUCC(ret) && op.is_multivalue_index_scan()) { ObDASIRAuxLookupCtDef *aux_lookup_ctdef = nullptr; - if (OB_FAIL(generate_doc_id_lookup_ctdef(op, tsc_ctdef, root_ctdef, aux_lookup_ctdef))) { + ObExpr *doc_id_col_expr = scan_ctdef.result_output_.at(scan_ctdef.result_output_.count() - 1); + if (OB_FAIL(generate_doc_id_lookup_ctdef(op, tsc_ctdef, root_ctdef, doc_id_col_expr, aux_lookup_ctdef))) { LOG_WARN("failed to generate text ir ctdef", K(ret)); } else { root_ctdef = aux_lookup_ctdef; @@ -194,14 +198,15 @@ int ObTscCgService::generate_tsc_ctdef(ObLogTableScan &op, ObTableScanCtDef &tsc return ret; } -int ObTscCgService::generate_table_param(const ObLogTableScan &op, ObDASScanCtDef &scan_ctdef) +int ObTscCgService::generate_table_param(const ObLogTableScan &op, + ObDASScanCtDef &scan_ctdef, + common::ObIArray &tsc_out_cols) { int ret = OB_SUCCESS; ObTableID index_id = scan_ctdef.ref_table_id_; const ObTableSchema *table_schema = NULL; const bool pd_agg = scan_ctdef.pd_expr_spec_.pd_storage_flag_.is_aggregate_pushdown(); const bool pd_group_by = scan_ctdef.pd_expr_spec_.pd_storage_flag_.is_group_by_pushdown(); - ObArray tsc_out_cols; ObSqlSchemaGuard *schema_guard = cg_.opt_ctx_->get_sql_schema_guard(); CK(OB_NOT_NULL(schema_guard)); if (OB_UNLIKELY(pd_agg && 0 == scan_ctdef.aggregate_column_ids_.count()) || @@ -242,8 +247,6 @@ int ObTscCgService::generate_table_param(const ObLogTableScan &op, ObDASScanCtDe scan_ctdef.pd_expr_spec_.pd_storage_flag_))) { LOG_WARN("convert group by failed", K(ret), K(*table_schema), K(scan_ctdef.aggregate_column_ids_), K(scan_ctdef.group_by_column_ids_)); - } else if (OB_FAIL(generate_das_result_output(tsc_out_cols, scan_ctdef, op.get_trans_info_expr(), pd_agg))) { - LOG_WARN("failed to init result outputs", K(ret)); } return ret; } @@ -297,6 +300,7 @@ int ObTscCgService::generate_das_result_output(const ObIArray &output_ scan_ctdef.trans_info_expr_ = e; } } + return ret; } @@ -594,15 +598,20 @@ int ObTscCgService::extract_das_access_exprs(const ObLogTableScan &op, LOG_WARN("append the data table rowkey expr failed", K(ret), K(op.get_rowkey_exprs())); } else if (OB_FAIL(append_array_no_dup(access_exprs, op.get_part_exprs()))) { LOG_WARN("append the data table part expr failed", K(ret), K(op.get_part_exprs())); - } else if (NULL != op.get_group_id_expr() && op.use_batch() - && OB_FAIL(add_var_to_array_no_dup(access_exprs, - const_cast(op.get_group_id_expr())))) { - LOG_WARN("fail to add group id", K(ret)); } } } else if (OB_FAIL(access_exprs.assign(op.get_access_exprs()))) { LOG_WARN("assign access exprs failed", K(ret)); } + // store group_id_expr when use group id + if (OB_SUCC(ret) && op.use_group_id()) { + const ObRawExpr* group_id_expr = op.get_group_id_expr(); + if (group_id_expr != nullptr && + OB_FAIL(add_var_to_array_no_dup(access_exprs, const_cast(group_id_expr)))) { + LOG_WARN("failed to push back group id expr", K(ret)); + } + } + if (OB_SUCC(ret) && is_oracle_mapping_real_virtual_table(op.get_ref_table_id())) { //the access exprs are the agent virtual table columns, but das need the real table columns //now to replace the real table column @@ -897,7 +906,7 @@ int ObTscCgService::generate_das_scan_ctdef(const ObLogTableScan &op, } //4. generate batch scan ctdef - if (OB_SUCC(ret) && op.use_batch()) { + if (OB_SUCC(ret) && op.use_group_id()) { if (OB_FAIL(cg_.generate_rt_expr(*op.get_group_id_expr(), scan_ctdef.group_id_expr_))) { LOG_WARN("generate group id expr failed", K(ret)); } @@ -911,11 +920,19 @@ int ObTscCgService::generate_das_scan_ctdef(const ObLogTableScan &op, } } //6. generate table param + ObArray tsc_out_cols; if (OB_SUCC(ret)) { - if (OB_FAIL(generate_table_param(op, scan_ctdef))) { + if (OB_FAIL(generate_table_param(op, scan_ctdef, tsc_out_cols))) { LOG_WARN("generate table param failed", K(ret)); } } + //7. generate das result output + if (OB_SUCC(ret)) { + const bool pd_agg = scan_ctdef.pd_expr_spec_.pd_storage_flag_.is_aggregate_pushdown(); + if (OB_FAIL(generate_das_result_output(tsc_out_cols, scan_ctdef, op.get_trans_info_expr(), pd_agg))) { + LOG_WARN("failed to init result outputs", K(ret)); + } + } return ret; } @@ -945,7 +962,7 @@ int ObTscCgService::extract_das_output_column_ids(const ObLogTableScan &op, LOG_WARN("unexpected nullptr to table schema", K(ret)); } else if (OB_FAIL(table_schema->get_rowkey_column_ids(output_cids))) { LOG_WARN("get rowkey column ids failed", K(ret)); - } else if (nullptr != op.get_group_id_expr() && op.use_batch()) { + } else if (nullptr != op.get_group_id_expr() && op.use_group_id()) { if (OB_FAIL(output_cids.push_back(OB_HIDDEN_GROUP_IDX_COLUMN_ID))) { LOG_WARN("store group column id failed", K(ret)); } @@ -986,6 +1003,7 @@ int ObTscCgService::extract_das_output_column_ids(const ObLogTableScan &op, LOG_WARN("store colum id failed", K(ret)); } } + //column expr in non-pushdown filter need to be output, //because filter_row will use it in TSC operator } else if (OB_FAIL(extract_tsc_access_columns(op, das_output_cols))) { @@ -1003,7 +1021,7 @@ int ObTscCgService::extract_das_output_column_ids(const ObLogTableScan &op, LOG_WARN("store real output colum id failed", K(ret), KPC(mapping_expr)); } } - } else if (nullptr != op.get_group_id_expr() && op.use_batch() && + } else if (nullptr != op.get_group_id_expr() && op.use_group_id() && OB_FAIL(das_output_cols.push_back(const_cast(op.get_group_id_expr())))) { LOG_WARN("store group id expr failed", K(ret)); } else if (OB_FAIL(extract_das_column_ids(das_output_cols, output_cids))) { @@ -1114,6 +1132,7 @@ int ObTscCgService::generate_text_ir_ctdef(const ObLogTableScan &op, ObSqlSchemaGuard *schema_guard = cg_.opt_ctx_->get_sql_schema_guard(); ObDASIRScanCtDef *ir_scan_ctdef = nullptr; ObDASSortCtDef *sort_ctdef = nullptr; + ObExpr *index_back_doc_id_column = nullptr; const bool use_approx_pre_agg = true; // TODO: support differentiate use approx agg or not if (OB_ISNULL(match_against) || OB_ISNULL(schema_guard)) { ret = OB_ERR_UNEXPECTED; @@ -1221,6 +1240,7 @@ int ObTscCgService::generate_text_ir_ctdef(const ObLogTableScan &op, partition_row_cnt = est_cost_info->table_meta_info_->table_row_count_ / est_cost_info->table_meta_info_->part_count_; } ir_scan_ctdef->estimated_total_doc_cnt_ = partition_row_cnt; + index_back_doc_id_column = ir_scan_ctdef->inv_scan_doc_id_col_; } } @@ -1245,7 +1265,8 @@ int ObTscCgService::generate_text_ir_ctdef(const ObLogTableScan &op, ObDASIRAuxLookupCtDef *aux_lookup_ctdef = nullptr; ObDASBaseCtDef *ir_output_ctdef = nullptr == sort_ctdef ? static_cast(ir_scan_ctdef) : static_cast(sort_ctdef); - if (OB_FAIL(generate_doc_id_lookup_ctdef(op, tsc_ctdef, ir_output_ctdef, aux_lookup_ctdef))) { + if (OB_FAIL(generate_doc_id_lookup_ctdef( + op, tsc_ctdef, ir_output_ctdef, index_back_doc_id_column, aux_lookup_ctdef))) { LOG_WARN("generate doc id lookup ctdef failed", K(ret)); } else if (OB_FAIL(append_fts_relavence_project_col(aux_lookup_ctdef, ir_scan_ctdef))) { LOG_WARN("failed to append fts relavence project col", K(ret)); @@ -1511,6 +1532,7 @@ int ObTscCgService::generate_text_ir_spec_exprs(const ObLogTableScan &op, int ObTscCgService::generate_doc_id_lookup_ctdef(const ObLogTableScan &op, ObTableScanCtDef &tsc_ctdef, ObDASBaseCtDef *ir_scan_ctdef, + ObExpr *doc_id_expr, ObDASIRAuxLookupCtDef *&aux_lookup_ctdef) { int ret = OB_SUCCESS; @@ -1560,6 +1582,13 @@ int ObTscCgService::generate_doc_id_lookup_ctdef(const ObLogTableScan &op, LOG_WARN("generate das lookup scan ctdef failed", K(ret)); } else if (OB_FAIL(result_outputs.assign(scan_ctdef->result_output_))) { LOG_WARN("construct aux lookup ctdef failed", K(ret)); + } else if (OB_ISNULL(doc_id_expr) || OB_UNLIKELY(!scan_ctdef->rowkey_exprs_.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected doc id expr status", K(ret), KPC(doc_id_expr), KPC(scan_ctdef)); + } else if (OB_FAIL(scan_ctdef->rowkey_exprs_.reserve(1))) { + LOG_WARN("failed to reserve doc id lookup rowkey exprs", K(ret)); + } else if (OB_FAIL(scan_ctdef->rowkey_exprs_.push_back(doc_id_expr))) { + LOG_WARN("failed to append doc id rowkey expr", K(ret)); } else if (OB_FAIL(generate_table_loc_meta(op.get_table_id(), *op.get_stmt(), *index_schema, @@ -1637,6 +1666,17 @@ int ObTscCgService::generate_table_lookup_ctdef(const ObLogTableScan &op, LOG_WARN("fail to generate rowkey exprs", K(ret)); } } + + if (OB_SUCC(ret) && op.get_index_back()) { + ObArray rowkey_exprs; + if (OB_FAIL(rowkey_exprs.assign(op.get_rowkey_exprs()))) { + LOG_WARN("failed to assign rowkey exprs", K(ret)); + } else if (!op.get_is_index_global() && OB_FAIL(mapping_oracle_real_agent_virtual_exprs(op, rowkey_exprs))) { + LOG_WARN("failed to mapping oracle real virtual exprs", K(ret)); + } else if (OB_FAIL(cg_.generate_rt_exprs(rowkey_exprs, tsc_ctdef.lookup_ctdef_->rowkey_exprs_))) { + LOG_WARN("failed to generate main table rowkey exprs", K(ret)); + } + } } if (OB_SUCC(ret)) { if (OB_FAIL(ObDASTaskFactory::alloc_das_ctdef(DAS_OP_TABLE_LOOKUP, allocator, lookup_ctdef))) { @@ -1828,5 +1868,33 @@ int ObTscCgService::generate_das_sort_ctdef( return ret; } +int ObTscCgService::mapping_oracle_real_agent_virtual_exprs(const ObLogTableScan &op, + common::ObIArray &access_exprs) +{ + int ret = OB_SUCCESS; + if (is_oracle_mapping_real_virtual_table(op.get_ref_table_id())) { + //the access exprs are the agent virtual table columns, but das need the real table columns + //now to replace the real table column + for (int64_t i = 0; OB_SUCC(ret) && i < access_exprs.count(); ++i) { + ObRawExpr *expr = access_exprs.at(i); + ObRawExpr *mapping_expr = nullptr; + uint64_t column_id = UINT64_MAX; + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr expr", K(expr), K(ret)); + } else if (T_ORA_ROWSCN == expr->get_expr_type()) { + // keep orign expr as access expr + } else if (OB_ISNULL(mapping_expr = op.get_real_expr(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mapping expr is null", K(ret), KPC(expr)); + } else { + //replace the agent virtual table column expr + access_exprs.at(i) = mapping_expr; + } + } + } + return ret; +} + } // namespace sql } // namespace oceanbase diff --git a/src/sql/code_generator/ob_tsc_cg_service.h b/src/sql/code_generator/ob_tsc_cg_service.h index 3b0b82a65..ab2b465e3 100644 --- a/src/sql/code_generator/ob_tsc_cg_service.h +++ b/src/sql/code_generator/ob_tsc_cg_service.h @@ -53,7 +53,7 @@ private: int generate_access_ctdef(const ObLogTableScan &op, ObDASScanCtDef &scan_ctdef, bool &has_rowscn); int generate_pushdown_aggr_ctdef(const ObLogTableScan &op, ObDASScanCtDef &scan_ctdef); int generate_das_scan_ctdef(const ObLogTableScan &op, ObDASScanCtDef &scan_ctdef, bool &has_rowscn); - int generate_table_param(const ObLogTableScan &op, ObDASScanCtDef &scan_ctdef); + int generate_table_param(const ObLogTableScan &op, ObDASScanCtDef &scan_ctdef, common::ObIArray &tsc_out_cols); int extract_das_output_column_ids(const ObLogTableScan &op, ObDASScanCtDef &scan_ctdef, const ObTableSchema &index_schema, @@ -79,6 +79,7 @@ private: int generate_doc_id_lookup_ctdef(const ObLogTableScan &op, ObTableScanCtDef &tsc_ctdef, ObDASBaseCtDef *ir_scan_ctdef, + ObExpr *doc_id_expr, ObDASIRAuxLookupCtDef *&aux_lookup_ctdef); int generate_table_lookup_ctdef(const ObLogTableScan &op, ObTableScanCtDef &tsc_ctdef, @@ -98,6 +99,9 @@ private: ObRawExpr *topk_offset_expr, ObDASBaseCtDef *child_ctdef, ObDASSortCtDef *&sort_ctdef); + int mapping_oracle_real_agent_virtual_exprs(const ObLogTableScan &op, + common::ObIArray &access_exprs); + private: ObStaticEngineCG &cg_; }; diff --git a/src/sql/das/iter/ob_das_global_lookup_iter.cpp b/src/sql/das/iter/ob_das_global_lookup_iter.cpp new file mode 100644 index 000000000..34dc8d8df --- /dev/null +++ b/src/sql/das/iter/ob_das_global_lookup_iter.cpp @@ -0,0 +1,241 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_DAS +#include "sql/das/iter/ob_das_global_lookup_iter.h" +#include "sql/das/iter/ob_das_merge_iter.h" +#include "sql/engine/ob_exec_context.h" + +namespace oceanbase +{ +using namespace common; +namespace sql +{ + +int ObDASGlobalLookupIter::inner_init(ObDASIterParam ¶m) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObDASLookupIter::inner_init(param))) { + LOG_WARN("failed to init das lookup iter", K(ret)); + } else if (param.type_ != ObDASIterType::DAS_ITER_GLOBAL_LOOKUP) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("inner init das iter with bad param type", K(param), K(ret)); + } else { + ObDASGlobalLookupIterParam &lookup_param = static_cast(param); + can_retry_ = lookup_param.can_retry_; + calc_part_id_ = lookup_param.calc_part_id_; + // use lookup_memctx for lookup to avoid memory expansion during global index lookup + lookup_rtdef_->scan_allocator_.set_alloc(&get_arena_allocator()); + lookup_rtdef_->stmt_allocator_.set_alloc(&get_arena_allocator()); + if (lookup_param.rowkey_exprs_->empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected empty global rowkey exprs", K(ret)); + } else if (OB_FAIL(rowkey_exprs_.assign(*lookup_param.rowkey_exprs_))) { + LOG_WARN("failed to assign rowkey exprs", K(ret)); + } + } + return ret; +} + +int ObDASGlobalLookupIter::inner_reuse() +{ + int ret = OB_SUCCESS; + // for global lookup, the reuse of index table will handled in TSC. + if (OB_FAIL(data_table_iter_->reuse())) { + LOG_WARN("failed to reuse data table iter", K(ret)); + } else if (OB_FAIL(ObDASLookupIter::inner_reuse())) { + LOG_WARN("failed to reuse das lookup iter", K(ret)); + } + index_ordered_idx_ = 0; + return ret; +} + +int ObDASGlobalLookupIter::inner_release() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObDASLookupIter::inner_release())) { + LOG_WARN("failed to release das lookup iter", K(ret)); + } + return ret; +} + +void ObDASGlobalLookupIter::reset_lookup_state() +{ + index_ordered_idx_ = 0; + ObDASLookupIter::reset_lookup_state(); +} + +int ObDASGlobalLookupIter::add_rowkey() +{ + int ret = OB_SUCCESS; + ObObjectID partition_id = OB_INVALID_ID; + ObTabletID tablet_id(OB_INVALID_ID); + ObDASScanOp *das_scan_op = nullptr; + ObDASTabletLoc *tablet_loc = nullptr; + ObDASCtx *das_ctx = nullptr; + bool reuse_das_op = false; + OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_MERGE); + if (OB_ISNULL(exec_ctx_) || OB_ISNULL(das_ctx = &exec_ctx_->get_das_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get das ctx", KPC_(exec_ctx)); + } else { + ObDASMergeIter *merge_iter = static_cast(data_table_iter_); + if (OB_FAIL(ObExprCalcPartitionBase::calc_part_and_tablet_id(calc_part_id_, + *eval_ctx_, + partition_id, + tablet_id))) { + LOG_WARN("failed to calc part id", K(ret), KPC(calc_part_id_)); + } else if (OB_FAIL(das_ctx->extended_tablet_loc(*lookup_rtdef_->table_loc_, + tablet_id, + tablet_loc))) { + LOG_WARN("failed to get tablet loc by tablet_id", K(ret)); + } else if (OB_FAIL(merge_iter->create_das_task(tablet_loc, das_scan_op, reuse_das_op))) { + LOG_WARN("failed to create das task", K(ret)); + } else if (!reuse_das_op) { + das_scan_op->set_scan_ctdef(lookup_ctdef_); + das_scan_op->set_scan_rtdef(lookup_rtdef_); + das_scan_op->set_can_part_retry(can_retry_); + } + } + if (OB_SUCC(ret)) { + storage::ObTableScanParam &scan_param = das_scan_op->get_scan_param(); + ObNewRange lookup_range; + int64_t group_id = 0; + OB_ASSERT(DAS_OP_TABLE_SCAN == index_ctdef_->op_type_); + const ObDASScanCtDef *index_ctdef = static_cast(index_ctdef_); + if (nullptr != index_ctdef->group_id_expr_) { + group_id = index_ctdef->group_id_expr_->locate_expr_datum(*eval_ctx_).get_int(); + } + if (nullptr != index_ctdef->trans_info_expr_) { + ObDatum *datum_ptr = nullptr; + if (OB_FAIL(build_trans_info_datum(index_ctdef->trans_info_expr_, datum_ptr))) { + LOG_WARN("failed to build trans info datum", K(ret)); + } else if (OB_ISNULL(datum_ptr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", K(ret)); + } else if (OB_FAIL(das_scan_op->trans_info_array_.push_back(datum_ptr))) { + LOG_WARN("failed to push back trans info array", K(ret), KPC(datum_ptr)); + } + } + + int64_t group_idx = ObNewRange::get_group_idx(group_id); + index_ordered_idx_++; + if (OB_FAIL(ret)) { + } else if (OB_FAIL(build_lookup_range(lookup_range))) { + LOG_WARN("failed to build lookup range", K(ret)); + } else if (FALSE_IT(lookup_range.group_idx_ = group_idx)) { + } else if (FALSE_IT(lookup_range.index_ordered_idx_ = index_ordered_idx_)) { + } else if (OB_FAIL(scan_param.key_ranges_.push_back(lookup_range))) { + LOG_WARN("failed to push back lookup range", K(ret)); + } else { + scan_param.is_get_ = true; + } + LOG_DEBUG("build global lookup range", K(lookup_range), K(ret)); + } + + return ret; +} + +int ObDASGlobalLookupIter::add_rowkeys(int64_t count) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(eval_ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", K_(eval_ctx)); + } else { + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*eval_ctx_); + batch_info_guard.set_batch_size(count); + for(int i = 0; OB_SUCC(ret) && i < count; i++) { + batch_info_guard.set_batch_idx(i); + if(OB_FAIL(add_rowkey())) { + LOG_WARN("failed to add rowkey", K(ret), K(i)); + } + } + } + + return ret; +} + +int ObDASGlobalLookupIter::do_index_lookup() +{ + int ret = OB_SUCCESS; + OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_MERGE); + ObDASMergeIter *merge_iter = static_cast(data_table_iter_); + if (OB_FAIL(merge_iter->do_table_scan())) { + LOG_WARN("failed to do global index lookup", K(ret)); + } else { + merge_iter->set_merge_status(merge_iter->get_merge_type()); + } + return ret; +} + +int ObDASGlobalLookupIter::check_index_lookup() +{ + int ret = OB_SUCCESS; + OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_MERGE); + ObDASMergeIter *merge_iter = static_cast(data_table_iter_); + if (GCONF.enable_defensive_check() && + lookup_ctdef_->pd_expr_spec_.pushdown_filters_.empty()) { + if (OB_UNLIKELY(lookup_rowkey_cnt_ != lookup_row_cnt_)) { + ret = OB_ERR_DEFENSIVE_CHECK; + ObSQLSessionInfo *my_session = exec_ctx_->get_my_session(); + ObString func_name = ObString::make_string("check_lookup_row_cnt"); + LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr()); + LOG_ERROR("Fatal Error!!! Catch a defensive error!", + K(ret), K(lookup_rowkey_cnt_), K(lookup_row_cnt_), + "main table id", lookup_ctdef_->ref_table_id_, + KPC(my_session->get_tx_desc())); + + int64_t row_num = 0; + for (DASTaskIter task_iter = merge_iter->begin_task_iter(); !task_iter.is_end(); ++task_iter) { + ObDASScanOp *das_op = static_cast(*task_iter); + if (das_op->trans_info_array_.count() == das_op->get_scan_param().key_ranges_.count()) { + for (int64_t i = 0; i < das_op->trans_info_array_.count(); i++) { + row_num++; + ObDatum *datum = das_op->trans_info_array_.at(i); + LOG_ERROR("dump lookup range and trans info of global lookup das task", + K(row_num), KPC(datum), + K(das_op->get_scan_param().key_ranges_.at(i)), + K(das_op->get_tablet_id())); + } + } else { + for (int64_t i = 0; i < das_op->get_scan_param().key_ranges_.count(); i++) { + row_num++; + LOG_ERROR("dump lookup range of global lookup das task", + K(row_num), + K(das_op->get_scan_param().key_ranges_.at(i)), + K(das_op->get_tablet_id())); + } + } + } + } + } + + int simulate_error = EVENT_CALL(EventTable::EN_DAS_SIMULATE_DUMP_WRITE_BUFFER); + if (0 != simulate_error) { + for (DASTaskIter task_iter = merge_iter->begin_task_iter(); !task_iter.is_end(); ++task_iter) { + ObDASScanOp *das_op = static_cast(*task_iter); + for (int64_t i = 0; i < das_op->trans_info_array_.count(); i++) { + ObDatum *datum = das_op->trans_info_array_.at(i); + LOG_INFO("dump trans info of global lookup das task", K(i), + KPC(das_op->trans_info_array_.at(i)), + K(das_op->get_scan_param().key_ranges_.at(i)), + K(das_op->get_tablet_id())); + } + } + } + + return ret; +} + +} // namespace sql +} // namespace oceanbase diff --git a/src/sql/das/iter/ob_das_global_lookup_iter.h b/src/sql/das/iter/ob_das_global_lookup_iter.h new file mode 100644 index 000000000..abaa8772a --- /dev/null +++ b/src/sql/das/iter/ob_das_global_lookup_iter.h @@ -0,0 +1,72 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OBDEV_SRC_SQL_DAS_ITER_OB_DAS_GLOBAL_LOOKUP_ITER_H_ +#define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_GLOBAL_LOOKUP_ITER_H_ + +#include "sql/das/iter/ob_das_lookup_iter.h" +namespace oceanbase +{ +using namespace common; +namespace sql +{ + +struct ObDASGlobalLookupIterParam : public ObDASLookupIterParam +{ +public: + ObDASGlobalLookupIterParam() + : ObDASLookupIterParam(true /*global lookup*/), + can_retry_(false), + calc_part_id_(nullptr) + {} + bool can_retry_; + const ObExpr *calc_part_id_; + + virtual bool is_valid() const override + { + return ObDASLookupIterParam::is_valid() && calc_part_id_ != nullptr; + } +}; + +class ObDASGlobalLookupIter : public ObDASLookupIter +{ +public: + ObDASGlobalLookupIter() + : ObDASLookupIter(ObDASIterType::DAS_ITER_GLOBAL_LOOKUP), + can_retry_(false), + calc_part_id_(nullptr), + index_ordered_idx_(0) + {} + virtual ~ObDASGlobalLookupIter() {} + +protected: + virtual int inner_init(ObDASIterParam ¶m) override; + virtual int inner_reuse() override; + virtual int inner_release() override; + virtual int add_rowkey() override; + virtual int add_rowkeys(int64_t count) override; + virtual int do_index_lookup() override; + virtual int check_index_lookup() override; + virtual void reset_lookup_state(); + +private: + bool can_retry_; + const ObExpr *calc_part_id_; + int64_t index_ordered_idx_; // used for keep order of global lookup +}; + +} // namespace sql +} // namespace oceanbase + + + +#endif /* OBDEV_SRC_SQL_DAS_ITER_OB_DAS_GLOBAL_LOOKUP_ITER_H_ */ diff --git a/src/sql/das/iter/ob_das_group_fold_iter.cpp b/src/sql/das/iter/ob_das_group_fold_iter.cpp index 04838b64d..0625dc317 100644 --- a/src/sql/das/iter/ob_das_group_fold_iter.cpp +++ b/src/sql/das/iter/ob_das_group_fold_iter.cpp @@ -101,8 +101,8 @@ int ObGroupResultSaveRows::to_expr(bool is_vectorized, int64_t start_pos, int64_ int64_t ObGroupResultSaveRows::cur_group_idx() { - return start_pos_ >= saved_size_ ? - OB_INVALID_INDEX : store_rows_[start_pos_].store_row_->cells()[group_id_idx_].get_int(); + return start_pos_ >= saved_size_ ? OB_INVALID_INDEX : + ObNewRange::get_group_idx(store_rows_[start_pos_].store_row_->cells()[group_id_idx_].get_int()); } @@ -140,7 +140,7 @@ int ObDASGroupFoldIter::set_scan_group(int64_t group_id) cur_group_idx_ = group_id; } if (group_save_rows_.need_check_output_datum_) { - reset_expr_datum_ptr(); + reset_expr_datum_ptr(); } if (cur_group_idx_ >= group_size_) { ret = OB_ITER_END; @@ -277,7 +277,7 @@ int ObDASGroupFoldIter::inner_get_next_rows(int64_t &count, int64_t capacity) PRINT_VECTORIZED_ROWS(SQL, DEBUG, *eval_ctx_, *output_, storage_count, skip); ObDatum *group_idx_batch = group_id_expr_->locate_batch_datums(*group_save_rows_.eval_ctx_); for (int64_t i = 0; OB_SUCC(ret) && i < storage_count; i++) { - group_idx = group_idx_batch[i].get_int(); + group_idx = ObNewRange::get_group_idx(group_idx_batch[i].get_int()); if (group_idx >= cur_group_idx_) { if (OB_FAIL(group_save_rows_.save(true, i, storage_count - i))) { LOG_WARN("das group fold iter failed to save batch result", K(ret)); @@ -352,7 +352,7 @@ int ObDASGroupFoldIter::inner_get_next_row() } else if (OB_FAIL(group_id_expr_->eval(*group_save_rows_.eval_ctx_, group_idx))) { LOG_WARN("failed to eval group id", K(ret)); } else { - available_group_idx_ = group_idx->get_int(); + available_group_idx_ = ObNewRange::get_group_idx(group_idx->get_int()); } } // while end diff --git a/src/sql/das/iter/ob_das_group_fold_iter.h b/src/sql/das/iter/ob_das_group_fold_iter.h index d87eadaf5..fa8b14913 100644 --- a/src/sql/das/iter/ob_das_group_fold_iter.h +++ b/src/sql/das/iter/ob_das_group_fold_iter.h @@ -75,6 +75,10 @@ public: struct ObDASGroupFoldIterParam : public ObDASIterParam { +public: + ObDASGroupFoldIterParam() + : ObDASIterParam(ObDASIterType::DAS_ITER_GROUP_FOLD) + {} bool need_check_output_datum_; ObDASIter *iter_tree_; @@ -88,7 +92,8 @@ class ObDASGroupFoldIter : public ObDASIter { public: ObDASGroupFoldIter() - : cur_group_idx_(0), + : ObDASIter(ObDASIterType::DAS_ITER_GROUP_FOLD), + cur_group_idx_(0), available_group_idx_(MIN_GROUP_INDEX), group_size_(0), need_check_output_datum_(false), diff --git a/src/sql/das/iter/ob_das_iter.cpp b/src/sql/das/iter/ob_das_iter.cpp index 684d4e6ae..4f7e7fee1 100644 --- a/src/sql/das/iter/ob_das_iter.cpp +++ b/src/sql/das/iter/ob_das_iter.cpp @@ -24,9 +24,11 @@ namespace sql int ObDASIter::set_merge_status(MergeType merge_type) { int ret = OB_SUCCESS; - ObDASIter *child = child_; - for (; child != nullptr && OB_SUCC(ret); child = child->right_) { - if (OB_FAIL(child->set_merge_status(merge_type))) { + for (uint32_t i = 0; i < children_cnt_; i++) { + if (OB_ISNULL(children_[i])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr das iter child", K(i), K_(children_cnt), K(ret)); + } else if (OB_FAIL(children_[i]->set_merge_status(merge_type))) { LOG_WARN("failed to set merge status", K(ret)); } } @@ -50,8 +52,6 @@ int ObDASIter::init(ObDASIterParam ¶m) exec_ctx_ = param.exec_ctx_; output_ = param.output_; group_id_expr_ = param.group_id_expr_; - child_ = param.child_; - right_ = param.right_; if (OB_FAIL(inner_init(param))) { LOG_WARN("failed to inner init das iter", K(param), K(ret)); } @@ -60,6 +60,7 @@ int ObDASIter::init(ObDASIterParam ¶m) return ret; } +// NOTE: unlike release(), reuse() does not recursively call the reuse() of its children. int ObDASIter::reuse() { int ret = OB_SUCCESS; @@ -76,15 +77,15 @@ int ObDASIter::release() { int ret = OB_SUCCESS; int child_ret = OB_SUCCESS; - ObDASIter *child = child_; int tmp_ret = OB_SUCCESS; - while (child != nullptr) { - ObDASIter *right = child->right_; - if (OB_TMP_FAIL(child->release())) { - LOG_WARN("failed to release child iter", K(tmp_ret), KPC(child)); - child_ret = tmp_ret; + for (uint32_t i = 0; i < children_cnt_; i++) { + if (OB_ISNULL(children_[i])) { + tmp_ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr das iter child", K(i), K_(children_cnt), K(tmp_ret)); + } else if (OB_TMP_FAIL(children_[i]->release())) { + LOG_WARN("failed to release child iter", K(tmp_ret), KPC(children_[i])); } - child = right; + child_ret = tmp_ret; } if (OB_FAIL(inner_release())) { LOG_WARN("failed to inner release das iter", K(ret), KPC(this)); @@ -92,8 +93,8 @@ int ObDASIter::release() ret = child_ret; } inited_ = false; - right_ = nullptr; - child_ = nullptr; + children_cnt_ = 0; + children_ = nullptr; group_id_expr_ = nullptr; output_ = nullptr; exec_ctx_ = nullptr; diff --git a/src/sql/das/iter/ob_das_iter.h b/src/sql/das/iter/ob_das_iter.h index a8cc473a6..ffb88c15d 100644 --- a/src/sql/das/iter/ob_das_iter.h +++ b/src/sql/das/iter/ob_das_iter.h @@ -13,9 +13,9 @@ #ifndef OBDEV_SRC_SQL_DAS_ITER_OB_DAS_ITER_H_ #define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_ITER_H_ #include "sql/engine/expr/ob_expr.h" -#include "sql/engine/ob_exec_context.h" #include "lib/container/ob_fixed_array.h" -#include "sql/das/ob_das_context.h" +#include "common/row/ob_row_iterator.h" +#include "sql/das/iter/ob_das_iter_define.h" namespace oceanbase { @@ -23,36 +23,18 @@ using namespace common; namespace sql { -class ObDASIter; - -enum ObDASIterType : uint32_t -{ - DAS_ITER_INVALID = 0, - DAS_ITER_SCAN, - DAS_ITER_MERGE, - DAS_ITER_GROUP_FOLD, - DAS_ITER_LOOKUP, - // append DASIterType before me - DAS_ITER_MAX -}; - -enum MergeType : uint32_t { - SEQUENTIAL_MERGE = 0, - SORT_MERGE -}; - +class ObEvalCtx; +class ObExecContext; struct ObDASIterParam { public: - ObDASIterParam() - : type_(ObDASIterType::DAS_ITER_INVALID), + ObDASIterParam(ObDASIterType type=ObDASIterType::DAS_ITER_INVALID) + : type_(type), max_size_(0), eval_ctx_(nullptr), exec_ctx_(nullptr), output_(nullptr), - group_id_expr_(nullptr), - child_(nullptr), - right_(nullptr) + group_id_expr_(nullptr) {} virtual ~ObDASIterParam() {} @@ -65,8 +47,6 @@ public: exec_ctx_ = param.exec_ctx_; output_ = param.output_; group_id_expr_ = param.group_id_expr_; - child_ = param.child_; - right_ = param.right_; } virtual bool is_valid() const @@ -80,33 +60,34 @@ public: ObExecContext *exec_ctx_; const ObIArray *output_; const ObExpr *group_id_expr_; - ObDASIter *child_; - ObDASIter *right_; - TO_STRING_KV(K_(type), K_(max_size), K_(eval_ctx), K_(exec_ctx), KPC_(output), K_(group_id_expr), - K_(child), K_(right)); + TO_STRING_KV(K_(type), K_(max_size), K_(eval_ctx), K_(exec_ctx), KPC_(output), K_(group_id_expr)); }; -class ObDASIter +class ObDASIter : public common::ObNewRowIterator { public: - ObDASIter() - : type_(ObDASIterType::DAS_ITER_INVALID), - max_size_(0), + ObDASIter(const ObDASIterType type = ObDASIterType::DAS_ITER_INVALID) + : type_(type), + max_size_(1), eval_ctx_(nullptr), exec_ctx_(nullptr), output_(nullptr), group_id_expr_(nullptr), - child_(nullptr), - right_(nullptr), + children_(nullptr), + children_cnt_(0), inited_(false) {} virtual ~ObDASIter() { release(); } VIRTUAL_TO_STRING_KV(K_(type), K_(max_size), K_(eval_ctx), K_(exec_ctx), K_(output), - K_(group_id_expr), K_(child), K_(right), K_(inited)); + K_(group_id_expr), K_(children_cnt), K_(inited)); void set_type(ObDASIterType type) { type_ = type; } ObDASIterType get_type() const { return type_; } + ObDASIter **&get_children() { return children_; } + void set_children_cnt(uint32_t children_cnt) { children_cnt_ = children_cnt; } + int64_t get_children_cnt() const { return children_cnt_; } + const ObIArray *get_output() { return output_; } // The state of ObDASMergeIter may change many times during execution, e.g., the merge_type // changing from SEQUENTIAL_MERGE to SORT_MERGE, or the creation of a new batch of DAS tasks. @@ -123,7 +104,17 @@ public: // get_next_row(s) should be called after init(). int get_next_row(); int get_next_rows(int64_t &count, int64_t capacity); + virtual void clear_evaluated_flag() {} + // required by iters related to DAS SCAN OP + virtual int do_table_scan() { return OB_NOT_IMPLEMENT; } + virtual int rescan() { return OB_NOT_IMPLEMENT; } + // required by iters related to DAS SCAN OP + + // for compatibility with ObNewRowIterator + virtual int get_next_row(ObNewRow *&row) override { return OB_NOT_IMPLEMENT; } + virtual void reset() override {} + // for compatibility with ObNewRowIterator protected: virtual int inner_init(ObDASIterParam ¶m) = 0; @@ -138,8 +129,8 @@ protected: ObExecContext *exec_ctx_; const ObIArray *output_; const ObExpr *group_id_expr_; - ObDASIter *child_; - ObDASIter *right_; + ObDASIter **children_; + uint32_t children_cnt_; private: bool inited_; diff --git a/src/sql/das/iter/ob_das_iter_define.h b/src/sql/das/iter/ob_das_iter_define.h new file mode 100644 index 000000000..4e9b851a0 --- /dev/null +++ b/src/sql/das/iter/ob_das_iter_define.h @@ -0,0 +1,89 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OBDEV_SRC_SQL_DAS_ITER_OB_DAS_ITER_DEFINE_H +#define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_ITER_DEFINE_H + +#include "common/ob_tablet_id.h" + +namespace oceanbase +{ +namespace sql +{ + +enum ObDASIterType : uint32_t +{ + DAS_ITER_INVALID = 0, + DAS_ITER_SCAN, + DAS_ITER_MERGE, + DAS_ITER_GROUP_FOLD, + DAS_ITER_LOCAL_LOOKUP, + DAS_ITER_GLOBAL_LOOKUP, + DAS_ITER_TEXT_RETRIEVAL, + DAS_ITER_SORT, + DAS_ITER_TEXT_RETRIEVAL_MERGE, + // append DASIterType before me + DAS_ITER_MAX +}; + +#define IS_LOOKUP_ITER(_iter_type) \ +({ \ + DAS_ITER_LOCAL_LOOKUP == (_iter_type) || \ + DAS_ITER_GLOBAL_LOOKUP == (_iter_type); \ +}) + +enum MergeType : uint32_t { + SEQUENTIAL_MERGE = 0, + SORT_MERGE +}; + +// group fold iter is used on demand and not considered as a part of iter tree, +// it is placed above iter tree when in use. +enum ObDASIterTreeType : uint32_t +{ + ITER_TREE_INVALID = 0, + ITER_TREE_TABLE_SCAN, + ITER_TREE_GLOBAL_LOOKUP, + ITER_TREE_PARTITION_SCAN, + ITER_TREE_LOCAL_LOOKUP, + ITER_TREE_GIS_LOOKUP, + ITER_TREE_DOMAIN_LOOKUP, + ITER_TREE_TEXT_RETRIEVAL, + // append iter tree type before me + ITER_TREE_MAX +}; + +struct ObDASRelatedTabletID +{ +public: + common::ObTabletID lookup_tablet_id_; + common::ObTabletID aux_lookup_tablet_id_; + + /* used by fulltext index */ + common::ObTabletID inv_idx_tablet_id_; + common::ObTabletID fwd_idx_tablet_id_; + common::ObTabletID doc_id_idx_tablet_id_; + /* used by fulltext index */ + void reset() + { + lookup_tablet_id_.reset(); + aux_lookup_tablet_id_.reset(); + inv_idx_tablet_id_.reset(); + fwd_idx_tablet_id_.reset(); + doc_id_idx_tablet_id_.reset(); + } +}; + +} // namespace sql +} // namespace oceanbase + +#endif /* OBDEV_SRC_SQL_DAS_ITER_OB_DAS_ITER_DEFINE_H */ diff --git a/src/sql/das/iter/ob_das_iter_utils.cpp b/src/sql/das/iter/ob_das_iter_utils.cpp index d97191819..979d033e6 100644 --- a/src/sql/das/iter/ob_das_iter_utils.cpp +++ b/src/sql/das/iter/ob_das_iter_utils.cpp @@ -17,142 +17,81 @@ namespace oceanbase { namespace sql { - -/***************** public begin *****************/ -int ObDASIterUtils::create_table_scan_iter_tree(const ObTableScanCtDef &tsc_ctdef, - ObTableScanRtDef &tsc_rtdef, - ObEvalCtx &eval_ctx, - ObExecContext &exec_ctx, - ObFixedArray &eval_infos, - const ObTableScanSpec &spec, - ObDASMergeIter *&scan_iter, - ObDASIter *&iter_tree) +/***************** PUBLIC BEGIN *****************/ +int ObDASIterUtils::create_das_scan_iter_tree(ObDASIterTreeType tree_type, + storage::ObTableScanParam &scan_param, + const ObDASScanCtDef *scan_ctdef, + ObDASScanRtDef *scan_rtdef, + const ObDASScanCtDef *lookup_ctdef, + ObDASScanRtDef *lookup_rtdef, + const ObDASBaseCtDef *attach_ctdef, + ObDASBaseRtDef *attach_rtdef, + const ObDASRelatedTabletID &related_tablet_ids, + transaction::ObTxDesc *trans_desc, + transaction::ObTxReadSnapshot *snapshot, + common::ObIAllocator &alloc, + ObDASIter *&iter_tree) { int ret = OB_SUCCESS; - ObDASMergeIter *iter = nullptr; - const ObDASScanCtDef *scan_ctdef = &tsc_ctdef.scan_ctdef_; - ObDASScanRtDef *scan_rtdef = &tsc_rtdef.scan_rtdef_; - common::ObIAllocator &alloc = exec_ctx.get_allocator(); - ObDASIterParam param; - param.max_size_ = eval_ctx.is_vectorized() ? eval_ctx.max_batch_size_ : 1; - param.eval_ctx_ = &eval_ctx; - param.exec_ctx_ = &exec_ctx; - param.output_ = &tsc_ctdef.get_das_output_exprs(); - param.group_id_expr_ = scan_ctdef->group_id_expr_; - param.child_ = nullptr; - param.right_ = nullptr; - if (OB_FAIL(create_das_merge_iter_help(param, - alloc, - false, - eval_infos, - spec, - iter))) { - LOG_WARN("failed to create das merge iter", K(ret)); - } else { - scan_iter = iter; - iter_tree = iter; + switch (tree_type) { + case ITER_TREE_PARTITION_SCAN: { + ret = create_partition_scan_tree(scan_param, alloc, scan_ctdef, scan_rtdef, iter_tree); + break; + } + case ITER_TREE_LOCAL_LOOKUP: { + ret = create_local_lookup_tree(scan_param, alloc, scan_ctdef, scan_rtdef, lookup_ctdef, lookup_rtdef, related_tablet_ids, trans_desc, snapshot, iter_tree); + break; + } + case ITER_TREE_DOMAIN_LOOKUP: { + // ret = create_domain_lookup_tree(scan_param, alloc, attach_ctdef, attach_rtdef, iter_tree); + break; + } + case ITER_TREE_TEXT_RETRIEVAL: { + ret = create_text_retrieval_tree(scan_param, alloc, attach_ctdef, attach_rtdef, related_tablet_ids, trans_desc, snapshot, iter_tree); + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + } + } + if (OB_FAIL(ret)) { + LOG_WARN("failed to create das scan iter tree", K(ret)); } + LOG_DEBUG("create das scan iter tree", K(tree_type), K(ret)); return ret; } -int ObDASIterUtils::create_local_lookup_iter_tree(const ObTableScanCtDef &tsc_ctdef, - ObTableScanRtDef &tsc_rtdef, - ObEvalCtx &eval_ctx, - ObExecContext &exec_ctx, - ObFixedArray &eval_infos, - const ObTableScanSpec &spec, - ObDASMergeIter *&scan_iter, - ObDASIter *&iter_tree) +int ObDASIterUtils::create_tsc_iter_tree(ObDASIterTreeType tree_type, + const ObTableScanCtDef &tsc_ctdef, + ObTableScanRtDef &tsc_rtdef, + ObEvalCtx &eval_ctx, + ObExecContext &exec_ctx, + ObFixedArray &eval_infos, + const ObTableScanSpec &spec, + bool can_retry, + ObDASMergeIter *&scan_iter, + ObDASIter *&iter_tree) { int ret = OB_SUCCESS; - // Currently, the iter tree of local index lookup is the same as that of table scan, - // this is because local index lookup is executed within a single DAS task. - // TODO bingfan: unify local index lookup and global index lookup. - if (OB_FAIL(create_table_scan_iter_tree(tsc_ctdef, - tsc_rtdef, - eval_ctx, - exec_ctx, - eval_infos, - spec, - scan_iter, - iter_tree))) { - LOG_WARN("failed to create local index lookup iter tree", K(ret)); - } - - return ret; -} - -int ObDASIterUtils::create_global_lookup_iter_tree(const ObTableScanCtDef &tsc_ctdef, - ObTableScanRtDef &tsc_rtdef, - ObEvalCtx &eval_ctx, - ObExecContext &exec_ctx, - ObFixedArray &eval_infos, - const ObTableScanSpec &spec, - bool can_retry, - ObDASMergeIter *&scan_iter, - ObDASIter *&iter_tree) -{ - int ret = OB_SUCCESS; - ObDASMergeIter *index_table_iter = nullptr; - ObDASMergeIter *data_table_iter = nullptr; - ObDASGlobalLookupIter *lookup_iter = nullptr; - /********* create index table iter *********/ - const ObDASScanCtDef *scan_ctdef = &tsc_ctdef.scan_ctdef_; - ObDASScanRtDef *scan_rtdef = &tsc_rtdef.scan_rtdef_; - common::ObIAllocator &alloc = exec_ctx.get_allocator(); - ObDASIterParam param; - param.max_size_ = eval_ctx.is_vectorized() ? eval_ctx.max_batch_size_ : 1; - param.eval_ctx_ = &eval_ctx; - param.exec_ctx_ = &exec_ctx; - param.output_ = &scan_ctdef->result_output_; - param.group_id_expr_ = scan_ctdef->group_id_expr_; - param.child_ = nullptr; - param.right_ = nullptr; - if (OB_FAIL(create_das_merge_iter_help(param, - alloc, - false, - eval_infos, - spec, - index_table_iter))) { - LOG_WARN("failed to create index table iter", K(ret)); - } - /********* create data table iter *********/ - if (OB_SUCC(ret)) { - param.output_ = &tsc_ctdef.lookup_ctdef_->result_output_; - param.child_ = nullptr; - param.right_ = index_table_iter; - if (OB_FAIL(create_das_merge_iter_help(param, - alloc, - true, - eval_infos, - spec, - data_table_iter))) { - LOG_WARN("failed to create data table iter", K(ret)); + switch (tree_type) { + case ITER_TREE_TABLE_SCAN: { + ret = create_table_scan_iter_tree(tsc_ctdef, eval_ctx, exec_ctx, eval_infos, spec, scan_iter, iter_tree); + break; + } + case ITER_TREE_GLOBAL_LOOKUP: { + ret = create_global_lookup_iter_tree(tsc_ctdef, tsc_rtdef, eval_ctx, exec_ctx, eval_infos, spec, can_retry, scan_iter, iter_tree); + break; + } + default: { + ret = OB_ERR_UNEXPECTED; } } - /********* create global lookup iter *********/ - if (OB_SUCC(ret)) { - index_table_iter->set_global_lookup_iter(data_table_iter); - param.child_ = data_table_iter; - param.right_ = nullptr; - if (OB_FAIL(create_das_global_lookup_iter_help(param, - alloc, - 10000, // hard code 10000 - index_table_iter, - data_table_iter, - can_retry, - tsc_ctdef, - tsc_rtdef, - lookup_iter))) { - LOG_WARN("failed to create das global lookup iter", K(ret)); - } - } - if (OB_SUCC(ret)) { - scan_iter = index_table_iter; - iter_tree = lookup_iter; + if (OB_FAIL(ret)) { + LOG_WARN("failed to create table scan iter tree", K(ret)); } + LOG_DEBUG("create table scan iter tree", K(tree_type), K(ret)); return ret; } @@ -169,190 +108,701 @@ int ObDASIterUtils::create_group_fold_iter(const ObTableScanCtDef &tsc_ctdef, ObDASGroupFoldIter *iter = nullptr; const ObDASScanCtDef *scan_ctdef = &tsc_ctdef.scan_ctdef_; ObDASScanRtDef *scan_rtdef = &tsc_rtdef.scan_rtdef_; - common::ObIAllocator &alloc = exec_ctx.get_allocator(); - ObDASIterParam param; + ObDASGroupFoldIterParam param; param.max_size_ = eval_ctx.is_vectorized() ? eval_ctx.max_batch_size_ : 1; param.eval_ctx_ = &eval_ctx; param.exec_ctx_ = &exec_ctx; param.output_ = &tsc_ctdef.get_das_output_exprs(); param.group_id_expr_ = tsc_ctdef.scan_ctdef_.group_id_expr_; - param.child_ = iter_tree; - param.right_ = nullptr; - if (OB_SUCC(ret)) { - if (OB_FAIL(create_das_group_fold_iter_help(param, - alloc, - scan_rtdef->need_check_output_datum_, - iter_tree, - iter))) { - LOG_WARN("failed to create das group fold iter", K(ret)); - } + param.need_check_output_datum_ = scan_rtdef->need_check_output_datum_; + param.iter_tree_ = iter_tree; + if (OB_FAIL(create_das_iter(exec_ctx.get_allocator(), param, iter))) { + LOG_WARN("failed to create das group fold iter", K(ret)); + } else if (OB_FAIL(create_iter_children_array(1, exec_ctx.get_allocator(), iter))) { + LOG_WARN("failed to create iter children array", K(ret)); + } else { + iter->get_children()[0] = iter_tree; } + if (OB_SUCC(ret)) { fold_iter = iter; } - return ret; } -/***************** private begin *****************/ -int ObDASIterUtils::create_das_merge_iter_help(ObDASIterParam ¶m, +int ObDASIterUtils::set_text_retrieval_related_ids(const ObDASBaseCtDef *attach_ctdef, + const ObDASRelatedTabletID &related_tablet_ids, + const ObLSID &ls_id, + ObDASIter *root_iter) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(attach_ctdef) || OB_ISNULL(root_iter)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", K(ret), KP(attach_ctdef), KP(root_iter)); + } else { + bool need_set_child = false; + const ObDASIterType &iter_type = root_iter->get_type(); + switch (attach_ctdef->op_type_) { + case ObDASOpType::DAS_OP_TABLE_LOOKUP: { + if (OB_UNLIKELY(iter_type != ObDASIterType::DAS_ITER_LOCAL_LOOKUP)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type)); + } else { + ObDASLocalLookupIter *lookup_iter = static_cast(root_iter); + lookup_iter->set_ls_id(ls_id); + lookup_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_); + need_set_child = true; + } + break; + } + case ObDASOpType::DAS_OP_IR_AUX_LOOKUP: { + if (OB_UNLIKELY(iter_type != ObDASIterType::DAS_ITER_LOCAL_LOOKUP)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type)); + } else { + ObDASLocalLookupIter *aux_lookup_iter = static_cast(root_iter); + aux_lookup_iter->set_ls_id(ls_id); + aux_lookup_iter->set_tablet_id(related_tablet_ids.aux_lookup_tablet_id_); + need_set_child = true; + } + break; + } + case ObDASOpType::DAS_OP_SORT: { + if (OB_UNLIKELY(iter_type != ObDASIterType::DAS_ITER_SORT)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type)); + } else { + need_set_child = true; + } + break; + } + case ObDASOpType::DAS_OP_IR_SCAN: { + if (OB_UNLIKELY(iter_type != ObDASIterType::DAS_ITER_TEXT_RETRIEVAL_MERGE)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type)); + } else { + ObDASTextRetrievalMergeIter *tr_merge_iter = static_cast(root_iter); + need_set_child = false; + if (OB_FAIL(tr_merge_iter->set_related_tablet_ids(ls_id, related_tablet_ids))) { + LOG_WARN("failed to set related tablet ids", K(ret)); + } + } + break; + } + default: { + need_set_child = false; + break; + } + } + + if (OB_FAIL(ret) || !need_set_child) { + } else if (OB_UNLIKELY(attach_ctdef->children_cnt_ != root_iter->get_children_cnt())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected iter children count not equal to ctdef children count", + K(ret), K(attach_ctdef->children_cnt_), K(root_iter->get_children_cnt())); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < attach_ctdef->children_cnt_; ++i) { + if (OB_FAIL(set_text_retrieval_related_ids( + attach_ctdef->children_[i], + related_tablet_ids, + ls_id, + root_iter->get_children()[i]))) { + LOG_WARN("failed to set text retrieval related ids", K(ret)); + } + } + } + } + return ret; +} +/***************** PUBLIC END *****************/ + +int ObDASIterUtils::create_partition_scan_tree(storage::ObTableScanParam &scan_param, common::ObIAllocator &alloc, - bool is_global_lookup, - ObFixedArray &eval_infos, - const ObTableScanSpec &spec, - ObDASMergeIter *&result) + const ObDASScanCtDef *scan_ctdef, + ObDASScanRtDef *scan_rtdef, + ObDASIter *&iter_tree) { int ret = OB_SUCCESS; - void *iter_buf = nullptr; - ObDASMergeIter *iter = nullptr; - if (OB_UNLIKELY(!param.is_valid())) { + ObDASScanIterParam param; + param.scan_ctdef_ = scan_ctdef; + ObDASScanIter *scan_iter = nullptr; + if (OB_FAIL(create_das_iter(alloc, param, scan_iter))) { + LOG_WARN("failed to create das scan iter", K(ret)); + } else { + scan_iter->set_scan_param(scan_param); + iter_tree = scan_iter; + } + return ret; +} + +int ObDASIterUtils::create_local_lookup_tree(ObTableScanParam &scan_param, + common::ObIAllocator &alloc, + const ObDASScanCtDef *scan_ctdef, + ObDASScanRtDef *scan_rtdef, + const ObDASScanCtDef *lookup_ctdef, + ObDASScanRtDef *lookup_rtdef, + const ObDASRelatedTabletID &related_tablet_ids, + transaction::ObTxDesc *trans_desc, + transaction::ObTxReadSnapshot *snapshot, + ObDASIter *&iter_tree) +{ + int ret = OB_SUCCESS; + ObDASScanIter *index_table_iter = nullptr; + ObDASScanIter *data_table_iter = nullptr; + ObDASLocalLookupIter *lookup_iter = nullptr; + ObDASScanIterParam index_table_param; + index_table_param.scan_ctdef_ = scan_ctdef; + ObDASScanIterParam data_table_param; + data_table_param.scan_ctdef_ = lookup_ctdef; + if (OB_FAIL(create_das_iter(alloc, index_table_param, index_table_iter))) { + LOG_WARN("failed to create index table iter", K(ret)); + } else if (OB_FAIL(create_das_iter(alloc, data_table_param, data_table_iter))) { + LOG_WARN("failed to create data table iter", K(ret)); + } else { + ObDASLocalLookupIterParam lookup_param; + lookup_param.max_size_ = lookup_rtdef->eval_ctx_->is_vectorized() ? lookup_rtdef->eval_ctx_->max_batch_size_ : 1; + lookup_param.eval_ctx_ = lookup_rtdef->eval_ctx_; + lookup_param.exec_ctx_ = &lookup_rtdef->eval_ctx_->exec_ctx_; + lookup_param.output_ = &lookup_ctdef->result_output_; + lookup_param.default_batch_row_count_ = 1000; // hard code 1000 for local lookup + lookup_param.index_ctdef_ = scan_ctdef; + lookup_param.index_rtdef_ = scan_rtdef; + lookup_param.lookup_ctdef_ = lookup_ctdef; + lookup_param.lookup_rtdef_ = lookup_rtdef; + lookup_param.index_table_iter_ = index_table_iter; + lookup_param.data_table_iter_ = data_table_iter; + lookup_param.trans_desc_ = trans_desc; + lookup_param.snapshot_ = snapshot; + lookup_param.rowkey_exprs_ = &lookup_ctdef->rowkey_exprs_; + if (OB_FAIL(create_das_iter(alloc, lookup_param, lookup_iter))) { + LOG_WARN("failed to create local lookup iter", K(ret)); + } else if (OB_FAIL(create_iter_children_array(2, alloc, lookup_iter))) { + LOG_WARN("failed to create iter children array", K(ret)); + } else { + lookup_iter->get_children()[0] = index_table_iter; + lookup_iter->get_children()[1] = data_table_iter; + index_table_iter->set_scan_param(scan_param); + data_table_iter->set_scan_param(lookup_iter->get_lookup_param()); + lookup_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_); + lookup_iter->set_ls_id(scan_param.ls_id_); + iter_tree = lookup_iter; + } + } + return ret; +} + +int ObDASIterUtils::create_text_retrieval_tree(ObTableScanParam &scan_param, + common::ObIAllocator &alloc, + const ObDASBaseCtDef *attach_ctdef, + ObDASBaseRtDef *attach_rtdef, + const ObDASRelatedTabletID &related_tablet_ids, + transaction::ObTxDesc *trans_desc, + transaction::ObTxReadSnapshot *snapshot, + ObDASIter *&iter_tree) +{ + int ret = OB_SUCCESS; + const ObDASIRScanCtDef *ir_scan_ctdef = nullptr; + ObDASIRScanRtDef *ir_scan_rtdef = nullptr; + const ObDASTableLookupCtDef *table_lookup_ctdef = nullptr; + ObDASTableLookupRtDef *table_lookup_rtdef = nullptr; + const ObDASSortCtDef *sort_ctdef = nullptr; + ObDASSortRtDef *sort_rtdef = nullptr; + ObDASIter *text_retrieval_result = nullptr; + ObDASIter *sort_result = nullptr; + ObDASIter *root_iter = nullptr; + const bool has_lookup = ObDASOpType::DAS_OP_TABLE_LOOKUP == attach_ctdef->op_type_; + if (OB_UNLIKELY(attach_ctdef->op_type_ != ObDASOpType::DAS_OP_IR_SCAN + && attach_ctdef->op_type_ != ObDASOpType::DAS_OP_TABLE_LOOKUP + && attach_ctdef->op_type_ != ObDASOpType::DAS_OP_SORT)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid das iter param", K(param), K(ret)); - } else if (OB_ISNULL(iter_buf = alloc.alloc(sizeof(ObDASMergeIter)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate memory", K(ret)); + LOG_WARN("unexpected text retrieval root attach def type", K(ret), KPC(attach_ctdef)); + } else if (OB_FAIL(ObDASUtils::find_target_das_def( + attach_ctdef, + attach_rtdef, + ObDASOpType::DAS_OP_IR_SCAN, + ir_scan_ctdef, + ir_scan_rtdef))) { + LOG_WARN("fail to find ir scan definition", K(ret)); + } else if (OB_FAIL(create_text_retrieval_sub_tree( + scan_param.ls_id_, + alloc, + ir_scan_ctdef, + ir_scan_rtdef, + related_tablet_ids, + trans_desc, + snapshot, + text_retrieval_result))) { + LOG_WARN("failed to create text retrieval sub tree", K(ret)); } else { - iter = new (iter_buf) ObDASMergeIter(); - } - - if (OB_SUCC(ret) && OB_NOT_NULL(iter)) { - ObDASMergeIterParam merge_param; - merge_param.assgin(param); - merge_param.type_ = DAS_ITER_MERGE; - merge_param.eval_infos_ = &eval_infos; - merge_param.need_update_partition_id_ = !is_global_lookup; - merge_param.pdml_partition_id_ = spec.pdml_partition_id_; - merge_param.partition_id_calc_type_ = spec.partition_id_calc_type_; - merge_param.should_scan_index_ = spec.should_scan_index(); - merge_param.ref_table_id_ = spec.ref_table_id_; - merge_param.is_vectorized_ = spec.is_vectorized(); - merge_param.frame_info_ = &spec.plan_->get_expr_frame_info(); - merge_param.execute_das_directly_ = !is_global_lookup && !spec.use_dist_das_; - merge_param.enable_rich_format_ = !is_global_lookup && spec.use_rich_format_; - - if (OB_FAIL(iter->init(merge_param))) { - LOG_WARN("failed to init das merge iter", K(ret)); + root_iter = text_retrieval_result; + if (has_lookup) { + table_lookup_ctdef = static_cast(attach_ctdef); + table_lookup_rtdef = static_cast(attach_rtdef); + if (table_lookup_ctdef->get_rowkey_scan_ctdef()->op_type_ == ObDASOpType::DAS_OP_IR_AUX_LOOKUP) { + const ObDASIRAuxLookupCtDef *aux_lookup_ctdef = static_cast( + table_lookup_ctdef->get_rowkey_scan_ctdef()); + ObDASIRAuxLookupRtDef *aux_lookup_rtdef = static_cast( + table_lookup_rtdef->get_rowkey_scan_rtdef()); + if (aux_lookup_ctdef->get_doc_id_scan_ctdef()->op_type_ == ObDASOpType::DAS_OP_SORT) { + sort_ctdef = static_cast(aux_lookup_ctdef->get_doc_id_scan_ctdef()); + sort_rtdef = static_cast(aux_lookup_rtdef->get_doc_id_scan_rtdef()); + } + } + } else { + if (attach_ctdef->op_type_ == ObDASOpType::DAS_OP_SORT) { + sort_ctdef = static_cast(attach_ctdef); + sort_rtdef = static_cast(attach_rtdef); + } + } + const bool has_sort = nullptr != sort_ctdef; + if (!has_sort) { + // skip + } else if (OB_FAIL(create_sort_sub_tree(alloc, sort_ctdef, sort_rtdef, text_retrieval_result, sort_result))) { + LOG_WARN("failed to create sort sub tree", K(ret)); + } else { + root_iter = sort_result; } } + + if (OB_SUCC(ret) && has_lookup) { + ObDASIter *domain_lookup_result = nullptr; + if (OB_FAIL(create_domain_lookup_sub_tree( + scan_param.ls_id_, + alloc, + table_lookup_ctdef, + table_lookup_rtdef, + related_tablet_ids, + trans_desc, + snapshot, + root_iter, + domain_lookup_result))) { + LOG_WARN("failed to create domain index lookup iters", K(ret)); + } else { + root_iter = domain_lookup_result; + } + } + if (OB_SUCC(ret)) { - result = iter; + iter_tree = root_iter; + } + return ret; +} + +int ObDASIterUtils::create_text_retrieval_sub_tree(const ObLSID &ls_id, + common::ObIAllocator &alloc, + const ObDASIRScanCtDef *ir_scan_ctdef, + ObDASIRScanRtDef *ir_scan_rtdef, + const ObDASRelatedTabletID &related_tablet_ids, + transaction::ObTxDesc *trans_desc, + transaction::ObTxReadSnapshot *snapshot, + ObDASIter *&retrieval_result) + { + int ret = OB_SUCCESS; + ObDASTextRetrievalMergeIterParam merge_iter_param; + ObDASTextRetrievalMergeIter *tr_merge_iter = nullptr; + ObDASScanIterParam doc_cnt_agg_param; + ObDASScanIter *doc_cnt_agg_iter = nullptr; + + merge_iter_param.max_size_ = ir_scan_rtdef->eval_ctx_->max_batch_size_; + merge_iter_param.eval_ctx_ = ir_scan_rtdef->eval_ctx_; + merge_iter_param.exec_ctx_ = &ir_scan_rtdef->eval_ctx_->exec_ctx_; + merge_iter_param.output_ = &ir_scan_ctdef->result_output_; + merge_iter_param.ir_ctdef_ = ir_scan_ctdef; + merge_iter_param.ir_rtdef_ = ir_scan_rtdef; + merge_iter_param.tx_desc_ = trans_desc; + merge_iter_param.snapshot_ = snapshot; + if (ir_scan_ctdef->need_do_total_doc_cnt()) { + doc_cnt_agg_param.scan_ctdef_ = ir_scan_ctdef->get_doc_id_idx_agg_ctdef(); + if (OB_FAIL(create_das_iter(alloc, doc_cnt_agg_param, doc_cnt_agg_iter))) { + LOG_WARN("failed to create doc cnt agg scan iter", K(ret)); + } else { + merge_iter_param.doc_cnt_iter_ = doc_cnt_agg_iter; + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(create_das_iter(alloc, merge_iter_param, tr_merge_iter))) { + LOG_WARN("failed to create text retrieval merge iter", K(ret)); } else { - if (OB_NOT_NULL(iter)) { - iter->release(); - iter = nullptr; + ObSEArray iters; + const ObIArray &query_tokens = tr_merge_iter->get_query_tokens(); + for (int64_t i = 0; OB_SUCC(ret) && i < query_tokens.count(); ++i) { + ObDASTextRetrievalIterParam retrieval_param; + ObDASTextRetrievalIter *retrieval_iter = nullptr; + retrieval_param.max_size_ = ir_scan_rtdef->eval_ctx_->max_batch_size_; + retrieval_param.eval_ctx_ = ir_scan_rtdef->eval_ctx_; + retrieval_param.exec_ctx_ = &ir_scan_rtdef->eval_ctx_->exec_ctx_; + retrieval_param.output_ = &ir_scan_ctdef->result_output_; + retrieval_param.ir_ctdef_ = ir_scan_ctdef; + retrieval_param.ir_rtdef_ = ir_scan_rtdef; + retrieval_param.tx_desc_ = trans_desc; + retrieval_param.snapshot_ = snapshot; + + ObDASScanIterParam inv_idx_scan_iter_param; + ObDASScanIter *inv_idx_scan_iter = nullptr; + inv_idx_scan_iter_param.scan_ctdef_ = ir_scan_ctdef->get_inv_idx_scan_ctdef(); + ObDASScanIterParam inv_idx_agg_iter_param; + ObDASScanIter *inv_idx_agg_iter = nullptr; + inv_idx_agg_iter_param.scan_ctdef_ = ir_scan_ctdef->get_inv_idx_agg_ctdef(); + ObDASScanIterParam fwd_idx_iter_param; + ObDASScanIter *fwd_idx_iter = nullptr; + fwd_idx_iter_param.scan_ctdef_ = ir_scan_ctdef->get_fwd_idx_agg_ctdef(); + if (OB_FAIL(create_das_iter(alloc, inv_idx_scan_iter_param, inv_idx_scan_iter))) { + LOG_WARN("failed to create inv idx iter", K(ret)); + } else if (ir_scan_ctdef->need_inv_idx_agg() + && OB_FAIL(create_das_iter(alloc, inv_idx_agg_iter_param, inv_idx_agg_iter))) { + LOG_WARN("failed to create inv idx agg iter", K(ret)); + } else if (ir_scan_ctdef->need_fwd_idx_agg() + && OB_FAIL(create_das_iter(alloc, fwd_idx_iter_param, fwd_idx_iter))) { + LOG_WARN("failed to create fwd idx iter", K(ret)); + } else { + retrieval_param.inv_idx_scan_iter_ = inv_idx_scan_iter; + retrieval_param.inv_idx_agg_iter_ = inv_idx_agg_iter; + retrieval_param.fwd_idx_iter_ = fwd_idx_iter; + const int64_t inv_idx_iter_cnt = ir_scan_ctdef->need_inv_idx_agg() ? 2 : 1; + const int64_t fwd_idx_iter_cnt = ir_scan_ctdef->need_fwd_idx_agg() ? 1 : 0; + const int64_t tr_children_cnt = inv_idx_iter_cnt + fwd_idx_iter_cnt; + if (OB_FAIL(create_das_iter(alloc, retrieval_param, retrieval_iter))) { + LOG_WARN("failed to create text retrieval iter", K(ret)); + } else if (OB_FAIL(retrieval_iter->set_query_token(query_tokens.at(i)))) { + LOG_WARN("failed to set query token for text retrieval iter", K(ret)); + } else if (OB_FAIL(create_iter_children_array(tr_children_cnt, alloc, retrieval_iter))) { + LOG_WARN("failed to create iter children array", K(ret)); + } else { + retrieval_iter->get_children()[0] = inv_idx_scan_iter; + if (ir_scan_ctdef->need_inv_idx_agg()) { + retrieval_iter->get_children()[1] = inv_idx_agg_iter; + } + if (ir_scan_ctdef->need_fwd_idx_agg()) { + retrieval_iter->get_children()[2] = fwd_idx_iter; + } + retrieval_iter->set_ls_tablet_ids( + ls_id, + related_tablet_ids.inv_idx_tablet_id_, + related_tablet_ids.fwd_idx_tablet_id_); + if (OB_FAIL(iters.push_back(retrieval_iter))) { + LOG_WARN("failed append retrieval iter to array", K(ret)); + } + } + } } - if (OB_NOT_NULL(iter_buf)) { - alloc.free(iter_buf); - iter_buf = nullptr; + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(tr_merge_iter->set_merge_iters(iters))) { + LOG_WARN("failed to set merge iters for text retrieval", K(ret)); + } else if (OB_FAIL(tr_merge_iter->set_related_tablet_ids(ls_id, related_tablet_ids))) { + LOG_WARN("failed to set related tabelt ids", K(ret)); + } else { + ObDASIter **&tr_merge_children = tr_merge_iter->get_children(); + const int64_t tr_merge_children_cnt = ir_scan_ctdef->need_do_total_doc_cnt() ? iters.count() + 1 : iters.count(); + if (0 != tr_merge_children_cnt + && OB_FAIL(create_iter_children_array(tr_merge_children_cnt, alloc, tr_merge_iter))) { + LOG_WARN("failed to alloc text retrieval merge iter children", K(ret), K(tr_merge_children_cnt)); + } else { + for (int64_t i = 0; i < iters.count(); ++i) { + tr_merge_children[i] = iters.at(i); + } + if (ir_scan_ctdef->need_do_total_doc_cnt()) { + tr_merge_children[iters.count()] = doc_cnt_agg_iter; + } + tr_merge_iter->set_doc_id_idx_tablet_id(related_tablet_ids.doc_id_idx_tablet_id_); + tr_merge_iter->set_ls_id(ls_id); + retrieval_result = tr_merge_iter; + } } - result = nullptr; } return ret; } -int ObDASIterUtils::create_das_global_lookup_iter_help(ObDASIterParam ¶m, - common::ObIAllocator &alloc, - int64_t default_batch_row_count, - ObDASMergeIter *index_table_iter, - ObDASMergeIter *data_table_iter, - bool can_retry, - const ObTableScanCtDef &tsc_ctdef, - ObTableScanRtDef &tsc_rtdef, - ObDASGlobalLookupIter *&result) +int ObDASIterUtils::create_domain_lookup_sub_tree(const ObLSID &ls_id, + common::ObIAllocator &alloc, + const ObDASTableLookupCtDef *table_lookup_ctdef, + ObDASTableLookupRtDef *table_lookup_rtdef, + const ObDASRelatedTabletID &related_tablet_ids, + transaction::ObTxDesc *trans_desc, + transaction::ObTxReadSnapshot *snapshot, + ObDASIter *doc_id_iter, + ObDASIter *&domain_lookup_result) { int ret = OB_SUCCESS; - void *iter_buf = nullptr; - ObDASGlobalLookupIter *iter = nullptr; - if (!param.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid das iter param", K(param), K(ret)); - } else if (OB_ISNULL(iter_buf = alloc.alloc(sizeof(ObDASGlobalLookupIter)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate memory", K(ret)); + const ObDASIRAuxLookupCtDef *aux_lookup_ctdef = nullptr; + ObDASIRAuxLookupRtDef *aux_lookup_rtdef = nullptr; + ObDASLocalLookupIter *doc_id_lookup_iter = nullptr; + ObDASLocalLookupIterParam doc_id_lookup_param; + ObDASLocalLookupIter *main_lookup_iter = nullptr; + ObDASLocalLookupIterParam main_lookup_param; + if (OB_UNLIKELY(table_lookup_ctdef->get_rowkey_scan_ctdef()->op_type_ != ObDASOpType::DAS_OP_IR_AUX_LOOKUP)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected rowkey scan is not an aux lookup", K(ret)); } else { - iter = new (iter_buf) ObDASGlobalLookupIter(); + aux_lookup_ctdef = static_cast(table_lookup_ctdef->get_rowkey_scan_ctdef()); + aux_lookup_rtdef = static_cast(table_lookup_rtdef->get_rowkey_scan_rtdef()); + ObDASScanIter *doc_id_table_iter = nullptr; + ObDASScanIterParam doc_id_table_param; + doc_id_table_param.scan_ctdef_ = aux_lookup_ctdef->get_lookup_scan_ctdef(); + + if (OB_FAIL(create_das_iter(alloc, doc_id_table_param, doc_id_table_iter))) { + LOG_WARN("failed to create doc id table iter", K(ret)); + } else { + // TODO: set to batch size after support formal vectorization + doc_id_lookup_param.max_size_ = 1; + // doc_id_lookup_param.max_size_ = aux_lookup_rtdef->eval_ctx_->is_vectorized() + // ? aux_lookup_rtdef->eval_ctx_->max_batch_size_ : 1; + doc_id_lookup_param.eval_ctx_ = aux_lookup_rtdef->eval_ctx_; + doc_id_lookup_param.exec_ctx_ = &aux_lookup_rtdef->eval_ctx_->exec_ctx_; + doc_id_lookup_param.output_ = &aux_lookup_ctdef->result_output_; + doc_id_lookup_param.default_batch_row_count_ = 1; + doc_id_lookup_param.index_ctdef_ = aux_lookup_ctdef->get_doc_id_scan_ctdef(); + doc_id_lookup_param.index_rtdef_ = aux_lookup_rtdef->get_doc_id_scan_rtdef(); + doc_id_lookup_param.lookup_ctdef_ = aux_lookup_ctdef->get_lookup_scan_ctdef(); + doc_id_lookup_param.lookup_rtdef_ = aux_lookup_rtdef->get_lookup_scan_rtdef(); + doc_id_lookup_param.index_table_iter_ = doc_id_iter; + doc_id_lookup_param.data_table_iter_ = doc_id_table_iter; + doc_id_lookup_param.trans_desc_ = trans_desc; + doc_id_lookup_param.snapshot_ = snapshot; + doc_id_lookup_param.rowkey_exprs_ = &aux_lookup_ctdef->get_lookup_scan_ctdef()->rowkey_exprs_; + if (OB_FAIL(create_das_iter(alloc, doc_id_lookup_param, doc_id_lookup_iter))) { + LOG_WARN("failed to create doc id lookup iter", K(ret)); + } else if (OB_FAIL(create_iter_children_array(2, alloc, doc_id_lookup_iter))) { + LOG_WARN("failed to create iter children array", K(ret)); + } else { + doc_id_lookup_iter->get_children()[0] = doc_id_iter; + doc_id_lookup_iter->get_children()[1] = doc_id_table_iter; + doc_id_table_iter->set_scan_param(doc_id_lookup_iter->get_lookup_param()); + doc_id_lookup_iter->set_tablet_id(related_tablet_ids.doc_id_idx_tablet_id_); + doc_id_lookup_iter->set_ls_id(ls_id); + } + } } - if (OB_SUCC(ret) && OB_NOT_NULL(iter)) { - ObDASLookupIterParam lookup_param; + if (OB_SUCC(ret)) { + ObDASScanIter *data_table_iter = nullptr; + ObDASScanIterParam data_table_param; + data_table_param.scan_ctdef_ = table_lookup_ctdef->get_lookup_scan_ctdef(); + if (OB_FAIL(create_das_iter(alloc, data_table_param, data_table_iter))) { + LOG_WARN("failed to create data table lookup scan iter", K(ret)); + } else { + // TODO: set to batch size after support formal vectorization + main_lookup_param.max_size_ = 1; + // main_lookup_param.max_size_ = table_lookup_rtdef->eval_ctx_->is_vectorized() + // ? aux_lookup_rtdef->eval_ctx_->max_batch_size_ : 1; + main_lookup_param.eval_ctx_ = table_lookup_rtdef->eval_ctx_; + main_lookup_param.exec_ctx_ = &table_lookup_rtdef->eval_ctx_->exec_ctx_; + main_lookup_param.output_ = &table_lookup_ctdef->result_output_; + main_lookup_param.default_batch_row_count_ = 1; + main_lookup_param.index_ctdef_ = table_lookup_ctdef->get_rowkey_scan_ctdef(); + main_lookup_param.index_rtdef_ = table_lookup_rtdef->get_rowkey_scan_rtdef(); + main_lookup_param.lookup_ctdef_ = table_lookup_ctdef->get_lookup_scan_ctdef(); + main_lookup_param.lookup_rtdef_ = table_lookup_rtdef->get_lookup_scan_rtdef(); + main_lookup_param.index_table_iter_ = doc_id_lookup_iter; + main_lookup_param.data_table_iter_ = data_table_iter; + main_lookup_param.trans_desc_ = trans_desc; + main_lookup_param.snapshot_ = snapshot; + main_lookup_param.rowkey_exprs_ = &table_lookup_ctdef->get_lookup_scan_ctdef()->rowkey_exprs_; + if (OB_FAIL(create_das_iter(alloc, main_lookup_param, main_lookup_iter))) { + LOG_WARN("failed to create das iter", K(ret)); + } else if (OB_FAIL(create_iter_children_array(2, alloc, main_lookup_iter))) { + LOG_WARN("failed to create iter children array", K(ret)); + } else { + main_lookup_iter->get_children()[0] = doc_id_lookup_iter; + main_lookup_iter->get_children()[1] = data_table_iter; + data_table_iter->set_scan_param(main_lookup_iter->get_lookup_param()); + main_lookup_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_); + main_lookup_iter->set_ls_id(ls_id); + } + } + } + + if (OB_SUCC(ret)) { + domain_lookup_result = main_lookup_iter; + } + + return ret; +} + +int ObDASIterUtils::create_sort_sub_tree(common::ObIAllocator &alloc, + const ObDASSortCtDef *sort_ctdef, + ObDASSortRtDef *sort_rtdef, + ObDASIter *sort_input, + ObDASIter *&sort_result) +{ + int ret = OB_SUCCESS; + ObDASSortIterParam sort_iter_param; + ObDASSortIter *sort_iter = nullptr; + ObEvalCtx *eval_ctx = sort_rtdef->eval_ctx_; + sort_iter_param.max_size_ = eval_ctx->is_vectorized() ? eval_ctx->max_batch_size_ : 1; + sort_iter_param.eval_ctx_ = eval_ctx; + sort_iter_param.exec_ctx_ = &eval_ctx->exec_ctx_; + sort_iter_param.output_ = &sort_ctdef->result_output_; + sort_iter_param.sort_ctdef_ = sort_ctdef; + sort_iter_param.child_ = sort_input; + if (OB_FAIL(create_das_iter(alloc, sort_iter_param, sort_iter))) { + LOG_WARN("failed to create sort iter", K(ret)); + } else if (OB_FAIL(create_iter_children_array(1, alloc, sort_iter))) { + LOG_WARN("failed to create iter children array", K(ret)); + } else { + sort_iter->get_children()[0] = sort_input; + sort_result = sort_iter; + } + return ret; +} + +int ObDASIterUtils::create_table_scan_iter_tree(const ObTableScanCtDef &tsc_ctdef, + ObEvalCtx &eval_ctx, + ObExecContext &exec_ctx, + ObFixedArray &eval_infos, + const ObTableScanSpec &spec, + ObDASMergeIter *&scan_iter, + ObDASIter *&iter_tree) +{ + int ret = OB_SUCCESS; + ObDASMergeIter *iter = nullptr; + ObDASMergeIterParam param; + const ObDASScanCtDef *scan_ctdef = &tsc_ctdef.scan_ctdef_; + param.max_size_ = eval_ctx.is_vectorized() ? eval_ctx.max_batch_size_ : 1; + param.eval_ctx_ = &eval_ctx; + param.exec_ctx_ = &exec_ctx; + param.output_ = &tsc_ctdef.get_das_output_exprs(); + param.group_id_expr_ = scan_ctdef->group_id_expr_; + param.eval_infos_ = &eval_infos; + param.need_update_partition_id_ = true; + param.pdml_partition_id_ = spec.pdml_partition_id_; + param.partition_id_calc_type_ = spec.partition_id_calc_type_; + param.should_scan_index_ = spec.should_scan_index(); + param.ref_table_id_ = spec.ref_table_id_; + param.is_vectorized_ = spec.is_vectorized(); + param.frame_info_ = &spec.plan_->get_expr_frame_info(); + param.execute_das_directly_ = !spec.use_dist_das_; + param.enable_rich_format_ = spec.use_rich_format_; + param.used_for_keep_order_ = false; + + if (OB_FAIL(create_das_iter(exec_ctx.get_allocator(), param, iter))) { + LOG_WARN("failed to create das merge iter", K(ret)); + } else { + scan_iter = iter; + iter_tree = iter; + } + + return ret; +} + + +int ObDASIterUtils::create_global_lookup_iter_tree(const ObTableScanCtDef &tsc_ctdef, + ObTableScanRtDef &tsc_rtdef, + ObEvalCtx &eval_ctx, + ObExecContext &exec_ctx, + ObFixedArray &eval_infos, + const ObTableScanSpec &spec, + bool can_retry, + ObDASMergeIter *&scan_iter, + ObDASIter *&iter_tree) +{ + int ret = OB_SUCCESS; + ObDASMergeIter *index_table_iter = nullptr; + ObDASMergeIter *data_table_iter = nullptr; + ObDASGlobalLookupIter *lookup_iter = nullptr; + + /********* create index table iter *********/ + const ObDASScanCtDef *scan_ctdef = &tsc_ctdef.scan_ctdef_; + const ObDASScanCtDef *lookup_ctdef = tsc_ctdef.lookup_ctdef_; + ObDASMergeIterParam param; + param.max_size_ = eval_ctx.is_vectorized() ? eval_ctx.max_batch_size_ : 1; + param.eval_ctx_ = &eval_ctx; + param.exec_ctx_ = &exec_ctx; + param.output_ = &scan_ctdef->result_output_; + param.group_id_expr_ = tsc_ctdef.scan_ctdef_.group_id_expr_; + param.eval_infos_ = &eval_infos; + param.need_update_partition_id_ = true; + param.pdml_partition_id_ = spec.pdml_partition_id_; + param.partition_id_calc_type_ = spec.partition_id_calc_type_; + param.should_scan_index_ = spec.should_scan_index(); + param.ref_table_id_ = scan_ctdef->ref_table_id_; + param.is_vectorized_ = spec.is_vectorized(); + param.frame_info_ = &spec.plan_->get_expr_frame_info(); + param.execute_das_directly_ = !spec.use_dist_das_; + param.enable_rich_format_ = spec.use_rich_format_; + param.used_for_keep_order_ = false; + + if (OB_FAIL(create_das_iter(exec_ctx.get_allocator(), param, index_table_iter))) { + LOG_WARN("failed to create global index table iter", K(ret)); + } + + /********* create data table iter *********/ + if (OB_SUCC(ret)) { + param.output_ = &lookup_ctdef->result_output_; + param.ref_table_id_ = lookup_ctdef->ref_table_id_; + param.need_update_partition_id_ = false; + param.execute_das_directly_ = false; + param.enable_rich_format_ = false; + param.used_for_keep_order_ = tsc_ctdef.is_das_keep_order_; + if (OB_FAIL(create_das_iter(exec_ctx.get_allocator(), param, data_table_iter))) { + LOG_WARN("failed to create global data table iter", K(ret)); + } else { + index_table_iter->set_global_lookup_iter(data_table_iter); + } + } + + /********* create global lookup iter *********/ + if (OB_SUCC(ret)) { + ObDASGlobalLookupIterParam lookup_param; lookup_param.assgin(param); - lookup_param.type_ = ObDASIterType::DAS_ITER_LOOKUP; - lookup_param.default_batch_row_count_ = default_batch_row_count; + lookup_param.type_ = DAS_ITER_GLOBAL_LOOKUP; + lookup_param.default_batch_row_count_ = 10000; // hard code 10000 for global lookup + lookup_param.index_ctdef_ = scan_ctdef; + lookup_param.index_rtdef_ = &tsc_rtdef.scan_rtdef_; + lookup_param.lookup_ctdef_ = lookup_ctdef; + lookup_param.lookup_rtdef_ = tsc_rtdef.lookup_rtdef_; + lookup_param.rowkey_exprs_ = !lookup_ctdef->rowkey_exprs_.empty() ? &lookup_ctdef->rowkey_exprs_ + : &tsc_ctdef.global_index_rowkey_exprs_; lookup_param.index_table_iter_ = index_table_iter; lookup_param.data_table_iter_ = data_table_iter; lookup_param.can_retry_ = can_retry; lookup_param.calc_part_id_ = tsc_ctdef.calc_part_id_expr_; - lookup_param.lookup_ctdef_ = tsc_ctdef.lookup_ctdef_; - lookup_param.lookup_rtdef_ = tsc_rtdef.lookup_rtdef_; - lookup_param.rowkey_exprs_ = &tsc_ctdef.global_index_rowkey_exprs_; - if (OB_FAIL(iter->init(lookup_param))) { - LOG_WARN("failed to init das global lookup iter", K(ret)); + + if (OB_FAIL(create_das_iter(exec_ctx.get_allocator(), lookup_param, lookup_iter))) { + LOG_WARN("failed to create global lookup iter", K(ret)); } } + if (OB_SUCC(ret)) { - result = iter; - } else { - if (OB_NOT_NULL(iter)) { - iter->release(); - iter = nullptr; + ObDASIter **&children = lookup_iter->get_children(); + if (OB_ISNULL(children = OB_NEW_ARRAY(ObDASIter*, &exec_ctx.get_allocator(), 2))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc for das iter children", K(ret)); + } else { + lookup_iter->set_children_cnt(2); + children[0] = index_table_iter; + children[1] = data_table_iter; } - if (OB_NOT_NULL(iter_buf)) { - alloc.free(iter_buf); - iter_buf = nullptr; - } - result = nullptr; + } + + if (OB_SUCC(ret)) { + scan_iter = index_table_iter; + iter_tree = lookup_iter; } return ret; } -int ObDASIterUtils::create_das_group_fold_iter_help(ObDASIterParam ¶m, - common::ObIAllocator &alloc, - bool need_check_output_datum, - ObDASIter *iter_tree, - ObDASGroupFoldIter *&result) +int ObDASIterUtils::create_iter_children_array(const int64_t children_cnt, + common::ObIAllocator &alloc, + ObDASIter *iter) { int ret = OB_SUCCESS; - void *iter_buf = nullptr; - ObDASGroupFoldIter *iter = nullptr; - if (!param.is_valid()) { + if (OB_UNLIKELY(children_cnt <= 0) || OB_ISNULL(iter)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid das iter param", K(param), K(ret)); - } else if (OB_ISNULL(iter_buf = alloc.alloc(sizeof(ObDASGroupFoldIter)))) { + LOG_WARN("invalid children array args", K(ret), K(children_cnt), KP(iter)); + } else if (OB_NOT_NULL(iter->get_children())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected das iter already has an children array", K(ret), KPC(iter)); + } else if (OB_ISNULL(iter->get_children() = OB_NEW_ARRAY(ObDASIter *, &alloc, children_cnt))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate memory", K(ret)); + LOG_WARN("failed to alloc das iter children array", K(ret), K(children_cnt)); } else { - iter = new (iter_buf) ObDASGroupFoldIter(); + iter->set_children_cnt(children_cnt); } - - if (OB_SUCC(ret) && OB_NOT_NULL(iter)) { - ObDASGroupFoldIterParam group_fold_param; - group_fold_param.assgin(param); - group_fold_param.type_ = ObDASIterType::DAS_ITER_GROUP_FOLD; - group_fold_param.need_check_output_datum_ = need_check_output_datum; - group_fold_param.iter_tree_ = iter_tree; - - if (OB_FAIL(iter->init(group_fold_param))) { - LOG_WARN("failed to init das group fold iter", K(ret)); - } - } - if (OB_SUCC(ret)) { - result = iter; - } else { - if (OB_NOT_NULL(iter)) { - iter->release(); - iter = nullptr; - } - if (OB_NOT_NULL(iter_buf)) { - alloc.free(iter_buf); - iter_buf = nullptr; - } - result = nullptr; - } - return ret; } - } // namespace sql } // namespace oceanbase diff --git a/src/sql/das/iter/ob_das_iter_utils.h b/src/sql/das/iter/ob_das_iter_utils.h index 5a0bb2e2d..84c814e65 100644 --- a/src/sql/das/iter/ob_das_iter_utils.h +++ b/src/sql/das/iter/ob_das_iter_utils.h @@ -13,20 +13,124 @@ #ifndef OBDEV_SRC_SQL_DAS_ITER_OB_DAS_ITER_UTILS_H_ #define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_ITER_UTILS_H_ +#include "sql/das/iter/ob_das_iter_define.h" +#include "sql/das/iter/ob_das_scan_iter.h" #include "sql/das/iter/ob_das_merge_iter.h" -#include "sql/das/iter/ob_das_lookup_iter.h" +#include "sql/das/iter/ob_das_local_lookup_iter.h" +#include "sql/das/iter/ob_das_global_lookup_iter.h" #include "sql/das/iter/ob_das_group_fold_iter.h" +#include "sql/das/iter/ob_das_sort_iter.h" +#include "sql/das/iter/ob_das_text_retrieval_iter.h" +#include "sql/das/iter/ob_das_text_retrieval_merge_iter.h" #include "sql/engine/table/ob_table_scan_op.h" namespace oceanbase { namespace sql { + class ObDASIterUtils { + public: + static int create_das_scan_iter_tree(ObDASIterTreeType tree_type, + storage::ObTableScanParam &scan_param, + const ObDASScanCtDef *scan_ctdef, + ObDASScanRtDef *scan_rtdef, + const ObDASScanCtDef *lookup_ctdef, + ObDASScanRtDef *lookup_rtdef, + const ObDASBaseCtDef *attach_ctdef, + ObDASBaseRtDef *attach_rtdef, + const ObDASRelatedTabletID &related_tablet_ids, + transaction::ObTxDesc *trans_desc, + transaction::ObTxReadSnapshot *snapshot, + common::ObIAllocator &alloc, + ObDASIter *&iter_tree); + + static int create_tsc_iter_tree(ObDASIterTreeType tree_type, + const ObTableScanCtDef &tsc_ctdef, + ObTableScanRtDef &tsc_rtdef, + ObEvalCtx &eval_ctx, + ObExecContext &exec_ctx, + ObFixedArray &eval_infos, + const ObTableScanSpec &spec, + bool can_retry, + ObDASMergeIter *&scan_iter, + ObDASIter *&iter_tree); + + static int create_group_fold_iter(const ObTableScanCtDef &tsc_ctdef, + ObTableScanRtDef &tsc_rtdef, + ObEvalCtx &eval_ctx, + ObExecContext &exec_ctx, + ObFixedArray &eval_infos, + const ObTableScanSpec &spec, + ObDASIter *iter_tree, + ObDASGroupFoldIter *&fold_iter); + + static int set_text_retrieval_related_ids(const ObDASBaseCtDef *attach_ctdef, + const ObDASRelatedTabletID &related_tablet_ids, + const ObLSID &ls_id, + ObDASIter *root_iter); + +private: + static int create_partition_scan_tree(ObTableScanParam &scan_param, + common::ObIAllocator &alloc, + const ObDASScanCtDef *scan_ctdef, + ObDASScanRtDef *scan_rtdef, + ObDASIter *&iter_tree); + + static int create_local_lookup_tree(ObTableScanParam &scan_param, + common::ObIAllocator &alloc, + const ObDASScanCtDef *scan_ctdef, + ObDASScanRtDef *scan_rtdef, + const ObDASScanCtDef *lookup_ctdef, + ObDASScanRtDef *lookup_rtdef, + const ObDASRelatedTabletID &related_tablet_ids, + transaction::ObTxDesc *trans_desc, + transaction::ObTxReadSnapshot *snapshot, + ObDASIter *&iter_tree); + + static int create_domain_lookup_tree(ObTableScanParam &scan_param, + common::ObIAllocator &alloc, + const ObDASBaseCtDef *attach_ctdef, + ObDASBaseRtDef *attach_rtdef, + ObDASIter *&iter_tree); + + static int create_text_retrieval_tree(ObTableScanParam &scan_param, + common::ObIAllocator &alloc, + const ObDASBaseCtDef *attach_ctdef, + ObDASBaseRtDef *attach_rtdef, + const ObDASRelatedTabletID &related_tablet_ids, + transaction::ObTxDesc *trans_desc, + transaction::ObTxReadSnapshot *snapshot, + ObDASIter *&iter_tree); + + static int create_domain_lookup_sub_tree(const ObLSID &ls_id, + common::ObIAllocator &alloc, + const ObDASTableLookupCtDef *table_lookup_ctdef, + ObDASTableLookupRtDef *table_lookup_rtdef, + const ObDASRelatedTabletID &related_tablet_ids, + transaction::ObTxDesc *trans_desc, + transaction::ObTxReadSnapshot *snapshot, + ObDASIter *doc_id_iter, + ObDASIter *&domain_lookup_result); + + static int create_text_retrieval_sub_tree(const ObLSID &ls_id, + common::ObIAllocator &alloc, + const ObDASIRScanCtDef *ir_scan_ctdef, + ObDASIRScanRtDef *ir_scan_rtdef, + const ObDASRelatedTabletID &related_tablet_ids, + transaction::ObTxDesc *trans_desc, + transaction::ObTxReadSnapshot *snapshot, + ObDASIter *&retrieval_result); + + static int create_sort_sub_tree(common::ObIAllocator &alloc, + const ObDASSortCtDef *sort_ctdef, + ObDASSortRtDef *sort_rtdef, + ObDASIter *sort_input, + ObDASIter *&sort_result); + static int create_table_scan_iter_tree(const ObTableScanCtDef &tsc_ctdef, - ObTableScanRtDef &tsc_rtdef, ObEvalCtx &eval_ctx, ObExecContext &exec_ctx, ObFixedArray &eval_infos, @@ -34,15 +138,6 @@ public: ObDASMergeIter *&scan_iter, ObDASIter *&iter_tree); - static int create_local_lookup_iter_tree(const ObTableScanCtDef &tsc_ctdef, - ObTableScanRtDef &tsc_rtdef, - ObEvalCtx &eval_ctx, - ObExecContext &exec_ctx, - ObFixedArray &eval_infos, - const ObTableScanSpec &spec, - ObDASMergeIter *&scan_iter, - ObDASIter *&iter_tree); - static int create_global_lookup_iter_tree(const ObTableScanCtDef &tsc_ctdef, ObTableScanRtDef &tsc_rtdef, ObEvalCtx &eval_ctx, @@ -53,38 +148,32 @@ public: ObDASMergeIter *&scan_iter, ObDASIter *&iter_tree); - static int create_group_fold_iter(const ObTableScanCtDef &tsc_ctdef, - ObTableScanRtDef &tsc_rtdef, - ObEvalCtx &eval_ctx, - ObExecContext &exec_ctx, - ObFixedArray &eval_infos, - const ObTableScanSpec &spec, - ObDASIter *iter_tree, - ObDASGroupFoldIter *&fold_iter); - -private: - static int create_das_merge_iter_help(ObDASIterParam ¶m, + static int create_iter_children_array(const int64_t children_cnt, common::ObIAllocator &alloc, - bool is_global_lookup, - ObFixedArray &eval_infos, - const ObTableScanSpec &spec, - ObDASMergeIter *&result); + ObDASIter *iter); - static int create_das_global_lookup_iter_help(ObDASIterParam ¶m, - common::ObIAllocator &alloc, - int64_t default_batch_row_count, - ObDASMergeIter *index_table_iter, - ObDASMergeIter *data_table_iter, - bool can_retry, - const ObTableScanCtDef &tsc_ctdef, - ObTableScanRtDef &tsc_rtdef, - ObDASGlobalLookupIter *&result); - - static int create_das_group_fold_iter_help(ObDASIterParam ¶m, - common::ObIAllocator &alloc, - bool need_check_output_datum, - ObDASIter *iter_tree, - ObDASGroupFoldIter *&result); + template + static int create_das_iter(common::ObIAllocator &alloc, IterParamType ¶m, IterType *&result) + { + int ret = OB_SUCCESS; + IterType *iter = nullptr; + if (OB_ISNULL(iter = OB_NEWx(IterType, &alloc))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to new a das iter", K(ret)); + } else if (OB_FAIL(iter->init(param))) { + LOG_WARN("failed to init das iter", K(param), K(ret)); + } + if (OB_SUCC(ret)) { + result = iter; + } else { + if (OB_NOT_NULL(iter)) { + iter->release(); + alloc.free(iter); + iter = nullptr; + } + } + return ret; + } ObDASIterUtils() = delete; ~ObDASIterUtils() = delete; diff --git a/src/sql/das/iter/ob_das_local_lookup_iter.cpp b/src/sql/das/iter/ob_das_local_lookup_iter.cpp new file mode 100644 index 000000000..4e56b5589 --- /dev/null +++ b/src/sql/das/iter/ob_das_local_lookup_iter.cpp @@ -0,0 +1,366 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_DAS +#include "sql/das/iter/ob_das_local_lookup_iter.h" +#include "sql/das/iter/ob_das_scan_iter.h" +#include "sql/das/ob_das_scan_op.h" +#include "sql/das/ob_das_ir_define.h" +#include "storage/concurrency_control/ob_data_validation_service.h" + +namespace oceanbase +{ +using namespace common; +namespace sql +{ + +int ObDASLocalLookupIter::inner_init(ObDASIterParam ¶m) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObDASLookupIter::inner_init(param))) { + LOG_WARN("failed to init das lookup iter", K(ret)); + } else if (param.type_ != ObDASIterType::DAS_ITER_LOCAL_LOOKUP) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("inner init das iter with bad param type", K(param), K(ret)); + } else { + ObDASLocalLookupIterParam &lookup_param = static_cast(param); + trans_desc_ = lookup_param.trans_desc_; + snapshot_ = lookup_param.snapshot_; + if (lookup_param.rowkey_exprs_->empty()) { + // for compatibility + if (OB_FAIL(init_rowkey_exprs_for_compat())) { + LOG_WARN("failed eto init rowkeys exprs for compat", K(ret)); + } + } else if (OB_FAIL(rowkey_exprs_.assign(*lookup_param.rowkey_exprs_))) { + LOG_WARN("failed to assign rowkey exprs", K(ret)); + } + } + + return ret; +} + +int ObDASLocalLookupIter::inner_reuse() +{ + int ret = OB_SUCCESS; + // index_scan_param is maintained by das scan op. + if (OB_FAIL(index_table_iter_->reuse())) { + LOG_WARN("failed to reuse index table iter", K(ret)); + } else { + const ObTabletID &old_tablet_id = lookup_param_.tablet_id_; + lookup_param_.need_switch_param_ = lookup_param_.need_switch_param_ || + ((old_tablet_id.is_valid() && old_tablet_id != lookup_tablet_id_) ? true : false); + if (OB_FAIL(data_table_iter_->reuse())) { + LOG_WARN("failed to reuse data table iter", K(ret)); + } + } + + if (OB_SUCC(ret) && OB_FAIL(ObDASLookupIter::inner_reuse())) { + LOG_WARN("failed to reuse das lookup iter", K(ret)); + } + trans_info_array_.reuse(); + return ret; +} + +int ObDASLocalLookupIter::inner_release() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObDASLookupIter::inner_release())) { + LOG_WARN("failed to release lookup iter", K(ret)); + } + lookup_param_.destroy_schema_guard(); + lookup_param_.snapshot_.reset(); + lookup_param_.destroy(); + trans_info_array_.reset(); + return ret; +} + +int ObDASLocalLookupIter::rescan() +{ + int ret = OB_SUCCESS; + // only rescan index table, data table will be rescan in do_lookup. + if (OB_FAIL(index_table_iter_->rescan())) { + LOG_WARN("failed to rescan index table iter", K(ret)); + } + return ret; +} + +int ObDASLocalLookupIter::init_scan_param(ObTableScanParam ¶m, const ObDASScanCtDef *ctdef, ObDASScanRtDef *rtdef) +{ + int ret = OB_SUCCESS; + uint64_t tenant_id = MTL_ID(); + param.tenant_id_ = tenant_id; + param.key_ranges_.set_attr(ObMemAttr(tenant_id, "ScanParamKR")); + param.ss_key_ranges_.set_attr(ObMemAttr(tenant_id, "ScanParamSSKR")); + if (OB_ISNULL(ctdef) || OB_ISNULL(rtdef)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr ctdef or rtdef", K(ctdef), K(rtdef)); + } else { + param.tablet_id_ = lookup_tablet_id_; + param.ls_id_ = lookup_ls_id_; + param.scan_allocator_ = &get_arena_allocator(); + param.allocator_ = &rtdef->stmt_allocator_; + param.tx_lock_timeout_ = rtdef->tx_lock_timeout_; + param.index_id_ = ctdef->ref_table_id_; + param.is_get_ = ctdef->is_get_; + param.is_for_foreign_check_ = rtdef->is_for_foreign_check_; + param.timeout_ = rtdef->timeout_ts_; + param.scan_flag_ = rtdef->scan_flag_; + param.reserved_cell_count_ = ctdef->access_column_ids_.count(); + param.sql_mode_ = rtdef->sql_mode_; + param.frozen_version_ = rtdef->frozen_version_; + param.force_refresh_lc_ = rtdef->force_refresh_lc_; + param.output_exprs_ = &(ctdef->pd_expr_spec_.access_exprs_); + param.aggregate_exprs_ = &(ctdef->pd_expr_spec_.pd_storage_aggregate_output_); + param.ext_file_column_exprs_ = &(ctdef->pd_expr_spec_.ext_file_column_exprs_); + param.ext_column_convert_exprs_ = &(ctdef->pd_expr_spec_.ext_column_convert_exprs_); + param.calc_exprs_ = &(ctdef->pd_expr_spec_.calc_exprs_); + param.table_param_ = &(ctdef->table_param_); + param.op_ = rtdef->p_pd_expr_op_; + param.row2exprs_projector_ = rtdef->p_row2exprs_projector_; + param.schema_version_ = ctdef->schema_version_; + param.tenant_schema_version_ = rtdef->tenant_schema_version_; + param.limit_param_ = rtdef->limit_param_; + param.need_scn_ = rtdef->need_scn_; + param.pd_storage_flag_ = ctdef->pd_expr_spec_.pd_storage_flag_.pd_flag_; + param.fb_snapshot_ = rtdef->fb_snapshot_; + param.fb_read_tx_uncommitted_ = rtdef->fb_read_tx_uncommitted_; + if (rtdef->is_for_foreign_check_) { + param.trans_desc_ = trans_desc_; + } + if (OB_NOT_NULL(snapshot_)) { + param.snapshot_ = *snapshot_; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null snapshot", K(ret), KPC(this)); + } + if (OB_NOT_NULL(trans_desc_)) { + param.tx_id_ = trans_desc_->get_tx_id(); + } else { + param.tx_id_.reset(); + } + if (!ctdef->pd_expr_spec_.pushdown_filters_.empty()) { + param.op_filters_ = &ctdef->pd_expr_spec_.pushdown_filters_; + } + param.pd_storage_filters_ = rtdef->p_pd_expr_op_->pd_storage_filters_; + if (OB_FAIL(param.column_ids_.assign(ctdef->access_column_ids_))) { + LOG_WARN("failed to assign column ids", K(ret)); + } + if (rtdef->sample_info_ != nullptr) { + param.sample_info_ = *rtdef->sample_info_; + } + } + + LOG_DEBUG("init local index lookup param finished", K(param), K(ret)); + return ret; +} + +void ObDASLocalLookupIter::reset_lookup_state() +{ + ObDASLookupIter::reset_lookup_state(); + trans_info_array_.reuse(); +} + +int ObDASLocalLookupIter::add_rowkey() +{ + int ret = OB_SUCCESS; + OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_SCAN); + ObDASScanIter *scan_iter = static_cast(data_table_iter_); + storage::ObTableScanParam &scan_param = scan_iter->get_scan_param(); + ObNewRange lookup_range; + int64 group_id = 0; + + if (DAS_OP_TABLE_SCAN == index_ctdef_->op_type_) { + const ObDASScanCtDef *index_ctdef = static_cast(index_ctdef_); + if (nullptr != index_ctdef->group_id_expr_) { + group_id = index_ctdef->group_id_expr_->locate_expr_datum(*eval_ctx_).get_int(); + } + if (nullptr != index_ctdef->trans_info_expr_) { + ObDatum *datum_ptr = nullptr; + if (OB_FAIL(build_trans_info_datum(index_ctdef->trans_info_expr_, datum_ptr))) { + LOG_WARN("failed to build trans info datum", K(ret)); + } else if (OB_ISNULL(datum_ptr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", K(ret)); + } else if (OB_FAIL(trans_info_array_.push_back(datum_ptr))) { + LOG_WARN("failed to push back trans info array", K(ret), KPC(datum_ptr)); + } + } + } + + int64_t group_idx = ObNewRange::get_group_idx(group_id); + if (OB_FAIL(ret)) { + } else if (OB_FAIL(build_lookup_range(lookup_range))) { + LOG_WARN("failed to build lookup range", K(ret)); + } else if (FALSE_IT(lookup_range.group_idx_ = group_idx)) { + } else if (OB_FAIL(scan_param.key_ranges_.push_back(lookup_range))) { + LOG_WARN("failed to push back lookup range", K(ret)); + } else { + scan_param.is_get_ = true; + } + LOG_DEBUG("build local lookup range", K(lookup_range), K(ret)); + + return ret; +} + +int ObDASLocalLookupIter::do_table_scan() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(index_table_iter_->do_table_scan())) { + LOG_WARN("failed to scan index table", K(ret)); + } + return ret; +} + +int ObDASLocalLookupIter::add_rowkeys(int64_t count) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(eval_ctx_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", K_(eval_ctx)); + } else { + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*eval_ctx_); + batch_info_guard.set_batch_size(count); + for(int i = 0; OB_SUCC(ret) && i < count; i++) { + batch_info_guard.set_batch_idx(i); + if(OB_FAIL(add_rowkey())) { + LOG_WARN("failed to add rowkey", K(ret), K(i)); + } + } + } + + return ret; +} + +int ObDASLocalLookupIter::do_index_lookup() +{ + int ret = OB_SUCCESS; + OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_SCAN); + if (is_first_lookup_) { + is_first_lookup_ = false; + if (OB_FAIL(init_scan_param(lookup_param_, lookup_ctdef_, lookup_rtdef_))) { + LOG_WARN("failed to init scan param", K(ret)); + } else if (OB_FAIL(data_table_iter_->do_table_scan())) { + if (OB_SNAPSHOT_DISCARDED == ret && lookup_param_.fb_snapshot_.is_valid()) { + ret = OB_INVALID_QUERY_TIMESTAMP; + } else if (OB_TRY_LOCK_ROW_CONFLICT != ret) { + LOG_WARN("failed to do partition scan", K(lookup_param_), K(ret)); + } + } + } else { + // reuse -> real rescan + // reuse: store next tablet_id, ls_id and reuse storage iter; + // rescan: bind tablet_id, ls_id to scan_param and rescan; + lookup_param_.tablet_id_ = lookup_tablet_id_; + lookup_param_.ls_id_ = lookup_ls_id_; + if (OB_FAIL(data_table_iter_->rescan())) { + LOG_WARN("failed to rescan data table", K(ret)); + } + } + return ret; +} + +int ObDASLocalLookupIter::check_index_lookup() +{ + int ret = OB_SUCCESS; + OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_SCAN); + ObDASScanIter *scan_iter = static_cast(data_table_iter_); + if (GCONF.enable_defensive_check() && + lookup_ctdef_->pd_expr_spec_.pushdown_filters_.empty()) { + if (OB_UNLIKELY(lookup_rowkey_cnt_ != lookup_row_cnt_)) { + ret = OB_ERR_DEFENSIVE_CHECK; + ObString func_name = ObString::make_string("check_lookup_row_cnt"); + LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr()); + LOG_ERROR("Fatal Error!!! Catch a defensive error!", + K(ret), K_(lookup_rowkey_cnt), K_(lookup_row_cnt), + "main table id", lookup_ctdef_->ref_table_id_, + "main tablet id", lookup_tablet_id_, + KPC_(trans_desc), KPC_(snapshot)); + concurrency_control::ObDataValidationService::set_delay_resource_recycle(lookup_ls_id_); + const ObTableScanParam &scan_param = scan_iter->get_scan_param(); + if (trans_info_array_.count() == scan_param.key_ranges_.count()) { + for (int64_t i = 0; i < trans_info_array_.count(); i++) { + ObDatum *datum = trans_info_array_.at(i); + LOG_ERROR("dump lookup range and trans info of local lookup das task", + K(i), KPC(trans_info_array_.at(i)), K(scan_param.key_ranges_.at(i))); + } + } else { + for (int64_t i = 0; i < scan_param.key_ranges_.count(); i++) { + LOG_ERROR("dump lookup range of local lookup das task", + K(i), K(scan_param.key_ranges_.at(i))); + } + } + } + } + + int simulate_error = EVENT_CALL(EventTable::EN_DAS_SIMULATE_DUMP_WRITE_BUFFER); + if (0 != simulate_error) { + for (int64_t i = 0; i < trans_info_array_.count(); i++) { + LOG_INFO("dump trans info of local lookup das task", K(i), KPC(trans_info_array_.at(i))); + } + } + + return ret; +} + +int ObDASLocalLookupIter::init_rowkey_exprs_for_compat() +{ + int ret = OB_SUCCESS; + if (ObDASOpType::DAS_OP_TABLE_SCAN == index_ctdef_->op_type_ + || ObDASOpType::DAS_OP_IR_AUX_LOOKUP == index_ctdef_->op_type_) { + const ObDASScanCtDef *scan_ctdef = static_cast(index_ctdef_); + int64_t rowkey_cnt = scan_ctdef->result_output_.count(); + if (nullptr != scan_ctdef->group_id_expr_) { + rowkey_cnt -= 1; + } + if (nullptr != scan_ctdef->trans_info_expr_) { + rowkey_cnt -= 1; + } + if (OB_FAIL(rowkey_exprs_.reserve(rowkey_cnt))) { + LOG_WARN("failed to reserve rowkey exprs cnt", K(rowkey_cnt), K(ret)); + } else { + for (int64_t i = 0; i < scan_ctdef->result_output_.count(); i++) { + ObExpr* const expr = scan_ctdef->result_output_.at(i); + if (T_PSEUDO_GROUP_ID == expr->type_ || T_PSEUDO_ROW_TRANS_INFO_COLUMN == expr->type_) { + // skip + } else if (OB_FAIL(rowkey_exprs_.push_back(expr))) { + LOG_WARN("failed to push back expr", K(ret)); + } + } + } + } else if (ObDASOpType::DAS_OP_IR_SCAN == index_ctdef_->op_type_ + || ObDASOpType::DAS_OP_SORT == index_ctdef_->op_type_) { + // only doc_id as rowkey for text retrieval index back + const ObDASIRScanCtDef *ir_ctdef = nullptr; + if (ObDASOpType::DAS_OP_SORT == index_ctdef_->op_type_) { + if (OB_UNLIKELY(ObDASOpType::DAS_OP_IR_SCAN != index_ctdef_->children_[0]->op_type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected child of sort iter is not an ir scan iter for compatible scan", K(ret)); + } else { + ir_ctdef = static_cast(index_ctdef_->children_[0]); + } + } else { + ir_ctdef = static_cast(index_ctdef_); + } + if (OB_SUCC(ret)) { + if (OB_FAIL(rowkey_exprs_.push_back(ir_ctdef->inv_scan_doc_id_col_))) { + LOG_WARN("gailed to add rowkey exprs", K(ret)); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected compatible das scan op type", K(ret), K(index_ctdef_->op_type_)); + } + return ret; +} + +} // namespace sql +} // namespace oceanbase diff --git a/src/sql/das/iter/ob_das_local_lookup_iter.h b/src/sql/das/iter/ob_das_local_lookup_iter.h new file mode 100644 index 000000000..d48a7bfaf --- /dev/null +++ b/src/sql/das/iter/ob_das_local_lookup_iter.h @@ -0,0 +1,94 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OBDEV_SRC_SQL_DAS_ITER_OB_DAS_LOCAL_LOOKUP_ITER_H_ +#define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_LOCAL_LOOKUP_ITER_H_ + +#include "sql/das/iter/ob_das_lookup_iter.h" +#include "storage/access/ob_dml_param.h" + +namespace oceanbase +{ +using namespace common; +namespace sql +{ + +struct ObDASLocalLookupIterParam : public ObDASLookupIterParam +{ +public: + ObDASLocalLookupIterParam() + : ObDASLookupIterParam(false /*local lookup*/), + trans_desc_(nullptr), + snapshot_(nullptr) + {} + transaction::ObTxDesc *trans_desc_; + transaction::ObTxReadSnapshot *snapshot_; + virtual bool is_valid() const override + { + return true; + } +}; + +class ObDASScanCtDef; +class ObDASScanRtDef; +class ObDASLocalLookupIter : public ObDASLookupIter +{ +public: + ObDASLocalLookupIter() + : ObDASLookupIter(ObDASIterType::DAS_ITER_LOCAL_LOOKUP), + trans_info_array_(), + lookup_param_(), + lookup_tablet_id_(), + lookup_ls_id_(), + trans_desc_(nullptr), + snapshot_(nullptr), + is_first_lookup_(true) + {} + virtual ~ObDASLocalLookupIter() {} + + storage::ObTableScanParam &get_lookup_param() { return lookup_param_; } + void set_tablet_id(const ObTabletID &tablet_id) { lookup_tablet_id_ = tablet_id; } + void set_ls_id(const share::ObLSID &ls_id) { lookup_ls_id_ = ls_id; } + int init_scan_param(storage::ObTableScanParam ¶m, const ObDASScanCtDef *ctdef, ObDASScanRtDef *rtdef); + +protected: + virtual int inner_init(ObDASIterParam ¶m) override; + virtual int inner_reuse() override; + virtual int inner_release() override; + virtual int do_table_scan() override; + virtual int rescan() override; + virtual void reset_lookup_state(); + + virtual int add_rowkey() override; + virtual int add_rowkeys(int64_t count) override; + virtual int do_index_lookup() override; + virtual int check_index_lookup() override; +private: + int init_rowkey_exprs_for_compat(); + +private: + ObSEArray trans_info_array_; + // Local lookup das task could rescan multiple times during execution, lookup_tablet_id_ and + // lookup_ls_id_ store the lookup parameter for this time. + storage::ObTableScanParam lookup_param_; + ObTabletID lookup_tablet_id_; + share::ObLSID lookup_ls_id_; + transaction::ObTxDesc *trans_desc_; + transaction::ObTxReadSnapshot *snapshot_; + bool is_first_lookup_; +}; + +} // namespace sql +} // namespace oceanbase + + +#endif /* OBDEV_SRC_SQL_DAS_ITER_OB_DAS_LOOKUP_ITER_H_ */ diff --git a/src/sql/das/iter/ob_das_lookup_iter.cpp b/src/sql/das/iter/ob_das_lookup_iter.cpp index 579d3dc30..887e0232d 100644 --- a/src/sql/das/iter/ob_das_lookup_iter.cpp +++ b/src/sql/das/iter/ob_das_lookup_iter.cpp @@ -23,9 +23,9 @@ namespace sql int ObDASLookupIter::inner_init(ObDASIterParam ¶m) { int ret = OB_SUCCESS; - if (param.type_ != ObDASIterType::DAS_ITER_LOOKUP) { + if (!IS_LOOKUP_ITER(param.type_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("specific_init with bad param type", K(param.type_)); + LOG_WARN("inner init das iter with bad param type", K(param), K(ret)); } else { ObDASLookupIterParam &lookup_param = static_cast(param); state_ = LookupState::INDEX_SCAN; @@ -35,15 +35,16 @@ int ObDASLookupIter::inner_init(ObDASIterParam ¶m) lookup_row_cnt_ = 0; index_table_iter_ = lookup_param.index_table_iter_; data_table_iter_ = lookup_param.data_table_iter_; - can_retry_ = lookup_param.can_retry_; - calc_part_id_ = lookup_param.calc_part_id_; + index_ctdef_ = lookup_param.index_ctdef_; + index_rtdef_ = lookup_param.index_rtdef_; lookup_ctdef_ = lookup_param.lookup_ctdef_; lookup_rtdef_ = lookup_param.lookup_rtdef_; - rowkey_exprs_ = lookup_param.rowkey_exprs_; - iter_alloc_ = new (iter_alloc_buf_) common::ObArenaAllocator(); - iter_alloc_->set_attr(ObMemAttr(MTL_ID(), "TableLookup")); - lookup_rtdef_->scan_allocator_.set_alloc(iter_alloc_); - lookup_rtdef_->stmt_allocator_.set_alloc(iter_alloc_); + lib::ContextParam param; + param.set_mem_attr(MTL_ID(), ObModIds::OB_SQL_TABLE_LOOKUP, ObCtxIds::DEFAULT_CTX_ID) + .set_properties(lib::USE_TL_PAGE_OPTIONAL); + if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(lookup_memctx_, param))) { + LOG_WARN("failed to create lookup memctx", K(ret)); + } } return ret; } @@ -51,12 +52,8 @@ int ObDASLookupIter::inner_init(ObDASIterParam ¶m) int ObDASLookupIter::inner_reuse() { int ret = OB_SUCCESS; - // the reuse() of index table iter will be handled in TSC. - if (OB_FAIL(data_table_iter_->reuse())) { - LOG_WARN("failed to reuse data table iter", K(ret)); - } - if (OB_NOT_NULL(iter_alloc_)) { - iter_alloc_->reset_remain_one_page(); + if (OB_NOT_NULL(lookup_memctx_)) { + lookup_memctx_->reset_remain_one_page(); } lookup_row_cnt_ = 0; lookup_rowkey_cnt_ = 0; @@ -68,25 +65,27 @@ int ObDASLookupIter::inner_reuse() int ObDASLookupIter::inner_release() { int ret = OB_SUCCESS; - if (OB_NOT_NULL(iter_alloc_)) { - iter_alloc_->reset(); - iter_alloc_->~ObArenaAllocator(); - iter_alloc_ = nullptr; + if (OB_NOT_NULL(lookup_memctx_)) { + DESTROY_CONTEXT(lookup_memctx_); + lookup_memctx_ = nullptr; } index_table_iter_ = nullptr; data_table_iter_ = nullptr; + rowkey_exprs_.reset(); return ret; } void ObDASLookupIter::reset_lookup_state() { - lookup_rowkey_cnt_ = 0; lookup_row_cnt_ = 0; + lookup_rowkey_cnt_ = 0; + index_end_ = false; + state_ = LookupState::INDEX_SCAN; if (OB_NOT_NULL(data_table_iter_)) { data_table_iter_->reuse(); } - if (OB_NOT_NULL(iter_alloc_)) { - iter_alloc_->reset_remain_one_page(); + if (OB_NOT_NULL(lookup_memctx_)) { + lookup_memctx_->reset_remain_one_page(); } } @@ -95,13 +94,15 @@ int ObDASLookupIter::inner_get_next_row() int ret = OB_SUCCESS; bool got_next_row = false; int64_t simulate_batch_row_cnt = - EVENT_CALL(EventTable::EN_TABLE_LOOKUP_BATCH_ROW_COUNT); - int64_t default_row_batch_cnt = simulate_batch_row_cnt > 0 ? simulate_batch_row_cnt : default_batch_row_count_; + const bool use_simulate_batch_row_cnt = simulate_batch_row_cnt > 0 && simulate_batch_row_cnt < default_batch_row_count_; + int64_t default_row_batch_cnt = use_simulate_batch_row_cnt ? simulate_batch_row_cnt : default_batch_row_count_; LOG_DEBUG("simulate lookup row batch count", K(simulate_batch_row_cnt), K(default_row_batch_cnt)); do { switch (state_) { case INDEX_SCAN: { reset_lookup_state(); while (OB_SUCC(ret) && !index_end_ && lookup_rowkey_cnt_ < default_row_batch_cnt) { + index_table_iter_->clear_evaluated_flag(); if (OB_FAIL(index_table_iter_->get_next_row())) { if(OB_UNLIKELY(OB_ITER_END != ret)) { LOG_WARN("failed to get next row from index table", K(ret)); @@ -136,6 +137,7 @@ int ObDASLookupIter::inner_get_next_row() } case OUTPUT_ROWS: { + data_table_iter_->clear_evaluated_flag(); if (OB_FAIL(data_table_iter_->get_next_row())) { if (OB_LIKELY(OB_ITER_END == ret)) { ret = OB_SUCCESS; @@ -173,7 +175,8 @@ int ObDASLookupIter::inner_get_next_rows(int64_t &count, int64_t capacity) int ret = OB_SUCCESS; bool get_next_rows = false; int64_t simulate_batch_row_cnt = - EVENT_CALL(EventTable::EN_TABLE_LOOKUP_BATCH_ROW_COUNT); - int64_t default_row_batch_cnt = simulate_batch_row_cnt > 0 ? simulate_batch_row_cnt : default_batch_row_count_; + const bool use_simulate_batch_row_cnt = simulate_batch_row_cnt > 0 && simulate_batch_row_cnt < default_batch_row_count_; + int64_t default_row_batch_cnt = use_simulate_batch_row_cnt ? simulate_batch_row_cnt : default_batch_row_count_; LOG_DEBUG("simulate lookup row batch count", K(simulate_batch_row_cnt), K(default_row_batch_cnt)); do { switch (state_) { @@ -184,6 +187,7 @@ int ObDASLookupIter::inner_get_next_rows(int64_t &count, int64_t capacity) while (OB_SUCC(ret) && !index_end_ && lookup_rowkey_cnt_ < default_row_batch_cnt) { storage_count = 0; index_capacity = std::min(max_size_, default_row_batch_cnt - lookup_rowkey_cnt_); + index_table_iter_->clear_evaluated_flag(); if (OB_FAIL(index_table_iter_->get_next_rows(storage_count, index_capacity))) { if (OB_UNLIKELY(OB_ITER_END != ret)) { LOG_WARN("failed to get next rows from index table", K(ret)); @@ -224,6 +228,7 @@ int ObDASLookupIter::inner_get_next_rows(int64_t &count, int64_t capacity) case OUTPUT_ROWS: { count = 0; + data_table_iter_->clear_evaluated_flag(); if (OB_FAIL(data_table_iter_->get_next_rows(count, capacity))) { if (OB_LIKELY(OB_ITER_END == ret)) { ret = OB_SUCCESS; @@ -260,46 +265,39 @@ int ObDASLookupIter::inner_get_next_rows(int64_t &count, int64_t capacity) int ObDASLookupIter::build_lookup_range(ObNewRange &range) { int ret = OB_SUCCESS; - if (OB_ISNULL(rowkey_exprs_) || OB_ISNULL(eval_ctx_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected nullptr", K(ret), K(rowkey_exprs_), K(eval_ctx_)); + if (OB_ISNULL(eval_ctx_) || OB_UNLIKELY(rowkey_exprs_.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid eval ctx or rowkey exprs", K_(eval_ctx), K_(rowkey_exprs), K(ret)); } else { - int64_t rowkey_cnt = rowkey_exprs_->count(); ObObj *obj_ptr = nullptr; void *buf = nullptr; - if (OB_ISNULL(buf = iter_alloc_->alloc(sizeof(ObObj) * rowkey_cnt))) { + int64_t rowkey_cnt = rowkey_exprs_.count(); + common::ObArenaAllocator& lookup_alloc = lookup_memctx_->get_arena_allocator(); + if (OB_ISNULL(buf = lookup_alloc.alloc(sizeof(ObObj) * rowkey_cnt))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate enough memory", K(ret), K(rowkey_cnt)); + LOG_WARN("failed to allocate enough memory", K(rowkey_cnt), K(ret)); } else { obj_ptr = new (buf) ObObj[rowkey_cnt]; } - for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt; ++i) { + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt; i++) { ObObj tmp_obj; - ObExpr *expr = rowkey_exprs_->at(i); + const ObExpr *expr = rowkey_exprs_.at(i); ObDatum &col_datum = expr->locate_expr_datum(*eval_ctx_); - if (OB_FAIL(col_datum.to_obj(tmp_obj, expr->obj_meta_, expr->obj_datum_map_))) { + if (OB_UNLIKELY(T_PSEUDO_GROUP_ID == expr->type_ || T_PSEUDO_ROW_TRANS_INFO_COLUMN == expr->type_)) { + // skip. + } else if (OB_FAIL(col_datum.to_obj(tmp_obj, expr->obj_meta_, expr->obj_datum_map_))) { LOG_WARN("failed to convert datum to obj", K(ret)); - } else if (OB_FAIL(ob_write_obj(*iter_alloc_, tmp_obj, obj_ptr[i]))) { + } else if (OB_FAIL(ob_write_obj(lookup_alloc, tmp_obj, obj_ptr[i]))) { LOG_WARN("failed to deep copy rowkey", K(ret), K(tmp_obj)); } } - int64_t group_id = 0; - if (OB_NOT_NULL(group_id_expr_)) { - ObDatum &group_datum = group_id_expr_->locate_expr_datum(*eval_ctx_); - OB_ASSERT(T_PSEUDO_GROUP_ID == group_id_expr_->type_); - group_id = group_datum.get_int(); - } - if (OB_SUCC(ret)) { ObRowkey row_key(obj_ptr, rowkey_cnt); if (OB_FAIL(range.build_range(lookup_ctdef_->ref_table_id_, row_key))) { LOG_WARN("failed to build lookup range", K(ret), K(lookup_ctdef_->ref_table_id_), K(row_key)); - } else { - range.group_idx_ = group_id; } - LOG_DEBUG("build lookup range", K(ret), K(row_key), K(range)); } } @@ -318,7 +316,7 @@ int ObDASLookupIter::build_trans_info_datum(const ObExpr *trans_info_expr, ObDat ObDatum &col_datum = trans_info_expr->locate_expr_datum(*eval_ctx_); int64_t pos = sizeof(ObDatum); int64_t len = sizeof(ObDatum) + col_datum.len_; - if (OB_ISNULL(buf = iter_alloc_->alloc(len))) { + if (OB_ISNULL(buf = lookup_memctx_->get_arena_allocator().alloc(len))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to allocate enough memory", K(ret)); } else if (FALSE_IT(datum_ptr = new (buf) ObDatum)) { @@ -330,158 +328,5 @@ int ObDASLookupIter::build_trans_info_datum(const ObExpr *trans_info_expr, ObDat return ret; } -int ObDASGlobalLookupIter::add_rowkey() -{ - int ret = OB_SUCCESS; - ObObjectID partition_id = OB_INVALID_ID; - ObTabletID tablet_id(OB_INVALID_ID); - ObDASScanOp *das_scan_op = nullptr; - ObDASTabletLoc *tablet_loc = nullptr; - ObDASCtx *das_ctx = nullptr; - bool reuse_das_op = false; - OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_MERGE); - if (OB_ISNULL(exec_ctx_) || OB_ISNULL(das_ctx = &exec_ctx_->get_das_ctx())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get das ctx", KPC_(exec_ctx)); - } else { - ObDASMergeIter *merge_iter = static_cast(data_table_iter_); - if (OB_FAIL(ObExprCalcPartitionBase::calc_part_and_tablet_id(calc_part_id_, - *eval_ctx_, - partition_id, - tablet_id))) { - LOG_WARN("failed to calc part id", K(ret), KPC(calc_part_id_)); - } else if (OB_FAIL(das_ctx->extended_tablet_loc(*lookup_rtdef_->table_loc_, - tablet_id, - tablet_loc))) { - LOG_WARN("failed to get tablet loc by tablet_id", K(ret)); - } else if (OB_FAIL(merge_iter->create_das_task(tablet_loc, das_scan_op, reuse_das_op))) { - LOG_WARN("failed to create das task", K(ret)); - } else if (!reuse_das_op) { - das_scan_op->set_scan_ctdef(lookup_ctdef_); - das_scan_op->set_scan_rtdef(lookup_rtdef_); - das_scan_op->set_can_part_retry(can_retry_); - } - } - if (OB_SUCC(ret)) { - storage::ObTableScanParam &scan_param = das_scan_op->get_scan_param(); - ObNewRange lookup_range; - if (OB_FAIL(build_lookup_range(lookup_range))) { - LOG_WARN("failed to build lookup range", K(ret)); - } else if (OB_FAIL(scan_param.key_ranges_.push_back(lookup_range))) { - LOG_WARN("failed to push back lookup range", K(ret)); - } else { - scan_param.is_get_ = true; - } - } - - const ObExpr *trans_info_expr = lookup_ctdef_->trans_info_expr_; - if (OB_SUCC(ret) && OB_NOT_NULL(trans_info_expr)) { - void *buf = nullptr; - ObDatum *datum_ptr = nullptr; - if (OB_FAIL(build_trans_info_datum(trans_info_expr, datum_ptr))) { - LOG_WARN("failed to build trans info datum", K(ret)); - } else if (OB_ISNULL(datum_ptr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected nullptr", K(ret)); - } else if (OB_FAIL(das_scan_op->trans_info_array_.push_back(datum_ptr))) { - LOG_WARN("failed to push back trans info array", K(ret), KPC(datum_ptr)); - } - } - - return ret; -} - -int ObDASGlobalLookupIter::add_rowkeys(int64_t count) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(eval_ctx_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected nullptr", K_(eval_ctx)); - } else { - ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*eval_ctx_); - batch_info_guard.set_batch_size(count); - for(int i = 0; OB_SUCC(ret) && i < count; i++) { - batch_info_guard.set_batch_idx(i); - if(OB_FAIL(add_rowkey())) { - LOG_WARN("failed to add rowkey", K(ret), K(i)); - } - } - } - - return ret; -} - -int ObDASGlobalLookupIter::do_index_lookup() -{ - int ret = OB_SUCCESS; - OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_MERGE); - ObDASMergeIter *merge_iter = static_cast(data_table_iter_); - if (OB_FAIL(merge_iter->do_table_scan())) { - LOG_WARN("failed to do global index lookup", K(ret)); - } else if (OB_FAIL(merge_iter->set_merge_status(merge_iter->get_merge_type()))) { - LOG_WARN("failed to set merge status for das iter", K(ret)); - } - return ret; -} - -int ObDASGlobalLookupIter::check_index_lookup() -{ - int ret = OB_SUCCESS; - OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_MERGE); - ObDASMergeIter *merge_iter = static_cast(data_table_iter_); - if (GCONF.enable_defensive_check() && - lookup_ctdef_->pd_expr_spec_.pushdown_filters_.empty()) { - if (OB_UNLIKELY(lookup_rowkey_cnt_ != lookup_row_cnt_)) { - ret = OB_ERR_DEFENSIVE_CHECK; - ObSQLSessionInfo *my_session = exec_ctx_->get_my_session(); - ObString func_name = ObString::make_string("check_lookup_row_cnt"); - LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr()); - LOG_ERROR("Fatal Error!!! Catch a defensive error!", - K(ret), K(lookup_rowkey_cnt_), K(lookup_row_cnt_), - "main table id", lookup_ctdef_->ref_table_id_, - KPC(my_session->get_tx_desc())); - - int64_t row_num = 0; - for (DASTaskIter task_iter = merge_iter->begin_task_iter(); !task_iter.is_end(); ++task_iter) { - ObDASScanOp *das_op = static_cast(*task_iter); - if (das_op->trans_info_array_.count() == das_op->get_scan_param().key_ranges_.count()) { - for (int64_t i = 0; i < das_op->trans_info_array_.count(); i++) { - row_num++; - ObDatum *datum = das_op->trans_info_array_.at(i); - LOG_ERROR("dump GLobalIndexBack das task lookup range and trans info", - K(row_num), KPC(datum), - K(das_op->get_scan_param().key_ranges_.at(i)), - K(das_op->get_tablet_id())); - } - } else { - for (int64_t i = 0; i < das_op->get_scan_param().key_ranges_.count(); i++) { - row_num++; - LOG_ERROR("dump GLobalIndexBack das task lookup range", - K(row_num), - K(das_op->get_scan_param().key_ranges_.at(i)), - K(das_op->get_tablet_id())); - } - } - } - } - } - - int simulate_error = EVENT_CALL(EventTable::EN_DAS_SIMULATE_DUMP_WRITE_BUFFER); - if (0 != simulate_error) { - for (DASTaskIter task_iter = merge_iter->begin_task_iter(); !task_iter.is_end(); ++task_iter) { - ObDASScanOp *das_op = static_cast(*task_iter); - for (int64_t i = 0; i < das_op->trans_info_array_.count(); i++) { - ObDatum *datum = das_op->trans_info_array_.at(i); - LOG_INFO("dump GLobalIndexBack das task trans info", K(i), - KPC(das_op->trans_info_array_.at(i)), - K(das_op->get_scan_param().key_ranges_.at(i)), - K(das_op->get_tablet_id())); - } - } - } - - return ret; -} - } // namespace sql } // namespace oceanbase diff --git a/src/sql/das/iter/ob_das_lookup_iter.h b/src/sql/das/iter/ob_das_lookup_iter.h index 5e9b54541..57291d3c6 100644 --- a/src/sql/das/iter/ob_das_lookup_iter.h +++ b/src/sql/das/iter/ob_das_lookup_iter.h @@ -21,47 +21,64 @@ using namespace common; namespace sql { +class ObDASBaseCtDef; +class ObDASBaseRtDef; +class ObDASScanCtDef; +class ObDASScanRtDef; struct ObDASLookupIterParam : public ObDASIterParam { +public: + ObDASLookupIterParam(bool is_global_index) + : ObDASIterParam(is_global_index ? DAS_ITER_GLOBAL_LOOKUP : DAS_ITER_LOCAL_LOOKUP), + default_batch_row_count_(0), + index_ctdef_(nullptr), + index_rtdef_(nullptr), + lookup_ctdef_(nullptr), + lookup_rtdef_(nullptr), + index_table_iter_(nullptr), + data_table_iter_(nullptr), + rowkey_exprs_(nullptr) + {} int64_t default_batch_row_count_; - bool can_retry_; - const ObExpr *calc_part_id_; + const ObDASBaseCtDef *index_ctdef_; + ObDASBaseRtDef *index_rtdef_; const ObDASScanCtDef *lookup_ctdef_; ObDASScanRtDef *lookup_rtdef_; - const ExprFixedArray *rowkey_exprs_; - ObTableID ref_table_id_; ObDASIter *index_table_iter_; ObDASIter *data_table_iter_; + const ExprFixedArray *rowkey_exprs_; virtual bool is_valid() const override { - return ObDASIterParam::is_valid() && - index_table_iter_ != nullptr && data_table_iter_ != nullptr && calc_part_id_ != nullptr && - lookup_ctdef_ != nullptr && lookup_rtdef_ != nullptr && rowkey_exprs_ != nullptr; + return ObDASIterParam::is_valid() + && index_table_iter_ != nullptr && data_table_iter_ != nullptr + && index_ctdef_ != nullptr && index_rtdef_ != nullptr + && lookup_ctdef_ != nullptr && lookup_rtdef_ != nullptr && rowkey_exprs_ != nullptr; } }; class ObDASLookupIter : public ObDASIter { public: - ObDASLookupIter() - : calc_part_id_(nullptr), + ObDASLookupIter(const ObDASIterType type = ObDASIterType::DAS_ITER_INVALID) + : ObDASIter(type), + index_ctdef_(nullptr), + index_rtdef_(nullptr), lookup_ctdef_(nullptr), lookup_rtdef_(nullptr), - rowkey_exprs_(nullptr), + rowkey_exprs_(), index_table_iter_(nullptr), data_table_iter_(nullptr), lookup_rowkey_cnt_(0), lookup_row_cnt_(0), - can_retry_(false), state_(INDEX_SCAN), index_end_(false), default_batch_row_count_(0), - iter_alloc_(nullptr) + lookup_memctx_() {} virtual ~ObDASLookupIter() {} - INHERIT_TO_STRING_KV("ObDASIter", ObDASIter, K_(state), K_(index_end), K(lookup_ctdef_->ref_table_id_)); + INHERIT_TO_STRING_KV("ObDASIter", ObDASIter, K_(state), K_(index_end), K_(default_batch_row_count)); protected: virtual int inner_init(ObDASIterParam ¶m) override; @@ -70,24 +87,24 @@ protected: virtual int inner_get_next_row() override; virtual int inner_get_next_rows(int64_t &count, int64_t capacity) override; virtual void reset_lookup_state(); - virtual int add_rowkey() = 0; virtual int add_rowkeys(int64_t count) = 0; virtual int do_index_lookup() = 0; virtual int check_index_lookup() = 0; protected: - const ObExpr *calc_part_id_; + const ObDASBaseCtDef *index_ctdef_; + ObDASBaseRtDef *index_rtdef_; const ObDASScanCtDef *lookup_ctdef_; ObDASScanRtDef *lookup_rtdef_; - const ExprFixedArray *rowkey_exprs_; + common::ObSEArray rowkey_exprs_; ObDASIter *index_table_iter_; ObDASIter *data_table_iter_; int64_t lookup_rowkey_cnt_; int64_t lookup_row_cnt_; - bool can_retry_; int build_lookup_range(ObNewRange &range); int build_trans_info_datum(const ObExpr *trans_info_expr, ObDatum *&datum_ptr); + common::ObArenaAllocator &get_arena_allocator() { return lookup_memctx_->get_arena_allocator(); } private: enum LookupState : uint32_t @@ -101,28 +118,10 @@ private: LookupState state_; bool index_end_; int64_t default_batch_row_count_; - common::ObArenaAllocator *iter_alloc_; - char iter_alloc_buf_[sizeof(common::ObArenaAllocator)]; -}; - -class ObDASGlobalLookupIter : public ObDASLookupIter -{ -public: - ObDASGlobalLookupIter() - : ObDASLookupIter() - {} - virtual ~ObDASGlobalLookupIter() {} - -protected: - virtual int add_rowkey() override; - virtual int add_rowkeys(int64_t count) override; - virtual int do_index_lookup() override; - virtual int check_index_lookup() override; + lib::MemoryContext lookup_memctx_; }; } // namespace sql } // namespace oceanbase - - #endif /* OBDEV_SRC_SQL_DAS_ITER_OB_DAS_LOOKUP_ITER_H_ */ diff --git a/src/sql/das/iter/ob_das_merge_iter.cpp b/src/sql/das/iter/ob_das_merge_iter.cpp index 441170382..1cb197823 100644 --- a/src/sql/das/iter/ob_das_merge_iter.cpp +++ b/src/sql/das/iter/ob_das_merge_iter.cpp @@ -91,7 +91,7 @@ int MergeStoreRows::to_expr(bool is_vectorized, int64_t size) int64_t MergeStoreRows::get_group_idx(int64_t idx) { OB_ASSERT(idx < saved_size_); - return store_rows_[idx].store_row_->cells()[group_id_idx_].get_int(); + return ObNewRange::get_group_idx(store_rows_[idx].store_row_->cells()[group_id_idx_].get_int()); } int64_t MergeStoreRows::cur_group_idx() @@ -111,6 +111,12 @@ int64_t MergeStoreRows::row_cnt_with_cur_group_idx() return end_idx - cur_idx_; } +const ObDatum *MergeStoreRows::cur_datums() +{ + OB_ASSERT(cur_idx_ < saved_size_); + return store_rows_[cur_idx_].store_row_->cells(); +} + void MergeStoreRows::reuse() { cur_idx_ = OB_INVALID_INDEX; @@ -136,7 +142,7 @@ void MergeStoreRows::reset() int ObDASMergeIter::set_merge_status(MergeType merge_type) { int ret = OB_SUCCESS; - merge_type_ = merge_type; + merge_type_ = used_for_keep_order_ ? MergeType::SORT_MERGE : merge_type; if (merge_type == MergeType::SEQUENTIAL_MERGE) { get_next_row_ = &ObDASMergeIter::get_next_seq_row; get_next_rows_ = &ObDASMergeIter::get_next_seq_rows; @@ -270,7 +276,7 @@ int ObDASMergeIter::inner_init(ObDASIterParam ¶m) int ret = OB_SUCCESS; if (param.type_ != ObDASIterType::DAS_ITER_MERGE) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("inner init das iter with bad param type", K(param)); + LOG_WARN("inner init das iter with bad param type", K(param), K(ret)); } else { ObDASMergeIterParam &merge_param = static_cast(param); eval_infos_ = merge_param.eval_infos_; @@ -287,6 +293,8 @@ int ObDASMergeIter::inner_init(ObDASIterParam ¶m) das_ref_->set_expr_frame_info(merge_param.frame_info_); das_ref_->set_execute_directly(merge_param.execute_das_directly_); das_ref_->set_enable_rich_format(merge_param.enable_rich_format_); + used_for_keep_order_ = merge_param.used_for_keep_order_; + merge_type_ = used_for_keep_order_ ? SORT_MERGE : SEQUENTIAL_MERGE; if (group_id_expr_ != nullptr) { for (int64_t i = 0; i < output_->count(); i++) { @@ -589,7 +597,9 @@ int ObDASMergeIter::get_next_sorted_row() LOG_WARN("failed to save store row", K(ret)); } else { merge_state_arr_[i].row_store_have_data_ = true; - compare(i, output_idx); + if (OB_FAIL(compare(i, output_idx))) { + LOG_WARN("failed to compare two rows", K(ret)); + } } } else if (OB_ITER_END == ret) { ret = OB_SUCCESS; @@ -597,8 +607,8 @@ int ObDASMergeIter::get_next_sorted_row() } else { LOG_WARN("das iter failed to get next row", K(ret)); } - } else { - compare(i, output_idx); + } else if (OB_FAIL(compare(i, output_idx))) { + LOG_WARN("failed to compare two rows", K(ret)); } } } // for end @@ -679,7 +689,9 @@ int ObDASMergeIter::get_next_sorted_rows(int64_t &count, int64_t capacity) LOG_WARN("failed to save store row", K(ret)); } else { merge_state_arr_[i].row_store_have_data_ = true; - compare(i, output_idx); + if (OB_FAIL(compare(i, output_idx))) { + LOG_WARN("failed to compare two rows", K(ret)); + } } } else if (OB_ITER_END == ret) { ret = OB_SUCCESS; @@ -689,7 +701,9 @@ int ObDASMergeIter::get_next_sorted_rows(int64_t &count, int64_t capacity) } } } else { - compare(i, output_idx); + if (OB_FAIL(compare(i, output_idx))) { + LOG_WARN("failed to compare two rows", K(ret)); + } } } } @@ -708,7 +722,7 @@ int ObDASMergeIter::get_next_sorted_rows(int64_t &count, int64_t capacity) } } MergeStoreRows &store_rows = merge_store_rows_arr_.at(output_idx); - int64_t ret_count = store_rows.row_cnt_with_cur_group_idx(); + int64_t ret_count = used_for_keep_order_ ? 1 : store_rows.row_cnt_with_cur_group_idx(); ret = store_rows.to_expr(true, ret_count); if (OB_SUCC(ret)) { count = ret_count; @@ -719,7 +733,6 @@ int ObDASMergeIter::get_next_sorted_rows(int64_t &count, int64_t capacity) } } } - return ret; } @@ -770,18 +783,34 @@ int ObDASMergeIter::prepare_sort_merge_info() return ret; } -void ObDASMergeIter::compare(int64_t cur_idx, int64_t &output_idx) + +// [GROUP_ID] is composed of group_idx and index_ordered_idx now, +// we should compare group_idx first and then index_ordered_idx, +// group_idx and index_ordered_idx should always be sorted in ascending order. +int ObDASMergeIter::compare(int64_t cur_idx, int64_t &output_idx) { + int ret = OB_SUCCESS; if (OB_INVALID_INDEX == output_idx) { output_idx = cur_idx; } else { - // compare the values of group_idx. - int64_t output_group_idx = merge_store_rows_arr_[output_idx].cur_group_idx(); - int64_t cur_group_idx = merge_store_rows_arr_[cur_idx].cur_group_idx(); - if (output_group_idx > cur_group_idx) { - output_idx = cur_idx; + const ObDatum *cur_datums = merge_store_rows_arr_[cur_idx].cur_datums(); + const ObDatum *output_datums = merge_store_rows_arr_[output_idx].cur_datums(); + if (nullptr != group_id_expr_) { + int64_t cur_group_idx = ObNewRange::get_group_idx(cur_datums[group_id_idx_].get_int()); + int64_t output_group_idx = ObNewRange::get_group_idx(output_datums[group_id_idx_].get_int()); + if (cur_group_idx != output_group_idx) { + output_idx = cur_group_idx < output_group_idx ? cur_idx : output_idx; + } else { + int64_t cur_order_idx = ObNewRange::get_index_ordered_idx(cur_datums[group_id_idx_].get_int()); + int64_t output_order_idx = ObNewRange::get_index_ordered_idx(output_datums[group_id_idx_].get_int()); + if (cur_order_idx != output_order_idx) { + output_idx = cur_order_idx < output_order_idx ? cur_idx : output_idx; + } + } } } + LOG_DEBUG("das merge iter compare finished", K(cur_idx), K(output_idx), K(used_for_keep_order_)); + return ret; } }//end namespace sql diff --git a/src/sql/das/iter/ob_das_merge_iter.h b/src/sql/das/iter/ob_das_merge_iter.h index 530875434..04df78554 100644 --- a/src/sql/das/iter/ob_das_merge_iter.h +++ b/src/sql/das/iter/ob_das_merge_iter.h @@ -14,6 +14,8 @@ #define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_MERGE_ITER_H_ #include "sql/das/ob_das_utils.h" #include "sql/das/iter/ob_das_iter.h" +#include "sql/das/ob_das_ref.h" +#include "sql/das/ob_das_scan_op.h" namespace oceanbase { @@ -21,9 +23,12 @@ using namespace common; namespace sql { -class ObDASMergeIterParam : public ObDASIterParam +struct ObDASMergeIterParam : public ObDASIterParam { public: + ObDASMergeIterParam() + : ObDASIterParam(ObDASIterType::DAS_ITER_MERGE) + {} ObFixedArray *eval_infos_; bool need_update_partition_id_; ObExpr *pdml_partition_id_; @@ -34,6 +39,7 @@ public: const ObExprFrameInfo *frame_info_; bool execute_das_directly_; bool enable_rich_format_; + bool used_for_keep_order_; virtual bool is_valid() const override { @@ -73,6 +79,8 @@ public: int64_t get_group_idx(int64_t idx); int64_t cur_group_idx(); int64_t row_cnt_with_cur_group_idx(); + + const ObDatum *cur_datums(); void reuse(); void reset(); TO_STRING_KV(K_(saved_size), @@ -93,7 +101,8 @@ class ObDASMergeIter : public ObDASIter { public: ObDASMergeIter() - : wild_datum_info_(), + : ObDASIter(ObDASIterType::DAS_ITER_MERGE), + wild_datum_info_(), merge_type_(SEQUENTIAL_MERGE), eval_infos_(nullptr), need_update_partition_id_(false), @@ -111,11 +120,13 @@ public: group_id_idx_(OB_INVALID_INDEX), need_prepare_sort_merge_info_(false), merge_state_arr_(), - merge_store_rows_arr_() + merge_store_rows_arr_(), + used_for_keep_order_(false) {} virtual ~ObDASMergeIter() {} virtual int set_merge_status(MergeType merge_type) override; + virtual int do_table_scan() override; MergeType get_merge_type() const { return merge_type_; } void set_global_lookup_iter(ObDASMergeIter *global_lookup_iter); INHERIT_TO_STRING_KV("ObDASIter", ObDASIter, K_(merge_type), K_(ref_table_id)); @@ -128,8 +139,6 @@ public: DASTaskIter begin_task_iter(); bool is_all_local_task() const; int rescan_das_task(ObDASScanOp *scan_op); - // do_table_scan() need be called before get_next_row(s). - int do_table_scan(); /********* DAS REF END *********/ protected: @@ -151,7 +160,7 @@ private: int get_next_sorted_row(); int get_next_sorted_rows(int64_t &count, int64_t capacity); int prepare_sort_merge_info(); - void compare(int64_t cur_idx, int64_t &output_idx); + int compare(int64_t cur_idx, int64_t &output_idx); private: @@ -218,6 +227,7 @@ private: typedef common::ObSEArray MergeStoreRowsArray; MergeStateArray merge_state_arr_; MergeStoreRowsArray merge_store_rows_arr_; + bool used_for_keep_order_; /********* SORT MERGE END *********/ }; diff --git a/src/sql/das/iter/ob_das_scan_iter.cpp b/src/sql/das/iter/ob_das_scan_iter.cpp new file mode 100644 index 000000000..4c0b10620 --- /dev/null +++ b/src/sql/das/iter/ob_das_scan_iter.cpp @@ -0,0 +1,141 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_DAS +#include "sql/das/iter/ob_das_scan_iter.h" +#include "sql/das/ob_das_scan_op.h" +#include "storage/tx_storage/ob_access_service.h" + +namespace oceanbase +{ +using namespace common; +namespace sql +{ + +int ObDASScanIter::inner_init(ObDASIterParam ¶m) +{ + int ret = OB_SUCCESS; + if (param.type_ != ObDASIterType::DAS_ITER_SCAN) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("inner init das iter with bad param type", K(param), K(ret)); + } else { + const ObDASScanCtDef *scan_ctdef = (static_cast(param)).scan_ctdef_; + output_ = &scan_ctdef->result_output_; + tsc_service_ = is_virtual_table(scan_ctdef->ref_table_id_) ? GCTX.vt_par_ser_ + : scan_ctdef->is_external_table_ ? GCTX.et_access_service_ + : MTL(ObAccessService *); + } + + return ret; +} + +int ObDASScanIter::inner_reuse() +{ + int ret = OB_SUCCESS; + // NOTE: need_switch_param_ should have been set before call reuse(). + if (OB_ISNULL(scan_param_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr scan param", K(ret)); + } else if (OB_FAIL(tsc_service_->reuse_scan_iter(scan_param_->need_switch_param_, result_))) { + LOG_WARN("failed to reuse storage scan iter", K(ret)); + } else { + scan_param_->key_ranges_.reuse(); + scan_param_->ss_key_ranges_.reuse(); + scan_param_->mbr_filters_.reuse(); + } + return ret; +} + +int ObDASScanIter::inner_release() +{ + int ret = OB_SUCCESS; + if (OB_NOT_NULL(result_)) { + if (OB_FAIL(tsc_service_->revert_scan_iter(result_))) { + LOG_WARN("failed to revert storage scan iter", K(ret)); + } + result_ = nullptr; + } + return ret; +} + +int ObDASScanIter::do_table_scan() +{ + int ret = OB_SUCCESS; + result_ = nullptr; + if (OB_ISNULL(scan_param_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr scan param", K(ret)); + } else if (OB_FAIL(tsc_service_->table_scan(*scan_param_, result_))) { + if (OB_SNAPSHOT_DISCARDED == ret && scan_param_->fb_snapshot_.is_valid()) { + ret = OB_INVALID_QUERY_TIMESTAMP; + } else if (OB_TRY_LOCK_ROW_CONFLICT != ret) { + LOG_WARN("fail to scan table", KPC_(scan_param), K(ret)); + } + } + LOG_DEBUG("das scan iter do table scan", KPC_(scan_param), K(ret)); + + return ret; +} + +int ObDASScanIter::rescan() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(scan_param_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr scan param", K(ret)); + } else if (OB_FAIL(tsc_service_->table_rescan(*scan_param_, result_))) { + LOG_WARN("failed to rescan tablet", K(scan_param_->tablet_id_), K(ret)); + } else { + // reset need_switch_param_ after real rescan. + scan_param_->need_switch_param_ = false; + } + LOG_DEBUG("das scan iter rescan", KPC_(scan_param), K(ret)); + + return ret; +} + +int ObDASScanIter::inner_get_next_row() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(result_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr scan iter", K(ret)); + } else if (OB_FAIL(result_->get_next_row())) { + if (ret != OB_ITER_END) { + LOG_WARN("failed to get next row", K(ret)); + } + } + return ret; +} + +int ObDASScanIter::inner_get_next_rows(int64_t &count, int64_t capacity) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(result_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr scan iter", K(ret)); + } else if (OB_FAIL(result_->get_next_rows(count, capacity))) { + if (ret != OB_ITER_END) { + LOG_WARN("failed to get next row", K(ret)); + } + } + return ret; +} + +void ObDASScanIter::clear_evaluated_flag() +{ + OB_ASSERT(nullptr != scan_param_); + scan_param_->op_->clear_evaluated_flag(); +} + +} // namespace sql +} // namespace oceanbase diff --git a/src/sql/das/iter/ob_das_scan_iter.h b/src/sql/das/iter/ob_das_scan_iter.h new file mode 100644 index 000000000..c3053a41f --- /dev/null +++ b/src/sql/das/iter/ob_das_scan_iter.h @@ -0,0 +1,77 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OBDEV_SRC_SQL_DAS_ITER_OB_DAS_SCAN_ITER_H_ +#define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_SCAN_ITER_H_ + +#include "sql/das/iter/ob_das_iter.h" +namespace oceanbase +{ +using namespace common; +namespace sql +{ + +// DASScanIter is a wrapper class for storage iter, it doesn't require eval_ctx or exprs like other iters. +struct ObDASScanIterParam : public ObDASIterParam +{ +public: + ObDASScanIterParam() + : ObDASIterParam(ObDASIterType::DAS_ITER_SCAN), + scan_ctdef_(nullptr) + {} + const ObDASScanCtDef *scan_ctdef_; + virtual bool is_valid() const override + { + return nullptr != scan_ctdef_; + } +}; + +class ObDASScanIter : public ObDASIter +{ +public: + ObDASScanIter() + : ObDASIter(ObDASIterType::DAS_ITER_SCAN), + tsc_service_(nullptr), + result_(nullptr), + scan_param_(nullptr) + {} + virtual ~ObDASScanIter() {} + common::ObNewRowIterator *&get_output_result_iter() { return result_; } + + void set_scan_param(storage::ObTableScanParam &scan_param) { scan_param_ = &scan_param; } + storage::ObTableScanParam &get_scan_param() { return *scan_param_; } + + virtual int do_table_scan() override; + virtual int rescan() override; + virtual void clear_evaluated_flag() override; + +protected: + virtual int inner_init(ObDASIterParam ¶m) override; + virtual int inner_reuse() override; + virtual int inner_release() override; + virtual int inner_get_next_row() override; + virtual int inner_get_next_rows(int64_t &count, int64_t capacity) override; + +private: + common::ObITabletScan *tsc_service_; + common::ObNewRowIterator *result_; + // must ensure the lifecycle of scan param is longer than scan iter. + storage::ObTableScanParam *scan_param_; +}; + + +} // namespace sql +} // namespace oceanbase + + + +#endif /* OBDEV_SRC_SQL_DAS_ITER_OB_DAS_SCAN_ITER_H_ */ diff --git a/src/sql/das/iter/ob_das_sort_iter.cpp b/src/sql/das/iter/ob_das_sort_iter.cpp new file mode 100644 index 000000000..df34d33f9 --- /dev/null +++ b/src/sql/das/iter/ob_das_sort_iter.cpp @@ -0,0 +1,272 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_DAS +#include "sql/das/iter/ob_das_sort_iter.h" +#include "sql/das/ob_das_attach_define.h" +#include "sql/engine/ob_bit_vector.h" + +namespace oceanbase +{ +using namespace common; +namespace sql +{ + +int ObDASSortIter::inner_init(ObDASIterParam ¶m) +{ + int ret = OB_SUCCESS; + if (param.type_ != ObDASIterType::DAS_ITER_SORT) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("inner init das iter with bad param type", K(param), K(ret)); + } else { + lib::ContextParam context_param; + context_param.set_mem_attr(MTL_ID(), "DASSortIter", ObCtxIds::DEFAULT_CTX_ID) + .set_properties(lib::USE_TL_PAGE_OPTIONAL); + if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(sort_memctx_, context_param))) { + LOG_WARN("failed to create lookup memctx", K(ret)); + } else { + ObDASSortIterParam &sort_param = static_cast(param); + sort_ctdef_ = sort_param.sort_ctdef_; + child_ = sort_param.child_; + // init top-n parameter + if ((nullptr != sort_ctdef_->limit_expr_ || nullptr != sort_ctdef_->offset_expr_) + && sort_param.limit_param_.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected both limit offset expr and limit param", K(ret)); + } else if (sort_param.limit_param_.is_valid()) { + limit_param_ = sort_param.limit_param_; + } else { + if (nullptr != sort_ctdef_->limit_expr_) { + ObDatum *limit_datum = nullptr; + if (OB_FAIL(sort_ctdef_->limit_expr_->eval(*eval_ctx_, limit_datum))) { + LOG_WARN("failed to eval limit expr", K(ret)); + } else { + limit_param_.limit_ = (limit_datum->is_null() || limit_datum->get_int() < 0) ? 0 : limit_datum->get_int(); + } + } + if (nullptr != sort_ctdef_->offset_expr_) { + ObDatum *offset_datum = nullptr; + if (OB_FAIL(sort_ctdef_->offset_expr_->eval(*eval_ctx_, offset_datum))) { + LOG_WARN("failed to eval limit expr", K(ret)); + } else if (offset_datum->is_null()) { + limit_param_.limit_ = 0; + limit_param_.offset_ = 0; + } else { + limit_param_.offset_ = offset_datum->get_int() < 0 ? 0 : offset_datum->get_int(); + } + } + } + + if (OB_SUCC(ret)) { + const bool top_k_overflow = INT64_MAX - limit_param_.offset_ < limit_param_.limit_; + const int64_t top_k = (limit_param_.is_valid() && !top_k_overflow) + ? (limit_param_.limit_ + limit_param_.offset_) : INT64_MAX; + if (OB_FAIL(sort_impl_.init(MTL_ID(), + &sort_ctdef_->sort_collations_, + &sort_ctdef_->sort_cmp_funcs_, + eval_ctx_, + exec_ctx_, + false, // enable encode sort key + false, // is local order + false, // need rewind + 0, // part cnt + top_k, + sort_ctdef_->fetch_with_ties_))) { + LOG_WARN("failed to init sort impl", K(ret)); + } else if (OB_FAIL(append(sort_row_, sort_ctdef_->sort_exprs_))) { + LOG_WARN("failed to append sort exprs", K(ret)); + } else if (OB_FAIL(append_array_no_dup(sort_row_, *child_->get_output()))) { + LOG_WARN("failed to append sort rows", K(ret)); + } + } + } + } + + return ret; +} + +int ObDASSortIter::inner_reuse() +{ + int ret = OB_SUCCESS; + if (OB_NOT_NULL(child_)) { + if (OB_FAIL(child_->reuse())) { + LOG_WARN("failed to reuse child iter", K(ret)); + } + } + if (OB_NOT_NULL(sort_memctx_)) { + sort_memctx_->reset_remain_one_page(); + } + sort_finished_ = false; + output_row_cnt_ = 0; + input_row_cnt_ = 0; + // TODO: check if we can reuse sort impl here + // reset sort impl here since ObSortOpImpl::outputted_rows_cnt_ is not reset in reuse() + sort_impl_.reset(); + fake_skip_ = nullptr; + return ret; +} + +int ObDASSortIter::inner_release() +{ + int ret = OB_SUCCESS; + sort_impl_.reset(); + if (OB_NOT_NULL(sort_memctx_)) { + sort_memctx_->reset_remain_one_page(); + DESTROY_CONTEXT(sort_memctx_); + sort_memctx_ = nullptr; + } + sort_row_.reset(); + return ret; +} + +int ObDASSortIter::do_table_scan() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(child_->do_table_scan())) { + LOG_WARN("failed to do table scan", K(ret)); + } + return ret; +} + +int ObDASSortIter::rescan() +{ + int ret = OB_SUCCESS; + const bool top_k_overflow = INT64_MAX - limit_param_.offset_ < limit_param_.limit_; + const int64_t top_k = (limit_param_.is_valid() && !top_k_overflow) + ? (limit_param_.limit_ + limit_param_.offset_) : INT64_MAX; + if (OB_FAIL(child_->rescan())) { + LOG_WARN("failed to rescan child", K(ret)); + } else if (OB_FAIL(sort_impl_.init(MTL_ID(), + &sort_ctdef_->sort_collations_, + &sort_ctdef_->sort_cmp_funcs_, + eval_ctx_, + exec_ctx_, + false, // enable encode sort key + false, // is local order + false, // need rewind + 0, // part cnt + top_k, + sort_ctdef_->fetch_with_ties_))) { + LOG_WARN("failed to init sort impl", K(ret)); + } + return ret; +} + +int ObDASSortIter::inner_get_next_row() +{ + int ret = OB_SUCCESS; + if (limit_param_.limit_ > 0 && output_row_cnt_ >= limit_param_.limit_) { + ret = OB_ITER_END; + LOG_DEBUG("das sort iter got enough rows", K_(limit_param), K_(output_row_cnt), K_(input_row_cnt), K(ret)); + } else if (!sort_finished_ && OB_FAIL(do_sort(false))) { + LOG_WARN("failed to do sort", K(ret)); + } else { + bool got_row = false; + while (OB_SUCC(ret) && !got_row) { + if (OB_FAIL(sort_impl_.get_next_row(sort_row_))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next row from sort impl", K(ret)); + } + } else { + ++input_row_cnt_; + if (input_row_cnt_ > limit_param_.offset_) { + got_row = true; + ++output_row_cnt_; + } + } + } + } + return ret; +} + +int ObDASSortIter::inner_get_next_rows(int64_t &count, int64_t capacity) +{ + int ret = OB_SUCCESS; + if (limit_param_.limit_ > 0 && output_row_cnt_ >= limit_param_.limit_) { + ret = OB_ITER_END; + LOG_DEBUG("das sort iter got enough rows", K_(limit_param), K_(output_row_cnt), K_(input_row_cnt), K(ret)); + } else if (!sort_finished_ && OB_FAIL(do_sort(true))) { + LOG_WARN("failed to do sort", K(ret)); + } else { + bool got_rows = false; + // TODO: @bingfan use vectorized interface instead. + while (OB_SUCC(ret) && !got_rows) { + if (OB_FAIL(sort_impl_.get_next_row(sort_row_))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next row from sort impl", K(ret)); + } + } else { + ++input_row_cnt_; + if (input_row_cnt_ > limit_param_.offset_) { + got_rows = true; + count = 1; + ++output_row_cnt_; + } + } + } + } + return ret; +} + +int ObDASSortIter::do_sort(bool is_vectorized) +{ + int ret = OB_SUCCESS; + if (OB_LIKELY(is_vectorized)) { + int64_t read_size = 0; + fake_skip_ = to_bit_vector(sort_memctx_->get_arena_allocator().alloc(ObBitVector::memory_size(max_size_))); + fake_skip_->init(max_size_); + while (OB_SUCC(ret)) { + read_size = 0; + if (OB_FAIL(child_->get_next_rows(read_size, max_size_))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed ro get next rows from child iter", K(ret)); + } else if (OB_FAIL(sort_impl_.add_batch(sort_row_, *fake_skip_, read_size, 0, nullptr))) { + LOG_WARN("failed to add batch to sort impl", K(ret)); + } else { + ret = OB_ITER_END; + } + } else if (OB_FAIL(sort_impl_.add_batch(sort_row_, *fake_skip_, read_size, 0, nullptr))) { + LOG_WARN("failed to add batch to sort impl", K(ret)); + } + } + } else { + while (OB_SUCC(ret)) { + if (OB_FAIL(child_->get_next_row())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed ro get next rows from child iter", K(ret)); + } + } else if (OB_FAIL(sort_impl_.add_row(sort_row_))) { + LOG_WARN("failed to add row to sort impl", K(ret)); + } + } + } + + if (OB_LIKELY(OB_ITER_END == ret)) { + ret = OB_SUCCESS; + if (OB_FAIL(sort_impl_.sort())) { + LOG_WARN("failed to do sort", K(ret)); + } else { + sort_finished_ = true; + } + } + + return ret; +} + +void ObDASSortIter::clear_evaluated_flag() +{ + OB_ASSERT(nullptr != child_); + child_->clear_evaluated_flag(); +} + +} // namespace sql +} // namespace oceanbase diff --git a/src/sql/das/iter/ob_das_sort_iter.h b/src/sql/das/iter/ob_das_sort_iter.h new file mode 100644 index 000000000..c9ca621c3 --- /dev/null +++ b/src/sql/das/iter/ob_das_sort_iter.h @@ -0,0 +1,95 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OBDEV_SRC_SQL_DAS_ITER_OB_DAS_SORT_ITER_H_ +#define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_SORT_ITER_H_ + +#include "sql/das/iter/ob_das_iter.h" +#include "sql/engine/sort/ob_sort_op_impl.h" + +namespace oceanbase +{ +using namespace common; +namespace sql +{ + +struct ObDASSortIterParam : public ObDASIterParam +{ +public: + ObDASSortIterParam() + : ObDASIterParam(ObDASIterType::DAS_ITER_SORT), + sort_ctdef_(nullptr), + child_(nullptr), + limit_param_() {} + virtual ~ObDASSortIterParam() {} + const ObDASSortCtDef *sort_ctdef_; + ObDASIter *child_; + common::ObLimitParam limit_param_; + + virtual bool is_valid() const override + { + return ObDASIterParam::is_valid() && nullptr != sort_ctdef_ && nullptr != child_; + } +}; + +class ObDASSortIter : public ObDASIter +{ +public: + ObDASSortIter() + : ObDASIter(ObDASIterType::DAS_ITER_SORT), + sort_impl_(), + sort_memctx_(), + sort_ctdef_(nullptr), + sort_row_(), + child_(nullptr), + sort_finished_(false), + limit_param_(), + input_row_cnt_(0), + output_row_cnt_(0), + fake_skip_(nullptr) + {} + virtual ~ObDASSortIter() {} + + virtual int do_table_scan() override; + virtual int rescan() override; + virtual void clear_evaluated_flag() override; + +protected: + virtual int inner_init(ObDASIterParam ¶m) override; + virtual int inner_reuse() override; + virtual int inner_release() override; + virtual int inner_get_next_row() override; + virtual int inner_get_next_rows(int64_t &count, int64_t capacity) override; + +private: + int do_sort(bool is_vectorized); + +private: + ObSortOpImpl sort_impl_; + lib::MemoryContext sort_memctx_; + const ObDASSortCtDef *sort_ctdef_; + ObSEArray sort_row_; + ObDASIter *child_; + bool sort_finished_; + // limit param was set only once at do_table_scan of TSC, which means it should not be reset at reuse, + // input row cnt and output row cnt are the same as well. + common::ObLimitParam limit_param_; + int64_t input_row_cnt_; + int64_t output_row_cnt_; + ObBitVector *fake_skip_; +}; + + +} // namespace sql +} // namespace oceanbase + +#endif /* OBDEV_SRC_SQL_DAS_ITER_OB_DAS_SORT_ITER_H_ */ diff --git a/src/storage/fts/ob_text_retrieval_iterator.cpp b/src/sql/das/iter/ob_das_text_retrieval_iter.cpp similarity index 54% rename from src/storage/fts/ob_text_retrieval_iterator.cpp rename to src/sql/das/iter/ob_das_text_retrieval_iter.cpp index df9dbfe23..373814be0 100644 --- a/src/storage/fts/ob_text_retrieval_iterator.cpp +++ b/src/sql/das/iter/ob_das_text_retrieval_iter.cpp @@ -10,86 +10,33 @@ * See the Mulan PubL v2 for more details. */ -#define USING_LOG_PREFIX STORAGE_FTS - +#define USING_LOG_PREFIX SQL_DAS +#include "ob_das_text_retrieval_iter.h" +#include "ob_das_scan_iter.h" +#include "sql/das/ob_das_ir_define.h" #include "sql/engine/expr/ob_expr_bm25.h" -#include "sql/das/ob_text_retrieval_op.h" -#include "storage/fts/ob_text_retrieval_iterator.h" -#include "storage/tx_storage/ob_access_service.h" namespace oceanbase { -namespace storage +namespace sql { -bool ObTokenRetrievalParam::need_relevance() const -{ - OB_ASSERT(nullptr != ir_ctdef_); - return ir_ctdef_->need_calc_relevance(); -} -const share::ObLSID &ObTokenRetrievalParam::get_ls_id() const -{ - return ls_id_; -} - -const sql::ObDASIRScanCtDef *ObTokenRetrievalParam::get_ir_ctdef() const -{ - return ir_ctdef_; -} - -sql::ObDASIRScanRtDef *ObTokenRetrievalParam::get_ir_rtdef() -{ - return ir_rtdef_; -} -const sql::ObDASScanCtDef *ObTokenRetrievalParam::get_inv_idx_scan_ctdef() const -{ - OB_ASSERT(nullptr != ir_ctdef_); - return ir_ctdef_->get_inv_idx_scan_ctdef(); -} - -const sql::ObDASScanCtDef *ObTokenRetrievalParam::get_inv_idx_agg_ctdef() const -{ - OB_ASSERT(nullptr != ir_ctdef_); - return ir_ctdef_->get_inv_idx_agg_ctdef(); -} - -const sql::ObDASScanCtDef *ObTokenRetrievalParam::get_fwd_idx_agg_ctdef() const -{ - OB_ASSERT(nullptr != ir_ctdef_); - return ir_ctdef_->get_fwd_idx_agg_ctdef(); -} - -const sql::ObDASScanCtDef *ObTokenRetrievalParam::get_doc_id_idx_agg_ctdef() const -{ - OB_ASSERT(nullptr != ir_ctdef_); - return ir_ctdef_->get_doc_id_idx_agg_ctdef(); -} - -const common::ObTabletID &ObTokenRetrievalParam::get_inv_idx_tablet_id() const -{ - return inv_idx_tablet_id_; -} -const common::ObTabletID &ObTokenRetrievalParam::get_fwd_idx_tablet_id() const -{ - return fwd_idx_tablet_id_; -} - -const common::ObTabletID &ObTokenRetrievalParam::get_doc_id_idx_tablet_id() const -{ - return doc_id_idx_tablet_id_; -} - -ObTextRetrievalIterator::ObTextRetrievalIterator() - : ObNewRowIterator(), +ObDASTextRetrievalIter::ObDASTextRetrievalIter() + : ObDASIter(ObDASIterType::DAS_ITER_TEXT_RETRIEVAL), mem_context_(nullptr), - retrieval_param_(nullptr), + ir_ctdef_(nullptr), + ir_rtdef_(nullptr), tx_desc_(nullptr), snapshot_(nullptr), + ls_id_(), + inv_idx_tablet_id_(), + fwd_idx_tablet_id_(), inv_idx_scan_param_(), inv_idx_agg_param_(), fwd_idx_scan_param_(), calc_exprs_(), - inverted_idx_iter_(nullptr), + inverted_idx_scan_iter_(nullptr), + inverted_idx_agg_iter_(nullptr), forward_idx_iter_(nullptr), fwd_range_objs_(nullptr), doc_token_cnt_expr_(nullptr), @@ -97,34 +44,59 @@ ObTextRetrievalIterator::ObTextRetrievalIterator() need_fwd_idx_agg_(false), need_inv_idx_agg_(false), inv_idx_agg_evaluated_(false), + not_first_fwd_agg_(false), is_inited_(false) { } -ObTextRetrievalIterator::~ObTextRetrievalIterator() +int ObDASTextRetrievalIter::set_query_token(const ObString &query_token) { - reset(); + int ret = OB_SUCCESS; + ObNewRange inv_idx_scan_range; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("text retrieval iter not inited", K(ret)); + } else if (OB_UNLIKELY(!inv_idx_scan_param_.key_ranges_.empty() || + (need_inv_idx_agg_ && !inv_idx_agg_param_.key_ranges_.empty()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected set query token with not null query range", K(ret), K(query_token), + K(inv_idx_scan_param_.key_ranges_), K_(need_inv_idx_agg), K(inv_idx_agg_param_.key_ranges_)); + } else if (OB_FAIL(gen_inv_idx_scan_range(query_token, inv_idx_scan_range))) { + LOG_WARN("failed to generate inverted index scan range", K(ret), K(query_token)); + } else if (OB_FAIL(inv_idx_scan_param_.key_ranges_.push_back(inv_idx_scan_range))) { + LOG_WARN("failed to add scan range for inv idx scan", K(ret)); + } else if (need_inv_idx_agg_ && OB_FAIL(inv_idx_agg_param_.key_ranges_.push_back(inv_idx_scan_range))) { + LOG_WARN("failed to add scan range for inv idx agg", K(ret)); + } + return ret; } -int ObTextRetrievalIterator::init( - ObTokenRetrievalParam &retrieval_param, - const ObString &query_token, - transaction::ObTxDesc *tx_desc, - transaction::ObTxReadSnapshot *snapshot) +int ObDASTextRetrievalIter::inner_init(ObDASIterParam ¶m) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; - LOG_WARN("double initialization", K(ret), KPC(this)); - } else if (OB_UNLIKELY(nullptr == tx_desc || nullptr == snapshot || !tx_desc->is_valid())) { + LOG_WARN("double initialization", K(ret)); + } else if (OB_UNLIKELY(ObDASIterType::DAS_ITER_TEXT_RETRIEVAL != param.type_)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), KPC(tx_desc), KPC(snapshot)); + LOG_WARN("invalid das iter param type for text retrieval iter", K(ret), K(param)); } else { - retrieval_param_ = &retrieval_param; - tx_desc_ = tx_desc; - snapshot_ = snapshot; - need_fwd_idx_agg_ = retrieval_param.get_ir_ctdef()->has_fwd_agg_ && retrieval_param.need_relevance(); - need_inv_idx_agg_ = retrieval_param.need_relevance(); + ObDASTextRetrievalIterParam &retrieval_param = static_cast(param); + inverted_idx_scan_iter_ = static_cast(retrieval_param.inv_idx_scan_iter_); + ir_ctdef_ = retrieval_param.ir_ctdef_; + ir_rtdef_ = retrieval_param.ir_rtdef_; + tx_desc_ = retrieval_param.tx_desc_; + snapshot_ = retrieval_param.snapshot_; + need_fwd_idx_agg_ = ir_ctdef_->need_fwd_idx_agg(); + need_inv_idx_agg_ = ir_ctdef_->need_inv_idx_agg(); + + if (need_inv_idx_agg_) { + inverted_idx_agg_iter_ = static_cast(retrieval_param.inv_idx_agg_iter_); + } + + if (need_fwd_idx_agg_) { + forward_idx_iter_ = static_cast(retrieval_param.fwd_idx_iter_); + } if (OB_ISNULL(mem_context_)) { lib::ContextParam param; @@ -134,10 +106,7 @@ int ObTextRetrievalIterator::init( } } - if (FAILEDx(init_inv_idx_scan_param(query_token))) { - LOG_WARN("failed to init inverted index scan param", K(ret), K_(inv_idx_scan_param), K_(inv_idx_agg_param)); - } else if (need_fwd_idx_agg_ && OB_FAIL(init_fwd_idx_scan_param())) { - LOG_WARN("failed to init forward index scan param", K(ret), K_(fwd_idx_scan_param)); + if (OB_FAIL(ret)) { } else if (OB_FAIL(init_calc_exprs())) { LOG_WARN("failed to init row-wise calc exprs", K(ret)); } else { @@ -147,94 +116,148 @@ int ObTextRetrievalIterator::init( return ret; } -void ObTextRetrievalIterator::reset() +int ObDASTextRetrievalIter::inner_reuse() { int ret = OB_SUCCESS; - ObAccessService *tsc_service = MTL(ObAccessService *); - if (nullptr == tsc_service) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("failed to get access service when reset text retrieval iterator", K(ret)); + if (nullptr != mem_context_) { + mem_context_->reset_remain_one_page(); + } + inv_idx_agg_evaluated_ = false; + const ObTabletID &old_inv_scan_id = inv_idx_scan_param_.tablet_id_; + inverted_idx_scan_iter_->set_scan_param(inv_idx_scan_param_); + inv_idx_scan_param_.need_switch_param_ = inv_idx_scan_param_.need_switch_param_ || + ((old_inv_scan_id.is_valid() && old_inv_scan_id != inv_idx_tablet_id_) ? true : false); + if (OB_FAIL(inverted_idx_scan_iter_->reuse())) { + LOG_WARN("failed to reuse inverted index iter", K(ret)); } else { - if (nullptr != inverted_idx_iter_) { - if (OB_FAIL(tsc_service->revert_scan_iter(inverted_idx_iter_))) { - LOG_ERROR("failed to revert inverted index iter", K(ret)); + if (need_inv_idx_agg_) { + const ObTabletID &old_inv_agg_id = inv_idx_agg_param_.tablet_id_; + inverted_idx_agg_iter_->set_scan_param(inv_idx_agg_param_); + inv_idx_agg_param_.need_switch_param_ = inv_idx_agg_param_.need_switch_param_ || + ((old_inv_agg_id.is_valid() && old_inv_agg_id != inv_idx_tablet_id_) ? true : false); + if (OB_FAIL(inverted_idx_agg_iter_->reuse())) { + LOG_WARN("failed to reuse inverted index agg iter", K(ret)); } - inverted_idx_iter_ = nullptr; } - if (nullptr != forward_idx_iter_) { - if (OB_FAIL(tsc_service->revert_scan_iter(forward_idx_iter_))) { - LOG_ERROR("failed to revert forward index iter", K(ret)); + + if (OB_SUCC(ret) && need_fwd_idx_agg_) { + const ObTabletID &old_fwd_agg_id = fwd_idx_scan_param_.tablet_id_; + forward_idx_iter_->set_scan_param(fwd_idx_scan_param_); + fwd_idx_scan_param_.need_switch_param_ = fwd_idx_scan_param_.need_switch_param_ || + ((old_fwd_agg_id.is_valid() && old_fwd_agg_id != fwd_idx_tablet_id_) ? true : false); + if (OB_FAIL(forward_idx_iter_->reuse())) { + LOG_WARN("failed to reuse forward index iter", K(ret)); } - forward_idx_iter_ = nullptr; } } - inv_idx_scan_param_.need_switch_param_ = false; - inv_idx_scan_param_.destroy_schema_guard(); - inv_idx_agg_param_.need_switch_param_ = false; - inv_idx_agg_param_.destroy_schema_guard(); - fwd_idx_scan_param_.need_switch_param_ = false; - fwd_idx_scan_param_.destroy_schema_guard(); - calc_exprs_.reset(); + if (OB_SUCC(ret)) { + inv_idx_agg_evaluated_ = false; + } + + return ret; +} + +int ObDASTextRetrievalIter::inner_release() +{ + int ret = OB_SUCCESS; + + inv_idx_scan_param_.destroy_schema_guard(); + inv_idx_scan_param_.snapshot_.reset(); + inv_idx_scan_param_.destroy(); + inv_idx_agg_param_.destroy_schema_guard(); + inv_idx_agg_param_.snapshot_.reset(); + inv_idx_agg_param_.destroy(); + fwd_idx_scan_param_.destroy_schema_guard(); + fwd_idx_scan_param_.snapshot_.reset(); + fwd_idx_scan_param_.destroy(); + calc_exprs_.reset(); if (nullptr != mem_context_) { mem_context_->reset_remain_one_page(); DESTROY_CONTEXT(mem_context_); mem_context_ = nullptr; } + ir_ctdef_ = nullptr; + ir_rtdef_ = nullptr; + inverted_idx_scan_iter_ = nullptr; + inverted_idx_agg_iter_ = nullptr; + forward_idx_iter_ = nullptr; fwd_range_objs_ = nullptr; doc_token_cnt_expr_ = nullptr; - retrieval_param_ = nullptr; tx_desc_ = nullptr; snapshot_ = nullptr; token_doc_cnt_ = 0; need_fwd_idx_agg_ = false; need_inv_idx_agg_ = false; inv_idx_agg_evaluated_ = false; + not_first_fwd_agg_ = false; is_inited_ = false; + + return ret; } -int ObTextRetrievalIterator::get_next_row(ObNewRow *&row) -{ - UNUSED(row); - return OB_NOT_IMPLEMENT; -} - -int ObTextRetrievalIterator::get_next_row() +int ObDASTextRetrievalIter::do_table_scan() +{ + int ret = OB_SUCCESS; + inverted_idx_scan_iter_->set_scan_param(inv_idx_scan_param_); + if (need_inv_idx_agg_) { + inverted_idx_agg_iter_->set_scan_param(inv_idx_agg_param_); + } + + if (OB_FAIL(init_inv_idx_scan_param())) { + LOG_WARN("failed to init inv idx scan param", K(ret)); + } else if (OB_FAIL(inverted_idx_scan_iter_->do_table_scan())) { + LOG_WARN("failed to do inverted index table scan", K(ret)); + } else if (need_inv_idx_agg_ && OB_FAIL(inverted_idx_agg_iter_->do_table_scan())) { + LOG_WARN("failed to do inverted index agg", K(ret)); + } + return ret; +} + +int ObDASTextRetrievalIter::rescan() +{ + int ret = OB_SUCCESS; + + inv_idx_scan_param_.tablet_id_ = inv_idx_tablet_id_; + inv_idx_scan_param_.ls_id_ = ls_id_; + if (need_inv_idx_agg_) { + inv_idx_agg_param_.tablet_id_ = inv_idx_tablet_id_; + inv_idx_agg_param_.ls_id_ = ls_id_; + } + if (OB_FAIL(inverted_idx_scan_iter_->rescan())) { + LOG_WARN("failed to rescan inverted scan iter", K(ret)); + } else if (need_inv_idx_agg_ && OB_FAIL(inverted_idx_agg_iter_->rescan())) { + LOG_WARN("failed to rescan inverted index agg iter", K(ret)); + } + return ret; +} + +int ObDASTextRetrievalIter::inner_get_next_row() { int ret = OB_SUCCESS; - ObAccessService *tsc_service = MTL(ObAccessService *); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("retrieval iterator not inited", K(ret)); - } else if (!inv_idx_agg_evaluated_ && retrieval_param_->need_relevance()) { + } else if (!inv_idx_agg_evaluated_ && need_inv_idx_agg_) { if (OB_FAIL(do_doc_cnt_agg())) { if (OB_UNLIKELY(OB_ITER_END != ret)) { LOG_WARN("Fail to do document count aggregation", K(ret), K_(inv_idx_agg_param)); } - } else if (OB_FAIL(tsc_service->revert_scan_iter(inverted_idx_iter_))) { - LOG_WARN("Fail to revert inverted index scan iterator after count aggregation", K(ret)); } else { - inverted_idx_iter_ = nullptr; inv_idx_agg_evaluated_ = true; } } - if (OB_SUCC(ret) && nullptr == inverted_idx_iter_) { - if (OB_FAIL(tsc_service->table_scan(inv_idx_scan_param_, inverted_idx_iter_))) { - LOG_WARN("failed to init inverted index scan iterator", K(ret)); - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(get_next_single_row(inv_idx_scan_param_.op_->is_vectorized(), inverted_idx_iter_))) { + } else if (OB_FAIL(get_next_single_row(inv_idx_scan_param_.op_->is_vectorized(), inverted_idx_scan_iter_))) { if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("failed to get next row from inverted index", K(ret), K_(inv_idx_scan_param), KPC_(inverted_idx_iter)); + LOG_WARN("failed to get next row from inverted index", K(ret), K_(inv_idx_scan_param), KPC_(inverted_idx_scan_iter)); } } else { LOG_DEBUG("get one invert index scan row", "row", - ROWEXPR2STR(*retrieval_param_->get_ir_rtdef()->get_inv_idx_scan_rtdef()->eval_ctx_, + ROWEXPR2STR(*ir_rtdef_->get_inv_idx_scan_rtdef()->eval_ctx_, *inv_idx_scan_param_.output_exprs_)); - if (retrieval_param_->need_relevance()) { + if (ir_ctdef_->need_calc_relevance()) { clear_row_wise_evaluated_flag(); if (OB_FAIL(get_next_doc_token_cnt(need_fwd_idx_agg_))) { LOG_WARN("failed to get next doc token count", K(ret)); @@ -248,67 +271,39 @@ int ObTextRetrievalIterator::get_next_row() return ret; } -int ObTextRetrievalIterator::get_next_rows(int64_t &count, int64_t capacity) +int ObDASTextRetrievalIter::inner_get_next_rows(int64_t &count, int64_t capacity) { UNUSEDx(count, capacity); return OB_NOT_IMPLEMENT; } -int ObTextRetrievalIterator::get_curr_iter_row( - const sql::ExprFixedArray *&curr_row, - sql::ObEvalCtx *&curr_eval_ctx) -{ - UNUSEDx(curr_row, curr_eval_ctx); - return OB_NOT_IMPLEMENT; -} - -int ObTextRetrievalIterator::get_curr_doc_id() -{ - return OB_NOT_IMPLEMENT; -} - -int ObTextRetrievalIterator::forward_to_doc(const ObDocId &doc_id) -{ - UNUSED(doc_id); - return OB_NOT_IMPLEMENT; -} - -int ObTextRetrievalIterator::init_inv_idx_scan_param(const ObString &query_token) +int ObDASTextRetrievalIter::init_inv_idx_scan_param() { int ret = OB_SUCCESS; - ObNewRange inv_idx_scan_range; - if (OB_FAIL(gen_inv_idx_scan_range(query_token, inv_idx_scan_range))) { - LOG_WARN("failed to generate inverted index scan range", K(ret), K(query_token)); - } else if (OB_FAIL(init_base_idx_scan_param( - retrieval_param_->get_ls_id(), - retrieval_param_->get_inv_idx_tablet_id(), - retrieval_param_->get_inv_idx_scan_ctdef(), - retrieval_param_->get_ir_rtdef()->get_inv_idx_scan_rtdef(), + if (OB_FAIL(init_base_idx_scan_param( + ls_id_, + inv_idx_tablet_id_, + ir_ctdef_->get_inv_idx_scan_ctdef(), + ir_rtdef_->get_inv_idx_scan_rtdef(), tx_desc_, snapshot_, inv_idx_scan_param_))) { - LOG_WARN("fail to init inverted index scan param", K(ret), KPC_(retrieval_param)); - } else if (OB_FAIL(inv_idx_scan_param_.key_ranges_.push_back(inv_idx_scan_range))) { - LOG_WARN("failed to append scan range", K(ret)); - } - - if (OB_SUCC(ret) && need_inv_idx_agg_) { + LOG_WARN("fail to init inverted index scan param", K(ret), KPC_(ir_ctdef)); + } else if (need_inv_idx_agg_) { if (OB_FAIL(init_base_idx_scan_param( - retrieval_param_->get_ls_id(), - retrieval_param_->get_inv_idx_tablet_id(), - retrieval_param_->get_inv_idx_agg_ctdef(), - retrieval_param_->get_ir_rtdef()->get_inv_idx_agg_rtdef(), + ls_id_, + inv_idx_tablet_id_, + ir_ctdef_->get_inv_idx_agg_ctdef(), + ir_rtdef_->get_inv_idx_agg_rtdef(), tx_desc_, snapshot_, inv_idx_agg_param_))) { - LOG_WARN("fail to init inverted index count aggregate param", K(ret), KPC_(retrieval_param)); - } else if (OB_FAIL(inv_idx_agg_param_.key_ranges_.push_back(inv_idx_scan_range))) { - LOG_WARN("failed to append scan range", K(ret)); + LOG_WARN("fail to init inverted index count aggregate param", K(ret), KPC_(ir_ctdef)); } else { if (OB_UNLIKELY(!static_cast( - retrieval_param_->get_inv_idx_agg_ctdef()->pd_expr_spec_.pd_storage_flag_).is_aggregate_pushdown())) { + ir_ctdef_->get_inv_idx_agg_ctdef()->pd_expr_spec_.pd_storage_flag_).is_aggregate_pushdown())) { ret = OB_NOT_IMPLEMENT; - LOG_ERROR("not pushdown aggregate not supported", K(ret), K_(retrieval_param)); + LOG_ERROR("not pushdown aggregate not supported", K(ret), KPC(ir_ctdef_->get_inv_idx_agg_ctdef())); } } } @@ -316,25 +311,25 @@ int ObTextRetrievalIterator::init_inv_idx_scan_param(const ObString &query_token return ret; } -int ObTextRetrievalIterator::init_fwd_idx_scan_param() +int ObDASTextRetrievalIter::init_fwd_idx_scan_param() { int ret = OB_SUCCESS; - if (!retrieval_param_->need_relevance()) { + if (!ir_ctdef_->need_calc_relevance()) { } else if (OB_FAIL(init_base_idx_scan_param( - retrieval_param_->get_ls_id(), - retrieval_param_->get_fwd_idx_tablet_id(), - retrieval_param_->get_fwd_idx_agg_ctdef(), - retrieval_param_->get_ir_rtdef()->get_fwd_idx_agg_rtdef(), + ls_id_, + fwd_idx_tablet_id_, + ir_ctdef_->get_fwd_idx_agg_ctdef(), + ir_rtdef_->get_fwd_idx_agg_rtdef(), tx_desc_, snapshot_, fwd_idx_scan_param_))) { - LOG_WARN("Fail to init foward index scan param", K(ret), KPC_(retrieval_param)); + LOG_WARN("Fail to init foward index scan param", K(ret), KPC_(ir_ctdef)); } return ret; } -int ObTextRetrievalIterator::init_base_idx_scan_param( +int ObDASTextRetrievalIter::init_base_idx_scan_param( const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const sql::ObDASScanCtDef *ctdef, @@ -403,42 +398,30 @@ int ObTextRetrievalIterator::init_base_idx_scan_param( return ret; } -int ObTextRetrievalIterator::do_doc_cnt_agg() +int ObDASTextRetrievalIter::do_doc_cnt_agg() { int ret = OB_SUCCESS; - ObAccessService *tsc_service = MTL(ObAccessService *); - if (OB_ISNULL(tsc_service)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get table access service", K(ret)); - } else if (OB_FAIL(tsc_service->table_scan(inv_idx_agg_param_, inverted_idx_iter_))) { - if (OB_SNAPSHOT_DISCARDED == ret && inv_idx_agg_param_.fb_snapshot_.is_valid()) { - ret = OB_INVALID_QUERY_TIMESTAMP; - } else if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to do table scan for document count aggregation", K(ret)); + if (OB_UNLIKELY(!static_cast(inv_idx_agg_param_.pd_storage_flag_).is_aggregate_pushdown())) { + ret = OB_NOT_SUPPORTED; + LOG_ERROR("aggregate without pushdown not supported", K(ret)); + } else if (OB_FAIL(get_next_single_row(inv_idx_agg_param_.op_->is_vectorized(), inverted_idx_agg_iter_))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get aggregated row from iter", K(ret)); } } else { - if (OB_UNLIKELY(!static_cast(inv_idx_agg_param_.pd_storage_flag_).is_aggregate_pushdown())) { - ret = OB_NOT_SUPPORTED; - LOG_ERROR("aggregate without pushdown not supported", K(ret)); - } else if (OB_FAIL(get_next_single_row(inv_idx_agg_param_.op_->is_vectorized(), inverted_idx_iter_))) { - if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("failed to get aggregated row from iter", K(ret)); - } + const sql::ObExpr *inv_idx_agg_expr = inv_idx_agg_param_.aggregate_exprs_->at(0); + sql::ObEvalCtx *eval_ctx = ir_rtdef_->get_inv_idx_agg_rtdef()->eval_ctx_; + ObDatum *doc_cnt_datum = nullptr; + if (OB_FAIL(inv_idx_agg_expr->eval(*eval_ctx, doc_cnt_datum))) { + LOG_WARN("failed to evaluate aggregated expr", K(ret)); } else { - const sql::ObExpr *inv_idx_agg_expr = inv_idx_agg_param_.aggregate_exprs_->at(0); - sql::ObEvalCtx *eval_ctx = retrieval_param_->get_ir_rtdef()->get_inv_idx_agg_rtdef()->eval_ctx_; - ObDatum *doc_cnt_datum = nullptr; - if (OB_FAIL(inv_idx_agg_expr->eval(*eval_ctx, doc_cnt_datum))) { - LOG_WARN("failed to evaluate aggregated expr", K(ret)); - } else { - token_doc_cnt_ = doc_cnt_datum->get_int(); - } + token_doc_cnt_ = doc_cnt_datum->get_int(); } } return ret; } -int ObTextRetrievalIterator::get_next_doc_token_cnt(const bool use_fwd_idx_agg) +int ObDASTextRetrievalIter::get_next_doc_token_cnt(const bool use_fwd_idx_agg) { int ret = OB_SUCCESS; if (use_fwd_idx_agg) { @@ -457,11 +440,11 @@ int ObTextRetrievalIterator::get_next_doc_token_cnt(const bool use_fwd_idx_agg) return ret; } -int ObTextRetrievalIterator::get_inv_idx_scan_doc_id(ObDocId &doc_id) +int ObDASTextRetrievalIter::get_inv_idx_scan_doc_id(ObDocId &doc_id) { int ret = OB_SUCCESS; - sql::ObExpr *doc_id_expr = retrieval_param_->get_ir_ctdef()->inv_scan_doc_id_col_; - sql::ObEvalCtx *eval_ctx = retrieval_param_->get_ir_rtdef()->get_inv_idx_scan_rtdef()->eval_ctx_; + sql::ObExpr *doc_id_expr = ir_ctdef_->inv_scan_doc_id_col_; + sql::ObEvalCtx *eval_ctx = ir_rtdef_->get_inv_idx_scan_rtdef()->eval_ctx_; ObDatum &doc_id_datum = doc_id_expr->locate_expr_datum(*eval_ctx); if (OB_FAIL(doc_id.from_string(doc_id_datum.get_string()))) { LOG_WARN("failed to get ObDocId from datum", K(ret)); @@ -470,59 +453,51 @@ int ObTextRetrievalIterator::get_inv_idx_scan_doc_id(ObDocId &doc_id) return ret; } -int ObTextRetrievalIterator::do_token_cnt_agg(const ObDocId &doc_id, int64_t &token_count) +int ObDASTextRetrievalIter::do_token_cnt_agg(const ObDocId &doc_id, int64_t &token_count) { int ret = OB_SUCCESS; token_count = 0; ObNewRange scan_range; - if (OB_FAIL(reuse_fwd_idx_iter())) { - LOG_WARN("failed to reuse forward index iterator", K(ret)); - } else if (OB_UNLIKELY(!fwd_idx_scan_param_.key_ranges_.empty())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected non empty forward index scan range", K(ret)); - } else if (OB_FAIL(gen_fwd_idx_scan_range(doc_id, scan_range))) { + if (OB_FAIL(gen_fwd_idx_scan_range(doc_id, scan_range))) { LOG_WARN("failed to generate forward index scan range", K(ret)); - } else if (OB_FAIL(fwd_idx_scan_param_.key_ranges_.push_back(scan_range))) { - LOG_WARN("failed to add forward index scan range", K(ret), K(scan_range)); } if (OB_SUCC(ret)) { - ObAccessService *tsc_service = MTL(ObAccessService *); - if (OB_ISNULL(tsc_service)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get table access service", K(ret)); - } else if (nullptr == forward_idx_iter_) { - if (OB_FAIL(tsc_service->table_scan(fwd_idx_scan_param_, forward_idx_iter_))) { - if (OB_SNAPSHOT_DISCARDED == ret && fwd_idx_scan_param_.fb_snapshot_.is_valid()) { - ret = OB_INVALID_QUERY_TIMESTAMP; - } else if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("failed to init forward index scan iterator", K(ret), K_(fwd_idx_scan_param)); - } + if (not_first_fwd_agg_) { + fwd_idx_scan_param_.tablet_id_ = fwd_idx_tablet_id_; + fwd_idx_scan_param_.ls_id_ = ls_id_; + if (OB_FAIL(reuse_fwd_idx_iter())) { + LOG_WARN("failed to reuse forward index iterator", K(ret)); + } else if (OB_FAIL(fwd_idx_scan_param_.key_ranges_.push_back(scan_range))) { + LOG_WARN("failed to add forward index scan range", K(ret), K(scan_range)); + } else if (OB_FAIL(forward_idx_iter_->rescan())) { + LOG_WARN("failed to rescan forward index", K(ret)); } } else { - const ObTabletID &storage_tablet_id = fwd_idx_scan_param_.tablet_id_; - const bool need_switch_param = - storage_tablet_id.is_valid() && storage_tablet_id != retrieval_param_->get_fwd_idx_tablet_id(); - fwd_idx_scan_param_.need_switch_param_ = need_switch_param; - if (OB_FAIL(tsc_service->reuse_scan_iter(need_switch_param, forward_idx_iter_))) { - LOG_WARN("failed to reuse scan iter", K(ret)); - } else if (OB_FAIL(tsc_service->table_rescan(fwd_idx_scan_param_, forward_idx_iter_))) { - LOG_WARN("failed to rescan forward index table", K(ret), K_(fwd_idx_scan_param)); + if (OB_FAIL(init_fwd_idx_scan_param())) { + LOG_WARN("failed to init forward index scan param", K(ret)); + } else if (OB_FAIL(fwd_idx_scan_param_.key_ranges_.push_back(scan_range))) { + LOG_WARN("failed to add forward index scan range", K(ret), K(scan_range)); + } else if (FALSE_IT(forward_idx_iter_->set_scan_param(fwd_idx_scan_param_))) { + } else if (OB_FAIL(forward_idx_iter_->do_table_scan())) { + LOG_WARN("failed to do forward index scan", K(ret)); + } else { + not_first_fwd_agg_ = true; } } if (OB_SUCC(ret)) { if (!static_cast( - retrieval_param_->get_fwd_idx_agg_ctdef()->pd_expr_spec_.pd_storage_flag_).is_aggregate_pushdown()) { + ir_ctdef_->get_fwd_idx_agg_ctdef()->pd_expr_spec_.pd_storage_flag_).is_aggregate_pushdown()) { ret = OB_NOT_IMPLEMENT; LOG_ERROR("aggregate without pushdown not implemented", K(ret)); } else { if (OB_FAIL(forward_idx_iter_->get_next_row())) { LOG_WARN("failed to get next row from forward index iterator", K(ret)); } else { - const sql::ObExpr *agg_expr = retrieval_param_->get_fwd_idx_agg_ctdef()->pd_expr_spec_.pd_storage_aggregate_output_.at(0); - sql::ObEvalCtx *eval_ctx = retrieval_param_->get_ir_rtdef()->get_fwd_idx_agg_rtdef()->eval_ctx_; + const sql::ObExpr *agg_expr = ir_ctdef_->get_fwd_idx_agg_ctdef()->pd_expr_spec_.pd_storage_aggregate_output_.at(0); + sql::ObEvalCtx *eval_ctx = ir_rtdef_->get_fwd_idx_agg_rtdef()->eval_ctx_; const ObDatum &word_cnt_datum = agg_expr->locate_expr_datum(*eval_ctx); token_count = word_cnt_datum.get_int(); LOG_DEBUG("retrieval iterator get token cnt for doc", K(ret), K(doc_id), K(token_count)); @@ -534,12 +509,12 @@ int ObTextRetrievalIterator::do_token_cnt_agg(const ObDocId &doc_id, int64_t &to return ret; } -int ObTextRetrievalIterator::fill_token_cnt_with_doc_len() +int ObDASTextRetrievalIter::fill_token_cnt_with_doc_len() { int ret = OB_SUCCESS; const sql::ObExpr *agg_expr = doc_token_cnt_expr_; - const sql::ObExpr *doc_length_expr = retrieval_param_->get_ir_ctdef()->inv_scan_doc_length_col_; - sql::ObEvalCtx *eval_ctx = retrieval_param_->get_ir_rtdef()->eval_ctx_; + const sql::ObExpr *doc_length_expr = ir_ctdef_->inv_scan_doc_length_col_; + sql::ObEvalCtx *eval_ctx = ir_rtdef_->eval_ctx_; ObDatum *doc_length_datum = nullptr; if (OB_ISNULL(agg_expr) || OB_ISNULL(doc_length_expr) || OB_ISNULL(eval_ctx) || OB_UNLIKELY(agg_expr->datum_meta_.get_type() != ObDecimalIntType)) { @@ -554,11 +529,11 @@ int ObTextRetrievalIterator::fill_token_cnt_with_doc_len() return ret; } -int ObTextRetrievalIterator::fill_token_doc_cnt() +int ObDASTextRetrievalIter::fill_token_doc_cnt() { int ret = OB_SUCCESS; const sql::ObExpr *inv_idx_agg_expr = inv_idx_agg_param_.aggregate_exprs_->at(0); - sql::ObEvalCtx *eval_ctx = retrieval_param_->get_ir_rtdef()->get_inv_idx_agg_rtdef()->eval_ctx_; + sql::ObEvalCtx *eval_ctx = ir_rtdef_->get_inv_idx_agg_rtdef()->eval_ctx_; if (OB_ISNULL(inv_idx_agg_expr) || OB_ISNULL(eval_ctx) || OB_UNLIKELY(inv_idx_agg_expr->datum_meta_.get_type() != ObIntType)) { ret = OB_ERR_UNEXPECTED; @@ -570,31 +545,32 @@ int ObTextRetrievalIterator::fill_token_doc_cnt() return ret; } -int ObTextRetrievalIterator::project_relevance_expr() +int ObDASTextRetrievalIter::project_relevance_expr() { int ret = OB_SUCCESS; - const sql::ObDASIRScanRtDef *ir_rtdef = retrieval_param_->get_ir_rtdef(); - sql::ObExpr *relevance_expr = retrieval_param_->get_ir_ctdef()->relevance_expr_; + sql::ObExpr *relevance_expr = ir_ctdef_->relevance_expr_; ObDatum *relevance_datum = nullptr; if (OB_ISNULL(relevance_expr)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid relevance expr", K(ret)); - } else if (OB_FAIL(relevance_expr->eval(*ir_rtdef->eval_ctx_, relevance_datum))) { + } else if (OB_FAIL(relevance_expr->eval(*ir_rtdef_->eval_ctx_, relevance_datum))) { LOG_WARN("failed to evaluate relevance", K(ret)); } return ret; } -int ObTextRetrievalIterator::reuse_fwd_idx_iter() +int ObDASTextRetrievalIter::reuse_fwd_idx_iter() { int ret = OB_SUCCESS; if (nullptr != forward_idx_iter_) { - fwd_idx_scan_param_.key_ranges_.reuse(); + if (OB_FAIL(forward_idx_iter_->reuse())) { + LOG_WARN("failed to reuse forward index iter", K(ret)); + } } return ret; } -int ObTextRetrievalIterator::gen_inv_idx_scan_range(const ObString &query_token, ObNewRange &scan_range) +int ObDASTextRetrievalIter::gen_inv_idx_scan_range(const ObString &query_token, ObNewRange &scan_range) { int ret = OB_SUCCESS; void *buf = nullptr; @@ -604,7 +580,7 @@ int ObTextRetrievalIterator::gen_inv_idx_scan_range(const ObString &query_token, ObObj tmp_obj; tmp_obj.set_string(ObVarcharType, query_token); // We need to ensure collation type / level between query text and token column is compatible - tmp_obj.set_meta_type(retrieval_param_->get_ir_ctdef()->search_text_->obj_meta_); + tmp_obj.set_meta_type(ir_ctdef_->search_text_->obj_meta_); if (OB_ISNULL(buf = ctx_alloc.alloc(sizeof(ObObj) * obj_cnt))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -621,7 +597,7 @@ int ObTextRetrievalIterator::gen_inv_idx_scan_range(const ObString &query_token, obj_ptr[3].set_max_value(); ObRowkey start_key(obj_ptr, INV_IDX_ROWKEY_COL_CNT); ObRowkey end_key(&obj_ptr[2], INV_IDX_ROWKEY_COL_CNT); - common::ObTableID inv_table_id = retrieval_param_->get_inv_idx_scan_ctdef()->ref_table_id_; + common::ObTableID inv_table_id = ir_ctdef_->get_inv_idx_scan_ctdef()->ref_table_id_; scan_range.table_id_ = inv_table_id; scan_range.start_key_.assign(obj_ptr, INV_IDX_ROWKEY_COL_CNT); scan_range.end_key_.assign(&obj_ptr[2], INV_IDX_ROWKEY_COL_CNT); @@ -631,7 +607,7 @@ int ObTextRetrievalIterator::gen_inv_idx_scan_range(const ObString &query_token, return ret; } -int ObTextRetrievalIterator::gen_fwd_idx_scan_range(const ObDocId &doc_id, ObNewRange &scan_range) +int ObDASTextRetrievalIter::gen_fwd_idx_scan_range(const ObDocId &doc_id, ObNewRange &scan_range) { int ret = OB_SUCCESS; if (nullptr == fwd_range_objs_) { @@ -651,7 +627,7 @@ int ObTextRetrievalIterator::gen_fwd_idx_scan_range(const ObDocId &doc_id, ObNew fwd_range_objs_[1].set_min_value(); fwd_range_objs_[2].set_varbinary(doc_id.get_string()); fwd_range_objs_[3].set_max_value(); - scan_range.table_id_ = retrieval_param_->get_fwd_idx_agg_ctdef()->ref_table_id_; + scan_range.table_id_ = ir_ctdef_->get_fwd_idx_agg_ctdef()->ref_table_id_; scan_range.start_key_.assign(fwd_range_objs_, FWD_IDX_ROWKEY_COL_CNT); scan_range.end_key_.assign(&fwd_range_objs_[2], FWD_IDX_ROWKEY_COL_CNT); scan_range.border_flag_.set_inclusive_start(); @@ -660,12 +636,12 @@ int ObTextRetrievalIterator::gen_fwd_idx_scan_range(const ObDocId &doc_id, ObNew return ret; } -int ObTextRetrievalIterator::init_calc_exprs() +int ObDASTextRetrievalIter::init_calc_exprs() { int ret = OB_SUCCESS; - if (retrieval_param_->get_ir_ctdef()->need_calc_relevance()) { - sql::ObExpr *relevance_expr = retrieval_param_->get_ir_ctdef()->relevance_expr_; - sql::ObEvalCtx *eval_ctx = retrieval_param_->get_ir_rtdef()->eval_ctx_; + if (ir_ctdef_->need_calc_relevance()) { + sql::ObExpr *relevance_expr = ir_ctdef_->relevance_expr_; + sql::ObEvalCtx *eval_ctx = ir_rtdef_->eval_ctx_; if (OB_ISNULL(relevance_expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null relevance expr", K(ret)); @@ -701,9 +677,9 @@ int ObTextRetrievalIterator::init_calc_exprs() return ret; } -void ObTextRetrievalIterator::clear_row_wise_evaluated_flag() +void ObDASTextRetrievalIter::clear_row_wise_evaluated_flag() { - sql::ObEvalCtx *eval_ctx = retrieval_param_->get_ir_rtdef()->eval_ctx_; + sql::ObEvalCtx *eval_ctx = ir_rtdef_->eval_ctx_; for (int64_t i = 0; i < calc_exprs_.count(); ++i) { sql::ObExpr *expr = calc_exprs_.at(i); if (expr->is_batch_result()) { @@ -714,5 +690,5 @@ void ObTextRetrievalIterator::clear_row_wise_evaluated_flag() } } -} // end storage -} // end oceanbase +} // namespace sql +} // namespace oceanbase diff --git a/src/sql/das/iter/ob_das_text_retrieval_iter.h b/src/sql/das/iter/ob_das_text_retrieval_iter.h new file mode 100644 index 000000000..57490c411 --- /dev/null +++ b/src/sql/das/iter/ob_das_text_retrieval_iter.h @@ -0,0 +1,151 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_DAS_TEXT_RETRIEVAL_ITER_H_ +#define OB_DAS_TEXT_RETRIEVAL_ITER_H_ + +#include "ob_das_iter.h" +#include "lib/container/ob_se_array.h" + +namespace oceanbase +{ +namespace sql +{ +struct ObDASIRScanCtDef; +struct ObDASIRScanRtDef; + +struct ObDASTextRetrievalIterParam : public ObDASIterParam +{ +public: + ObDASTextRetrievalIterParam() + : ObDASIterParam(DAS_ITER_TEXT_RETRIEVAL), + ir_ctdef_(nullptr), + ir_rtdef_(nullptr), + inv_idx_scan_iter_(nullptr), + inv_idx_agg_iter_(nullptr), + fwd_idx_iter_(nullptr), + tx_desc_(nullptr), + snapshot_(nullptr) + {} + + virtual bool is_valid() const override + { + return nullptr != ir_ctdef_ && nullptr != ir_rtdef_ && nullptr != inv_idx_scan_iter_; + } + + const ObDASIRScanCtDef *ir_ctdef_; + ObDASIRScanRtDef *ir_rtdef_; + ObDASIter *inv_idx_scan_iter_; + ObDASIter *inv_idx_agg_iter_; + ObDASIter *fwd_idx_iter_; + transaction::ObTxDesc *tx_desc_; + transaction::ObTxReadSnapshot *snapshot_; +}; + +// single token +class ObDASTextRetrievalIter : public ObDASIter +{ +public: + ObDASTextRetrievalIter(); + virtual ~ObDASTextRetrievalIter() {} + virtual int do_table_scan() override; + virtual int rescan() override; + + int set_query_token(const ObString &query_token); + void set_ls_tablet_ids( + const share::ObLSID &ls_id, + const ObTabletID &inv_tablet_id, + const ObTabletID &fwd_tablet_id) + { + ls_id_ = ls_id; + inv_idx_tablet_id_ = inv_tablet_id; + fwd_idx_tablet_id_ = fwd_tablet_id; + } + + INHERIT_TO_STRING_KV("ObDASIter", ObDASIter, K_(calc_exprs), K_(need_fwd_idx_agg), + K_(need_inv_idx_agg), K_(inv_idx_agg_evaluated), K_(not_first_fwd_agg), K_(is_inited)); +protected: + virtual int inner_init(ObDASIterParam ¶m) override; + virtual int inner_reuse() override; + virtual int inner_release() override; + virtual int inner_get_next_row() override; + virtual int inner_get_next_rows(int64_t &count, int64_t capacity) override; +private: + int init_inv_idx_scan_param(); + int init_fwd_idx_scan_param(); + static int init_base_idx_scan_param( + const share::ObLSID &ls_id, + const common::ObTabletID &tablet_id, + const sql::ObDASScanCtDef *ctdef, + sql::ObDASScanRtDef *rtdef, + transaction::ObTxDesc *tx_desc, + transaction::ObTxReadSnapshot *snapshot, + ObTableScanParam &scan_param); + int get_next_doc_token_cnt(const bool use_fwd_idx_agg); + int do_doc_cnt_agg(); + int do_token_cnt_agg(const ObDocId &doc_id, int64_t &token_count); + int get_inv_idx_scan_doc_id(ObDocId &doc_id); + int fill_token_cnt_with_doc_len(); + int fill_token_doc_cnt(); + int project_relevance_expr(); + int reuse_fwd_idx_iter(); + int gen_inv_idx_scan_range(const ObString &query_token, ObNewRange &scan_range); + int gen_fwd_idx_scan_range(const ObDocId &doc_id, ObNewRange &scan_range); + inline bool need_calc_relevance() { return true; } // TODO: reduce tsc ops if no need to calc relevance + int init_calc_exprs(); + void clear_row_wise_evaluated_flag(); + + // TODO: delete this after enable standard vectorized execution + inline int get_next_single_row(const bool is_vectorized, ObNewRowIterator *iter) + { + int ret = OB_SUCCESS; + if (is_vectorized) { + int64_t scan_row_cnt = 0; + ret = iter->get_next_rows(scan_row_cnt, 1); + } else { + ret = iter->get_next_row(); + } + return ret; + } +private: + static const int64_t FWD_IDX_ROWKEY_COL_CNT = 2; + static const int64_t INV_IDX_ROWKEY_COL_CNT = 2; +private: + lib::MemoryContext mem_context_; + const ObDASIRScanCtDef *ir_ctdef_; + ObDASIRScanRtDef *ir_rtdef_; + transaction::ObTxDesc *tx_desc_; + transaction::ObTxReadSnapshot *snapshot_; + share::ObLSID ls_id_; + common::ObTabletID inv_idx_tablet_id_; + common::ObTabletID fwd_idx_tablet_id_; + ObTableScanParam inv_idx_scan_param_; + ObTableScanParam inv_idx_agg_param_; + ObTableScanParam fwd_idx_scan_param_; + common::ObSEArray calc_exprs_; + ObDASScanIter *inverted_idx_scan_iter_; + ObDASScanIter *inverted_idx_agg_iter_; + ObDASScanIter *forward_idx_iter_; + ObObj *fwd_range_objs_; + sql::ObExpr *doc_token_cnt_expr_; + int64_t token_doc_cnt_; + bool need_fwd_idx_agg_; + bool need_inv_idx_agg_; + bool inv_idx_agg_evaluated_; + bool not_first_fwd_agg_; + bool is_inited_; +}; + +} // namespace sql +} // namespace oceanbase + +#endif // OB_DAS_TEXT_RETRIEVAL_ITER_H_ diff --git a/src/sql/das/iter/ob_das_text_retrieval_merge_iter.cpp b/src/sql/das/iter/ob_das_text_retrieval_merge_iter.cpp new file mode 100644 index 000000000..d2f0061c4 --- /dev/null +++ b/src/sql/das/iter/ob_das_text_retrieval_merge_iter.cpp @@ -0,0 +1,708 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_DAS +#include "ob_das_scan_iter.h" +#include "ob_das_text_retrieval_merge_iter.h" +#include "ob_das_text_retrieval_iter.h" +#include "sql/das/ob_das_ir_define.h" +#include "share/text_analysis/ob_text_analyzer.h" +#include "storage/fts/ob_fts_plugin_helper.h" + +namespace oceanbase +{ +namespace sql +{ + +ObIRIterLoserTreeItem::ObIRIterLoserTreeItem() + : relevance_(0), doc_id_(), iter_idx_(-1) +{ +} + +ObIRIterLoserTreeCmp::ObIRIterLoserTreeCmp() + : cmp_func_(), is_inited_(false) +{ +} + +ObIRIterLoserTreeCmp::~ObIRIterLoserTreeCmp() +{ +} + +int ObIRIterLoserTreeCmp::init() +{ + int ret = OB_SUCCESS; + sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(ObVarcharType, CS_TYPE_BINARY); + cmp_func_ = lib::is_oracle_mode() ? basic_funcs->null_last_cmp_ : basic_funcs->null_first_cmp_; + if (OB_ISNULL(cmp_func_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to init IRIterLoserTreeCmp", K(ret)); + } else { + is_inited_ = true; + } + return ret; +} + +int ObIRIterLoserTreeCmp::cmp( + const ObIRIterLoserTreeItem &l, + const ObIRIterLoserTreeItem &r, + int64_t &cmp_ret) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else { + ObDatum l_datum; + ObDatum r_datum; + l_datum.set_string(l.doc_id_.get_string()); + r_datum.set_string(r.doc_id_.get_string()); + int tmp_ret = 0; + if (OB_FAIL(cmp_func_(l_datum, r_datum, tmp_ret))) { + LOG_WARN("failed to compare doc id by datum", K(ret)); + } else { + cmp_ret = tmp_ret; + } + } + return ret; +} + +ObDASTextRetrievalMergeIter::ObDASTextRetrievalMergeIter() + : ObDASIter(ObDASIterType::DAS_ITER_TEXT_RETRIEVAL_MERGE), + mem_context_(nullptr), + relation_type_(MAX_RELATION_TYPE), + processing_type_(MAX_PROC_TYPE), + ir_ctdef_(nullptr), + ir_rtdef_(nullptr), + tx_desc_(nullptr), + snapshot_(nullptr), + ls_id_(), + doc_id_idx_tablet_id_(), + query_tokens_(), + loser_tree_cmp_(), + iter_row_heap_(nullptr), + next_batch_iter_idxes_(), + next_batch_cnt_(0), + whole_doc_cnt_iter_(nullptr), + whole_doc_agg_param_(), + limit_param_(), + input_row_cnt_(0), + output_row_cnt_(0), + doc_cnt_calculated_(false), + is_inited_(false) +{ +} + +int ObDASTextRetrievalMergeIter::inner_init(ObDASIterParam ¶m) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("double initialization", K(ret)); + } else if (OB_UNLIKELY(ObDASIterType::DAS_ITER_TEXT_RETRIEVAL_MERGE != param.type_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid das iter param type for text retrieval iter", K(ret), K(param)); + } else { + ObDASTextRetrievalMergeIterParam &retrieval_param = static_cast(param); + ir_ctdef_ = retrieval_param.ir_ctdef_; + ir_rtdef_ = retrieval_param.ir_rtdef_; + whole_doc_cnt_iter_ = static_cast(retrieval_param.doc_cnt_iter_); + tx_desc_ = retrieval_param.tx_desc_; + snapshot_ = retrieval_param.snapshot_; + + relation_type_ = TokenRelationType::DISJUNCTIVE; + processing_type_ = RetrievalProcType::DAAT; + if (OB_ISNULL(mem_context_)) { + lib::ContextParam param; + param.set_mem_attr(MTL_ID(), "TextIRIter", ObCtxIds::DEFAULT_CTX_ID); + if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(mem_context_, param))) { + LOG_WARN("failed to create text retrieval iterator memory context", K(ret)); + } + } + + if (FAILEDx(loser_tree_cmp_.init())) { + LOG_WARN("failed to init loser tree comparator", K(ret)); + } else if (OB_ISNULL(iter_row_heap_ = OB_NEWx(ObIRIterLoserTree, &mem_context_->get_arena_allocator(), loser_tree_cmp_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate loser tree", K(ret)); + } else if (OB_FAIL(init_query_tokens(ir_ctdef_, ir_rtdef_))) { + LOG_WARN("failed to init query tokens", K(ret)); + } else if (0 == query_tokens_.count()) { + // empty token set + LOG_DEBUG("empty query token set after tokenization", K(ret), KPC_(ir_ctdef)); + is_inited_ = true; + } else if (OB_UNLIKELY(query_tokens_.count() > OB_MAX_TEXT_RETRIEVAL_TOKEN_CNT)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("too many query tokens in a single query not supported", K(ret), K_(query_tokens)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "text retrieval query with token count exceed limit"); + } else if (OB_FAIL(iter_row_heap_->init(query_tokens_.count(), mem_context_->get_arena_allocator()))) { + LOG_WARN("failed to init iter loser tree", K(ret)); + } else { + limit_param_ = ir_rtdef_->get_inv_idx_scan_rtdef()->limit_param_; + is_inited_ = true; + } + LOG_DEBUG("init text retrieval op", K(ret), KPC_(ir_ctdef), KPC_(ir_rtdef)); + + } + return ret; +} + +int ObDASTextRetrievalMergeIter::rescan() +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(query_tokens_.count() != token_iters_.count())) { + ret= OB_ERR_UNEXPECTED; + LOG_WARN("unexpected iter count mismatch with query tokens", + K(ret), K_(query_tokens), K_(token_iters)); + } else if (0 == query_tokens_.count()) { + } else if (nullptr != whole_doc_cnt_iter_ && OB_FAIL(whole_doc_cnt_iter_->rescan())) { + LOG_WARN("failed to rescan doc count iter", K(ret)); + } else if (OB_FAIL(next_batch_iter_idxes_.init(query_tokens_.count()))) { + LOG_WARN("failed to init next batch iter idxes array", K(ret)); + } else if (OB_FAIL(next_batch_iter_idxes_.prepare_allocate(query_tokens_.count()))) { + LOG_WARN("failed to prepare allocate next batch iter idxes array", K(ret)); + } else if (OB_FAIL(iter_row_heap_->open(query_tokens_.count()))) { + LOG_WARN("failed to open iter row heap", K(ret), K_(query_tokens)); + } else { + limit_param_ = ir_rtdef_->get_inv_idx_scan_rtdef()->limit_param_; + next_batch_cnt_ = token_iters_.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < token_iters_.count(); ++i) { + ObDASTextRetrievalIter *iter = token_iters_.at(i); + if (OB_FAIL(token_iters_.at(i)->set_query_token(query_tokens_.at(i)))) { + LOG_WARN("failed to set query token before rescan", K(ret), K(query_tokens_.at(i))); + } else if (OB_FAIL(iter->rescan())) { + LOG_WARN("failed to append token iter to array", K(ret)); + } else { + next_batch_iter_idxes_[i] = i; + } + } + } + return ret; +} + +int ObDASTextRetrievalMergeIter::do_table_scan() +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; i < token_iters_.count(); ++i) { + if (OB_FAIL(token_iters_.at(i)->do_table_scan())) { + LOG_WARN("failed to do table scan for token iter", K(ret)); + } + } + return ret; +} + +int ObDASTextRetrievalMergeIter::set_related_tablet_ids( + const ObLSID &ls_id, + const ObDASRelatedTabletID &related_tablet_ids) +{ + int ret = OB_SUCCESS; + ls_id_ = ls_id; + doc_id_idx_tablet_id_ = related_tablet_ids.doc_id_idx_tablet_id_; + for (int64_t i = 0; i < token_iters_.count(); ++i) { + token_iters_.at(i)->set_ls_tablet_ids( + ls_id, + related_tablet_ids.inv_idx_tablet_id_, + related_tablet_ids.fwd_idx_tablet_id_); + } + return ret; +} + +int ObDASTextRetrievalMergeIter::inner_reuse() +{ + int ret = OB_SUCCESS; + next_batch_iter_idxes_.reuse(); + iter_row_heap_->reuse(); + next_batch_cnt_ = 0; + doc_cnt_calculated_ = false; + input_row_cnt_ = 0; + output_row_cnt_ = 0; + const ObTabletID &old_doc_id_tablet_id = whole_doc_agg_param_.tablet_id_; + whole_doc_agg_param_.need_switch_param_ = whole_doc_agg_param_.need_switch_param_ || + ((old_doc_id_tablet_id.is_valid() && old_doc_id_tablet_id != doc_id_idx_tablet_id_) ? true : false); + if (nullptr != whole_doc_cnt_iter_) { + whole_doc_cnt_iter_->set_scan_param(whole_doc_agg_param_); + if (OB_FAIL(whole_doc_cnt_iter_->reuse())) { + LOG_WARN("failed to reuse whole doc cnt iter", K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < token_iters_.count(); ++i) { + if (OB_FAIL(token_iters_.at(i)->reuse())) { + LOG_WARN("failed to reuse token iters", K(ret)); + } + } + return ret; +} + +int ObDASTextRetrievalMergeIter::inner_release() +{ + int ret = OB_SUCCESS; + query_tokens_.reset(); + if (nullptr != iter_row_heap_) { + iter_row_heap_->~ObIRIterLoserTree(); + iter_row_heap_ = nullptr; + } + whole_doc_agg_param_.destroy_schema_guard(); + whole_doc_agg_param_.snapshot_.reset(); + whole_doc_agg_param_.destroy(); + whole_doc_cnt_iter_ = nullptr; + token_iters_.reset(); + next_batch_iter_idxes_.reset(); + if (nullptr != mem_context_) { + mem_context_->reset_remain_one_page(); + DESTROY_CONTEXT(mem_context_); + mem_context_ = nullptr; + } + next_batch_cnt_ = 0; + input_row_cnt_ = 0; + output_row_cnt_ = 0; + limit_param_.offset_ = 0; + limit_param_.limit_ = -1; + doc_cnt_calculated_ = false; + is_inited_ = false; + return ret; +} + +int ObDASTextRetrievalMergeIter::inner_get_next_row() +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (0 == query_tokens_.count()) { + ret = OB_ITER_END; + } else if (limit_param_.limit_ > 0 && output_row_cnt_ >= limit_param_.limit_) { + ret = OB_ITER_END; + LOG_DEBUG("get row with limit finished", + K(ret), K_(limit_param), K_(output_row_cnt), K_(input_row_cnt)); + } else if (!doc_cnt_calculated_) { + if (OB_FAIL(do_total_doc_cnt())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to do total document count", K(ret), KPC_(ir_ctdef)); + } + } else { + doc_cnt_calculated_ = true; + } + } + + bool filter_valid = false; + bool got_valid_document = false; + ObExpr *match_filter = ir_ctdef_->need_calc_relevance() ? ir_ctdef_->match_filter_ : nullptr; + ObDatum *filter_res = nullptr; + while (OB_SUCC(ret) && !got_valid_document) { + clear_evaluated_infos(); + filter_valid = false; + if (OB_FAIL(pull_next_batch_rows())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to pull next batch rows from iterator", K(ret)); + } + } else if (OB_FAIL(next_disjunctive_document())) { + LOG_WARN("failed to get next document with disjunctive tokens", K(ret)); + } else if (OB_ISNULL(match_filter)) { + filter_valid = true; + } else if (OB_FAIL(match_filter->eval(*ir_rtdef_->eval_ctx_, filter_res))) { + LOG_WARN("failed to evaluate match filter", K(ret)); + } else { + filter_valid = !(filter_res->is_null() || 0 == filter_res->get_int()); + } + + if (OB_SUCC(ret)) { + if (filter_valid) { + ++input_row_cnt_; + if (input_row_cnt_ > limit_param_.offset_) { + got_valid_document = true; + ++output_row_cnt_; + } + } + } + + } + + return ret; +} + +int ObDASTextRetrievalMergeIter::inner_get_next_rows(int64_t &count, int64_t capacity) +{ + // only one row at a time + // TODO: support batch vectorized execution later + int ret = OB_SUCCESS; + if (OB_FAIL(inner_get_next_row())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next row", K(ret)); + } + } else { + count += 1; + } + return ret; +} + +int ObDASTextRetrievalMergeIter::init_query_tokens(const ObDASIRScanCtDef *ir_ctdef, ObDASIRScanRtDef *ir_rtdef) +{ + int ret = OB_SUCCESS; + ObExpr *search_text = ir_ctdef->search_text_; + ObEvalCtx *eval_ctx = ir_rtdef->eval_ctx_; + ObDatum *search_text_datum = nullptr; + if (OB_ISNULL(search_text) || OB_ISNULL(eval_ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", K(ret), KP(search_text), KP(eval_ctx)); + } else if (OB_FAIL(search_text->eval(*eval_ctx, search_text_datum))) { + LOG_WARN("expr evaluation failed", K(ret)); + } else if (0 == search_text_datum->len_) { + // empty query text + } else { + // TODO: FTParseHelper currently does not support deduplicate tokens + // We should abstract such universal analyse functors into utility structs + const ObString &search_text_string = search_text_datum->get_string(); + const ObString &parser_name = ir_ctdef->get_inv_idx_scan_ctdef()->table_param_.get_parser_name(); + const ObCollationType &cs_type = search_text->datum_meta_.cs_type_; + int64_t doc_length = 0; + storage::ObFTParseHelper tokenize_helper; + common::ObSEArray tokens; + hash::ObHashMap token_map; + const int64_t ft_word_bkt_cnt = MAX(search_text_string.length() / 10, 2); + if (OB_FAIL(tokenize_helper.init(&mem_context_->get_arena_allocator(), parser_name))) { + LOG_WARN("failed to init tokenize helper", K(ret)); + } else if (OB_FAIL(token_map.create(ft_word_bkt_cnt, common::ObMemAttr(MTL_ID(), "FTWordMap")))) { + LOG_WARN("failed to create token map", K(ret)); + } else if (OB_FAIL(tokenize_helper.segment( + cs_type, search_text_string.ptr(), search_text_string.length(), doc_length, token_map))) { + LOG_WARN("failed to segment"); + } else { + for (hash::ObHashMap::const_iterator iter = token_map.begin(); + OB_SUCC(ret) && iter != token_map.end(); + ++iter) { + const ObFTWord &token = iter->first; + ObString token_string; + if (OB_FAIL(ob_write_string(mem_context_->get_arena_allocator(), token.get_word(), token_string))) { + LOG_WARN("failed to deep copy query token", K(ret)); + } else if (OB_FAIL(query_tokens_.push_back(token_string))) { + LOG_WARN("failed to append query token", K(ret)); + } + } + } + +// TODO: try use this interface instead +/* + share::ObITokenStream *token_stream = nullptr; + share::ObTextAnalysisCtx query_analysis_ctx; + query_analysis_ctx.need_grouping_ = true; + query_analysis_ctx.filter_stopword_ = true; + query_analysis_ctx.cs_ = common::ObCharset::get_charset(search_text->obj_meta_.get_collation_type()); + share::ObEnglishTextAnalyzer query_analyzer; + if (OB_FAIL(query_analyzer.init(query_analysis_ctx, token_analyze_alloc))) { + LOG_WARN("failed to init query text analyzer", K(ret)); + } else if (OB_FAIL(query_analyzer.analyze(*search_text_datum, token_stream))) { + LOG_WARN("failed to analyze search text", K(ret), K(query_analysis_ctx), KPC(search_text_datum)); + } + while (OB_SUCC(ret)) { + ObDatum token; + ObString token_string; + if (OB_FAIL(token_stream->get_next(token))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next query token", K(ret)); + } + } else if (OB_FAIL(ob_write_string(token_analyze_alloc, token.get_string(), token_string))) { + LOG_WARN("failed to deep copy query token", K(ret)); + } else if (OB_FAIL(query_tokens_.push_back(token_string))) { + LOG_WARN("failed to append query token", K(ret)); + } + } + + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to init query tokens", K(ret)); + } else { + ret = OB_SUCCESS; + } +*/ + + LOG_DEBUG("tokenized text query:", K(ret), KPC(search_text_datum), K_(query_tokens)); + } + return ret; +} + +int ObDASTextRetrievalMergeIter::set_merge_iters(const ObIArray &retrieval_iters) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (OB_UNLIKELY(retrieval_iters.count() != query_tokens_.count())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid retrieval iter count mismatch with query tokens", K(ret), K_(query_tokens), K(retrieval_iters)); + } else if (token_iters_.count() != 0) { + ret = OB_INIT_TWICE; + LOG_WARN("set retrieval iters to a non-empty merge iter", K(ret)); + } else if (0 == query_tokens_.count()) { + // no valid tokens + } else if (FALSE_IT(next_batch_iter_idxes_.set_allocator(&mem_context_->get_arena_allocator()))) { + } else if (OB_FAIL(next_batch_iter_idxes_.init(query_tokens_.count()))) { + LOG_WARN("failed to init next batch iter idxes array", K(ret)); + } else if (OB_FAIL(next_batch_iter_idxes_.prepare_allocate(query_tokens_.count()))) { + LOG_WARN("failed to prepare allocate next batch iter idxes array", K(ret)); + } else { + next_batch_cnt_ = query_tokens_.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < query_tokens_.count(); ++i) { + const ObString &query_token = query_tokens_.at(i); + ObDASTextRetrievalIter *iter = static_cast(retrieval_iters.at(i)); + if (OB_FAIL(token_iters_.push_back(iter))) { + LOG_WARN("failed to append token iter to array", K(ret)); + } else { + next_batch_iter_idxes_[i] = i; + } + } + } + + return ret; +} + +int ObDASTextRetrievalMergeIter::pull_next_batch_rows() +{ + int ret = OB_SUCCESS; + ObIRIterLoserTreeItem item; + for (int64_t i = 0; OB_SUCC(ret) && i < next_batch_cnt_; ++i) { + const int64_t iter_idx = next_batch_iter_idxes_[i]; + ObDASTextRetrievalIter *iter = nullptr; + if (OB_ISNULL(iter = token_iters_.at(iter_idx))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null token iter ptr", K(ret), K(iter_idx), K(token_iters_.count())); + } else if (OB_FAIL(iter->get_next_row())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get pull next batch rows from iterator", K(ret)); + } else { + ret = OB_SUCCESS; + } + } else if (OB_FAIL(fill_loser_tree_item(*iter, iter_idx, item))) { + LOG_WARN("fail to fill loser tree item", K(ret)); + } else if (OB_FAIL(iter_row_heap_->push(item))) { + LOG_WARN("fail to push item to loser tree", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (iter_row_heap_->empty()) { + ret = OB_ITER_END; + } else if (OB_FAIL(iter_row_heap_->rebuild())) { + LOG_WARN("fail to rebuild loser tree", K(ret), K_(next_batch_cnt)); + } else { + next_batch_cnt_ = 0; + } + } + return ret; +} + +int ObDASTextRetrievalMergeIter::next_disjunctive_document() +{ + int ret = OB_SUCCESS; + int64_t doc_cnt = 0; + bool curr_doc_end = false; + const ObIRIterLoserTreeItem *top_item = nullptr; + // Do we need to use ObExpr to collect relevance? + double cur_doc_relevance = 0.0; + while (OB_SUCC(ret) && !iter_row_heap_->empty() && !curr_doc_end) { + if (iter_row_heap_->is_unique_champion()) { + curr_doc_end = true; + } + if (OB_FAIL(iter_row_heap_->top(top_item))) { + LOG_WARN("failed to get top item from heap", K(ret)); + } else { + // consider to add an expr for collectiong conjunction result between query tokens here? + cur_doc_relevance += top_item->relevance_; + next_batch_iter_idxes_[next_batch_cnt_++] = top_item->iter_idx_; + if (OB_FAIL(iter_row_heap_->pop())) { + LOG_WARN("failed to pop top item in heap", K(ret)); + } + } + } + + if (OB_SUCC(ret)) { + const double project_relevance = ir_ctdef_->need_calc_relevance() ? cur_doc_relevance : 1; + if (OB_FAIL(project_result(*top_item, project_relevance))) { + LOG_WARN("failed to project relevance", K(ret)); + } + } + + return ret; +} + +int ObDASTextRetrievalMergeIter::project_result(const ObIRIterLoserTreeItem &item, const double relevance) +{ + int ret = OB_SUCCESS; + // TODO: usage of doc id column is somehow weird here, since in single token retrieval iterators, + // we use doc id expr to scan doc_id column for scan document. But here after DaaT processing, we use this expr + // to record current disjunctive documents. Though current implementation can make sure lifetime is + // safe, but it's tricky and indirect to read. + // P.S we cannot allocate multiple doc id expr at cg for every query token since tokenization now is an runtime operation + ObExpr *doc_id_col = ir_ctdef_->inv_scan_doc_id_col_; + ObEvalCtx *eval_ctx = ir_rtdef_->eval_ctx_; + if (OB_ISNULL(doc_id_col) || OB_ISNULL(eval_ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr to relevance proejction column", + K(ret), KP(doc_id_col), KP(eval_ctx)); + } else { + ObDatum &doc_id_proj_datum = doc_id_col->locate_datum_for_write(*eval_ctx); + doc_id_proj_datum.set_string(item.doc_id_.get_string()); + if (ir_ctdef_->need_proj_relevance_score()) { + ObExpr *relevance_proj_col = ir_ctdef_->relevance_proj_col_; + if (OB_ISNULL(relevance_proj_col)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null relevance proj col", K(ret)); + } else { + ObDatum &relevance_proj_datum = relevance_proj_col->locate_datum_for_write(*eval_ctx); + relevance_proj_datum.set_double(relevance); + } + } + LOG_DEBUG("project one fulltext search result", K(ret), K(item)); + } + return ret; +} + +int ObDASTextRetrievalMergeIter::fill_loser_tree_item( + ObDASTextRetrievalIter &iter, + const int64_t iter_idx, + ObIRIterLoserTreeItem &item) +{ + int ret = OB_SUCCESS; + item.iter_idx_ = iter_idx; + ObExpr *doc_id_expr = ir_ctdef_->inv_scan_doc_id_col_; + const ObDatum &doc_id_datum = doc_id_expr->locate_expr_datum(*ir_rtdef_->eval_ctx_); + if (OB_FAIL(item.doc_id_.from_string(doc_id_datum.get_string()))) { + LOG_WARN("failed to get ObDocId from string", K(ret), K(doc_id_datum), KPC(doc_id_expr)); + } else if (ir_ctdef_->need_calc_relevance()) { + ObExpr *relevance_expr = ir_ctdef_->relevance_expr_; + const ObDatum &relevance_datum = relevance_expr->locate_expr_datum(*ir_rtdef_->eval_ctx_); + item.relevance_ = relevance_datum.get_double(); + } + return ret; +} + +int ObDASTextRetrievalMergeIter::init_total_doc_cnt_param( + transaction::ObTxDesc *tx_desc, + transaction::ObTxReadSnapshot *snapshot) +{ + int ret = OB_SUCCESS; + const ObDASScanCtDef *ctdef = ir_ctdef_->get_doc_id_idx_agg_ctdef(); + ObDASScanRtDef *rtdef = ir_rtdef_->get_doc_id_idx_agg_rtdef(); + if (!ir_ctdef_->need_calc_relevance()) { + // no need to do total doc cnt + } else if (OB_ISNULL(ctdef) || OB_ISNULL(rtdef)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected scan descriptor", K(ret)); + } else { + ObTableScanParam &scan_param = whole_doc_agg_param_; + scan_param.tenant_id_ = MTL_ID(); + scan_param.tx_lock_timeout_ = rtdef->tx_lock_timeout_; + scan_param.index_id_ = ctdef->ref_table_id_; + scan_param.is_get_ = false; // scan + scan_param.is_for_foreign_check_ = false; + scan_param.timeout_ = rtdef->timeout_ts_; + scan_param.scan_flag_ = rtdef->scan_flag_; + scan_param.reserved_cell_count_ = ctdef->access_column_ids_.count(); + scan_param.allocator_ = &rtdef->stmt_allocator_; + scan_param.scan_allocator_ = &rtdef->scan_allocator_; + scan_param.sql_mode_ = rtdef->sql_mode_; + scan_param.frozen_version_ = rtdef->frozen_version_; + scan_param.force_refresh_lc_ = rtdef->force_refresh_lc_; + scan_param.output_exprs_ = &(ctdef->pd_expr_spec_.access_exprs_); + scan_param.calc_exprs_ = &(ctdef->pd_expr_spec_.calc_exprs_); + scan_param.aggregate_exprs_ = &(ctdef->pd_expr_spec_.pd_storage_aggregate_output_); + scan_param.table_param_ = &(ctdef->table_param_); + scan_param.op_ = rtdef->p_pd_expr_op_; + scan_param.row2exprs_projector_ = rtdef->p_row2exprs_projector_; + scan_param.schema_version_ = ctdef->schema_version_; + scan_param.tenant_schema_version_ = rtdef->tenant_schema_version_; + scan_param.limit_param_ = rtdef->limit_param_; + scan_param.need_scn_ = rtdef->need_scn_; + scan_param.pd_storage_flag_ = ctdef->pd_expr_spec_.pd_storage_flag_.pd_flag_; + scan_param.fb_snapshot_ = rtdef->fb_snapshot_; + scan_param.fb_read_tx_uncommitted_ = rtdef->fb_read_tx_uncommitted_; + scan_param.ls_id_ = ls_id_; + scan_param.tablet_id_ = doc_id_idx_tablet_id_; + if (ctdef->pd_expr_spec_.pushdown_filters_.empty()) { + scan_param.op_filters_ = &ctdef->pd_expr_spec_.pushdown_filters_; + } + scan_param.pd_storage_filters_ = rtdef->p_pd_expr_op_->pd_storage_filters_; + if (OB_NOT_NULL(tx_desc)) { + scan_param.tx_id_ = tx_desc->get_tx_id(); + } else { + scan_param.tx_id_.reset(); + } + + if (OB_NOT_NULL(snapshot)) { + scan_param.snapshot_ = *snapshot; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("null snapshot", K(ret), KP(snapshot)); + } + + if (FAILEDx(scan_param.column_ids_.assign(ctdef->access_column_ids_))) { + LOG_WARN("failed to init column ids", K(ret)); + } + } + return ret; +} + +int ObDASTextRetrievalMergeIter::do_total_doc_cnt() +{ + int ret = OB_SUCCESS; + + if (!ir_ctdef_->need_calc_relevance()) { + // skip + } else if (ir_ctdef_->need_do_total_doc_cnt()) { + // When estimation info not exist, or we found estimation info not accurate, calculate document count by scan + if (OB_FAIL(init_total_doc_cnt_param(tx_desc_, snapshot_))) { + LOG_WARN("failed to do total doc cnt", K(ret)); + } else if (FALSE_IT(whole_doc_cnt_iter_->set_scan_param(whole_doc_agg_param_))) { + } else if (OB_FAIL(whole_doc_cnt_iter_->do_table_scan())) { + LOG_WARN("failed to do table scan for document count aggregation", K(ret)); + } else { + if (OB_UNLIKELY(!static_cast(whole_doc_agg_param_.pd_storage_flag_).is_aggregate_pushdown())) { + ret = OB_NOT_IMPLEMENT; + LOG_ERROR("aggregate without pushdown not implemented", K(ret)); + } else if (OB_FAIL(whole_doc_cnt_iter_->get_next_row())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get aggregated row from iter", K(ret)); + } + } + } + } else { + // use estimated document count for relevance estimation + // Need to note that when total doc count is under estimated too much, the IDF component in BM25 + // would be invalidate and result to token frequence have major influence on final relevance score + ObExpr *total_doc_cnt_expr = ir_ctdef_->get_doc_id_idx_agg_ctdef()->pd_expr_spec_.pd_storage_aggregate_output_.at(0); + if (OB_ISNULL(total_doc_cnt_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null total doc cnt expr", K(ret)); + } else { + ObDatum &total_doc_cnt = total_doc_cnt_expr->locate_datum_for_write(*ir_rtdef_->eval_ctx_); + total_doc_cnt.set_int(ir_ctdef_->estimated_total_doc_cnt_); + LOG_TRACE("use estimated row count as partition document count", K(ret), K(total_doc_cnt)); + } + } + + return ret; +} + +void ObDASTextRetrievalMergeIter::clear_evaluated_infos() +{ + ObExpr *match_filter = ir_ctdef_->match_filter_; + ObEvalCtx *eval_ctx = ir_rtdef_->eval_ctx_; + if (nullptr != match_filter) { + if (match_filter->is_batch_result()) { + match_filter->get_evaluated_flags(*eval_ctx).unset(eval_ctx->get_batch_idx()); + } else { + match_filter->get_eval_info(*eval_ctx).clear_evaluated_flag(); + } + } +} + +} // namespace sql +} // namespace oceanbase diff --git a/src/sql/das/iter/ob_das_text_retrieval_merge_iter.h b/src/sql/das/iter/ob_das_text_retrieval_merge_iter.h new file mode 100644 index 000000000..8652a6481 --- /dev/null +++ b/src/sql/das/iter/ob_das_text_retrieval_merge_iter.h @@ -0,0 +1,165 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_DAS_TEXT_RETRIEVAL_MERGE_ITER_H_ +#define OB_DAS_TEXT_RETRIEVAL_MERGE_ITER_H_ + +#include "ob_das_iter.h" +#include "lib/container/ob_loser_tree.h" + +namespace oceanbase +{ +namespace sql +{ +struct ObDASIRScanCtDef; +struct ObDASIRScanRtDef; +class ObDASTextRetrievalIterator; + +static const int64_t OB_MAX_TEXT_RETRIEVAL_TOKEN_CNT = 256; + + +struct ObIRIterLoserTreeItem +{ + ObIRIterLoserTreeItem(); + ~ObIRIterLoserTreeItem() = default; + + TO_STRING_KV(K_(iter_idx), K_(relevance), K_(doc_id), K(doc_id_.get_string())); + + double relevance_; + ObDocId doc_id_; + int64_t iter_idx_; +}; + +struct ObIRIterLoserTreeCmp +{ + ObIRIterLoserTreeCmp(); + virtual ~ObIRIterLoserTreeCmp(); + + int init(); + int cmp(const ObIRIterLoserTreeItem &l, const ObIRIterLoserTreeItem &r, int64_t &cmp_ret); +private: + common::ObDatumCmpFuncType cmp_func_; + bool is_inited_; +}; +typedef common::ObLoserTree ObIRIterLoserTree; + + + +struct ObDASTextRetrievalMergeIterParam : public ObDASIterParam +{ +public: + ObDASTextRetrievalMergeIterParam() + : ObDASIterParam(DAS_ITER_TEXT_RETRIEVAL_MERGE), + ir_ctdef_(nullptr), + ir_rtdef_(nullptr), + doc_cnt_iter_(nullptr), + tx_desc_(nullptr), + snapshot_(nullptr) + {} + + virtual bool is_valid() const override + { + return nullptr != ir_ctdef_ && nullptr != ir_rtdef_; + } + + const ObDASIRScanCtDef *ir_ctdef_; + ObDASIRScanRtDef *ir_rtdef_; + ObDASIter *doc_cnt_iter_; + transaction::ObTxDesc *tx_desc_; + transaction::ObTxReadSnapshot *snapshot_; +}; + +class ObDASTextRetrievalMergeIter : public ObDASIter +{ +public: + enum TokenRelationType + { + DISJUNCTIVE = 0, + // CONJUNCTIVE = 1, + // BOOLEAN = 2, + MAX_RELATION_TYPE + }; + enum RetrievalProcType + { + DAAT = 0, + // TAAT = 1, + // VAAT = 2, + MAX_PROC_TYPE + }; +public: + ObDASTextRetrievalMergeIter(); + virtual ~ObDASTextRetrievalMergeIter() {} + + virtual int do_table_scan() override; + virtual int rescan() override; + void set_doc_id_idx_tablet_id(const ObTabletID &tablet_id) { doc_id_idx_tablet_id_ = tablet_id; } + void set_ls_id(const ObLSID &ls_id) { ls_id_ = ls_id; } + storage::ObTableScanParam &get_doc_agg_param() { return whole_doc_agg_param_; } + int set_related_tablet_ids(const ObLSID &ls_id, const ObDASRelatedTabletID &related_tablet_ids); + int set_merge_iters(const ObIArray &retrieval_iters); + const ObIArray &get_query_tokens() { return query_tokens_; } +protected: + virtual int inner_init(ObDASIterParam ¶m) override; + virtual int inner_reuse() override; + virtual int inner_release() override; + virtual int inner_get_next_row() override; + virtual int inner_get_next_rows(int64_t &count, int64_t capacity) override; + +private: + int init_iters( + transaction::ObTxDesc *tx_desc, + transaction::ObTxReadSnapshot *snapshot, + const ObIArray &query_tokens); + int init_query_tokens(const ObDASIRScanCtDef *ir_ctdef, ObDASIRScanRtDef *ir_rtdef); + void release_iters(); + int pull_next_batch_rows(); + int fill_loser_tree_item( + ObDASTextRetrievalIter &iter, + const int64_t iter_idx, + ObIRIterLoserTreeItem &item); + int next_disjunctive_document(); + int init_total_doc_cnt_param(transaction::ObTxDesc *tx_desc, transaction::ObTxReadSnapshot *snapshot); + int do_total_doc_cnt(); + int project_result(const ObIRIterLoserTreeItem &item, const double relevance); + void clear_evaluated_infos(); +private: + static const int64_t OB_DEFAULT_QUERY_TOKEN_ITER_CNT = 4; + typedef ObSEArray ObDASTokenRetrievalIterArray; + lib::MemoryContext mem_context_; + TokenRelationType relation_type_; + RetrievalProcType processing_type_; + ObIAllocator *allocator_; + const ObDASIRScanCtDef *ir_ctdef_; + ObDASIRScanRtDef *ir_rtdef_; + transaction::ObTxDesc *tx_desc_; + transaction::ObTxReadSnapshot *snapshot_; + share::ObLSID ls_id_; + common::ObTabletID doc_id_idx_tablet_id_; + ObArray query_tokens_; + ObDASTokenRetrievalIterArray token_iters_; + ObIRIterLoserTreeCmp loser_tree_cmp_; + ObIRIterLoserTree *iter_row_heap_; + ObFixedArray next_batch_iter_idxes_; + int64_t next_batch_cnt_; + ObDASScanIter *whole_doc_cnt_iter_; + ObTableScanParam whole_doc_agg_param_; + common::ObLimitParam limit_param_; + int64_t input_row_cnt_; + int64_t output_row_cnt_; + bool doc_cnt_calculated_; + bool is_inited_; +}; + +} // namespace sql +} // namespace oceanbase + +#endif // OB_DAS_TEXT_RETRIEVAL_MERGE_ITER_H_ diff --git a/src/sql/das/ob_das_factory.cpp b/src/sql/das/ob_das_factory.cpp index 9753737e4..61beb9700 100644 --- a/src/sql/das/ob_das_factory.cpp +++ b/src/sql/das/ob_das_factory.cpp @@ -23,7 +23,7 @@ #include "sql/das/ob_das_rpc_processor.h" #include "sql/das/ob_das_ref.h" #include "sql/das/ob_das_attach_define.h" -#include "sql/das/ob_text_retrieval_op.h" +#include "sql/das/ob_das_ir_define.h" #include "share/datum/ob_datum_util.h" #define STORE_DAS_OBJ(obj_store, das_obj, class_name) \ diff --git a/src/sql/das/ob_das_ir_define.cpp b/src/sql/das/ob_das_ir_define.cpp new file mode 100644 index 000000000..ab7448829 --- /dev/null +++ b/src/sql/das/ob_das_ir_define.cpp @@ -0,0 +1,39 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_DAS +#include "ob_das_ir_define.h" + +namespace oceanbase +{ +namespace sql +{ + +OB_SERIALIZE_MEMBER((ObDASIRScanCtDef, ObDASAttachCtDef), + flags_, + search_text_, + inv_scan_doc_id_col_, + inv_scan_doc_length_col_, + match_filter_, + relevance_expr_, + relevance_proj_col_, + estimated_total_doc_cnt_); + +OB_SERIALIZE_MEMBER(ObDASIRScanRtDef); + +OB_SERIALIZE_MEMBER((ObDASIRAuxLookupCtDef, ObDASAttachCtDef), + relevance_proj_col_); + +OB_SERIALIZE_MEMBER((ObDASIRAuxLookupRtDef, ObDASAttachRtDef)); + +} // sql +} // oceanbase diff --git a/src/sql/das/ob_text_retrieval_op.h b/src/sql/das/ob_das_ir_define.h similarity index 54% rename from src/sql/das/ob_text_retrieval_op.h rename to src/sql/das/ob_das_ir_define.h index a1bc85aac..ab9251c20 100644 --- a/src/sql/das/ob_text_retrieval_op.h +++ b/src/sql/das/ob_das_ir_define.h @@ -10,21 +10,15 @@ * See the Mulan PubL v2 for more details. */ -#ifndef OBDEV_SRC_SQL_DAS_OB_TEXT_RETRIEVAL_OP_H_ -#define OBDEV_SRC_SQL_DAS_OB_TEXT_RETRIEVAL_OP_H_ +#ifndef OB_DAS_IR_DEFINE_H_ +#define OB_DAS_IR_DEFINE_H_ -#include "lib/container/ob_loser_tree.h" -#include "sql/das/ob_das_task.h" -#include "sql/das/ob_das_scan_op.h" -#include "sql/das/ob_das_attach_define.h" -#include "sql/engine/sort/ob_sort_op_impl.h" -#include "storage/fts/ob_text_retrieval_iterator.h" +#include "ob_das_attach_define.h" namespace oceanbase { namespace sql { -static const int64_t OB_MAX_TEXT_RETRIEVAL_TOKEN_CNT = 256; struct ObDASIRScanCtDef : ObDASAttachCtDef { @@ -44,6 +38,8 @@ public: } bool need_calc_relevance() const { return nullptr != relevance_expr_; } bool need_proj_relevance_score() const { return nullptr != relevance_proj_col_; } + bool need_fwd_idx_agg() const { return has_fwd_agg_ && need_calc_relevance(); } + bool need_inv_idx_agg() const { return has_inv_agg_ && need_calc_relevance(); } const ObDASScanCtDef *get_inv_idx_scan_ctdef() const { const ObDASScanCtDef *idx_scan_ctdef = nullptr; @@ -92,7 +88,7 @@ public: int64_t get_inv_agg_idx() const { return has_inv_agg_ ? 1 : -1; } int64_t get_doc_agg_idx() const { return has_doc_id_agg_ ? (1 + has_inv_agg_) : -1; } int64_t get_fwd_agg_idx() const { return has_fwd_agg_ ? (1 + has_inv_agg_ + has_doc_id_agg_) : -1; } - bool need_do_total_doc_cnt() const { return 0 == estimated_total_doc_cnt_; } + bool need_do_total_doc_cnt() const { return need_calc_relevance() && 0 == estimated_total_doc_cnt_; } INHERIT_TO_STRING_KV("ObDASBaseCtDef", ObDASBaseCtDef, K_(flags), @@ -220,180 +216,6 @@ public: } }; -struct ObIRIterLoserTreeItem -{ - ObIRIterLoserTreeItem(); - ~ObIRIterLoserTreeItem() = default; - - TO_STRING_KV(K_(iter_idx), K_(relevance), K_(doc_id), K(doc_id_.get_string())); - - double relevance_; - ObDocId doc_id_; - int64_t iter_idx_; -}; - -struct ObIRIterLoserTreeCmp -{ - ObIRIterLoserTreeCmp(); - virtual ~ObIRIterLoserTreeCmp(); - - int init(); - int cmp(const ObIRIterLoserTreeItem &l, const ObIRIterLoserTreeItem &r, int64_t &cmp_ret); -private: - common::ObDatumCmpFuncType cmp_func_; - bool is_inited_; -}; - -typedef common::ObLoserTree ObIRIterLoserTree; - -class ObTextRetrievalMerge : public common::ObNewRowIterator -{ -public: - enum TokenRelationType - { - DISJUNCTIVE = 0, - // CONJUNCTIVE = 1, - // BOOLEAN = 2, - MAX_RELATION_TYPE - }; - enum RetrievalProcType - { - DAAT = 0, - // TAAT = 1, - // VAAT = 2, - MAX_PROC_TYPE - }; -public: - ObTextRetrievalMerge(); - virtual ~ObTextRetrievalMerge(); - - int init( - const share::ObLSID &ls_id, - const ObTabletID &inv_idx_tablet_id, - const ObTabletID &fwd_idx_tablet_id, - const ObTabletID &doc_id_idx_tablet_id, - const ObDASIRScanCtDef *ir_ctdef, - ObDASIRScanRtDef *ir_rtdef, - transaction::ObTxDesc *tx_desc, - transaction::ObTxReadSnapshot *snapshot, - ObIAllocator &allocator); - int rescan( - const share::ObLSID &ls_id, - const ObTabletID &inv_idx_tablet_id, - const ObTabletID &fwd_idx_tablet_id, - const ObTabletID &doc_id_idx_tablet_id, - const ObDASIRScanCtDef *ir_ctdef, - ObDASIRScanRtDef *ir_rtdef, - transaction::ObTxDesc *tx_desc, - transaction::ObTxReadSnapshot *snapshot, - ObIAllocator &allocator); - - virtual int get_next_row(ObNewRow *&row) override; - virtual int get_next_row() override { ObNewRow *r = nullptr; return get_next_row(r); } - virtual int get_next_rows(int64_t &count, int64_t capacity) override; - virtual void reset() override; -private: - int init_iter_params( - const share::ObLSID &ls_id, - const ObTabletID &inv_idx_tablet_id, - const ObTabletID &fwd_idx_tablet_id, - const ObTabletID &doc_id_idx_tablet_id, - const ObDASIRScanCtDef *ir_ctdef, - ObDASIRScanRtDef *ir_rtdef); - int init_iters( - transaction::ObTxDesc *tx_desc, - transaction::ObTxReadSnapshot *snapshot, - const ObIArray &query_tokens); - int init_query_tokens(const ObDASIRScanCtDef *ir_ctdef, ObDASIRScanRtDef *ir_rtdef); - void release_iters(); - int pull_next_batch_rows(); - int fill_loser_tree_item( - storage::ObTextRetrievalIterator &iter, - const int64_t iter_idx, - ObIRIterLoserTreeItem &item); - int next_disjunctive_document(); - int init_total_doc_cnt_param(transaction::ObTxDesc *tx_desc, transaction::ObTxReadSnapshot *snapshot); - int do_total_doc_cnt(); - int project_result(const ObIRIterLoserTreeItem &item, const double relevance); - void clear_evaluated_infos(); -private: - static const int64_t OB_DEFAULT_QUERY_TOKEN_ITER_CNT = 4; - typedef ObSEArray ObTokenRetrievalIterArray; - TokenRelationType relation_type_; - RetrievalProcType processing_type_; - ObIAllocator *allocator_; - ObTokenRetrievalParam retrieval_param_; - ObArray query_tokens_; - ObTokenRetrievalIterArray token_iters_; - ObIRIterLoserTreeCmp loser_tree_cmp_; - ObIRIterLoserTree *iter_row_heap_; - ObFixedArray next_batch_iter_idxes_; - int64_t next_batch_cnt_; - common::ObNewRowIterator *whole_doc_cnt_iter_; - ObTableScanParam whole_doc_agg_param_; - bool doc_cnt_calculated_; - bool is_inited_; -}; - - -class ObTextRetrievalOp : public common::ObNewRowIterator -{ -public: - ObTextRetrievalOp(); - virtual ~ObTextRetrievalOp(); - - int init( - const share::ObLSID &ls_id, - const ObTabletID &inv_idx_tablet_id, - const ObTabletID &fwd_idx_tablet_id, - const ObTabletID &doc_id_idx_tablet_id, - const ObDASIRScanCtDef *ir_ctdef, - ObDASIRScanRtDef *ir_rtdef, - const ObDASSortCtDef *sort_ctdef, - ObDASSortRtDef *sort_rtdef, - transaction::ObTxDesc *tx_desc, - transaction::ObTxReadSnapshot *snapshot); - int rescan( - const share::ObLSID &ls_id, - const ObTabletID &inv_idx_tablet_id, - const ObTabletID &fwd_idx_tablet_id, - const ObTabletID &doc_id_idx_tablet_id, - const ObDASIRScanCtDef *ir_ctdef, - ObDASIRScanRtDef *ir_rtdef, - const ObDASSortCtDef *sort_ctdef, - ObDASSortRtDef *sort_rtdef, - transaction::ObTxDesc *tx_desc, - transaction::ObTxReadSnapshot *snapshot); - - virtual int get_next_row(ObNewRow *&row) override; - virtual int get_next_row() override { ObNewRow *r = nullptr; return get_next_row(r); } - virtual int get_next_rows(int64_t &count, int64_t capacity) override; - virtual void reset() override; -private: - int inner_get_next_row_for_output(); - int init_sort( - const ObDASIRScanCtDef *ir_ctdef, - const ObDASSortCtDef *sort_ctdef, - ObDASSortRtDef *sort_rtdef); - int init_limit( - const ObDASIRScanCtDef *ir_ctdef, - ObDASIRScanRtDef *ir_rtdef, - const ObDASSortCtDef *sort_ctdef, - ObDASSortRtDef *sort_rtdef); - int do_sort(); -private: - lib::MemoryContext mem_context_; - ObTextRetrievalMerge token_merge_; - common::ObLimitParam limit_param_; - int64_t input_row_cnt_; - int64_t output_row_cnt_; - ObSortOpImpl *sort_impl_; - ObSEArray sort_row_; - bool sort_finished_; - bool is_inited_; -}; - - } // namespace sql } // namespace oceanbase diff --git a/src/sql/das/ob_das_scan_op.cpp b/src/sql/das/ob_das_scan_op.cpp index 29c164aac..e24635712 100644 --- a/src/sql/das/ob_das_scan_op.cpp +++ b/src/sql/das/ob_das_scan_op.cpp @@ -21,6 +21,7 @@ #include "sql/engine/ob_des_exec_context.h" #include "storage/access/ob_table_scan_iterator.h" #include "storage/concurrency_control/ob_data_validation_service.h" +#include "sql/das/iter/ob_das_iter_utils.h" namespace oceanbase { @@ -320,6 +321,42 @@ ObITabletScan &ObDASScanOp::get_tsc_service() : *(MTL(ObAccessService *)); } +ObDASIterTreeType ObDASScanOp::get_iter_tree_type() const +{ + ObDASIterTreeType tree_type = ObDASIterTreeType::ITER_TREE_INVALID; + bool is_fts_index = scan_param_.table_param_->is_fts_index() && attach_ctdef_ != nullptr; + bool is_spatial_index = scan_param_.table_param_->is_spatial_index(); + bool is_multivalue_index = scan_param_.table_param_->is_multivalue_index(); + if (is_fts_index) { + tree_type = ObDASIterTreeType::ITER_TREE_TEXT_RETRIEVAL; + } else if (is_spatial_index) { + tree_type = ObDASIterTreeType::ITER_TREE_GIS_LOOKUP; + } else if (is_multivalue_index) { + tree_type = ObDASIterTreeType::ITER_TREE_DOMAIN_LOOKUP; + } else { + tree_type = OB_ISNULL(get_lookup_ctdef()) ? ObDASIterTreeType::ITER_TREE_PARTITION_SCAN + : ObDASIterTreeType::ITER_TREE_LOCAL_LOOKUP; + } + return tree_type; +} + +int ObDASScanOp::init_related_tablet_ids(ObDASRelatedTabletID &related_tablet_ids) +{ + int ret = OB_SUCCESS; + related_tablet_ids.reset(); + if (OB_FAIL(get_table_lookup_tablet_id(related_tablet_ids.lookup_tablet_id_))) { + LOG_WARN("failed to get table lookup tablet id", K(ret)); + } else if (OB_ISNULL(attach_ctdef_) || OB_ISNULL(attach_rtdef_)) { // no attached task. + } else if (OB_FAIL(get_aux_lookup_tablet_id(related_tablet_ids.aux_lookup_tablet_id_))) { + LOG_WARN("failed to get aux lookup tablet id", K(ret)); + } else if (OB_FAIL(get_text_ir_tablet_ids(related_tablet_ids.inv_idx_tablet_id_, + related_tablet_ids.fwd_idx_tablet_id_, + related_tablet_ids.doc_id_idx_tablet_id_))) { + LOG_WARN("failed to get text ir tablet ids", K(ret)); + } + return ret; +} + int ObDASScanOp::open_op() { int ret = OB_SUCCESS; @@ -330,8 +367,35 @@ int ObDASScanOp::open_op() init_retry_alloc(); } reset_access_datums_ptr(); + ObDASIterTreeType tree_type = ITER_TREE_INVALID; if (OB_FAIL(init_scan_param())) { LOG_WARN("init scan param failed", K(ret)); + } else if (FALSE_IT(tree_type = get_iter_tree_type())) { + } else if (ITER_TREE_PARTITION_SCAN == tree_type || ITER_TREE_LOCAL_LOOKUP == tree_type + || ITER_TREE_TEXT_RETRIEVAL == tree_type) { + ObDASIter *result = nullptr; + if (OB_FAIL(init_related_tablet_ids(tablet_ids_))) { + LOG_WARN("failed to init related tablet ids", K(ret)); + } else if (OB_FAIL(ObDASIterUtils::create_das_scan_iter_tree(tree_type, + scan_param_, + scan_ctdef_, + scan_rtdef_, + get_lookup_ctdef(), + get_lookup_rtdef(), + attach_ctdef_, + attach_rtdef_, + tablet_ids_, + trans_desc_, + snapshot_, + op_alloc_, + result))) { + LOG_WARN("failed to create das scan iter tree", K(get_iter_tree_type()), K(ret)); + } else { + result_ = result; + if (OB_FAIL(result->do_table_scan())) { + LOG_WARN("iter tree failed to do table scan", K(ret)); + } + } } else if (OB_FAIL(do_table_scan())) { if (OB_SNAPSHOT_DISCARDED == ret && scan_param_.fb_snapshot_.is_valid()) { ret = OB_INVALID_QUERY_TIMESTAMP; @@ -343,46 +407,58 @@ int ObDASScanOp::open_op() LOG_WARN("do local index lookup failed", K(ret)); } } + + int simulate_error = EVENT_CALL(EventTable::EN_DAS_SIMULATE_LOOKUPOP_INIT_ERROR); + if (OB_UNLIKELY(OB_SUCCESS != simulate_error)) { + ret = simulate_error; + } return ret; } int ObDASScanOp::release_op() { int ret = OB_SUCCESS; - int lookup_ret = OB_SUCCESS; - ObITabletScan &tsc_service = get_tsc_service(); - if (result_ != nullptr) { - if (ObNewRowIterator::IterType::ObLocalIndexLookupIterator == result_->get_type() || - ObNewRowIterator::IterType::ObGroupLookupOp == result_->get_type()) { - ObLocalIndexLookupOp *lookup_op = static_cast(result_); + ObDASIterTreeType tree_type = get_iter_tree_type(); + if (ITER_TREE_PARTITION_SCAN == tree_type || ITER_TREE_LOCAL_LOOKUP == tree_type + || ITER_TREE_TEXT_RETRIEVAL == tree_type) { + if (OB_NOT_NULL(result_)) { + ObDASIter *result = static_cast(result_); + if (OB_FAIL(result->release())) { + LOG_WARN("failed to release das iter tree", K(ret)); + } + } + } else { + int lookup_ret = OB_SUCCESS; + ObITabletScan &tsc_service = get_tsc_service(); + if (result_ != nullptr) { + if (ObNewRowIterator::IterType::ObLocalIndexLookupIterator == result_->get_type() || + ObNewRowIterator::IterType::ObGroupLookupOp == result_->get_type()) { + ObLocalIndexLookupOp *lookup_op = static_cast(result_); - ret = tsc_service.revert_scan_iter(lookup_op->get_rowkey_iter()); - if (OB_SUCCESS != ret) { - LOG_WARN("revert scan iterator failed", K(ret)); - } + ret = tsc_service.revert_scan_iter(lookup_op->get_rowkey_iter()); + if (OB_SUCCESS != ret) { + LOG_WARN("revert scan iterator failed", K(ret)); + } - lookup_ret = lookup_op->revert_iter(); - if (OB_SUCCESS != lookup_ret) { - LOG_WARN("revert lookup iterator failed", K(lookup_ret)); - } + lookup_ret = lookup_op->revert_iter(); + if (OB_SUCCESS != lookup_ret) { + LOG_WARN("revert lookup iterator failed", K(lookup_ret)); + } - result_ = nullptr; - //if row_key revert is succ return look_up iter ret code. - //if row_key revert is fail return row_key iter ret code. - //In short if row_key and look_up iter all revert fail. - //We just ignore lookup_iter ret code. - if (OB_SUCCESS == ret) { - ret = lookup_ret; + result_ = nullptr; + //if row_key revert is succ return look_up iter ret code. + //if row_key revert is fail return row_key iter ret code. + //In short if row_key and look_up iter all revert fail. + //We just ignore lookup_iter ret code. + if (OB_SUCCESS == ret) { + ret = lookup_ret; + } + } else { + if (OB_FAIL(tsc_service.revert_scan_iter(result_))) { + LOG_WARN("revert scan iterator failed", K(ret)); + } + result_ = nullptr; } - } else if (ObNewRowIterator::IterType::ObTextRetrievalOp == result_->get_type()) { - ObTextRetrievalOp *text_retrieval_op = static_cast(result_); - text_retrieval_op->reset(); - result_ = nullptr; - } else { - if (OB_FAIL(tsc_service.revert_scan_iter(result_))) { - LOG_WARN("revert scan iterator failed", K(ret)); - } - result_ = nullptr; } } //need to clear the flag:need_switch_param_ @@ -440,12 +516,7 @@ ObLocalIndexLookupOp *ObDASScanOp::get_lookup_op() int ObDASScanOp::do_table_scan() { int ret = OB_SUCCESS; - if (scan_param_.table_param_->is_fts_index() && attach_ctdef_ != nullptr) { - // full text index retrieval scan - if (OB_FAIL(do_text_retrieve(result_))) { - LOG_WARN("fail to retrieve token from full text index", K(ret)); - } - } else if (OB_FAIL(get_tsc_service().table_scan(scan_param_, result_))) { + if (OB_FAIL(get_tsc_service().table_scan(scan_param_, result_))) { if (OB_SNAPSHOT_DISCARDED == ret && scan_param_.fb_snapshot_.is_valid()) { ret = OB_INVALID_QUERY_TIMESTAMP; } else if (OB_TRY_LOCK_ROW_CONFLICT != ret) { @@ -458,8 +529,8 @@ int ObDASScanOp::do_table_scan() int ObDASScanOp::do_local_index_lookup() { int ret = OB_SUCCESS; - if (scan_param_.table_param_->is_fts_index() || - scan_param_.table_param_->is_multivalue_index()) { + ObTabletID lookup_tablet_id; + if (scan_param_.table_param_->is_multivalue_index()) { if (OB_FAIL(do_domain_index_lookup())) { LOG_WARN("failed to do domain index lookup", K(ret)); } @@ -475,8 +546,9 @@ int ObDASScanOp::do_local_index_lookup() if (OB_FAIL(op->init(get_lookup_ctdef(), get_lookup_rtdef(), scan_ctdef_, scan_rtdef_, trans_desc_, snapshot_, scan_param_))) { LOG_WARN("init spatial lookup op failed", K(ret)); + } else if (FALSE_IT(get_table_lookup_tablet_id(lookup_tablet_id))) { } else { - op->set_tablet_id(get_table_lookup_tablet_id()); + op->set_tablet_id(lookup_tablet_id); op->set_ls_id(ls_id_); } } @@ -496,8 +568,9 @@ int ObDASScanOp::do_local_index_lookup() trans_desc_, snapshot_))) { LOG_WARN("init lookup op failed", K(ret)); + } else if (FALSE_IT(get_table_lookup_tablet_id(lookup_tablet_id))) { } else { - op->set_tablet_id(get_table_lookup_tablet_id()); + op->set_tablet_id(lookup_tablet_id); op->set_ls_id(ls_id_); } } @@ -509,27 +582,8 @@ int ObDASScanOp::do_domain_index_lookup() { int ret = OB_SUCCESS; ObTabletID doc_id_idx_tablet_id; - - if (scan_param_.table_param_->is_fts_index()) { - ObFullTextIndexLookupOp *op = nullptr; - ObTabletID doc_id_idx_tablet_id; - const ObDASTableLookupCtDef *table_lookup_ctdef = nullptr; - ObDASTableLookupRtDef *table_lookup_rtdef = nullptr; - if (OB_FAIL(get_aux_lookup_tablet_id(doc_id_idx_tablet_id))) { - LOG_WARN("failed to get doc id idx tablet id", K(ret), K_(related_tablet_ids)); - } else if (OB_ISNULL(op = OB_NEWx(ObFullTextIndexLookupOp, &op_alloc_, op_alloc_))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate full text index lookup op", K(ret)); - } else if (FALSE_IT(op->set_text_retrieval_iter(result_))) { - } else if (FALSE_IT(result_ = op)) { - } else if (OB_FAIL(op->init(attach_ctdef_, attach_rtdef_, trans_desc_, snapshot_, scan_param_))) { - LOG_WARN("failed to init full text index lookup op", K(ret)); - } else { - op->set_tablet_id(get_table_lookup_tablet_id()); - op->set_doc_id_idx_tablet_id(doc_id_idx_tablet_id); - op->set_ls_id(ls_id_); - } - } else if (scan_param_.table_param_->is_multivalue_index()) { + ObTabletID lookup_tablet_id; + if (scan_param_.table_param_->is_multivalue_index()) { ObMulValueIndexLookupOp* op = nullptr; if (OB_FAIL(get_aux_lookup_tablet_id(doc_id_idx_tablet_id))) { LOG_WARN("failed to get doc id idx tablet id", K(ret), K_(related_tablet_ids)); @@ -540,8 +594,9 @@ int ObDASScanOp::do_domain_index_lookup() } else if (FALSE_IT(result_ = op)) { } else if (OB_FAIL(op->init(attach_ctdef_, attach_rtdef_, trans_desc_, snapshot_, scan_param_))) { LOG_WARN("failed to init multivalue index lookup op", K(ret)); + } else if (FALSE_IT(get_table_lookup_tablet_id(lookup_tablet_id))) { } else { - op->set_tablet_id(get_table_lookup_tablet_id()); + op->set_tablet_id(lookup_tablet_id); op->set_doc_id_idx_tablet_id(doc_id_idx_tablet_id); op->set_ls_id(ls_id_); } @@ -769,19 +824,30 @@ int ObDASScanOp::rescan() "tablet_id", scan_param_.tablet_id_, "scan_range", scan_param_.key_ranges_, "range_pos", scan_param_.range_array_pos_); - ObLocalIndexLookupOp *lookup_op = get_lookup_op(); - if (scan_param_.table_param_->is_fts_index() && attach_ctdef_ != nullptr) { - if (OB_FAIL(do_text_retrieve_rescan())) { - LOG_WARN("failed to do text retrieval rescan", K(ret)); + ObDASIterTreeType tree_type = get_iter_tree_type(); + if (ITER_TREE_PARTITION_SCAN == tree_type || ITER_TREE_LOCAL_LOOKUP == tree_type + || ITER_TREE_TEXT_RETRIEVAL == tree_type) { + if (OB_ISNULL(result_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr das iter tree", K(ret)); + } else { + ObDASIter *result = static_cast(result_); + if (OB_FAIL(result->rescan())) { + LOG_WARN("failed to rescan das iter", K(ret)); + } + } + } else { + ObLocalIndexLookupOp *lookup_op = get_lookup_op(); + ObTabletID lookup_tablet_id; + if (OB_FAIL(tsc_service.table_rescan(scan_param_, get_storage_scan_iter()))) { + LOG_WARN("rescan the table iterator failed", K(ret)); + } else if (lookup_op != nullptr) { + OZ(get_table_lookup_tablet_id(lookup_tablet_id)); + lookup_op->set_tablet_id(lookup_tablet_id); + lookup_op->set_ls_id(ls_id_); } - } else if (OB_FAIL(tsc_service.table_rescan(scan_param_, get_storage_scan_iter()))) { - LOG_WARN("rescan the table iterator failed", K(ret)); - } else if (lookup_op != nullptr) { - lookup_op->set_tablet_id(get_table_lookup_tablet_id()); - lookup_op->set_ls_id(ls_id_); - //lookup op's table_rescan will be drive by its get_next_row() - //so will can not call it here } + return ret; } @@ -791,15 +857,54 @@ int ObDASScanOp::reuse_iter() //May be retry change to retry alloc. //Change back. scan_param_.scan_allocator_ = &scan_rtdef_->scan_allocator_; - - ObITabletScan &tsc_service = get_tsc_service(); - ObLocalIndexLookupOp *lookup_op = get_lookup_op(); const ObTabletID &storage_tablet_id = scan_param_.tablet_id_; scan_param_.need_switch_param_ = (storage_tablet_id.is_valid() && storage_tablet_id != tablet_id_ ? true : false); - if (scan_param_.table_param_->is_fts_index() && attach_ctdef_ != nullptr) { - if (nullptr != lookup_op - && OB_FAIL(static_cast(lookup_op)->reuse_scan_iter())) { - LOG_WARN("failed to reuse text lookup iters", K(ret)); + scan_param_.key_ranges_.reuse(); + scan_param_.ss_key_ranges_.reuse(); + scan_param_.mbr_filters_.reuse(); + ObDASIterTreeType tree_type = get_iter_tree_type(); + ObITabletScan &tsc_service = get_tsc_service(); + ObLocalIndexLookupOp *lookup_op = get_lookup_op(); + if (ITER_TREE_PARTITION_SCAN == tree_type || ITER_TREE_LOCAL_LOOKUP == tree_type + || ITER_TREE_TEXT_RETRIEVAL == tree_type) { + if (OB_ISNULL(result_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr das iter tree", K(ret)); + } else { + ObDASIter *result = static_cast(result_); + if (OB_FAIL(init_related_tablet_ids(tablet_ids_))) { + LOG_WARN("fail to init related tablet ids", K(ret)); + } else { + switch (get_iter_tree_type()) { + case ITER_TREE_PARTITION_SCAN: { + break; + } + case ITER_TREE_LOCAL_LOOKUP: { + ObDASLocalLookupIter *lookup_iter = static_cast(result_); + lookup_iter->set_tablet_id(tablet_ids_.lookup_tablet_id_); + lookup_iter->set_ls_id(ls_id_); + break; + } + case ITER_TREE_DOMAIN_LOOKUP: { + break; + } + case ITER_TREE_TEXT_RETRIEVAL: { + ObDASIter *result_iter = static_cast(result_); + if (OB_FAIL(ObDASIterUtils::set_text_retrieval_related_ids( + attach_ctdef_, tablet_ids_, ls_id_, result_iter))) { + LOG_WARN("failed to set text retrieval related ids", K(ret)); + } + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + } + } + } + + if (OB_SUCC(ret) && OB_FAIL(result->reuse())) { + LOG_WARN("failed to reuse das iter tree", K(ret)); + } } } else if (scan_param_.table_param_->is_multivalue_index() && attach_ctdef_ != nullptr) { if (nullptr != lookup_op @@ -811,11 +916,8 @@ int ObDASScanOp::reuse_iter() } else if (lookup_op != nullptr && OB_FAIL(lookup_op->reset_lookup_state())) { LOG_WARN("reuse lookup iterator failed", K(ret)); - } else { - scan_param_.key_ranges_.reuse(); - scan_param_.ss_key_ranges_.reuse(); - scan_param_.mbr_filters_.reuse(); } + return ret; } @@ -899,14 +1001,13 @@ int ObDASScanOp::get_aux_lookup_tablet_id(common::ObTabletID &tablet_id) const int ret = OB_SUCCESS; tablet_id.reset(); const ObDASIRAuxLookupCtDef *aux_lookup_ctdef = nullptr; - ObDASIRAuxLookupRtDef *aux_lookup_rtdef = nullptr; - if (OB_FAIL(ObDASUtils::find_target_das_def(attach_ctdef_, - attach_rtdef_, - DAS_OP_IR_AUX_LOOKUP, - aux_lookup_ctdef, - aux_lookup_rtdef))) { - LOG_WARN("find aux lookup definition failed", K(ret)); - } else { + if (nullptr != attach_ctdef_ && ObDASOpType::DAS_OP_TABLE_LOOKUP == attach_ctdef_->op_type_) { + const ObDASTableLookupCtDef *table_lookup_ctdef = static_cast(attach_ctdef_); + if (ObDASOpType::DAS_OP_IR_AUX_LOOKUP == table_lookup_ctdef->get_rowkey_scan_ctdef()->op_type_) { + aux_lookup_ctdef = static_cast(table_lookup_ctdef->get_rowkey_scan_ctdef()); + } else if (ObDASOpType::DAS_OP_SORT == table_lookup_ctdef->get_rowkey_scan_ctdef()->op_type_) { + aux_lookup_ctdef = static_cast(table_lookup_ctdef->get_rowkey_scan_ctdef()->children_[0]); + } for (int i = 0; !tablet_id.is_valid() && i < related_ctdefs_.count(); ++i) { if (aux_lookup_ctdef->get_lookup_scan_ctdef() == related_ctdefs_.at(i)) { tablet_id = related_tablet_ids_.at(i); @@ -916,9 +1017,10 @@ int ObDASScanOp::get_aux_lookup_tablet_id(common::ObTabletID &tablet_id) const return ret; } -ObTabletID ObDASScanOp::get_table_lookup_tablet_id() const +int ObDASScanOp::get_table_lookup_tablet_id(common::ObTabletID &tablet_id) const { - ObTabletID tablet_id; + int ret = OB_SUCCESS; + tablet_id.reset(); if (get_lookup_ctdef() != nullptr) { for (int i = 0; !tablet_id.is_valid() && i < related_ctdefs_.count(); ++i) { if (get_lookup_ctdef() == related_ctdefs_.at(i)) { @@ -926,158 +1028,6 @@ ObTabletID ObDASScanOp::get_table_lookup_tablet_id() const } } } - return tablet_id; -} - -int ObDASScanOp::do_text_retrieve(common::ObNewRowIterator *&retrieval_iter) -{ - int ret = OB_SUCCESS; - retrieval_iter = nullptr; - ObTextRetrievalOp *retrieval_op = nullptr; - ObTabletID inv_idx_tablet_id; - ObTabletID fwd_idx_tablet_id; - ObTabletID doc_id_idx_tablet_id; - const ObDASIRScanCtDef *ir_scan_ctdef = nullptr; - ObDASIRScanRtDef *ir_scan_rtdef = nullptr; - const ObDASIRAuxLookupCtDef *aux_lookup_ctdef = nullptr; - ObDASIRAuxLookupRtDef *aux_lookup_rtdef = nullptr; - const ObDASSortCtDef *sort_ctdef = nullptr; - ObDASSortRtDef *sort_rtdef = nullptr; - const bool has_lookup = nullptr != get_lookup_ctdef(); - if (OB_ISNULL(retrieval_op = OB_NEWx(ObTextRetrievalOp, &op_alloc_))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate text retrieval op", K(ret)); - } else if (FALSE_IT(retrieval_iter = retrieval_op)) { - } else if (OB_FAIL(get_text_ir_tablet_ids(inv_idx_tablet_id, fwd_idx_tablet_id, doc_id_idx_tablet_id))) { - LOG_WARN("failed to get text ir tablet ids", K(ret)); - } else if (OB_FAIL(ObDASUtils::find_target_das_def(attach_ctdef_, - attach_rtdef_, - DAS_OP_IR_SCAN, - ir_scan_ctdef, - ir_scan_rtdef))) { - LOG_WARN("find ir scan definition failed", K(ret)); - } else if (has_lookup && OB_FAIL(ObDASUtils::find_target_das_def(attach_ctdef_, - attach_rtdef_, - DAS_OP_IR_AUX_LOOKUP, - aux_lookup_ctdef, - aux_lookup_rtdef))) { - LOG_WARN("find aux lookup definition failed", K(ret)); - } - - if (OB_SUCC(ret)) { - if (has_lookup) { - // relevance sort would be child of aux lookup if aux lookup exists - if (DAS_OP_SORT == aux_lookup_ctdef->get_doc_id_scan_ctdef()->op_type_) { - sort_ctdef = static_cast(aux_lookup_ctdef->get_doc_id_scan_ctdef()); - sort_rtdef = static_cast(aux_lookup_rtdef->get_doc_id_scan_rtdef()); - } - } else { - // relevance sort would be the root attach ctdef if no aux/table lookup - if (DAS_OP_SORT == attach_ctdef_->op_type_) { - sort_ctdef = static_cast(attach_ctdef_); - sort_rtdef = static_cast(attach_rtdef_); - } - } - } - - if (FAILEDx(retrieval_op->init(ls_id_, - inv_idx_tablet_id, - fwd_idx_tablet_id, - doc_id_idx_tablet_id, - ir_scan_ctdef, - ir_scan_rtdef, - sort_ctdef, - sort_rtdef, - trans_desc_, - snapshot_))) { - LOG_WARN("failed to init text retrieval op", K(ret)); - } - return ret; -} - -int ObDASScanOp::do_text_retrieve_rescan() -{ - int ret = OB_SUCCESS; - if (nullptr == result_ || (result_->get_type() != ObNewRowIterator::IterType::ObLocalIndexLookupIterator - && result_->get_type() != ObNewRowIterator::IterType::ObTextRetrievalOp)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected text retrieve rescan status", K(ret), KP_(result)); - } else { - ObTabletID inv_idx_tablet_id; - ObTabletID fwd_idx_tablet_id; - ObTabletID doc_id_idx_tablet_id; - ObTabletID aux_lookup_tablet_id; - const ObDASIRScanCtDef *ir_scan_ctdef = nullptr; - ObDASIRScanRtDef *ir_scan_rtdef = nullptr; - const ObDASIRAuxLookupCtDef *aux_lookup_ctdef = nullptr; - ObDASIRAuxLookupRtDef *aux_lookup_rtdef = nullptr; - const ObDASSortCtDef *sort_ctdef = nullptr; - ObDASSortRtDef *sort_rtdef = nullptr; - const bool has_lookup = nullptr != get_lookup_ctdef(); - ObFullTextIndexLookupOp *text_lookup_op = has_lookup - ? static_cast(result_) - : nullptr; - ObTextRetrievalOp * text_retrieval_op = has_lookup - ? static_cast(text_lookup_op->get_text_retrieval_iter()) - : static_cast(result_); - if (OB_FAIL(get_text_ir_tablet_ids(inv_idx_tablet_id, fwd_idx_tablet_id, doc_id_idx_tablet_id))) { - LOG_WARN("failed to get text ir tablet ids", K(ret)); - } else if (OB_FAIL(ObDASUtils::find_target_das_def(attach_ctdef_, - attach_rtdef_, - DAS_OP_IR_SCAN, - ir_scan_ctdef, - ir_scan_rtdef))) { - LOG_WARN("find ir scan definition failed", K(ret)); - } else if (!has_lookup) { - // skip - } else if (OB_ISNULL(text_lookup_op)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected nullptr to text lookup op", K(ret), KPC(result_)); - } else if (OB_FAIL(get_aux_lookup_tablet_id(aux_lookup_tablet_id))) { - LOG_WARN("failed to get doc id idx tablet id", K(ret), K_(related_tablet_ids)); - } else if (OB_FAIL(ObDASUtils::find_target_das_def(attach_ctdef_, - attach_rtdef_, - DAS_OP_IR_AUX_LOOKUP, - aux_lookup_ctdef, - aux_lookup_rtdef))) { - LOG_WARN("find aux lookup definition failed", K(ret)); - } else { - text_lookup_op->set_tablet_id(get_table_lookup_tablet_id()); - text_lookup_op->set_ls_id(ls_id_); - text_lookup_op->set_doc_id_idx_tablet_id(aux_lookup_tablet_id); - } - - if (OB_SUCC(ret)) { - if (has_lookup) { - // relevance sort would be child of aux lookup if aux lookup exists - if (DAS_OP_SORT == aux_lookup_ctdef->get_doc_id_scan_ctdef()->op_type_) { - sort_ctdef = static_cast(aux_lookup_ctdef->get_doc_id_scan_ctdef()); - sort_rtdef = static_cast(aux_lookup_rtdef->get_doc_id_scan_rtdef()); - } - } else { - // relevance sort would be the root attach ctdef if no aux/table lookup - if (DAS_OP_SORT == attach_ctdef_->op_type_) { - sort_ctdef = static_cast(attach_ctdef_); - sort_rtdef = static_cast(attach_rtdef_); - } - } - } - - if (OB_FAIL(ret)) { - } else if (nullptr != text_retrieval_op - && OB_FAIL(text_retrieval_op->rescan(ls_id_, - inv_idx_tablet_id, - fwd_idx_tablet_id, - doc_id_idx_tablet_id, - ir_scan_ctdef, - ir_scan_rtdef, - sort_ctdef, - sort_rtdef, - trans_desc_, - snapshot_))) { - LOG_WARN("failed to do text retrieval rescan", K(ret)); - } - } return ret; } @@ -1350,10 +1300,6 @@ int ObLocalIndexLookupOp::init(const ObDASScanCtDef *lookup_ctdef, LOG_WARN("create lookup mem context entity failed", K(ret)); } } - int simulate_error = EVENT_CALL(EventTable::EN_DAS_SIMULATE_LOOKUPOP_INIT_ERROR); - if (OB_UNLIKELY(OB_SUCCESS != simulate_error)) { - ret = simulate_error; - } return ret; } @@ -1458,7 +1404,7 @@ int ObLocalIndexLookupOp::process_data_table_rowkey() ObObj tmp_obj; ObExpr *expr = index_ctdef_->result_output_.at(i); if (T_PSEUDO_GROUP_ID == expr->type_) { - group_idx = expr->locate_expr_datum(*lookup_rtdef_->eval_ctx_).get_int(); + group_idx = ObNewRange::get_group_idx(expr->locate_expr_datum(*lookup_rtdef_->eval_ctx_).get_int()); } else if (T_PSEUDO_ROW_TRANS_INFO_COLUMN == expr->type_) { // do nothing } else { diff --git a/src/sql/das/ob_das_scan_op.h b/src/sql/das/ob_das_scan_op.h index c480d2b8c..74f7990f3 100644 --- a/src/sql/das/ob_das_scan_op.h +++ b/src/sql/das/ob_das_scan_op.h @@ -17,6 +17,7 @@ #include "sql/engine/basic/ob_chunk_datum_store.h" #include "sql/engine/table/ob_index_lookup_op_impl.h" #include "sql/das/ob_group_scan_iter.h" +#include "sql/das/iter/ob_das_iter.h" namespace oceanbase { @@ -202,6 +203,8 @@ public: storage::ObTableScanParam &get_scan_param() { return scan_param_; } const storage::ObTableScanParam &get_scan_param() const { return scan_param_; } + int init_related_tablet_ids(ObDASRelatedTabletID &related_tablet_ids); + virtual int decode_task_result(ObIDASTaskResult *task_result) override; virtual int fill_task_result(ObIDASTaskResult &task_result, bool &has_more, int64_t &memory_limit) override; virtual int fill_extra_result() override; @@ -221,7 +224,7 @@ public: const ObDASScanCtDef *get_lookup_ctdef() const; ObDASScanRtDef *get_lookup_rtdef(); int get_aux_lookup_tablet_id(common::ObTabletID &tablet_id) const; - common::ObTabletID get_table_lookup_tablet_id() const; + int get_table_lookup_tablet_id(common::ObTabletID &tablet_id) const; int init_scan_param(); int rescan(); int reuse_iter(); @@ -230,8 +233,6 @@ public: bool is_contain_trans_info() {return NULL != scan_ctdef_->trans_info_expr_; } int do_table_scan(); int do_domain_index_lookup(); - int do_text_retrieve(common::ObNewRowIterator *&retrieval_iter); - int do_text_retrieve_rescan(); int get_text_ir_tablet_ids( common::ObTabletID &inv_idx_tablet_id, common::ObTabletID &fwd_idx_tablet_id, @@ -248,6 +249,7 @@ protected: int do_local_index_lookup(); common::ObNewRowIterator *get_storage_scan_iter(); common::ObNewRowIterator *get_output_result_iter() { return result_; } + ObDASIterTreeType get_iter_tree_type() const; public: ObSEArray trans_info_array_; protected: @@ -266,9 +268,12 @@ protected: storage::ObTableScanParam scan_param_; const ObDASScanCtDef *scan_ctdef_; ObDASScanRtDef *scan_rtdef_; + // result_ is actually a ObDASIter during execution common::ObNewRowIterator *result_; //Indicates the number of remaining rows currently that need to be sent through DTL int64_t remain_row_cnt_; + // only can be used in runner server + ObDASRelatedTabletID tablet_ids_; common::ObArenaAllocator *retry_alloc_; union { diff --git a/src/sql/das/ob_das_spatial_index_lookup_op.cpp b/src/sql/das/ob_das_spatial_index_lookup_op.cpp index bcfafb13e..852dd6cee 100644 --- a/src/sql/das/ob_das_spatial_index_lookup_op.cpp +++ b/src/sql/das/ob_das_spatial_index_lookup_op.cpp @@ -132,7 +132,7 @@ int ObSpatialIndexLookupOp::save_rowkeys() ObObj tmp_obj; ObExpr *expr = index_ctdef_->result_output_.at(i); if (T_PSEUDO_GROUP_ID == expr->type_) { - group_idx = expr->locate_expr_datum(*lookup_rtdef_->eval_ctx_).get_int(); + group_idx = ObNewRange::get_group_idx(expr->locate_expr_datum(*lookup_rtdef_->eval_ctx_).get_int()); } } if (OB_FAIL(lookup_range.build_range(ref_table_id, *idx_row))) { diff --git a/src/sql/das/ob_domain_index_lookup_op.cpp b/src/sql/das/ob_domain_index_lookup_op.cpp index 4550cce40..5ec5036fc 100644 --- a/src/sql/das/ob_domain_index_lookup_op.cpp +++ b/src/sql/das/ob_domain_index_lookup_op.cpp @@ -11,7 +11,7 @@ */ #define USING_LOG_PREFIX SQL_DAS #include "sql/das/ob_das_scan_op.h" -#include "sql/das/ob_text_retrieval_op.h" +#include "sql/das/ob_das_ir_define.h" #include "sql/das/ob_domain_index_lookup_op.h" #include "sql/das/ob_das_utils.h" #include "sql/engine/ob_exec_context.h" @@ -385,233 +385,6 @@ int ObDomainIndexLookupOp::reuse_scan_iter() return OB_SUCCESS; } -int ObFullTextIndexLookupOp::reuse_scan_iter() -{ - int ret = OB_SUCCESS; - - if (OB_FAIL(ObDomainIndexLookupOp::reuse_scan_iter())) { - LOG_WARN("failed to reuse scan iter", K(ret)); - } else { - ObITabletScan &tsc_service = get_tsc_service(); - const ObTabletID &scan_tablet_id = doc_id_scan_param_.tablet_id_; - doc_id_scan_param_.need_switch_param_ = scan_tablet_id.is_valid() && scan_tablet_id != doc_id_idx_tablet_id_; - if (OB_FAIL(tsc_service.reuse_scan_iter(doc_id_scan_param_.need_switch_param_, rowkey_iter_))) { - LOG_WARN("failed to reuse scan iter", K(ret)); - } else if (nullptr != lookup_iter_) { - doc_id_scan_param_.key_ranges_.reuse(); - doc_id_scan_param_.ss_key_ranges_.reuse(); - } - } - return ret; -} - -int ObFullTextIndexLookupOp::init(const ObDASBaseCtDef *table_lookup_ctdef, - ObDASBaseRtDef *table_lookup_rtdef, - transaction::ObTxDesc *tx_desc, - transaction::ObTxReadSnapshot *snapshot, - storage::ObTableScanParam &scan_param) -{ - int ret = OB_SUCCESS; - const ObDASTableLookupCtDef *tbl_lookup_ctdef = nullptr; - ObDASTableLookupRtDef *tbl_lookup_rtdef = nullptr; - const ObDASIRAuxLookupCtDef *aux_lookup_ctdef = nullptr; - ObDASIRAuxLookupRtDef *aux_lookup_rtdef = nullptr; - const ObDASIRScanCtDef *ir_scan_ctdef = nullptr; - ObDASIRScanRtDef *ir_scan_rtdef = nullptr; - if (OB_ISNULL(table_lookup_ctdef) || OB_ISNULL(table_lookup_rtdef)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table lookup param is nullptr", KP(table_lookup_ctdef), KP(table_lookup_rtdef)); - } else if (OB_FAIL(ObDASUtils::find_target_das_def(table_lookup_ctdef, - table_lookup_rtdef, - DAS_OP_TABLE_LOOKUP, - tbl_lookup_ctdef, - tbl_lookup_rtdef))) { - LOG_WARN("find data table lookup def failed", K(ret)); - } else if (OB_FAIL(ObDASUtils::find_target_das_def(tbl_lookup_ctdef, - tbl_lookup_rtdef, - DAS_OP_IR_AUX_LOOKUP, - aux_lookup_ctdef, - aux_lookup_rtdef))) { - LOG_WARN("find ir aux lookup def failed", K(ret)); - } else if (OB_FAIL(ObDASUtils::find_target_das_def(aux_lookup_ctdef, - aux_lookup_rtdef, - DAS_OP_IR_SCAN, - ir_scan_ctdef, - ir_scan_rtdef))) { - LOG_WARN("find ir scan def failed", K(ret)); - } else { - if (OB_FAIL(ObDomainIndexLookupOp::init(tbl_lookup_ctdef->get_lookup_scan_ctdef(), - tbl_lookup_rtdef->get_lookup_scan_rtdef(), - ir_scan_ctdef->get_inv_idx_scan_ctdef(), - ir_scan_rtdef->get_inv_idx_scan_rtdef(), - aux_lookup_ctdef->get_lookup_scan_ctdef(), - aux_lookup_rtdef->get_lookup_scan_rtdef(), - tx_desc, - snapshot, - scan_param))) { - LOG_WARN("failed to init domain index lookup op", K(ret)); - } else { - need_scan_aux_ = true; - doc_id_lookup_ctdef_ = aux_lookup_ctdef->get_lookup_scan_ctdef(); - doc_id_lookup_rtdef_ = aux_lookup_rtdef->get_lookup_scan_rtdef(); - doc_id_expr_ = ir_scan_ctdef->inv_scan_doc_id_col_; - retrieval_ctx_ = ir_scan_rtdef->eval_ctx_; - } - } - return ret; -} - -int ObFullTextIndexLookupOp::reset_lookup_state() -{ - int ret = OB_SUCCESS; - if (OB_FAIL(ObDomainIndexLookupOp::reset_lookup_state())) { - LOG_WARN("failed to reset lookup state for domain index lookup op", K(ret)); - } else { - if (nullptr != lookup_iter_) { - doc_id_scan_param_.key_ranges_.reuse(); - doc_id_scan_param_.ss_key_ranges_.reuse(); - } - } - return ret; -} - -void ObFullTextIndexLookupOp::do_clear_evaluated_flag() -{ - return ObDomainIndexLookupOp::do_clear_evaluated_flag(); -} - -int ObFullTextIndexLookupOp::revert_iter() -{ - int ret = OB_SUCCESS; - if (nullptr != text_retrieval_iter_) { - text_retrieval_iter_->reset(); - text_retrieval_iter_->~ObNewRowIterator(); - if (nullptr != allocator_) { - allocator_->free(text_retrieval_iter_); - } - text_retrieval_iter_ = nullptr; - } - - if (OB_FAIL(ObDomainIndexLookupOp::revert_iter())) { - LOG_WARN("failed to revert local index lookup op iter", K(ret)); - } - return ret; -} - -int ObFullTextIndexLookupOp::fetch_index_table_rowkey() -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(text_retrieval_iter_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null text retrieval iterator for index lookup", K(ret), KP(text_retrieval_iter_)); - } else if (OB_FAIL(text_retrieval_iter_->get_next_row())) { - if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("failed to get next next row from text retrieval iter", K(ret)); - } - } else if (OB_FAIL(set_lookup_doc_id_key(doc_id_expr_, retrieval_ctx_))) { - LOG_WARN("failed to set lookup doc id query key", K(ret)); - } - return ret; -} - -int ObFullTextIndexLookupOp::fetch_index_table_rowkeys(int64_t &count, const int64_t capacity) -{ - int ret = OB_SUCCESS; - int64_t index_scan_row_cnt = 0; - if (OB_ISNULL(text_retrieval_iter_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null text retrieval iterator for index lookup", K(ret), KP(text_retrieval_iter_)); - } else if (OB_FAIL(text_retrieval_iter_->get_next_rows(index_scan_row_cnt, capacity))) { - if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("failed to get next next row from text retrieval iter", K(ret)); - } else { - ret = OB_SUCCESS; - } - } - if (OB_SUCC(ret) && index_scan_row_cnt > 0) { - if (OB_FAIL(set_lookup_doc_id_keys(index_scan_row_cnt))) { - LOG_WARN("failed to set lookup doc id query key", K(ret)); - } else { - count += index_scan_row_cnt; - } - } - return ret; -} - -int ObFullTextIndexLookupOp::do_aux_table_lookup() -{ - int ret = OB_SUCCESS; - ObITabletScan &tsc_service = get_tsc_service(); - if (nullptr == rowkey_iter_) { - // init doc_id -> rowkey table iterator as rowkey iter - if (OB_FAIL(set_doc_id_idx_lookup_param( - doc_id_lookup_ctdef_, doc_id_lookup_rtdef_, doc_id_scan_param_, doc_id_idx_tablet_id_, ls_id_))) { - LOG_WARN("failed to init doc id lookup scan param", K(ret)); - } else if (tsc_service.table_scan(doc_id_scan_param_, rowkey_iter_)) { - if (OB_SNAPSHOT_DISCARDED == ret && scan_param_.fb_snapshot_.is_valid()) { - ret = OB_INVALID_QUERY_TIMESTAMP; - } else if (OB_TRY_LOCK_ROW_CONFLICT != ret) { - LOG_WARN("fail to scan table", K(scan_param_), K(ret)); - } - } - } else { - const ObTabletID &scan_tablet_id = doc_id_scan_param_.tablet_id_; - doc_id_scan_param_.need_switch_param_ = scan_tablet_id.is_valid() && (doc_id_idx_tablet_id_ != scan_tablet_id); - doc_id_scan_param_.tablet_id_ = doc_id_idx_tablet_id_; - doc_id_scan_param_.ls_id_ = ls_id_; - if (OB_FAIL(tsc_service.reuse_scan_iter(doc_id_scan_param_.need_switch_param_, rowkey_iter_))) { - LOG_WARN("failed to reuse doc id iterator", K(ret)); - } else if (OB_FAIL(tsc_service.table_rescan(doc_id_scan_param_, rowkey_iter_))) { - LOG_WARN("failed to rescan doc id rowkey table", K(ret), K_(doc_id_idx_tablet_id), K(scan_tablet_id)); - } - } - return ret; -} - -int ObFullTextIndexLookupOp::get_aux_table_rowkey() -{ - int ret = OB_SUCCESS; - doc_id_lookup_rtdef_->p_pd_expr_op_->clear_evaluated_flag(); - if (index_end_ && doc_id_scan_param_.key_ranges_.empty()) { - ret = OB_ITER_END; - } else if (OB_FAIL(do_aux_table_lookup())) { - LOG_WARN("failed to do aux table lookup", K(ret)); - } else if (OB_FAIL(rowkey_iter_->get_next_row())) { - LOG_WARN("failed to get rowkey by doc id", K(ret)); - } else if (OB_FAIL(set_main_table_lookup_key())) { - LOG_WARN("failed to set main table lookup key", K(ret)); - } - return ret; -} - -int ObFullTextIndexLookupOp::get_aux_table_rowkeys(const int64_t lookup_row_cnt) -{ - int ret = OB_SUCCESS; - doc_id_lookup_rtdef_->p_pd_expr_op_->clear_evaluated_flag(); - int64_t rowkey_cnt = 0; - if (index_end_ && doc_id_scan_param_.key_ranges_.empty()) { - ret = OB_ITER_END; - } else if (OB_FAIL(do_aux_table_lookup())) { - LOG_WARN("failed to do aux table lookup", K(ret)); - } else if (OB_FAIL(rowkey_iter_->get_next_rows(rowkey_cnt, lookup_row_cnt))) { - LOG_WARN("failed to get rowkey by doc id", K(ret), K(doc_id_scan_param_.key_ranges_)); - } else if (OB_UNLIKELY(lookup_row_cnt != rowkey_cnt)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected aux lookup row count not match", K(ret), K(rowkey_cnt), K(lookup_row_cnt)); - } else { - ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*doc_id_lookup_rtdef_->eval_ctx_); - batch_info_guard.set_batch_size(lookup_row_cnt); - for (int64_t i = 0; OB_SUCC(ret) && i < lookup_row_cnt; ++i) { - batch_info_guard.set_batch_idx(i); - if (OB_FAIL(set_main_table_lookup_key())) { - LOG_WARN("failed to set main table lookup key", K(ret)); - } - } - - } - return ret; -} - int ObMulValueIndexLookupOp::revert_iter() { int ret = OB_SUCCESS; @@ -979,70 +752,6 @@ int ObMulValueIndexLookupOp::save_rowkeys() return ret; } -int ObFullTextIndexLookupOp::set_lookup_doc_id_keys(const int64_t size) -{ - int ret = OB_SUCCESS; - ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*retrieval_ctx_); - batch_info_guard.set_batch_size(size); - for (int64_t i = 0; OB_SUCC(ret) && i < size; ++i) { - batch_info_guard.set_batch_idx(i); - if (OB_FAIL(set_lookup_doc_id_key(doc_id_expr_, retrieval_ctx_))) { - LOG_WARN("failed to set lookup doc id key", K(ret)); - } - } - return ret; -} - -int ObFullTextIndexLookupOp::set_main_table_lookup_key() -{ - int ret = OB_SUCCESS; - int64_t rowkey_cnt = doc_id_lookup_ctdef_->result_output_.count(); - void *buf = nullptr; - ObObj *obj_ptr = nullptr; - common::ObArenaAllocator &lookup_alloc = lookup_memctx_->get_arena_allocator(); - ObNewRange lookup_range; - if (nullptr != doc_id_lookup_ctdef_->trans_info_expr_) { - rowkey_cnt = rowkey_cnt - 1; - } - - if (OB_UNLIKELY(rowkey_cnt <= 0)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid rowkey cnt", K(ret), KPC(doc_id_lookup_ctdef_)); - } else if (OB_ISNULL(buf = lookup_alloc.alloc(sizeof(ObObj) * rowkey_cnt))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("allocate memory failed", K(ret), K(rowkey_cnt)); - } else { - obj_ptr = new (buf) ObObj[rowkey_cnt]; - } - - for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt; ++i) { - ObObj tmp_obj; - ObExpr *expr = doc_id_lookup_ctdef_->result_output_.at(i); - if (T_PSEUDO_GROUP_ID == expr->type_) { - // do nothing - } else { - ObDatum &col_datum = expr->locate_expr_datum(*doc_id_lookup_rtdef_->eval_ctx_); - if (OB_FAIL(col_datum.to_obj(tmp_obj, expr->obj_meta_, expr->obj_datum_map_))) { - LOG_WARN("convert datum to obj failed", K(ret)); - } else if (OB_FAIL(ob_write_obj(lookup_alloc, tmp_obj, obj_ptr[i]))) { - LOG_WARN("deep copy rowkey value failed", K(ret), K(tmp_obj)); - } - } - } - - if (OB_SUCC(ret)) { - ObRowkey table_rowkey(obj_ptr, rowkey_cnt); - if (OB_FAIL(lookup_range.build_range(lookup_ctdef_->ref_table_id_, table_rowkey))) { - LOG_WARN("failed to build lookup range", K(ret), K(table_rowkey)); - } else if (OB_FAIL(scan_param_.key_ranges_.push_back(lookup_range))) { - LOG_WARN("store lookup key range failed", K(ret), K(scan_param_)); - } else { - LOG_DEBUG("get rowkey from docid rowkey table", K(ret), K(table_rowkey), K(lookup_range)); - } - } - return ret; -} - int ObMulValueIndexLookupOp::get_aux_table_rowkey() { INIT_SUCC(ret); diff --git a/src/sql/das/ob_domain_index_lookup_op.h b/src/sql/das/ob_domain_index_lookup_op.h index 37d963fca..3858aebe4 100644 --- a/src/sql/das/ob_domain_index_lookup_op.h +++ b/src/sql/das/ob_domain_index_lookup_op.h @@ -116,44 +116,6 @@ protected: static const int64_t MAX_NUM_PER_BATCH = 1000; }; -class ObFullTextIndexLookupOp : public ObDomainIndexLookupOp -{ -public: - explicit ObFullTextIndexLookupOp(ObIAllocator &allocator) - : ObDomainIndexLookupOp(allocator), - text_retrieval_iter_(nullptr), - retrieval_ctx_(nullptr) {} - - virtual ~ObFullTextIndexLookupOp() {} - - int init(const ObDASBaseCtDef *table_lookup_ctdef, - ObDASBaseRtDef *table_lookup_rtdef, - transaction::ObTxDesc *tx_desc, - transaction::ObTxReadSnapshot *snapshot, - storage::ObTableScanParam &scan_param); - void set_text_retrieval_iter(common::ObNewRowIterator *text_retrieval_iter) - { - text_retrieval_iter_ = text_retrieval_iter; - } - common::ObNewRowIterator *get_text_retrieval_iter() { return text_retrieval_iter_; } - virtual int reset_lookup_state() override; - virtual int do_aux_table_lookup(); - virtual int revert_iter() override; - virtual int reuse_scan_iter(); - virtual void do_clear_evaluated_flag() override; -protected: - virtual int fetch_index_table_rowkey() override; - virtual int fetch_index_table_rowkeys(int64_t &count, const int64_t capacity) override; - virtual int get_aux_table_rowkey() override; - virtual int get_aux_table_rowkeys(const int64_t lookup_row_cnt) override; -private: - int set_lookup_doc_id_keys(const int64_t size); - int set_main_table_lookup_key(); -private: - ObNewRowIterator *text_retrieval_iter_; - ObEvalCtx *retrieval_ctx_; -}; - class ObMulValueIndexLookupOp : public ObDomainIndexLookupOp { public: diff --git a/src/sql/engine/table/ob_table_scan_op.cpp b/src/sql/engine/table/ob_table_scan_op.cpp index 4b7b65f56..7bfb76f69 100644 --- a/src/sql/engine/table/ob_table_scan_op.cpp +++ b/src/sql/engine/table/ob_table_scan_op.cpp @@ -136,6 +136,7 @@ OB_DEF_SERIALIZE(ObTableScanCtDef) OB_UNIS_ENCODE(abandoned_always_false_aux_lookup); OB_UNIS_ENCODE(abandoned_always_false_text_ir); OB_UNIS_ENCODE(attach_spec_); + OB_UNIS_ENCODE(flags_); return ret; } @@ -169,6 +170,7 @@ OB_DEF_SERIALIZE_SIZE(ObTableScanCtDef) OB_UNIS_ADD_LEN(abandoned_always_false_aux_lookup); OB_UNIS_ADD_LEN(abandoned_always_false_text_ir); OB_UNIS_ADD_LEN(attach_spec_); + OB_UNIS_ADD_LEN(flags_); return len; } @@ -219,6 +221,7 @@ OB_DEF_DESERIALIZE(ObTableScanCtDef) OB_UNIS_DECODE(abandoned_always_false_aux_lookup); OB_UNIS_DECODE(abandoned_always_false_text_ir); OB_UNIS_DECODE(attach_spec_); + OB_UNIS_DECODE(flags_); return ret; } @@ -801,6 +804,23 @@ int ObTableScanOp::prepare_pushdown_limit_param() need_final_limit_ = true; tsc_rtdef_.prepare_multi_part_limit_param(); } + + // NOTICE: TSC operator can not apply final limit when das need keep ordering for multi partitions, + // consider following: + // TSC (limit 10), need_keep_order + // / \ + // / \ + // / \ + // DAS SCAN DAS SCAN + // p0 p1 + // (limit 10) (limit 10) + // when das need keep ordering, TSC should get 10 rows from each partition, with the upper operator + // applying merge sort and final limit. + // However, if we apply final limit on TSC operator, it will exit after got 10 rows from p0. + // TODO: @bingfan remove need_final_limit_ from TSC operator + if (MY_CTDEF.is_das_keep_order_ && OB_NOT_NULL(scan_iter_) && scan_iter_->get_das_task_cnt() > 1) { + need_final_limit_ = false; + } return ret; } @@ -1102,60 +1122,6 @@ OB_INLINE int ObTableScanOp::init_das_scan_rtdef(const ObDASScanCtDef &das_ctdef return ret; } -int ObTableScanOp::update_output_tablet_id() -{ - int ret = OB_SUCCESS; - if (NULL != MY_SPEC.pdml_partition_id_) { - const ObDASTabletLoc *data_tablet_loc = nullptr; - int64_t output_id = OB_INVALID_ID; - if (MY_SPEC.partition_id_calc_type_ > 0) { - // partition id for gather statistics, index scan should output index partition id - data_tablet_loc = scan_result_.get_tablet_loc(); - } else if (MY_SPEC.should_scan_index()) { - data_tablet_loc = ObDASUtils::get_related_tablet_loc(*scan_result_.get_tablet_loc(), MY_SPEC.ref_table_id_); - } else { - data_tablet_loc = scan_result_.get_tablet_loc(); - } - if (OB_ISNULL(data_tablet_loc)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("data tablet loc is null, value of pdml partition id will not be set", K(ret), - K(MY_SPEC.should_scan_index()), K(MY_SPEC.ref_table_id_)); - } else { - if (MY_SPEC.partition_id_calc_type_ == 0) { - output_id = data_tablet_loc->tablet_id_.id(); - } else if (MY_SPEC.partition_id_calc_type_ == 1) { - output_id = data_tablet_loc->first_level_part_id_ != OB_INVALID_ID ? - data_tablet_loc->first_level_part_id_ : data_tablet_loc->partition_id_; - } else if (MY_SPEC.partition_id_calc_type_ == 2) { - output_id = data_tablet_loc->partition_id_; - } else { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("get invalid partition id cacl type", K(ret)); - } - if (OB_FAIL(ret)) { - } else if (is_vectorized()) { - const int64_t batch_size = MY_SPEC.max_batch_size_; - ObExpr *expr = MY_SPEC.pdml_partition_id_; - ObDatum *datums = expr->locate_datums_for_update(eval_ctx_, batch_size); - for (int64_t i = 0; i < batch_size; i++) { - datums[i].set_int(output_id); - } - expr->set_evaluated_projected(eval_ctx_); - LOG_TRACE("find the partition id expr in pdml table scan", K(ret), KPC(expr), KPC(data_tablet_loc), K(output_id)); - } else { - // handle PDML partition id: - // if partition id expr in TSC output_exprs, - // set the TSC partition id to the corresponding expr frame - ObExpr *expr = MY_SPEC.pdml_partition_id_; - expr->locate_datum_for_write(eval_ctx_).set_int(output_id); - expr->set_evaluated_projected(eval_ctx_); - LOG_TRACE("find the partition id expr in pdml table scan", K(ret), KPC(data_tablet_loc), K(output_id)); - } - } - } - return ret; -} - int ObTableScanOp::prepare_scan_range() { int ret = OB_SUCCESS; @@ -1196,6 +1162,7 @@ int ObTableScanOp::prepare_batch_scan_range() LOG_DEBUG("after prepare batch scan range", K(MY_INPUT.key_ranges_), K(MY_INPUT.ss_key_ranges_)); return ret; } + int ObTableScanOp::build_bnlj_params() { int ret = OB_SUCCESS; @@ -1492,10 +1459,9 @@ int ObTableScanOp::inner_open() // create and init iter_tree_. const ObTableScanSpec &spec = MY_SPEC; - if (OB_SUCC(ret)) { - if (spec.should_scan_index()) { - if (spec.is_global_index_back()) { - if (OB_FAIL(ObDASIterUtils::create_global_lookup_iter_tree(MY_CTDEF, + ObDASIterTreeType tree_type = spec.is_global_index_back() ? ITER_TREE_GLOBAL_LOOKUP : ITER_TREE_TABLE_SCAN; + if (OB_SUCC(ret) && OB_FAIL(ObDASIterUtils::create_tsc_iter_tree(tree_type, + spec.tsc_ctdef_, tsc_rtdef_, eval_ctx_, ctx_, @@ -1504,28 +1470,7 @@ int ObTableScanOp::inner_open() can_partition_retry(), scan_iter_, iter_tree_))) { - LOG_WARN("failed to create global lookup iter tree", K(ret)); - } - } else if (OB_FAIL(ObDASIterUtils::create_local_lookup_iter_tree(MY_CTDEF, - tsc_rtdef_, - eval_ctx_, - ctx_, - eval_infos_, - spec, - scan_iter_, - iter_tree_))) { - LOG_WARN("failed to create local lookup iter tree", K(ret)); - } - } else if (OB_FAIL(ObDASIterUtils::create_table_scan_iter_tree(MY_CTDEF, - tsc_rtdef_, - eval_ctx_, - ctx_, - eval_infos_, - spec, - scan_iter_, - iter_tree_))) { - LOG_WARN("failed to create table scan iter tree", K(ret)); - } + LOG_WARN("failed to create table scan iter tree", K(tree_type), K(ret)); } output_ = iter_tree_; return ret; @@ -1823,28 +1768,10 @@ int ObTableScanOp::local_iter_rescan() DASTaskIter task_iter = scan_iter_->begin_task_iter(); for (; OB_SUCC(ret) && !task_iter.is_end(); ++task_iter) { ObDASScanOp *scan_op = DAS_SCAN_OP(*task_iter); - if (OB_ISNULL(scan_op)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected nullptr das scan op", K(ret)); - } else if (MY_SPEC.gi_above_) { - if (!MY_SPEC.is_index_global_ && MY_CTDEF.lookup_ctdef_ != nullptr) { - //is local index lookup, need to set the lookup ctdef to the das scan op - if (OB_FAIL(pushdown_normal_lookup_to_das(*scan_op))) { - LOG_WARN("pushdown normal lookup to das failed", K(ret)); - } - } - if (OB_SUCC(ret) && MY_CTDEF.attach_spec_.attach_ctdef_ != nullptr) { - if (OB_FAIL(pushdown_attach_task_to_das(*scan_op))) { - LOG_WARN("pushdown attach task to das failed", K(ret)); - } - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(cherry_pick_range_by_tablet_id(scan_op))) { - LOG_WARN("prune query range by partition id failed", K(ret)); - } else if (OB_FAIL(scan_iter_->rescan_das_task(scan_op))) { - LOG_WARN("rescan das task failed", K(ret)); - } + if (OB_FAIL(cherry_pick_range_by_tablet_id(scan_op))) { + LOG_WARN("prune query range by partition id failed", K(ret)); + } else if (OB_FAIL(scan_iter_->rescan_das_task(scan_op))) { + LOG_WARN("rescan das task failed", K(ret)); } } } @@ -1882,9 +1809,23 @@ int ObTableScanOp::local_iter_reuse() scan_op->set_ls_id(MY_INPUT.tablet_loc_->ls_id_); scan_op->set_tablet_loc(MY_INPUT.tablet_loc_); } + if (MY_SPEC.gi_above_) { + if (!MY_SPEC.is_index_global_ && MY_CTDEF.lookup_ctdef_ != nullptr) { + //is local index lookup, need to set the lookup ctdef to the das scan op + if (OB_FAIL(pushdown_normal_lookup_to_das(*scan_op))) { + LOG_WARN("pushdown normal lookup to das failed", K(ret)); + } + } + if (OB_SUCC(ret) && MY_CTDEF.attach_spec_.attach_ctdef_ != nullptr) { + if (OB_FAIL(pushdown_attach_task_to_das(*scan_op))) { + LOG_WARN("pushdown attach task to das failed", K(ret)); + } + } + } scan_op->reuse_iter(); } } + if (OB_FAIL(ret)) { } else if (OB_FAIL(reuse_table_rescan_allocator())) { LOG_WARN("get table allocator", K(ret)); @@ -1975,7 +1916,9 @@ void ObTableScanOp::reset_iter_tree_for_rescan() if (OB_NOT_NULL(fold_iter_)) { fold_iter_->reuse(); } - if (iter_tree_->get_type() == DAS_ITER_LOOKUP) { + + // we cannot simply reuse iter tree due to local iter rescan optimization. + if (iter_tree_->get_type() == DAS_ITER_GLOBAL_LOOKUP) { iter_tree_->reuse(); } } @@ -1992,9 +1935,6 @@ int ObTableScanOp::set_batch_iter(int64_t group_id) return ret; } - - - int ObTableScanOp::get_next_row_with_das() { int ret = OB_SUCCESS; @@ -2464,6 +2404,7 @@ int ObTableScanOp::cherry_pick_range_by_tablet_id(ObDASScanOp *scan_op) ObNewRange whole_range; false_range.set_false_range(); false_range.group_idx_ = input_ranges.at(0).group_idx_; + false_range.index_ordered_idx_ = input_ranges.at(0).index_ordered_idx_; whole_range.set_whole_range(); if (OB_FAIL(scan_ranges.push_back(false_range))) { LOG_WARN("store false range to scan ranges failed", K(ret)); diff --git a/src/sql/engine/table/ob_table_scan_op.h b/src/sql/engine/table/ob_table_scan_op.h index dada33d49..24b67d61e 100644 --- a/src/sql/engine/table/ob_table_scan_op.h +++ b/src/sql/engine/table/ob_table_scan_op.h @@ -24,9 +24,8 @@ #include "sql/das/ob_das_ref.h" #include "sql/das/ob_data_access_service.h" #include "sql/das/ob_das_scan_op.h" -#include "sql/das/ob_text_retrieval_op.h" #include "sql/das/ob_das_attach_define.h" -#include "sql/das/ob_text_retrieval_op.h" +#include "sql/das/ob_das_ir_define.h" #include "sql/engine/basic/ob_pushdown_filter.h" #include "sql/engine/table/ob_index_lookup_op_impl.h" #include "sql/das/iter/ob_das_iter.h" @@ -147,7 +146,8 @@ public: allocator_(allocator), calc_part_id_expr_(NULL), global_index_rowkey_exprs_(allocator), - attach_spec_(allocator_, &scan_ctdef_) + attach_spec_(allocator_, &scan_ctdef_), + flags_(0) { } const ExprFixedArray &get_das_output_exprs() const { @@ -199,6 +199,13 @@ public: ExprFixedArray global_index_rowkey_exprs_; // end for Global Index Lookup ObDASAttachSpec attach_spec_; + union { + uint64_t flags_; + struct { + uint64_t is_das_keep_order_ : 1; // whether das need keep ordering + uint64_t reserved_ : 63; + }; + }; }; struct ObTableScanRtDef @@ -470,7 +477,6 @@ protected: int local_iter_rescan(); int close_and_reopen(); - int update_output_tablet_id(); int cherry_pick_range_by_tablet_id(ObDASScanOp *scan_op); int can_prune_by_tablet_id(const common::ObTabletID &tablet_id, diff --git a/src/sql/ob_optimizer_trace_impl.cpp b/src/sql/ob_optimizer_trace_impl.cpp index b161b0159..29100e49f 100644 --- a/src/sql/ob_optimizer_trace_impl.cpp +++ b/src/sql/ob_optimizer_trace_impl.cpp @@ -937,6 +937,7 @@ int ObOptimizerTraceImpl::trace_parameters() TRACE_PARAMETER(_optimizer_sortmerge_join_enabled, bool); TRACE_PARAMETER(_nested_loop_join_enabled, bool); TRACE_PARAMETER(_enable_var_assign_use_das, bool); + TRACE_PARAMETER(_enable_das_keep_order, bool); //for system variables TRACE_SYS_VAR(_PX_SHARED_HASH_JOIN, int64_t); TRACE_SYS_VAR(_ENABLE_PARALLEL_DML, int64_t); diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index 5556a1cd3..7e67550fc 100644 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -618,8 +618,15 @@ int ObJoinOrder::compute_base_table_path_ordering(AccessPath *path) ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(path), K(ret)); } else if (path->use_das_ && + !path->ordering_.empty() && path->table_partition_info_->get_phy_tbl_location_info().get_partition_cnt() > 1) { - path->ordering_.reset(); + if (get_plan()->get_optimizer_context().is_das_keep_order_enabled()) { + // when enable das keep order optimization, DAS layer can provide a guarantee of local order, + // otherwise the order is totally not guaranteed. + path->is_local_order_ = true; + } else { + path->ordering_.reset(); + } } else if (path->ordering_.empty() || is_at_most_one_row_ || !path->strong_sharding_->is_distributed()) { path->is_local_order_ = false; } else if (get_plan()->get_optimizer_context().is_online_ddl()) { @@ -1782,10 +1789,13 @@ int ObJoinOrder::create_one_access_path(const uint64_t table_id, LOG_WARN("failed to add access filters", K(*ap), K(ordering_info.get_index_keys()), K(ret)); } else if (OB_FAIL(get_plan()->get_stmt()->get_column_items(table_id, ap->est_cost_info_.access_column_items_))) { LOG_WARN("failed to get column items", K(ret)); - } else if ((!ap->is_global_index_ || !index_info_entry->is_index_back()) && - OB_FAIL(ObOptimizerUtil::make_sort_keys(ordering_info.get_ordering(), - ordering_info.get_scan_direction(), - ap->ordering_))) { + } else if ((!ap->is_global_index_ || + !index_info_entry->is_index_back() || + get_plan()->get_optimizer_context().is_das_keep_order_enabled()) + // for global index lookup without keep order, the ordering is wrong. + && OB_FAIL(ObOptimizerUtil::make_sort_keys(ordering_info.get_ordering(), + ordering_info.get_scan_direction(), + ap->ordering_))) { LOG_WARN("failed to create index keys expression array", K(index_id), K(ret)); } else if (ordering_info.get_index_keys().count() > 0) { ap->pre_query_range_ = const_cast(range_info.get_query_range()); @@ -2153,8 +2163,8 @@ int ObJoinOrder::get_access_path_ordering(const uint64_t table_id, LOG_WARN("NULL pointer error", K(ret), K(table_id), K(ref_table_id), K(index_id)); } else if (!index_schema->is_ordered()) { // for virtual table, we have HASH index which offers no ordering on index keys - } else if (index_schema->is_global_index_table() && is_index_back) { - // for global index lookup, the order is wrong. + } else if (index_schema->is_global_index_table() && is_index_back && !opt_ctx->is_das_keep_order_enabled()) { + // for global index lookup without keep order, the ordering is wrong. } else if (OB_FAIL(append(ordering, index_keys))) { LOG_WARN("failed to append index ordering expr", K(ret)); } else if (OB_FAIL(get_index_scan_direction(ordering, stmt, @@ -2616,7 +2626,7 @@ int ObJoinOrder::fill_index_info_entry(const uint64_t table_id, entry->get_ordering_info().get_ordering(), direction, is_index_back))) { - LOG_WARN("get_access_path_ordering ", K(ret)); + LOG_WARN("get access path ordering ", K(ret)); } else { entry->set_is_index_global(is_index_global); entry->set_is_index_geo(is_index_geo); diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index 7e9d7123c..6d037220d 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -7455,7 +7455,18 @@ int ObLogPlan::try_push_limit_into_table_scan(ObLogicalOperator *top, (NULL == table_scan->get_limit_expr() || ObOptimizerUtil::is_point_based_sub_expr(limit_expr, table_scan->get_limit_expr())) && table_scan->get_text_retrieval_info().topk_limit_expr_ == NULL) { - if (!top->is_distributed()) { + bool das_multi_partition = false; + if (table_scan->use_das() && NULL != table_scan->get_table_partition_info()) { + int64_t partition_count = table_scan->get_table_partition_info()-> + get_phy_tbl_location_info().get_phy_part_loc_info_list().count(); + if (1 != partition_count) { + das_multi_partition = true; + } + } + + if (das_multi_partition) { + new_limit_expr = pushed_expr; + } else if (!top->is_distributed()) { new_limit_expr = limit_expr; new_offset_expr = offset_expr; } else { @@ -7469,6 +7480,9 @@ int ObLogPlan::try_push_limit_into_table_scan(ObLogicalOperator *top, } else { is_pushed = true; } + if (das_multi_partition) { + is_pushed = false; + } } else if (OB_NOT_NULL(table_scan->get_text_retrieval_info().topk_limit_expr_)) { is_pushed = true; } @@ -10664,6 +10678,8 @@ int ObLogPlan::do_post_plan_processing() LOG_WARN("failed to collect table location", K(ret)); } else if (OB_FAIL(build_location_related_tablet_ids())) { LOG_WARN("build location related tablet ids failed", K(ret)); + } else if (OB_FAIL(check_das_need_keep_ordering(root))) { + LOG_WARN("failed to check das need keep ordering", K(ret)); } else { /*do nothing*/ } return ret; } @@ -11355,6 +11371,26 @@ int ObLogPlan::build_location_related_tablet_ids() return ret; } +int ObLogPlan::check_das_need_keep_ordering(ObLogicalOperator *op) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(op)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null param", K(ret)); + } else if (log_op_def::LOG_TABLE_SCAN == op->get_type()) { + ObLogTableScan *scan = static_cast(op); + if (OB_FAIL(scan->check_das_need_keep_ordering())) { + LOG_WARN("failed to check das need keep ordering", K(ret)); + } + } + for (int i = 0; OB_SUCC(ret) && i < op->get_num_of_child(); ++i) { + if (OB_FAIL(SMART_CALL(check_das_need_keep_ordering(op->get_child(i))))) { + LOG_WARN("failed to check das need keep ordering", K(ret)); + } + } + return ret; +} + int ObLogPlan::calc_plan_resource() { int ret = OB_SUCCESS; diff --git a/src/sql/optimizer/ob_log_plan.h b/src/sql/optimizer/ob_log_plan.h index ae40c206b..0593b4659 100644 --- a/src/sql/optimizer/ob_log_plan.h +++ b/src/sql/optimizer/ob_log_plan.h @@ -262,6 +262,7 @@ public: int collect_location_related_info(ObLogicalOperator &op); int build_location_related_tablet_ids(); + int check_das_need_keep_ordering(ObLogicalOperator *op); int gen_das_table_location_info(ObLogTableScan *table_scan, ObTablePartitionInfo *&table_partition_info); diff --git a/src/sql/optimizer/ob_log_table_scan.cpp b/src/sql/optimizer/ob_log_table_scan.cpp index 05a98979e..4d20949fc 100644 --- a/src/sql/optimizer/ob_log_table_scan.cpp +++ b/src/sql/optimizer/ob_log_table_scan.cpp @@ -196,6 +196,8 @@ int ObLogTableScan::get_op_exprs(ObIArray &all_exprs) LOG_WARN("failed to add lookup trans expr", K(ret)); } else if (NULL != trans_info_expr_ && OB_FAIL(all_exprs.push_back(trans_info_expr_))) { LOG_WARN("failed to push back expr", K(ret)); + } else if (NULL != group_id_expr_ && OB_FAIL(all_exprs.push_back(group_id_expr_))) { + LOG_WARN("failed to push back expr", K(ret)); } else if (is_text_retrieval_scan() && OB_FAIL(get_text_retrieval_calc_exprs(all_exprs))) { LOG_WARN("failed to get text retrieval exprs", K(ret)); } else if (OB_FAIL(append(all_exprs, access_exprs_))) { @@ -304,7 +306,7 @@ int ObLogTableScan::check_output_dependance(common::ObIArray &child LOG_WARN("failed to append exprs", K(ret)); } else if (OB_FAIL(append_array_no_dup(exprs, spatial_exprs_))) { LOG_WARN("failed to append exprs", K(ret)); - } else if (use_batch() && nullptr != group_id_expr_ + } else if (use_group_id() && nullptr != group_id_expr_ && OB_FAIL(add_var_to_array_no_dup(exprs, group_id_expr_))) { LOG_WARN("failed to push back group id expr", K(ret)); } else if (index_back_ && @@ -435,10 +437,9 @@ int ObLogTableScan::generate_access_exprs() LOG_WARN("failed to copy text retrieval aggr exprs", K(ret)); } else if (OB_FAIL(generate_necessary_rowkey_and_partkey_exprs())) { LOG_WARN("failed to generate rowkey and part exprs", K(ret)); - } else if (use_batch() - && OB_FAIL(ObOptimizerUtil::allocate_group_id_expr(get_plan(), group_id_expr_))) { - LOG_WARN("allocate group id expr failed", K(ret)); - } else if (nullptr != group_id_expr_ && OB_FAIL(access_exprs_.push_back(group_id_expr_))) { + } else if (OB_FAIL(allocate_group_id_expr())) { + LOG_WARN("failed to allocate group id expr", K(ret)); + } else if (NULL != group_id_expr_ && use_batch_ && OB_FAIL(access_exprs_.push_back(group_id_expr_))) { LOG_WARN("failed to push back expr", K(ret)); } else if (OB_FAIL(append_array_no_dup(access_exprs_, rowkey_exprs_))) { LOG_WARN("failed to push back exprs", K(ret)); @@ -923,6 +924,23 @@ int ObLogTableScan::allocate_lookup_trans_info_expr() return ret; } +int ObLogTableScan::allocate_group_id_expr() +{ + int ret = OB_SUCCESS; + // [GROUP_ID] expr is now used for group rescan and global lookup keep order, it is handled + // by DAS layer and transparent to TSC operator. + ObRawExpr *group_id_expr = nullptr; + if (OB_ISNULL(get_plan())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", K(ret)); + } else if (use_group_id() && OB_FAIL(ObOptimizerUtil::allocate_group_id_expr(get_plan(), group_id_expr))) { + LOG_WARN("failed to allocate group id expr", K(ret)); + } else { + group_id_expr_ = group_id_expr; + } + return ret; +} + int ObLogTableScan::generate_necessary_rowkey_and_partkey_exprs() { int ret = OB_SUCCESS; @@ -1370,6 +1388,12 @@ int ObLogTableScan::get_plan_item_info(PlanText &plan_text, LOG_WARN("BUF_PRINTF fails", K(ret)); } else if (OB_FAIL(BUF_PRINTF("is_global_index=%s", is_index_global_? "true" : "false"))) { LOG_WARN("BUF_PRINTF fails", K(ret)); + } else if (!das_keep_ordering_) { + //do nothing + } else if (OB_FAIL(BUF_PRINTF(", "))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else if (OB_FAIL(BUF_PRINTF("keep_ordering=%s", das_keep_ordering_ ? "true" : "false"))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); } else { /* Do nothing */ } if (OB_SUCC(ret) && (0 != filter_before_index_back_.count())) { @@ -1379,7 +1403,6 @@ int ObLogTableScan::get_plan_item_info(PlanText &plan_text, LOG_WARN("BUF_PRINTF fails", K(ret)); } else { /* Do nothing */ } } - //Print ranges if (OB_FAIL(ret) || is_text_retrieval_scan()) { } else if (OB_FAIL(BUF_PRINTF(", "))) { @@ -2598,6 +2621,20 @@ int ObLogTableScan::get_card_without_filter(double &card) return ret; } +int ObLogTableScan::check_das_need_keep_ordering() +{ + int ret = OB_SUCCESS; + das_keep_ordering_ = true; + bool ordering_be_used = true; + if (!use_das_ && !(is_index_global_ && index_back_)) { + das_keep_ordering_ = false; + } else if (OB_FAIL(check_op_orderding_used_by_parent(ordering_be_used))) { + LOG_WARN("failed to check op ordering used by parent", K(ret)); + } else if (!ordering_be_used) { + das_keep_ordering_ = false; + } + return ret; +} int ObLogTableScan::generate_filter_monotonicity() { int ret = OB_SUCCESS; @@ -2803,4 +2840,4 @@ int ObLogTableScan::get_filter_assist_exprs(ObIArray &assist_exprs) } } return ret; -} \ No newline at end of file +} diff --git a/src/sql/optimizer/ob_log_table_scan.h b/src/sql/optimizer/ob_log_table_scan.h index c0f7c3fbd..77f8aba7b 100644 --- a/src/sql/optimizer/ob_log_table_scan.h +++ b/src/sql/optimizer/ob_log_table_scan.h @@ -155,6 +155,7 @@ public: use_column_store_(false), doc_id_table_id_(common::OB_INVALID_ID), text_retrieval_info_(), + das_keep_ordering_(false), filter_monotonicity_() { } @@ -520,7 +521,8 @@ public: int copy_filter_before_index_back(); void set_use_batch(bool use_batch) { use_batch_ = use_batch; } bool use_batch() const { return use_batch_; } - // only use group_id_expr_ when use_batch() is true. + // use group_id_expr_ when batch rescan or keep order for global lookup. + bool use_group_id() const { return use_batch_ || (is_index_global_ && index_back_ && das_keep_ordering_); } inline const ObRawExpr *get_group_id_expr() const { return group_id_expr_; } int extract_bnlj_param_idxs(common::ObIArray &bnlj_params); @@ -582,6 +584,10 @@ public: } void set_identify_seq_expr(ObRawExpr *expr) { identify_seq_expr_ = expr; } + inline bool das_need_keep_ordering() const { return das_keep_ordering_; } + + int check_das_need_keep_ordering(); + const ObIArray& get_filter_monotonicity() const { return filter_monotonicity_; } int get_filter_monotonicity(const ObRawExpr *filter, @@ -603,6 +609,7 @@ private: // member functions int add_mapping_columns_for_vt(ObIArray &access_exprs); int get_mbr_column_exprs(const uint64_t table_id, ObIArray &mbr_exprs); int allocate_lookup_trans_info_expr(); + int allocate_group_id_expr(); int extract_doc_id_index_back_expr(ObIArray &exprs); int extract_text_retrieval_access_expr(ObIArray &exprs); int get_text_retrieval_calc_exprs(ObIArray &all_exprs); @@ -725,6 +732,7 @@ protected: // memeber variables ObTextRetrievalInfo text_retrieval_info_; ObPxRFStaticInfo px_rf_info_; + bool das_keep_ordering_; typedef common::ObSEArray FilterMonotonicity; FilterMonotonicity filter_monotonicity_; // disallow copy and assign diff --git a/src/sql/optimizer/ob_logical_operator.cpp b/src/sql/optimizer/ob_logical_operator.cpp index d50f91e10..e01d3fe22 100644 --- a/src/sql/optimizer/ob_logical_operator.cpp +++ b/src/sql/optimizer/ob_logical_operator.cpp @@ -6288,7 +6288,11 @@ int ObLogicalOperator::check_use_child_ordering(bool &used, int64_t &inherit_chi } else { used = true; } - inherit_child_ordering_index = first_child; + if (LOG_TEMP_TABLE_TRANSFORMATION == get_type()) { + inherit_child_ordering_index = get_num_of_child()-1; + } else { + inherit_child_ordering_index = first_child; + } } else { used = false; inherit_child_ordering_index = -1; @@ -6488,36 +6492,38 @@ int ObLogicalOperator::check_op_orderding_used_by_parent(bool &used) { int ret = OB_SUCCESS; used = true; - bool is_first_child = true; bool inherit_child_ordering = true; int64_t inherit_child_ordering_index = -1; ObLogicalOperator *parent = get_parent(); ObLogicalOperator *child = this; - while (OB_SUCC(ret) && NULL != parent) { - if (OB_FAIL(parent->check_use_child_ordering(used, inherit_child_ordering_index))) { - LOG_WARN("failed to check use child ordering", K(ret)); - } else { - inherit_child_ordering = child == parent->get_child(inherit_child_ordering_index); - if (!used && inherit_child_ordering && child->is_plan_root()) { - ObLogPlan *plan = child->get_plan(); - const ObDMLStmt *stmt = NULL; - if (OB_ISNULL(plan) || OB_ISNULL(stmt=plan->get_stmt())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null param", K(ret)); - } else if (0 == stmt->get_order_item_size()) { - //do nothing - } else { - used = true; - } + while (OB_SUCC(ret) && NULL != child) { + if (child->is_plan_root()) { + ObLogPlan *plan = child->get_plan(); + const ObDMLStmt *stmt = NULL; + if (OB_ISNULL(plan) || OB_ISNULL(stmt=plan->get_stmt())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null param", K(ret)); + } else if (0 == stmt->get_order_item_size()) { + //do nothing + } else { + used = true; + break; + } + if (NULL == parent) { + break; } } - if (OB_FAIL(ret)) { - } else if (used || !inherit_child_ordering) { - break; - } else { - child = parent; - parent = parent->get_parent(); + if (OB_SUCC(ret) && NULL != parent) { + if (OB_FAIL(parent->check_use_child_ordering(used, inherit_child_ordering_index))) { + LOG_WARN("failed to check use child ordering", K(ret)); + } else if (OB_FALSE_IT(inherit_child_ordering = child == parent->get_child(inherit_child_ordering_index))) { + } else if (used || !inherit_child_ordering) { + break; + } else { + child = parent; + parent = parent->get_parent(); + } } } return ret; -} +} \ No newline at end of file diff --git a/src/sql/optimizer/ob_optimizer.cpp b/src/sql/optimizer/ob_optimizer.cpp index 4b7be7fd1..7891ee80b 100644 --- a/src/sql/optimizer/ob_optimizer.cpp +++ b/src/sql/optimizer/ob_optimizer.cpp @@ -560,6 +560,8 @@ int ObOptimizer::extract_opt_ctx_basic_flags(const ObDMLStmt &stmt, ObSQLSession omt::ObTenantConfigGuard tenant_config(TENANT_CONF(session.get_effective_tenant_id())); bool rowsets_enabled = tenant_config.is_valid() && tenant_config->_rowsets_enabled; ctx_.set_is_online_ddl(session.get_ddl_info().is_ddl()); // set is online ddl first, is used by other extract operations + bool das_keep_order_enabled = tenant_config.is_valid() && tenant_config->_enable_das_keep_order; + const ObOptParamHint &opt_params = ctx_.get_global_hint().opt_params_; if (OB_FAIL(check_whether_contain_nested_sql(stmt))) { LOG_WARN("check whether contain nested sql failed", K(ret)); } else if (OB_FAIL(stmt.check_has_subquery_in_function_table(has_subquery_in_function_table))) { @@ -582,6 +584,8 @@ int ObOptimizer::extract_opt_ctx_basic_flags(const ObDMLStmt &stmt, ObSQLSession LOG_WARN("fail to check cursor expression info", K(ret)); } else if (OB_FAIL(session.is_storage_estimation_enabled(storage_estimation_enabled))) { LOG_WARN("fail to get storage_estimation_enabled", K(ret)); + } else if (OB_FAIL(opt_params.get_bool_opt_param(ObOptParamHint::ENABLE_DAS_KEEP_ORDER, das_keep_order_enabled))) { + LOG_WARN("failed to check das keep order enabled", K(ret)); } else { ctx_.set_storage_estimation_enabled(storage_estimation_enabled); ctx_.set_serial_set_order(force_serial_set_order); @@ -592,6 +596,7 @@ int ObOptimizer::extract_opt_ctx_basic_flags(const ObDMLStmt &stmt, ObSQLSession ctx_.set_has_dblink(has_dblink); ctx_.set_cost_model_type(rowsets_enabled ? ObOptEstCost::VECTOR_MODEL : ObOptEstCost::NORMAL_MODEL); ctx_.set_has_cursor_expression(has_cursor_expr); + ctx_.set_das_keep_order_enabled(GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_3_2_0 ? false : das_keep_order_enabled); if (!tenant_config.is_valid() || (!tenant_config->_hash_join_enabled && !tenant_config->_optimizer_sortmerge_join_enabled && diff --git a/src/sql/optimizer/ob_optimizer_context.h b/src/sql/optimizer/ob_optimizer_context.h index 8e569419e..c7ee0c02e 100644 --- a/src/sql/optimizer/ob_optimizer_context.h +++ b/src/sql/optimizer/ob_optimizer_context.h @@ -234,6 +234,7 @@ ObOptimizerContext(ObSQLSessionInfo *session_info, nested_loop_join_enabled_(true), system_stat_(), storage_estimation_enabled_(false), + das_keep_order_enabled_(true), generate_random_plan_(false) { } inline common::ObOptStatManager *get_opt_stat_manager() { return opt_stat_manager_; } @@ -311,6 +312,8 @@ ObOptimizerContext(ObSQLSessionInfo *session_info, inline bool is_storage_estimation_enabled() const { return storage_estimation_enabled_; } void set_storage_estimation_enabled(bool storage_estimation_enabled) { storage_estimation_enabled_ = storage_estimation_enabled; } + inline bool is_das_keep_order_enabled() const { return das_keep_order_enabled_; } + void set_das_keep_order_enabled(bool das_keep_order_enabled) { das_keep_order_enabled_ = das_keep_order_enabled; } inline int64_t get_parallel() const { return parallel_; } inline int64_t get_max_parallel() const { return max_parallel_; } inline int64_t get_parallel_degree_limit(const int64_t server_cnt) const { return auto_dop_params_.get_parallel_degree_limit(server_cnt); } @@ -690,6 +693,7 @@ private: bool nested_loop_join_enabled_; OptSystemStat system_stat_; bool storage_estimation_enabled_; + bool das_keep_order_enabled_; bool generate_random_plan_; }; diff --git a/src/sql/plan_cache/ob_plan_cache_util.cpp b/src/sql/plan_cache/ob_plan_cache_util.cpp index 99b4f0b33..a7ecf4c92 100644 --- a/src/sql/plan_cache/ob_plan_cache_util.cpp +++ b/src/sql/plan_cache/ob_plan_cache_util.cpp @@ -537,6 +537,7 @@ int ObConfigInfoInPC::load_influence_plan_config() min_cluster_version_ = GET_MIN_CLUSTER_VERSION(); enable_spf_batch_rescan_ = tenant_config->_enable_spf_batch_rescan; enable_var_assign_use_das_ = tenant_config->_enable_var_assign_use_das; + enable_das_keep_order_ = tenant_config->_enable_das_keep_order; } return ret; @@ -581,16 +582,19 @@ int ObConfigInfoInPC::serialize_configs(char *buf, int buf_len, int64_t &pos) "%lu,", min_cluster_version_))) { SQL_PC_LOG(WARN, "failed to databuff_printf", K(ret), K(min_cluster_version_)); } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, - "%d", is_enable_px_fast_reclaim_))) { + "%d,", is_enable_px_fast_reclaim_))) { SQL_PC_LOG(WARN, "failed to databuff_printf", K(ret), K(is_enable_px_fast_reclaim_)); } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, - "%d", enable_spf_batch_rescan_))) { + "%d,", enable_spf_batch_rescan_))) { SQL_PC_LOG(WARN, "failed to databuff_printf", K(ret), K(enable_spf_batch_rescan_)); } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, - "%d", enable_var_assign_use_das_))) { + "%d,", enable_var_assign_use_das_))) { SQL_PC_LOG(WARN, "failed to databuff_printf", K(ret), K(enable_var_assign_use_das_)); } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, - "%d", bloom_filter_ratio_))) { + "%d,", enable_das_keep_order_))) { + SQL_PC_LOG(WARN, "failed to databuff_printf", K(ret), K(enable_das_keep_order_)); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, + "%d,", bloom_filter_ratio_))) { SQL_PC_LOG(WARN, "failed to databuff_printf", K(ret), K(bloom_filter_ratio_)); } else { // do nothing diff --git a/src/sql/plan_cache/ob_plan_cache_util.h b/src/sql/plan_cache/ob_plan_cache_util.h index 5d68ae6f1..e1203591b 100644 --- a/src/sql/plan_cache/ob_plan_cache_util.h +++ b/src/sql/plan_cache/ob_plan_cache_util.h @@ -1026,6 +1026,7 @@ public: is_enable_px_fast_reclaim_(false), enable_spf_batch_rescan_(false), enable_var_assign_use_das_(false), + enable_das_keep_order_(false), bloom_filter_ratio_(0), cluster_config_version_(-1), tenant_config_version_(-1), @@ -1070,6 +1071,7 @@ public: bool is_enable_px_fast_reclaim_; bool enable_spf_batch_rescan_; bool enable_var_assign_use_das_; + bool enable_das_keep_order_; int bloom_filter_ratio_; private: diff --git a/src/sql/resolver/ddl/ob_create_index_resolver.cpp b/src/sql/resolver/ddl/ob_create_index_resolver.cpp index b5ba0b3f5..394ac6b0e 100644 --- a/src/sql/resolver/ddl/ob_create_index_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_index_resolver.cpp @@ -749,8 +749,20 @@ int ObCreateIndexResolver::set_table_option_to_stmt(bool is_partitioned) index_arg.index_type_ = INDEX_TYPE_SPATIAL_LOCAL; } } else if (FTS_KEY == index_keyname_) { - // TODO hanxuan - ret = OB_NOT_SUPPORTED; + uint64_t tenant_data_version = 0; + if (OB_FAIL(GET_MIN_DATA_VERSION(session_info_->get_effective_tenant_id(), tenant_data_version))) { + LOG_WARN("get tenant data version failed", K(ret)); + } else if (tenant_data_version < DATA_VERSION_4_3_2_0) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("tenant data version is less than 4.3.2, create fulltext index on existing table not supported", K(ret), K(tenant_data_version)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "tenant data version is less than 4.3.2, fulltext index"); + } else if (global_) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support global fts index now", K(ret)); + } else { + // set type to fts_index_aux first, append other fts arg later + index_arg.index_type_ = INDEX_TYPE_FTS_INDEX_LOCAL; + } } index_arg.data_table_id_ = data_table_id_; index_arg.index_table_id_ = index_table_id_; diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index c0a2ea1f2..749dc7108 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -182,18 +182,19 @@ int ObDDLResolver::append_fts_args( LOG_WARN("allocator is null", K(ret)); } else if (!fts_common_aux_table_exist) { const int64_t num_fts_args = 4; - if (OB_FAIL(ObFtsIndexBuilderUtil::append_fts_rowkey_doc_arg(index_arg, - allocator, - index_arg_list))) { + // append fts index aux arg first, keep same logic as build fts index on existing table + if (OB_FAIL(ObFtsIndexBuilderUtil::append_fts_index_arg(index_arg, + allocator, + index_arg_list))) { + LOG_WARN("failed to append fts_index arg", K(ret)); + } else if (OB_FAIL(ObFtsIndexBuilderUtil::append_fts_rowkey_doc_arg(index_arg, + allocator, + index_arg_list))) { LOG_WARN("failed to append fts_rowkey_doc arg", K(ret)); } else if (OB_FAIL(ObFtsIndexBuilderUtil::append_fts_doc_rowkey_arg(index_arg, allocator, index_arg_list))) { LOG_WARN("failed to append fts_doc_rowkey arg", K(ret)); - } else if (OB_FAIL(ObFtsIndexBuilderUtil::append_fts_index_arg(index_arg, - allocator, - index_arg_list))) { - LOG_WARN("failed to append fts_index arg", K(ret)); } else if (OB_FAIL(ObFtsIndexBuilderUtil::append_fts_doc_word_arg(index_arg, allocator, index_arg_list))) { diff --git a/src/sql/resolver/dml/ob_hint.cpp b/src/sql/resolver/dml/ob_hint.cpp index c9454004f..792f9b014 100644 --- a/src/sql/resolver/dml/ob_hint.cpp +++ b/src/sql/resolver/dml/ob_hint.cpp @@ -838,6 +838,11 @@ bool ObOptParamHint::is_param_val_valid(const OptParamType param_type, const ObO || 0 == val.get_varchar().case_compare("false")); break; } + case ENABLE_DAS_KEEP_ORDER : { + is_valid = val.is_varchar() && (0 == val.get_varchar().case_compare("true") + || 0 == val.get_varchar().case_compare("false")); + break; + } case SPILL_COMPRESSION_CODEC: { is_valid = val.is_varchar(); if (is_valid) { diff --git a/src/sql/resolver/dml/ob_hint.h b/src/sql/resolver/dml/ob_hint.h index 53872240c..eb6238cf5 100644 --- a/src/sql/resolver/dml/ob_hint.h +++ b/src/sql/resolver/dml/ob_hint.h @@ -156,6 +156,7 @@ struct ObOptParamHint DEF(ENABLE_RICH_VECTOR_FORMAT,) \ DEF(_ENABLE_STORAGE_CARDINALITY_ESTIMATION,) \ DEF(PRESERVE_ORDER_FOR_PAGINATION,) \ + DEF(ENABLE_DAS_KEEP_ORDER,) \ DEF(SPILL_COMPRESSION_CODEC,) \ DEF(INLIST_REWRITE_THRESHOLD,) \ diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 64fa57746..5819ae034 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -177,7 +177,6 @@ ob_set_subtarget(ob_storage fts fts/ob_fts_stop_word.cpp fts/ob_ngram_ft_parser.cpp fts/ob_whitespace_ft_parser.cpp - fts/ob_text_retrieval_iterator.cpp ) ob_set_subtarget(ob_storage high_availability diff --git a/src/storage/access/ob_table_scan_range.cpp b/src/storage/access/ob_table_scan_range.cpp index 2e31d6ce1..a33f2ce87 100644 --- a/src/storage/access/ob_table_scan_range.cpp +++ b/src/storage/access/ob_table_scan_range.cpp @@ -187,7 +187,7 @@ int ObTableScanRange::init_rowkeys(const common::ObIArray &r } else if (is_false) { } else if (OB_FAIL(datum_rowkey.from_rowkey(rowkey, *allocator_))) { STORAGE_LOG(WARN, "Failed to transfer rowkey to datum rowkey", K(ret)); - } else if (FALSE_IT(datum_rowkey.set_group_idx(ranges.at(i).get_group_idx()))) { + } else if (FALSE_IT(datum_rowkey.set_group_idx(ranges.at(i).get_group_id()))) { } else if (OB_FAIL(rowkeys_.push_back(datum_rowkey))) { STORAGE_LOG(WARN, "Failed to push back datum rowkey", K(ret)); } diff --git a/src/storage/blocksstable/ob_datum_range.h b/src/storage/blocksstable/ob_datum_range.h index b0efa83f8..87317716f 100644 --- a/src/storage/blocksstable/ob_datum_range.h +++ b/src/storage/blocksstable/ob_datum_range.h @@ -212,7 +212,7 @@ OB_INLINE int ObDatumRange::from_range(const common::ObNewRange &range, ObIAlloc STORAGE_LOG(WARN, "Failed to from end key", K(ret)); } else { border_flag_ = range.border_flag_; - group_idx_ = range.get_group_idx(); + group_idx_ = range.get_group_id(); } return ret; diff --git a/src/storage/blocksstable/ob_datum_rowkey.h b/src/storage/blocksstable/ob_datum_rowkey.h index 1672b842c..b6108c5d1 100644 --- a/src/storage/blocksstable/ob_datum_rowkey.h +++ b/src/storage/blocksstable/ob_datum_rowkey.h @@ -56,7 +56,7 @@ public: OB_INLINE void set_max_rowkey() { *this = MAX_ROWKEY; store_rowkey_.set_max(); } OB_INLINE void set_min_rowkey() { *this = MIN_ROWKEY; store_rowkey_.set_min(); } OB_INLINE bool is_static_rowkey() const { return datums_ == &MIN_DATUM || datums_ == &MAX_DATUM; } - OB_INLINE void set_group_idx(const int32_t group_idx) { group_idx_ = group_idx; } + OB_INLINE void set_group_idx(const int64_t group_idx) { group_idx_ = group_idx; } OB_INLINE int64_t get_group_idx() const { return group_idx_; } OB_INLINE const common::ObStoreRowkey &get_store_rowkey() const { return store_rowkey_; } //only for unittest @@ -97,7 +97,7 @@ public: DECLARE_TO_STRING; public: int32_t datum_cnt_; - int32_t group_idx_; + int64_t group_idx_; mutable uint64_t hash_; ObStorageDatum *datums_; common::ObStoreRowkey store_rowkey_; diff --git a/src/storage/fts/ob_text_retrieval_iterator.h b/src/storage/fts/ob_text_retrieval_iterator.h deleted file mode 100644 index 29324b8d7..000000000 --- a/src/storage/fts/ob_text_retrieval_iterator.h +++ /dev/null @@ -1,177 +0,0 @@ -/** - * Copyright (c) 2024 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OB_TEXT_RETRIEVAL_ITERATOR_H_ -#define OB_TEXT_RETRIEVAL_ITERATOR_H_ - -#include "common/row/ob_row_iterator.h" -#include "storage/access/ob_dml_param.h" - - -namespace oceanbase -{ -namespace sql -{ -struct ObDASScanCtDef; -struct ObDASScanRtDef; -struct ObDASIRScanCtDef; -struct ObDASIRScanRtDef; -} -namespace storage -{ - -struct ObTokenRetrievalParam -{ -public: - ObTokenRetrievalParam() - : app_avg_tablet_doc_token_cnt_(0), - ls_id_(), - inv_idx_tablet_id_(), - fwd_idx_tablet_id_(), - doc_id_idx_tablet_id_(), - ir_ctdef_(nullptr), - ir_rtdef_(nullptr) - {} - ~ObTokenRetrievalParam() {} - - bool need_relevance() const; - const share::ObLSID &get_ls_id() const; - const sql::ObDASIRScanCtDef *get_ir_ctdef() const; - sql::ObDASIRScanRtDef *get_ir_rtdef(); - const sql::ObDASScanCtDef *get_inv_idx_scan_ctdef() const; - const sql::ObDASScanCtDef *get_inv_idx_agg_ctdef() const; - const sql::ObDASScanCtDef *get_fwd_idx_agg_ctdef() const; - const sql::ObDASScanCtDef *get_doc_id_idx_agg_ctdef() const; - const common::ObTabletID &get_inv_idx_tablet_id() const; - const common::ObTabletID &get_fwd_idx_tablet_id() const; - const common::ObTabletID &get_doc_id_idx_tablet_id() const; - inline void set_param( - const share::ObLSID &ls_id, - const ObTabletID &inv_idx_tablet_id, - const ObTabletID &fwd_idx_tablet_id, - const ObTabletID &doc_id_idx_tablet_id, - const sql::ObDASIRScanCtDef *ir_ctdef, - sql::ObDASIRScanRtDef *ir_rtdef, - const int64_t approx_avg_token_cnt = 0) - { - ls_id_ = ls_id; - inv_idx_tablet_id_ = inv_idx_tablet_id; - fwd_idx_tablet_id_ = fwd_idx_tablet_id; - doc_id_idx_tablet_id_ = doc_id_idx_tablet_id; - ir_ctdef_ = ir_ctdef; - ir_rtdef_ = ir_rtdef; - } - - TO_STRING_KV(K_(app_avg_tablet_doc_token_cnt), - K_(ls_id), - K_(inv_idx_tablet_id), - K_(fwd_idx_tablet_id), - K_(doc_id_idx_tablet_id)); -private: - int64_t app_avg_tablet_doc_token_cnt_; // TODO: use app avg tablet doc token cnt to calc bm25 idf - share::ObLSID ls_id_; - common::ObTabletID inv_idx_tablet_id_; - common::ObTabletID fwd_idx_tablet_id_; - common::ObTabletID doc_id_idx_tablet_id_; - const sql::ObDASIRScanCtDef *ir_ctdef_; - sql::ObDASIRScanRtDef *ir_rtdef_; -}; - -// Single token retrieval iter -class ObTextRetrievalIterator : public common::ObNewRowIterator -{ -public: - ObTextRetrievalIterator(); - virtual ~ObTextRetrievalIterator(); - - int init( - ObTokenRetrievalParam &retrieval_param, - const ObString &query_token, - transaction::ObTxDesc *tx_desc, - transaction::ObTxReadSnapshot *snapshot); - - virtual int get_next_row(ObNewRow *&row) override; - virtual int get_next_row() override; - virtual int get_next_rows(int64_t &count, int64_t capacity) override; - virtual void reset() override; - - int get_curr_iter_row(const sql::ExprFixedArray *&curr_row, sql::ObEvalCtx *&curr_eval_ctx); - - int get_curr_doc_id(); - int forward_to_doc(const common::ObDocId &doc_id); // TODO: impl this primitive for skipping scan in conjunctive processing - - TO_STRING_KV(KPC_(retrieval_param), K_(is_inited)); -private: - int init_inv_idx_scan_param(const ObString &query_token); - int init_fwd_idx_scan_param(); - static int init_base_idx_scan_param( - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, - const sql::ObDASScanCtDef *ctdef, - sql::ObDASScanRtDef *rtdef, - transaction::ObTxDesc *tx_desc, - transaction::ObTxReadSnapshot *snapshot, - ObTableScanParam &scan_param); - int get_next_doc_token_cnt(const bool use_fwd_idx_agg); - int do_doc_cnt_agg(); - int do_token_cnt_agg(const ObDocId &doc_id, int64_t &token_count); - int get_inv_idx_scan_doc_id(ObDocId &doc_id); - int fill_token_cnt_with_doc_len(); - int fill_token_doc_cnt(); - int project_relevance_expr(); - int reuse_fwd_idx_iter(); - int gen_inv_idx_scan_range(const ObString &query_token, ObNewRange &scan_range); - int gen_fwd_idx_scan_range(const ObDocId &doc_id, ObNewRange &scan_range); - inline bool need_calc_relevance() { return true; } // TODO: reduce tsc ops if no need to calc relevance - int init_calc_exprs(); - void clear_row_wise_evaluated_flag(); - - // TODO: delete this after enable standard vectorized execution - inline int get_next_single_row(const bool is_vectorized, ObNewRowIterator *iter) - { - int ret = OB_SUCCESS; - if (is_vectorized) { - int64_t scan_row_cnt = 0; - ret = iter->get_next_rows(scan_row_cnt, 1); - } else { - ret = iter->get_next_row(); - } - return ret; - } -private: - static const int64_t FWD_IDX_ROWKEY_COL_CNT = 2; - static const int64_t INV_IDX_ROWKEY_COL_CNT = 2; -private: - lib::MemoryContext mem_context_; - ObTokenRetrievalParam *retrieval_param_; - transaction::ObTxDesc *tx_desc_; - transaction::ObTxReadSnapshot *snapshot_; - ObTableScanParam inv_idx_scan_param_; - ObTableScanParam inv_idx_agg_param_; - ObTableScanParam fwd_idx_scan_param_; - common::ObSEArray calc_exprs_; - common::ObNewRowIterator *inverted_idx_iter_; - common::ObNewRowIterator *forward_idx_iter_; - ObObj *fwd_range_objs_; - sql::ObExpr *doc_token_cnt_expr_; - int64_t token_doc_cnt_; - bool need_fwd_idx_agg_; - bool need_inv_idx_agg_; - bool inv_idx_agg_evaluated_; - bool is_inited_; -}; - -} -} - - -#endif //OB_TEXT_RETRIEVAL_ITERATOR_H_ diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result index 10a250f72..4922c5a6f 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result @@ -296,6 +296,7 @@ _enable_column_store _enable_compaction_diagnose _enable_compatible_monotonic _enable_convert_real_to_decimal +_enable_das_keep_order _enable_dblink_reuse_connection _enable_dbms_job_package _enable_dbms_lob_partial_update diff --git a/tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_business_mysql.result b/tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_business_mysql.result index 8ab7812ae..643b2eea4 100644 --- a/tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_business_mysql.result +++ b/tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_business_mysql.result @@ -711,7 +711,7 @@ Outputs & filters: ------------------------------------- 0 - output([wfsydp.WSJJDM], [wfsydp.WSJWSY]), filter(nil), rowset=16 access([wfsydp.WSJJDM], [wfsydp.WSJWSY]), partitions(p0) - limit(181819), offset(0), is_index_back=false, is_global_index=true, + limit(181819), offset(0), is_index_back=false, is_global_index=true, keep_ordering=true, range_key([wfsydp.WSJJDM], [wfsydp.WSJWSY], [wfsydp.WSJJZH], [wfsydp.WSDRDM], [wfsydp.WSJYZH]), range(666888,MIN,MIN,MIN,MIN ; 666888,MAX,MAX,MAX, MAX), range_cond([wfsydp.WSJJDM = '666888']) diff --git a/tools/deploy/mysql_test/test_suite/subquery/r/mysql/subquery.result b/tools/deploy/mysql_test/test_suite/subquery/r/mysql/subquery.result index 4d41a8ae8..3463ebcbb 100644 --- a/tools/deploy/mysql_test/test_suite/subquery/r/mysql/subquery.result +++ b/tools/deploy/mysql_test/test_suite/subquery/r/mysql/subquery.result @@ -760,14 +760,14 @@ Outputs & filters: group(nil), agg_func([T_FUN_SUM(t2.c1)], [T_FUN_COUNT(t2.c1)]) 4 - output([t2.c1]), filter(nil), rowset=16 access([t2.c1]), partitions(p0) - is_index_back=false, is_global_index=false, + is_index_back=false, is_global_index=false, keep_ordering=true, range_key([t2.c1]), range(MIN ; MAX)always true, range_cond([t2.c1 = :0]) 5 - output([cast(T_FUN_COUNT(*), DECIMAL(24, 4))]), filter(nil), rowset=16 group(nil), agg_func([T_FUN_COUNT(*)]) 6 - output(nil), filter(nil), rowset=16 access(nil), partitions(p0) - is_index_back=false, is_global_index=false, + is_index_back=false, is_global_index=false, keep_ordering=true, range_key([t3.c1]), range(MIN ; MAX)always true, range_cond([t3.c1 = :0]) explain select * from t1 where t1.c1 != (select c2 from t2 where t2.c1 = (select max(c2) from t3 where t3.c1 = t1.c1) order by t2.c2 limit 1); @@ -803,7 +803,7 @@ Outputs & filters: group(nil), agg_func([T_FUN_MAX(t3.c2)]) 6 - output([t3.c2]), filter(nil), rowset=16 access([t3.c2]), partitions(p0) - is_index_back=false, is_global_index=false, + is_index_back=false, is_global_index=false, keep_ordering=true, range_key([t3.c1]), range(MIN ; MAX)always true, range_cond([t3.c1 = :0]) 7 - output([t2.c1], [t2.c2]), filter(nil), rowset=16 diff --git a/tools/deploy/mysql_test/test_suite/update/r/mysql/update2.result b/tools/deploy/mysql_test/test_suite/update/r/mysql/update2.result index 9b68d8c01..aae0a93e0 100644 --- a/tools/deploy/mysql_test/test_suite/update/r/mysql/update2.result +++ b/tools/deploy/mysql_test/test_suite/update/r/mysql/update2.result @@ -338,7 +338,7 @@ Outputs & filters: ------------------------------------- 0 - output([t1.a], [t1.b]), filter(nil), rowset=16 access([t1.a], [t1.b]), partitions(p0) - is_index_back=false, is_global_index=true, + is_index_back=false, is_global_index=true, keep_ordering=true, range_key([t1.b], [t1.a]), range(MIN,MIN ; MAX,MAX)always true select * from t1; +---+------+ @@ -396,7 +396,7 @@ Outputs & filters: ------------------------------------- 0 - output([t1.a], [t1.b]), filter(nil), rowset=16 access([t1.a], [t1.b]), partitions(p0) - is_index_back=false, is_global_index=true, + is_index_back=false, is_global_index=true, keep_ordering=true, range_key([t1.b], [t1.a]), range(MIN,MIN ; MAX,MAX)always true select * from t1; +---+------+ @@ -3283,7 +3283,7 @@ Outputs & filters: [gf_ar_mthly_bill.chked_amt], [gf_ar_mthly_bill.writingoff_amt], [gf_ar_mthly_bill.svc_bill_amt], [gf_ar_mthly_bill.svc_ccy], [gf_ar_mthly_bill.env_source], [gf_ar_mthly_bill.metadata_source], [gf_ar_mthly_bill.setl_time_zone], [gf_ar_mthly_bill.actg_time_zone], [gf_ar_mthly_bill.inter_trade_flag], [gf_ar_mthly_bill.actg_bill_mth], [gf_ar_mthly_bill.auto_writeoff_group_no]), partitions(p0) - is_index_back=true, is_global_index=true, + is_index_back=true, is_global_index=true, keep_ordering=true, range_key([gf_ar_mthly_bill.tnt_inst_id], [gf_ar_mthly_bill.rcrd_id], [gf_ar_mthly_bill.shadow_pk_0], [gf_ar_mthly_bill.shadow_pk_1]), range(MIN,MIN, MIN,MIN ; MAX,MAX,MAX,MAX)always true select * from gf_ar_mthly_bill; @@ -3461,7 +3461,7 @@ Outputs & filters: [gf_ar_mthly_bill.chked_amt], [gf_ar_mthly_bill.writingoff_amt], [gf_ar_mthly_bill.svc_bill_amt], [gf_ar_mthly_bill.svc_ccy], [gf_ar_mthly_bill.env_source], [gf_ar_mthly_bill.metadata_source], [gf_ar_mthly_bill.setl_time_zone], [gf_ar_mthly_bill.actg_time_zone], [gf_ar_mthly_bill.inter_trade_flag], [gf_ar_mthly_bill.actg_bill_mth], [gf_ar_mthly_bill.auto_writeoff_group_no]), partitions(p0) - is_index_back=true, is_global_index=true, + is_index_back=true, is_global_index=true, keep_ordering=true, range_key([gf_ar_mthly_bill.tnt_inst_id], [gf_ar_mthly_bill.rcrd_id], [gf_ar_mthly_bill.shadow_pk_0], [gf_ar_mthly_bill.shadow_pk_1]), range(MIN,MIN, MIN,MIN ; MAX,MAX,MAX,MAX)always true select * from gf_ar_mthly_bill; diff --git a/unittest/sql/rewrite/test_query_range.cpp b/unittest/sql/rewrite/test_query_range.cpp index 0c4ec49bb..cc47b19be 100644 --- a/unittest/sql/rewrite/test_query_range.cpp +++ b/unittest/sql/rewrite/test_query_range.cpp @@ -823,7 +823,7 @@ TEST_F(ObQueryRangeTest, single_filed_key_whole_range1) OK(query_range.preliminary_extract_query_range(single_range_columns_, NULL, NULL, &exec_ctx_)); OK(query_range.final_extract_query_range(exec_ctx_, NULL)); OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params)); - ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,(MIN;MAX)\"}]")); + ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)\"}]")); } TEST_F(ObQueryRangeTest, single_filed_key_whole_range2) @@ -840,7 +840,7 @@ TEST_F(ObQueryRangeTest, single_filed_key_whole_range2) OK(query_range.preliminary_extract_query_range(single_range_columns_, exprs, dtc_params, &exec_ctx_)); OK(query_range.final_extract_query_range(exec_ctx_, dtc_params)); OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params)); - ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,(MIN;MAX)\"}]")); + ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)\"}]")); } TEST_F(ObQueryRangeTest, double_filed_key_whole_range1) @@ -856,7 +856,7 @@ TEST_F(ObQueryRangeTest, double_filed_key_whole_range1) OK(query_range.preliminary_extract_query_range(double_range_columns_, NULL, dtc_params, &exec_ctx_)); OK(query_range.final_extract_query_range(exec_ctx_, dtc_params)); OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params)); - ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)\"}]")); + ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)\"}]")); } TEST_F(ObQueryRangeTest, double_filed_key_whole_range2) @@ -873,7 +873,7 @@ TEST_F(ObQueryRangeTest, double_filed_key_whole_range2) OK(query_range.preliminary_extract_query_range(double_range_columns_, exprs, dtc_params, &exec_ctx_)); OK(query_range.final_extract_query_range(exec_ctx_, dtc_params)); OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params)); - ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)\"}]")); + ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)\"}]")); } TEST_F(ObQueryRangeTest, range_column_with_like) @@ -1067,8 +1067,8 @@ TEST_F(ObQueryRangeTest, basic_test) except_result(double_range_columns_, params, "(b = 6 and a < 5) or (a > 8 and b = 15)", - "[{\"range\":\"table_id:3003,group_idx:0,({\"NULL\":\"NULL\"},MAX;{\"BIGINT\":5},MIN)\"}, " - "{\"range\":\"table_id:3003,group_idx:0,({\"BIGINT\":8},MAX;MAX,{\"BIGINT\":15})\"}]", + "[{\"range\":\"table_id:3003,group_idx:0,index_ordered_idx:0,({\"NULL\":\"NULL\"},MAX;{\"BIGINT\":5},MIN)\"}, " + "{\"range\":\"table_id:3003,group_idx:0,index_ordered_idx:0,({\"BIGINT\":8},MAX;MAX,{\"BIGINT\":15})\"}]", false); query_range.reset(); } diff --git a/unittest/sql/rewrite/test_query_range_collation.result b/unittest/sql/rewrite/test_query_range_collation.result index 3cd62637b..bbf11b756 100644 --- a/unittest/sql/rewrite/test_query_range_collation.result +++ b/unittest/sql/rewrite/test_query_range_collation.result @@ -7,7 +7,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"VARCHAR":"a", collation:"utf8mb4_general_ci"};{"VARCHAR":"a", collation:"utf8mb4_general_ci"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"VARCHAR":"a", collation:"utf8mb4_general_ci"};{"VARCHAR":"a", collation:"utf8mb4_general_ci"}]"}] f = 'a'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -15,7 +15,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f = 'a'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -23,7 +23,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f = 'a'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -31,7 +31,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f = 'a'--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -39,7 +39,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"VARCHAR":"a", collation:"utf8mb4_bin"};{"VARCHAR":"a", collation:"utf8mb4_bin"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"VARCHAR":"a", collation:"utf8mb4_bin"};{"VARCHAR":"a", collation:"utf8mb4_bin"}]"}] f = 'a'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -47,7 +47,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f = 'a'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -55,7 +55,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f = 'a'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -63,7 +63,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f = 'a'--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -71,7 +71,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"VARCHAR":"a", collation:"binary"};{"VARCHAR":"a", collation:"binary"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"VARCHAR":"a", collation:"binary"};{"VARCHAR":"a", collation:"binary"}]"}] [2] f != 'a' f != 'a'--------------connection_collation = 45 col_type = 45 @@ -83,7 +83,7 @@ end_border_flag[0] = 0 star_border_flag[1] = 0 end_border_flag[1] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"utf8mb4_general_ci"})"}, {"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"utf8mb4_general_ci"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"utf8mb4_general_ci"})"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"utf8mb4_general_ci"};MAX)"}] f != 'a'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -91,7 +91,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f != 'a'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -99,7 +99,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f != 'a'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -107,7 +107,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f != 'a'--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 2 @@ -117,7 +117,7 @@ end_border_flag[0] = 0 star_border_flag[1] = 0 end_border_flag[1] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"utf8mb4_bin"})"}, {"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"utf8mb4_bin"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"utf8mb4_bin"})"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"utf8mb4_bin"};MAX)"}] f != 'a'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -125,7 +125,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f != 'a'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -133,7 +133,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f != 'a'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -141,7 +141,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f != 'a'--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 2 @@ -151,7 +151,7 @@ end_border_flag[0] = 0 star_border_flag[1] = 0 end_border_flag[1] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"binary"})"}, {"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"binary"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"binary"})"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"binary"};MAX)"}] [3] f > 'a' f > 'a'--------------connection_collation = 45 col_type = 45 @@ -161,7 +161,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"utf8mb4_general_ci"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"utf8mb4_general_ci"};MAX)"}] f > 'a'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -169,7 +169,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -177,7 +177,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -185,7 +185,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a'--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -193,7 +193,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"utf8mb4_bin"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"utf8mb4_bin"};MAX)"}] f > 'a'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -201,7 +201,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -209,7 +209,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -217,7 +217,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a'--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -225,7 +225,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"binary"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"binary"};MAX)"}] [4] f < 'a' f < 'a'--------------connection_collation = 45 col_type = 45 @@ -235,7 +235,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"utf8mb4_general_ci"})"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"utf8mb4_general_ci"})"}] f < 'a'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -243,7 +243,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f < 'a'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -251,7 +251,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f < 'a'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -259,7 +259,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f < 'a'--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -267,7 +267,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"utf8mb4_bin"})"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"utf8mb4_bin"})"}] f < 'a'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -275,7 +275,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f < 'a'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -283,7 +283,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f < 'a'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -291,7 +291,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f < 'a'--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -299,7 +299,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"binary"})"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"binary"})"}] [5] f >= 'a' f >= 'a'--------------connection_collation = 45 col_type = 45 @@ -309,7 +309,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 1 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"VARCHAR":"a", collation:"utf8mb4_general_ci"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"VARCHAR":"a", collation:"utf8mb4_general_ci"};MAX)"}] f >= 'a'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -317,7 +317,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f >= 'a'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -325,7 +325,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f >= 'a'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -333,7 +333,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f >= 'a'--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -341,7 +341,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 1 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"VARCHAR":"a", collation:"utf8mb4_bin"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"VARCHAR":"a", collation:"utf8mb4_bin"};MAX)"}] f >= 'a'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -349,7 +349,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f >= 'a'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -357,7 +357,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f >= 'a'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -365,7 +365,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f >= 'a'--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -373,7 +373,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 1 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"VARCHAR":"a", collation:"binary"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"VARCHAR":"a", collation:"binary"};MAX)"}] [6] f <= 'a' f <= 'a'--------------connection_collation = 45 col_type = 45 @@ -383,7 +383,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"utf8mb4_general_ci"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"utf8mb4_general_ci"}]"}] f <= 'a'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -391,7 +391,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f <= 'a'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -399,7 +399,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f <= 'a'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -407,7 +407,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f <= 'a'--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -415,7 +415,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"utf8mb4_bin"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"utf8mb4_bin"}]"}] f <= 'a'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -423,7 +423,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f <= 'a'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -431,7 +431,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f <= 'a'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -439,7 +439,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f <= 'a'--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -447,7 +447,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"binary"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"binary"}]"}] [7] f is TRUE f is TRUE--------------connection_collation = 45 col_type = 45 @@ -457,7 +457,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f is TRUE--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -465,7 +465,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f is TRUE--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -473,7 +473,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f is TRUE--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -481,7 +481,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f is TRUE--------------connection_collation = 46 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -489,7 +489,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f is TRUE--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -497,7 +497,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f is TRUE--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -505,7 +505,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f is TRUE--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -513,7 +513,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f is TRUE--------------connection_collation = 63 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -521,7 +521,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] [8] f is FALSE f is FALSE--------------connection_collation = 45 col_type = 45 @@ -531,7 +531,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f is FALSE--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -539,7 +539,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f is FALSE--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -547,7 +547,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f is FALSE--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -555,7 +555,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f is FALSE--------------connection_collation = 46 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -563,7 +563,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f is FALSE--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -571,7 +571,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f is FALSE--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -579,7 +579,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f is FALSE--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -587,7 +587,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f is FALSE--------------connection_collation = 63 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -595,7 +595,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] [9] f is NULL f is NULL--------------connection_collation = 45 col_type = 45 @@ -605,7 +605,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] f is NULL--------------connection_collation = 45 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -613,7 +613,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] f is NULL--------------connection_collation = 45 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -621,7 +621,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] f is NULL--------------connection_collation = 46 col_type = 45 is not min_to_max_range ranges.count() = 1 @@ -629,7 +629,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] f is NULL--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -637,7 +637,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] f is NULL--------------connection_collation = 46 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -645,7 +645,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] f is NULL--------------connection_collation = 63 col_type = 45 is not min_to_max_range ranges.count() = 1 @@ -653,7 +653,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] f is NULL--------------connection_collation = 63 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -661,7 +661,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] f is NULL--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -669,7 +669,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"NULL":"NULL"};{"NULL":"NULL"}]"}] [10] f > 'a' and false f > 'a' and false--------------connection_collation = 45 col_type = 45 @@ -679,7 +679,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"utf8mb4_general_ci"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"utf8mb4_general_ci"};MAX)"}] f > 'a' and false--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -687,7 +687,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a' and false--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -695,7 +695,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a' and false--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -703,7 +703,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a' and false--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -711,7 +711,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"utf8mb4_bin"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"utf8mb4_bin"};MAX)"}] f > 'a' and false--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -719,7 +719,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a' and false--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -727,7 +727,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a' and false--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -735,7 +735,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a' and false--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -743,7 +743,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"binary"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"binary"};MAX)"}] [11] false and f > 'a' false and f > 'a'--------------connection_collation = 45 col_type = 45 @@ -753,7 +753,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"utf8mb4_general_ci"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"utf8mb4_general_ci"};MAX)"}] false and f > 'a'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -761,7 +761,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] false and f > 'a'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -769,7 +769,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] false and f > 'a'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -777,7 +777,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] false and f > 'a'--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -785,7 +785,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"utf8mb4_bin"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"utf8mb4_bin"};MAX)"}] false and f > 'a'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -793,7 +793,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] false and f > 'a'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -801,7 +801,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] false and f > 'a'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -809,7 +809,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] false and f > 'a'--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -817,7 +817,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"binary"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"binary"};MAX)"}] [12] f < 'a' or f > 'b' or f > 'sda' and f > 'sds' or f = 'adas' f < 'a' or f > 'b' or f > 'sda' and f > 'sds' or f = 'adas'--------------connection_collation = 45 col_type = 45 @@ -831,7 +831,7 @@ end_border_flag[1] = 0 star_border_flag[2] = 1 end_border_flag[2] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"utf8mb4_general_ci"})"}, {"range":"table_id:3003,group_idx:0,({"VARCHAR":"b", collation:"utf8mb4_general_ci"};MAX)"}, {"range":"table_id:3003,group_idx:0,[{"VARCHAR":"adas", collation:"utf8mb4_general_ci"};{"VARCHAR":"adas", collation:"utf8mb4_general_ci"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"utf8mb4_general_ci"})"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"b", collation:"utf8mb4_general_ci"};MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"VARCHAR":"adas", collation:"utf8mb4_general_ci"};{"VARCHAR":"adas", collation:"utf8mb4_general_ci"}]"}] f < 'a' or f > 'b' or f > 'sda' and f > 'sds' or f = 'adas'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -839,7 +839,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f < 'a' or f > 'b' or f > 'sda' and f > 'sds' or f = 'adas'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -847,7 +847,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f < 'a' or f > 'b' or f > 'sda' and f > 'sds' or f = 'adas'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -855,7 +855,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f < 'a' or f > 'b' or f > 'sda' and f > 'sds' or f = 'adas'--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 3 @@ -867,7 +867,7 @@ end_border_flag[1] = 0 star_border_flag[2] = 1 end_border_flag[2] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"utf8mb4_bin"})"}, {"range":"table_id:3003,group_idx:0,({"VARCHAR":"b", collation:"utf8mb4_bin"};MAX)"}, {"range":"table_id:3003,group_idx:0,[{"VARCHAR":"adas", collation:"utf8mb4_bin"};{"VARCHAR":"adas", collation:"utf8mb4_bin"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"utf8mb4_bin"})"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"b", collation:"utf8mb4_bin"};MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"VARCHAR":"adas", collation:"utf8mb4_bin"};{"VARCHAR":"adas", collation:"utf8mb4_bin"}]"}] f < 'a' or f > 'b' or f > 'sda' and f > 'sds' or f = 'adas'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -875,7 +875,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f < 'a' or f > 'b' or f > 'sda' and f > 'sds' or f = 'adas'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -883,7 +883,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f < 'a' or f > 'b' or f > 'sda' and f > 'sds' or f = 'adas'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -891,7 +891,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f < 'a' or f > 'b' or f > 'sda' and f > 'sds' or f = 'adas'--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 3 @@ -903,7 +903,7 @@ end_border_flag[1] = 0 star_border_flag[2] = 1 end_border_flag[2] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"binary"})"}, {"range":"table_id:3003,group_idx:0,({"VARCHAR":"b", collation:"binary"};MAX)"}, {"range":"table_id:3003,group_idx:0,[{"VARCHAR":"adas", collation:"binary"};{"VARCHAR":"adas", collation:"binary"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"NULL":"NULL"};{"VARCHAR":"a", collation:"binary"})"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"b", collation:"binary"};MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"VARCHAR":"adas", collation:"binary"};{"VARCHAR":"adas", collation:"binary"}]"}] [13] f > 'a' or f > 'b' and (g > 'a' or g > 'k') and (f > 'c') f > 'a' or f > 'b' and (g > 'a' or g > 'k') and (f > 'c')--------------connection_collation = 45 col_type = 45 @@ -913,7 +913,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"utf8mb4_general_ci"},MAX;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"utf8mb4_general_ci"},MAX;MAX,MAX)"}] f > 'a' or f > 'b' and (g > 'a' or g > 'k') and (f > 'c')--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -921,7 +921,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'a' or f > 'b' and (g > 'a' or g > 'k') and (f > 'c')--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -929,7 +929,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'a' or f > 'b' and (g > 'a' or g > 'k') and (f > 'c')--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -937,7 +937,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'a' or f > 'b' and (g > 'a' or g > 'k') and (f > 'c')--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -945,7 +945,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"utf8mb4_bin"},MAX;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"utf8mb4_bin"},MAX;MAX,MAX)"}] f > 'a' or f > 'b' and (g > 'a' or g > 'k') and (f > 'c')--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -953,7 +953,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'a' or f > 'b' and (g > 'a' or g > 'k') and (f > 'c')--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -961,7 +961,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'a' or f > 'b' and (g > 'a' or g > 'k') and (f > 'c')--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -969,7 +969,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'a' or f > 'b' and (g > 'a' or g > 'k') and (f > 'c')--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -977,7 +977,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"binary"},MAX;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"binary"},MAX;MAX,MAX)"}] [14] f > 'a' and g > 'a' and h > 'a' and i > 'a' and j > 'a' f > 'a' and g > 'a' and h > 'a' and i > 'a' and j > 'a'--------------connection_collation = 45 col_type = 45 @@ -987,7 +987,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 5 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"utf8mb4_general_ci"},MAX,MAX,MAX,MAX;MAX,MAX,MAX,MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"utf8mb4_general_ci"},MAX,MAX,MAX,MAX;MAX,MAX,MAX,MAX,MAX)"}] f > 'a' and g > 'a' and h > 'a' and i > 'a' and j > 'a'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -995,7 +995,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 5 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN,MIN,MIN,MIN;MAX,MAX,MAX,MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN,MIN,MIN,MIN;MAX,MAX,MAX,MAX,MAX)"}] f > 'a' and g > 'a' and h > 'a' and i > 'a' and j > 'a'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1003,7 +1003,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 5 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN,MIN,MIN,MIN;MAX,MAX,MAX,MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN,MIN,MIN,MIN;MAX,MAX,MAX,MAX,MAX)"}] f > 'a' and g > 'a' and h > 'a' and i > 'a' and j > 'a'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1011,7 +1011,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 5 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN,MIN,MIN,MIN;MAX,MAX,MAX,MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN,MIN,MIN,MIN;MAX,MAX,MAX,MAX,MAX)"}] f > 'a' and g > 'a' and h > 'a' and i > 'a' and j > 'a'--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -1019,7 +1019,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 5 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"utf8mb4_bin"},MAX,MAX,MAX,MAX;MAX,MAX,MAX,MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"utf8mb4_bin"},MAX,MAX,MAX,MAX;MAX,MAX,MAX,MAX,MAX)"}] f > 'a' and g > 'a' and h > 'a' and i > 'a' and j > 'a'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1027,7 +1027,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 5 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN,MIN,MIN,MIN;MAX,MAX,MAX,MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN,MIN,MIN,MIN;MAX,MAX,MAX,MAX,MAX)"}] f > 'a' and g > 'a' and h > 'a' and i > 'a' and j > 'a'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1035,7 +1035,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 5 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN,MIN,MIN,MIN;MAX,MAX,MAX,MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN,MIN,MIN,MIN;MAX,MAX,MAX,MAX,MAX)"}] f > 'a' and g > 'a' and h > 'a' and i > 'a' and j > 'a'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1043,7 +1043,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 5 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN,MIN,MIN,MIN;MAX,MAX,MAX,MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN,MIN,MIN,MIN;MAX,MAX,MAX,MAX,MAX)"}] f > 'a' and g > 'a' and h > 'a' and i > 'a' and j > 'a'--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -1051,7 +1051,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 5 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"binary"},MAX,MAX,MAX,MAX;MAX,MAX,MAX,MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"binary"},MAX,MAX,MAX,MAX;MAX,MAX,MAX,MAX,MAX)"}] [15] f > 'a' and f < 'B' f > 'a' and f < 'B' --------------connection_collation = 45 col_type = 45 @@ -1061,7 +1061,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"a", collation:"utf8mb4_general_ci"};{"VARCHAR":"B", collation:"utf8mb4_general_ci"})"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"a", collation:"utf8mb4_general_ci"};{"VARCHAR":"B", collation:"utf8mb4_general_ci"})"}] f > 'a' and f < 'B' --------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1069,7 +1069,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a' and f < 'B' --------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1077,7 +1077,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a' and f < 'B' --------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1085,7 +1085,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a' and f < 'B' --------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -1093,7 +1093,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MAX;MIN)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MAX;MIN)"}] f > 'a' and f < 'B' --------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1101,7 +1101,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a' and f < 'B' --------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1109,7 +1109,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a' and f < 'B' --------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1117,7 +1117,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'a' and f < 'B' --------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -1125,7 +1125,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MAX;MIN)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MAX;MIN)"}] [16] f > 'B' and f < 'a' and g > 'a' and g < 'b' f > 'B' and f < 'a' and g > 'a' and g < 'b'--------------connection_collation = 45 col_type = 45 @@ -1135,7 +1135,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MAX,MAX;MIN,MIN)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MAX,MAX;MIN,MIN)"}] f > 'B' and f < 'a' and g > 'a' and g < 'b'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1143,7 +1143,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' and f < 'a' and g > 'a' and g < 'b'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1151,7 +1151,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' and f < 'a' and g > 'a' and g < 'b'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1159,7 +1159,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' and f < 'a' and g > 'a' and g < 'b'--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -1167,7 +1167,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"B", collation:"utf8mb4_bin"},MAX;{"VARCHAR":"a", collation:"utf8mb4_bin"},MIN)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"B", collation:"utf8mb4_bin"},MAX;{"VARCHAR":"a", collation:"utf8mb4_bin"},MIN)"}] f > 'B' and f < 'a' and g > 'a' and g < 'b'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1175,7 +1175,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' and f < 'a' and g > 'a' and g < 'b'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1183,7 +1183,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' and f < 'a' and g > 'a' and g < 'b'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1191,7 +1191,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' and f < 'a' and g > 'a' and g < 'b'--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -1199,7 +1199,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"B", collation:"binary"},MAX;{"VARCHAR":"a", collation:"binary"},MIN)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"B", collation:"binary"},MAX;{"VARCHAR":"a", collation:"binary"},MIN)"}] [17] f > 'B' or f < 'a' and g > 'a' or g < 'b' f > 'B' or f < 'a' and g > 'a' or g < 'b'--------------connection_collation = 45 col_type = 45 @@ -1209,7 +1209,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' or f < 'a' and g > 'a' or g < 'b'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1217,7 +1217,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' or f < 'a' and g > 'a' or g < 'b'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1225,7 +1225,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' or f < 'a' and g > 'a' or g < 'b'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1233,7 +1233,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' or f < 'a' and g > 'a' or g < 'b'--------------connection_collation = 46 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1241,7 +1241,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' or f < 'a' and g > 'a' or g < 'b'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1249,7 +1249,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' or f < 'a' and g > 'a' or g < 'b'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1257,7 +1257,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' or f < 'a' and g > 'a' or g < 'b'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1265,7 +1265,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' or f < 'a' and g > 'a' or g < 'b'--------------connection_collation = 63 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1273,7 +1273,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] [18] f > 'B' and f < 'a' and g > 'a' and g < 'B' f > 'B' and f < 'a' and g > 'a' and g < 'B'--------------connection_collation = 45 col_type = 45 @@ -1283,7 +1283,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MAX,MAX;MIN,MIN)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MAX,MAX;MIN,MIN)"}] f > 'B' and f < 'a' and g > 'a' and g < 'B'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1291,7 +1291,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' and f < 'a' and g > 'a' and g < 'B'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1299,7 +1299,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' and f < 'a' and g > 'a' and g < 'B'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1307,7 +1307,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' and f < 'a' and g > 'a' and g < 'B'--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -1315,7 +1315,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MAX,MAX;MIN,MIN)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MAX,MAX;MIN,MIN)"}] f > 'B' and f < 'a' and g > 'a' and g < 'B'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1323,7 +1323,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' and f < 'a' and g > 'a' and g < 'B'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1331,7 +1331,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' and f < 'a' and g > 'a' and g < 'B'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1339,7 +1339,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] f > 'B' and f < 'a' and g > 'a' and g < 'B'--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -1347,7 +1347,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MAX,MAX;MIN,MIN)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MAX,MAX;MIN,MIN)"}] [19] f > 'B' and 'a' = 'A' f > 'B' and 'a' = 'A'--------------connection_collation = 45 col_type = 45 @@ -1357,7 +1357,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"B", collation:"utf8mb4_general_ci"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"B", collation:"utf8mb4_general_ci"};MAX)"}] f > 'B' and 'a' = 'A'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1365,7 +1365,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'B' and 'a' = 'A'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1373,7 +1373,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'B' and 'a' = 'A'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1381,7 +1381,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'B' and 'a' = 'A'--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -1389,7 +1389,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"B", collation:"utf8mb4_bin"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"B", collation:"utf8mb4_bin"};MAX)"}] f > 'B' and 'a' = 'A'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1397,7 +1397,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'B' and 'a' = 'A'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1405,7 +1405,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'B' and 'a' = 'A'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1413,7 +1413,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f > 'B' and 'a' = 'A'--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -1421,7 +1421,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"VARCHAR":"B", collation:"binary"};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"VARCHAR":"B", collation:"binary"};MAX)"}] [20] #(f, g) in (('a', 'b'), ('B', 'A')) [21] #(f, g) > ('a', 'b') and (f, g) < ('B', 'A') @@ -1433,7 +1433,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"VARCHAR":"a", collation:"utf8mb4_general_ci"};{"VARCHAR":"b", collation:"utf8mb4_general_ci"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"VARCHAR":"a", collation:"utf8mb4_general_ci"};{"VARCHAR":"b", collation:"utf8mb4_general_ci"}]"}] f between 'a' and 'b'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1441,7 +1441,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f between 'a' and 'b'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1449,7 +1449,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f between 'a' and 'b'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1457,7 +1457,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f between 'a' and 'b'--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -1465,7 +1465,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"VARCHAR":"a", collation:"utf8mb4_bin"};{"VARCHAR":"b", collation:"utf8mb4_bin"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"VARCHAR":"a", collation:"utf8mb4_bin"};{"VARCHAR":"b", collation:"utf8mb4_bin"}]"}] f between 'a' and 'b'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1473,7 +1473,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f between 'a' and 'b'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1481,7 +1481,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f between 'a' and 'b'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1489,7 +1489,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f between 'a' and 'b'--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -1497,7 +1497,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"VARCHAR":"a", collation:"binary"};{"VARCHAR":"b", collation:"binary"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"VARCHAR":"a", collation:"binary"};{"VARCHAR":"b", collation:"binary"}]"}] [23] f between 'B' and 'a' f between 'B' and 'a'--------------connection_collation = 45 col_type = 45 @@ -1507,7 +1507,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MAX;MIN)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MAX;MIN)"}] f between 'B' and 'a'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1515,7 +1515,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f between 'B' and 'a'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1523,7 +1523,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f between 'B' and 'a'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1531,7 +1531,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f between 'B' and 'a'--------------connection_collation = 46 col_type = 46 is not min_to_max_range ranges.count() = 1 @@ -1539,7 +1539,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"VARCHAR":"B", collation:"utf8mb4_bin"};{"VARCHAR":"a", collation:"utf8mb4_bin"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"VARCHAR":"B", collation:"utf8mb4_bin"};{"VARCHAR":"a", collation:"utf8mb4_bin"}]"}] f between 'B' and 'a'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1547,7 +1547,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f between 'B' and 'a'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1555,7 +1555,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f between 'B' and 'a'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1563,7 +1563,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f between 'B' and 'a'--------------connection_collation = 63 col_type = 63 is not min_to_max_range ranges.count() = 1 @@ -1571,7 +1571,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"VARCHAR":"B", collation:"binary"};{"VARCHAR":"a", collation:"binary"}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"VARCHAR":"B", collation:"binary"};{"VARCHAR":"a", collation:"binary"}]"}] [24] f not between 'a' and 'b' f not between 'a' and 'b'--------------connection_collation = 45 col_type = 45 @@ -1581,7 +1581,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f not between 'a' and 'b'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1589,7 +1589,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f not between 'a' and 'b'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1597,7 +1597,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f not between 'a' and 'b'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1605,7 +1605,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f not between 'a' and 'b'--------------connection_collation = 46 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1613,7 +1613,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f not between 'a' and 'b'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1621,7 +1621,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f not between 'a' and 'b'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1629,7 +1629,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f not between 'a' and 'b'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1637,7 +1637,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f not between 'a' and 'b'--------------connection_collation = 63 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1645,7 +1645,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] [25] f = 'a' + 'b' f = 'a' + 'b'--------------connection_collation = 45 col_type = 45 @@ -1655,7 +1655,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f = 'a' + 'b'--------------connection_collation = 45 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1663,7 +1663,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f = 'a' + 'b'--------------connection_collation = 45 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1671,7 +1671,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f = 'a' + 'b'--------------connection_collation = 46 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1679,7 +1679,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f = 'a' + 'b'--------------connection_collation = 46 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1687,7 +1687,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f = 'a' + 'b'--------------connection_collation = 46 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1695,7 +1695,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f = 'a' + 'b'--------------connection_collation = 63 col_type = 45 is min_to_max_range ranges.count() = 1 @@ -1703,7 +1703,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f = 'a' + 'b'--------------connection_collation = 63 col_type = 46 is min_to_max_range ranges.count() = 1 @@ -1711,7 +1711,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] f = 'a' + 'b'--------------connection_collation = 63 col_type = 63 is min_to_max_range ranges.count() = 1 @@ -1719,5 +1719,5 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] diff --git a/unittest/sql/rewrite/test_query_range_filter.result b/unittest/sql/rewrite/test_query_range_filter.result index 6e0022dce..5ca924534 100644 --- a/unittest/sql/rewrite/test_query_range_filter.result +++ b/unittest/sql/rewrite/test_query_range_filter.result @@ -8,7 +8,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}] **rowkey num = 2** **filter count = 0** @@ -18,7 +18,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}] **rowkey num = 3** **filter count = 0** @@ -28,7 +28,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}] [1] a in (1,2,3) or (a = 4 and b = 4) @@ -46,7 +46,7 @@ end_border_flag[2] = 1 star_border_flag[3] = 1 end_border_flag[3] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}] **rowkey num = 2** **filter count = 0** @@ -62,7 +62,7 @@ end_border_flag[2] = 0 star_border_flag[3] = 1 end_border_flag[3] = 1 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}] **rowkey num = 3** **filter count = 0** @@ -78,7 +78,7 @@ end_border_flag[2] = 0 star_border_flag[3] = 0 end_border_flag[3] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN,MIN;{"BIGINT":2},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN,MIN;{"BIGINT":3},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN,MIN;{"BIGINT":2},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN,MIN;{"BIGINT":3},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}] [2] (a, b) in ((1,2),(2,3)) @@ -92,7 +92,7 @@ end_border_flag[0] = 1 star_border_flag[1] = 1 end_border_flag[1] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}] **rowkey num = 2** **filter count = 1** @@ -104,7 +104,7 @@ end_border_flag[0] = 1 star_border_flag[1] = 1 end_border_flag[1] = 1 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}] **rowkey num = 3** **filter count = 1** @@ -116,7 +116,7 @@ end_border_flag[0] = 0 star_border_flag[1] = 0 end_border_flag[1] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},{"BIGINT":3},MIN;{"BIGINT":2},{"BIGINT":3},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},{"BIGINT":3},MIN;{"BIGINT":2},{"BIGINT":3},MAX)"}] [3] a > 1 @@ -128,7 +128,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1};MAX)"}] **rowkey num = 2** **filter count = 1** @@ -138,7 +138,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MAX;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MAX;MAX,MAX)"}] **rowkey num = 3** **filter count = 1** @@ -148,7 +148,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MAX;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MAX;MAX,MAX)"}] [4] a > 1 and b = 1 @@ -160,7 +160,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1};MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1};MAX)"}] **rowkey num = 2** **filter count = 1** @@ -170,7 +170,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MAX;MAX,{"BIGINT":1})"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MAX;MAX,{"BIGINT":1})"}] **rowkey num = 3** **filter count = 1** @@ -180,7 +180,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MAX,MAX;MAX,{"BIGINT":1},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MAX,MAX;MAX,{"BIGINT":1},MAX)"}] [5] a in (1,2,3) @@ -196,7 +196,7 @@ end_border_flag[1] = 1 star_border_flag[2] = 1 end_border_flag[2] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}] **rowkey num = 2** **filter count = 1** @@ -210,7 +210,7 @@ end_border_flag[1] = 0 star_border_flag[2] = 0 end_border_flag[2] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}] **rowkey num = 3** **filter count = 1** @@ -224,7 +224,7 @@ end_border_flag[1] = 0 star_border_flag[2] = 0 end_border_flag[2] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}] [6] a in (?{1},?{2},?{3}) or (a = ?{4}) @@ -242,7 +242,7 @@ end_border_flag[2] = 1 star_border_flag[3] = 1 end_border_flag[3] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}] **rowkey num = 2** **filter count = 1** @@ -258,7 +258,7 @@ end_border_flag[2] = 0 star_border_flag[3] = 0 end_border_flag[3] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},MIN;{"BIGINT":4},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":4},MIN;{"BIGINT":4},MAX)"}] **rowkey num = 3** **filter count = 1** @@ -274,7 +274,7 @@ end_border_flag[2] = 0 star_border_flag[3] = 0 end_border_flag[3] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},MIN;{"BIGINT":4},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":4},MIN;{"BIGINT":4},MAX)"}] [7] a in (?{1},?{2},?{3}) or (a = ?{4} and b = ?{4}) @@ -292,7 +292,7 @@ end_border_flag[2] = 1 star_border_flag[3] = 1 end_border_flag[3] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}] **rowkey num = 2** **filter count = 0** @@ -308,7 +308,7 @@ end_border_flag[2] = 0 star_border_flag[3] = 1 end_border_flag[3] = 1 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}] **rowkey num = 3** **filter count = 0** @@ -324,7 +324,7 @@ end_border_flag[2] = 0 star_border_flag[3] = 0 end_border_flag[3] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN,MIN;{"BIGINT":2},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN,MIN;{"BIGINT":3},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN,MIN;{"BIGINT":2},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN,MIN;{"BIGINT":3},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}] [8] a in (1,2,3) and b = 2 @@ -340,7 +340,7 @@ end_border_flag[1] = 1 star_border_flag[2] = 1 end_border_flag[2] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}] **rowkey num = 2** **filter count = 2** @@ -354,7 +354,7 @@ end_border_flag[1] = 1 star_border_flag[2] = 1 end_border_flag[2] = 1 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3},{"BIGINT":2};{"BIGINT":3},{"BIGINT":2}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3},{"BIGINT":2};{"BIGINT":3},{"BIGINT":2}]"}] **rowkey num = 3** **filter count = 2** @@ -368,7 +368,7 @@ end_border_flag[1] = 0 star_border_flag[2] = 0 end_border_flag[2] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},{"BIGINT":2},MIN;{"BIGINT":2},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},{"BIGINT":2},MIN;{"BIGINT":3},{"BIGINT":2},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},{"BIGINT":2},MIN;{"BIGINT":2},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},{"BIGINT":2},MIN;{"BIGINT":3},{"BIGINT":2},MAX)"}] [9] (a,b) in ((1,2),(2,3)) or (a,b) in ((3,4),(4,5)) @@ -386,7 +386,7 @@ end_border_flag[2] = 1 star_border_flag[3] = 1 end_border_flag[3] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}] **rowkey num = 2** **filter count = 1** @@ -402,7 +402,7 @@ end_border_flag[2] = 1 star_border_flag[3] = 1 end_border_flag[3] = 1 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3},{"BIGINT":4};{"BIGINT":3},{"BIGINT":4}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":5};{"BIGINT":4},{"BIGINT":5}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3},{"BIGINT":4};{"BIGINT":3},{"BIGINT":4}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4},{"BIGINT":5};{"BIGINT":4},{"BIGINT":5}]"}] **rowkey num = 3** **filter count = 1** @@ -418,7 +418,7 @@ end_border_flag[2] = 0 star_border_flag[3] = 0 end_border_flag[3] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},{"BIGINT":3},MIN;{"BIGINT":2},{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},{"BIGINT":4},MIN;{"BIGINT":3},{"BIGINT":4},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},{"BIGINT":5},MIN;{"BIGINT":4},{"BIGINT":5},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},{"BIGINT":3},MIN;{"BIGINT":2},{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},{"BIGINT":4},MIN;{"BIGINT":3},{"BIGINT":4},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":4},{"BIGINT":5},MIN;{"BIGINT":4},{"BIGINT":5},MAX)"}] [10] (a,b) in ((1,2),(2,3)) and c > 1 @@ -432,7 +432,7 @@ end_border_flag[0] = 1 star_border_flag[1] = 1 end_border_flag[1] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}] **rowkey num = 2** **filter count = 1** @@ -444,7 +444,7 @@ end_border_flag[0] = 1 star_border_flag[1] = 1 end_border_flag[1] = 1 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}] **rowkey num = 3** **filter count = 0** @@ -456,7 +456,7 @@ end_border_flag[0] = 0 star_border_flag[1] = 0 end_border_flag[1] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},{"BIGINT":2},{"BIGINT":1};{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},{"BIGINT":3},{"BIGINT":1};{"BIGINT":2},{"BIGINT":3},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},{"BIGINT":2},{"BIGINT":1};{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},{"BIGINT":3},{"BIGINT":1};{"BIGINT":2},{"BIGINT":3},MAX)"}] [11] (a,b) in ((1,2),(2,3)) and c = 1 @@ -470,7 +470,7 @@ end_border_flag[0] = 1 star_border_flag[1] = 1 end_border_flag[1] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}] **rowkey num = 2** **filter count = 1** @@ -482,7 +482,7 @@ end_border_flag[0] = 1 star_border_flag[1] = 1 end_border_flag[1] = 1 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}] **rowkey num = 3** **filter count = 2** @@ -494,7 +494,7 @@ end_border_flag[0] = 1 star_border_flag[1] = 1 end_border_flag[1] = 1 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2},{"BIGINT":1};{"BIGINT":1},{"BIGINT":2},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":3},{"BIGINT":1};{"BIGINT":2},{"BIGINT":3},{"BIGINT":1}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2},{"BIGINT":1};{"BIGINT":1},{"BIGINT":2},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":3},{"BIGINT":1};{"BIGINT":2},{"BIGINT":3},{"BIGINT":1}]"}] [12] a = 1 and b = 1 @@ -506,7 +506,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}] **rowkey num = 2** **filter count = 2** @@ -516,7 +516,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}] **rowkey num = 3** **filter count = 2** @@ -526,7 +526,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},{"BIGINT":1},MIN;{"BIGINT":1},{"BIGINT":1},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},{"BIGINT":1},MIN;{"BIGINT":1},{"BIGINT":1},MAX)"}] [13] (a, b, c) in ((?{1},?{1},?{1}),(?{2},?{2},?{2})) or (a,b) in ((?{3},?{3}),(?{4},?{4})) @@ -544,7 +544,7 @@ end_border_flag[2] = 1 star_border_flag[3] = 1 end_border_flag[3] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}] **rowkey num = 2** **filter count = 0** @@ -560,7 +560,7 @@ end_border_flag[2] = 1 star_border_flag[3] = 1 end_border_flag[3] = 1 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3},{"BIGINT":3};{"BIGINT":3},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3},{"BIGINT":3};{"BIGINT":3},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}] **rowkey num = 3** **filter count = 0** @@ -576,7 +576,7 @@ end_border_flag[2] = 0 star_border_flag[3] = 0 end_border_flag[3] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},{"BIGINT":3},MIN;{"BIGINT":3},{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},{"BIGINT":3},MIN;{"BIGINT":3},{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}] [14] (a, b, c) in ((?{1},?{1},?{1}),(?{2},?{2},?{2})) or (a,b) in ((?{3},?{3}),(?{4},?{4})) and a = ?{1} @@ -590,7 +590,7 @@ end_border_flag[0] = 1 star_border_flag[1] = 1 end_border_flag[1] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}] **rowkey num = 2** **filter count = 0** @@ -602,7 +602,7 @@ end_border_flag[0] = 1 star_border_flag[1] = 1 end_border_flag[1] = 1 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}] **rowkey num = 3** **filter count = 0** @@ -614,7 +614,7 @@ end_border_flag[0] = 1 star_border_flag[1] = 1 end_border_flag[1] = 1 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}] [15] (a, b, c) in ((?{1},?{1},?{1}),(?{2},?{2},?{2})) or (a,b) in ((?{3},?{3}),(?{4},?{4})) and c = ?{1} @@ -632,7 +632,7 @@ end_border_flag[2] = 1 star_border_flag[3] = 1 end_border_flag[3] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}] **rowkey num = 2** **filter count = 0** @@ -648,7 +648,7 @@ end_border_flag[2] = 1 star_border_flag[3] = 1 end_border_flag[3] = 1 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3},{"BIGINT":3};{"BIGINT":3},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3},{"BIGINT":3};{"BIGINT":3},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}] **rowkey num = 3** **filter count = 1** @@ -664,7 +664,7 @@ end_border_flag[2] = 1 star_border_flag[3] = 1 end_border_flag[3] = 1 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3},{"BIGINT":3},{"BIGINT":1};{"BIGINT":3},{"BIGINT":3},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":4},{"BIGINT":1};{"BIGINT":4},{"BIGINT":4},{"BIGINT":1}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3},{"BIGINT":3},{"BIGINT":1};{"BIGINT":3},{"BIGINT":3},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4},{"BIGINT":4},{"BIGINT":1};{"BIGINT":4},{"BIGINT":4},{"BIGINT":1}]"}] [16] (a, b, c) in ((?{1},?{1},?{1}),(?{2},?{2},?{2})) or (a,b) in ((?{3},?{3}),(?{4},?{4})) or a = ?{1} @@ -682,7 +682,7 @@ end_border_flag[2] = 1 star_border_flag[3] = 1 end_border_flag[3] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}] **rowkey num = 2** **filter count = 0** @@ -698,7 +698,7 @@ end_border_flag[2] = 1 star_border_flag[3] = 1 end_border_flag[3] = 1 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3},{"BIGINT":3};{"BIGINT":3},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3},{"BIGINT":3};{"BIGINT":3},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}] **rowkey num = 3** **filter count = 0** @@ -714,7 +714,7 @@ end_border_flag[2] = 0 star_border_flag[3] = 0 end_border_flag[3] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},{"BIGINT":3},MIN;{"BIGINT":3},{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},{"BIGINT":3},MIN;{"BIGINT":3},{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}] [17] a = 1 and c = 1 or (a = 2 and b = 2) @@ -728,7 +728,7 @@ end_border_flag[0] = 1 star_border_flag[1] = 1 end_border_flag[1] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}] **rowkey num = 2** **filter count = 0** @@ -740,7 +740,7 @@ end_border_flag[0] = 0 star_border_flag[1] = 1 end_border_flag[1] = 1 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}] **rowkey num = 3** **filter count = 0** @@ -752,7 +752,7 @@ end_border_flag[0] = 0 star_border_flag[1] = 0 end_border_flag[1] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},{"BIGINT":2},MIN;{"BIGINT":2},{"BIGINT":2},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},{"BIGINT":2},MIN;{"BIGINT":2},{"BIGINT":2},MAX)"}] [18] a = 1 and b = 1 or (b = 2 and c = 2) @@ -764,7 +764,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}] **rowkey num = 2** **filter count = 0** @@ -774,7 +774,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}] **rowkey num = 3** **filter count = 0** @@ -784,7 +784,7 @@ all_single_value_ranges = 0 star_border_flag[0] = 0 end_border_flag[0] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,(MIN,MIN,MIN;MAX,MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN,MIN;MAX,MAX,MAX)"}] [19] (a,b) in ((1,2)) or (a,c) in ((2,3),(3,4)) @@ -800,7 +800,7 @@ end_border_flag[1] = 1 star_border_flag[2] = 1 end_border_flag[2] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}] **rowkey num = 2** **filter count = 0** @@ -814,7 +814,7 @@ end_border_flag[1] = 0 star_border_flag[2] = 0 end_border_flag[2] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}] **rowkey num = 3** **filter count = 0** @@ -828,7 +828,7 @@ end_border_flag[1] = 0 star_border_flag[2] = 0 end_border_flag[2] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN,MIN;{"BIGINT":2},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN,MIN;{"BIGINT":3},MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN,MIN;{"BIGINT":2},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN,MIN;{"BIGINT":3},MAX,MAX)"}] [20] (a,b) in ((1,2)) or (a = 3 and c = 3) @@ -842,7 +842,7 @@ end_border_flag[0] = 1 star_border_flag[1] = 1 end_border_flag[1] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}] **rowkey num = 2** **filter count = 0** @@ -854,7 +854,7 @@ end_border_flag[0] = 1 star_border_flag[1] = 0 end_border_flag[1] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}] **rowkey num = 3** **filter count = 0** @@ -866,7 +866,7 @@ end_border_flag[0] = 0 star_border_flag[1] = 0 end_border_flag[1] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN,MIN;{"BIGINT":3},MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN,MIN;{"BIGINT":3},MAX,MAX)"}] [21] (a,b,c) in ((1,2,3),(2,3,4)) or (a = 5 and b = 5) or (a = 7) @@ -884,7 +884,7 @@ end_border_flag[2] = 1 star_border_flag[3] = 1 end_border_flag[3] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":5};{"BIGINT":5}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":7};{"BIGINT":7}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":5};{"BIGINT":5}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":7};{"BIGINT":7}]"}] **rowkey num = 2** **filter count = 0** @@ -900,7 +900,7 @@ end_border_flag[2] = 1 star_border_flag[3] = 0 end_border_flag[3] = 0 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":5},{"BIGINT":5};{"BIGINT":5},{"BIGINT":5}]"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":7},MIN;{"BIGINT":7},MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":5},{"BIGINT":5};{"BIGINT":5},{"BIGINT":5}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":7},MIN;{"BIGINT":7},MAX)"}] **rowkey num = 3** **filter count = 0** @@ -916,7 +916,7 @@ end_border_flag[2] = 0 star_border_flag[3] = 0 end_border_flag[3] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2},{"BIGINT":3};{"BIGINT":1},{"BIGINT":2},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":3},{"BIGINT":4};{"BIGINT":2},{"BIGINT":3},{"BIGINT":4}]"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":5},{"BIGINT":5},MIN;{"BIGINT":5},{"BIGINT":5},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":7},MIN,MIN;{"BIGINT":7},MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2},{"BIGINT":3};{"BIGINT":1},{"BIGINT":2},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":3},{"BIGINT":4};{"BIGINT":2},{"BIGINT":3},{"BIGINT":4}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":5},{"BIGINT":5},MIN;{"BIGINT":5},{"BIGINT":5},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":7},MIN,MIN;{"BIGINT":7},MAX,MAX)"}] [22] a = 8 and c = 9 and b = 1 or (a,b,c) in ((1,2,3),(2,3,4)) and (a = 10) @@ -928,7 +928,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 1 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":8};{"BIGINT":8}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":8};{"BIGINT":8}]"}] **rowkey num = 2** **filter count = 0** @@ -938,7 +938,7 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":8},{"BIGINT":1};{"BIGINT":8},{"BIGINT":1}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":8},{"BIGINT":1};{"BIGINT":8},{"BIGINT":1}]"}] **rowkey num = 3** **filter count = 1** @@ -948,5 +948,5 @@ all_single_value_ranges = 1 star_border_flag[0] = 1 end_border_flag[0] = 1 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":8},{"BIGINT":1},{"BIGINT":9};{"BIGINT":8},{"BIGINT":1},{"BIGINT":9}]"}] +[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":8},{"BIGINT":1},{"BIGINT":9};{"BIGINT":8},{"BIGINT":1},{"BIGINT":9}]"}]