hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] DAS iterator refactor and keep order optimization

Browse Source 
Co-authored-by: saltonz <saltonzh@gmail.com>
Co-authored-by: zhenhan.gong@gmail.com <zhenhan.gong@gmail.com>
Co-authored-by: Tyshawn <tuyunshan@gmail.com>
This commit is contained in:
pe-99y 2024-06-24 13:57:14 +00:00 committed by ob-robot
parent 7f3ce430fb
commit 5c5e6da6ce
88 changed files with 7062 additions and 2432 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
deps/oblib/src/common/ob_range.cpp vendored
View File

@ -96,6 +96,7 @@ int64_t ObNewRange::to_simple_string(char *buffer, const int64_t length) const
databuff_printf(buffer, length, pos, "table_id:null,");
}
databuff_printf(buffer, length, pos, "group_idx:%d,", group_idx_);
databuff_printf(buffer, length, pos, "index_ordered_idx:%d,", index_ordered_idx_);
if (border_flag_.inclusive_start()) {
databuff_printf(buffer, length, pos, "[");
} else {

27
deps/oblib/src/common/ob_range.h vendored
View File

@ -17,6 +17,7 @@
#include "lib/utility/utility.h"
#include "common/rowkey/ob_rowkey.h"
#include "common/ob_string_buf.h"
#include "share/ob_cluster_version.h"
namespace oceanbase
@ -335,7 +336,8 @@ public:
struct {
int64_t group_idx_: 32;
int64_t is_physical_rowid_range_: 1;
int64_t reserved_: 31;
int64_t index_ordered_idx_ : 16; // used for keep order of global index lookup
int64_t reserved_: 15;
};
};
@ -382,6 +384,29 @@ public:
{
return group_idx_;
}
inline int32_t get_index_ordered_idx() const
{
return index_ordered_idx_;
}
// pseudo-column [GROUP_ID], with high 32 bits as group_idx_ and low 32 bits as index_ordered_idx_
// when cluster version < 4.3.2, the das keep order optimization is disabled, we should only fill
// group_idx to [GROUP_ID] for compatibility.
inline int64_t get_group_id() const
{
return GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_3_2_0 ? group_idx_ :
(static_cast<int64_t>(group_idx_) << 32) | (index_ordered_idx_ & 0xffffffff);
}
// get group_idx from [GROUP_ID]
static int64_t get_group_idx(int64_t group_id)
{
return GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_3_2_0 ? group_id : (group_id >> 32);
}
// get index_order_idx from [GROUP_ID]
static int64_t get_index_ordered_idx(int64_t group_id)
{
return group_id & 0xffffffff;
}
int build_range(uint64_t table_id, ObRowkey rowkey)
{

2
deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h vendored
View File

@ -1161,7 +1161,7 @@ PCODE_DEF(OB_CAL_STANDBY_TENANT_PHY_RESOURCE, 0x1623)
//PCODE_DEF(OB_UPDATE_MVIEW_REFERENCE_TABLE_STATUS, 0x1624)
//PCODE_DEF(OB_DO_EVENT_DDL, 0x1625)
//PCODE_DEF(OB_GENERATE_AUX_INDEX_SCHEMA, 0x1626)
PCODE_DEF(OB_GENERATE_AUX_INDEX_SCHEMA, 0x1626)
//PCODE_DEF(OB_DUMP_SS_PHY_BLOCK, 0x1627)
//PCODE_DEF(OB_DUMP_SS_MACRO_BLOCK, 0x1628)

1
src/observer/ob_srv_xlator_rootserver.cpp
View File

@ -111,6 +111,7 @@ void oceanbase::observer::init_srv_xlator_for_rootserver(ObSrvRpcXlator *xlator)
RPC_PROCESSOR(rootserver::ObRpcRenameTableP, *gctx_.root_service_);
RPC_PROCESSOR(rootserver::ObRpcTruncateTableP, *gctx_.root_service_);
RPC_PROCESSOR(rootserver::ObRpcTruncateTableV2P, *gctx_.root_service_);
RPC_PROCESSOR(rootserver::ObRpcGenerateAuxIndexSchemaP, *gctx_.root_service_);
RPC_PROCESSOR(rootserver::ObRpcCreateIndexP, *gctx_.root_service_);
RPC_PROCESSOR(rootserver::ObRpcDropIndexP, *gctx_.root_service_);
RPC_PROCESSOR(rootserver::ObRpcCreateMLogP, *gctx_.root_service_);

1
src/rootserver/CMakeLists.txt
View File

@ -125,6 +125,7 @@ ob_set_subtarget(ob_rootserver ddl_task
ddl_task/ob_drop_primary_key_task.cpp
ddl_task/ob_index_build_task.cpp
ddl_task/ob_build_mview_task.cpp
ddl_task/ob_fts_index_build_task.cpp
ddl_task/ob_modify_autoinc_task.cpp
ddl_task/ob_table_redefinition_task.cpp
ddl_task/ob_recover_restore_table_task.cpp

134
src/rootserver/ddl_task/ob_ddl_scheduler.cpp
View File

@ -26,6 +26,7 @@
#include "rootserver/ddl_task/ob_drop_primary_key_task.h"
#include "rootserver/ddl_task/ob_index_build_task.h"
#include "rootserver/ddl_task/ob_build_mview_task.h"
#include "rootserver/ddl_task/ob_fts_index_build_task.h"
#include "rootserver/ddl_task/ob_modify_autoinc_task.h"
#include "rootserver/ddl_task/ob_table_redefinition_task.h"
#include "rootserver/ddl_task/ob_recover_restore_table_task.h"
@ -1002,6 +1003,20 @@ int ObDDLScheduler::create_ddl_task(const ObCreateDDLTaskParam &param,
LOG_WARN("fail to create build index task", K(ret));
}
break;
case DDL_CREATE_FTS_INDEX:
create_index_arg = static_cast<const obrpc::ObCreateIndexArg *>(param.ddl_arg_);
if (OB_FAIL(create_build_fts_index_task(proxy,
param.src_table_schema_,
param.dest_table_schema_,
param.parallelism_,
param.parent_task_id_,
param.consumer_group_id_,
create_index_arg,
*param.allocator_,
task_record))) {
LOG_WARN("fail to create build fts index task", K(ret));
}
break;
case DDL_DROP_INDEX:
case DDL_DROP_MLOG:
// in this case, src_table_schema is data table, dest_table_schema is index table
@ -1519,6 +1534,50 @@ int ObDDLScheduler::start_redef_table(const obrpc::ObStartRedefTableArg &arg, ob
return ret;
}
int ObDDLScheduler::create_build_fts_index_task(
common::ObISQLClient &proxy,
const ObTableSchema *data_table_schema,
const ObTableSchema *index_schema,
const int64_t parallelism,
const int64_t parent_task_id,
const int64_t consumer_group_id,
const obrpc::ObCreateIndexArg *create_index_arg,
ObIAllocator &allocator,
ObDDLTaskRecord &task_record)
{
int ret = OB_SUCCESS;
int64_t task_id = 0;
SMART_VAR(ObFtsIndexBuildTask, index_task) {
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_ISNULL(create_index_arg) || OB_ISNULL(data_table_schema) || OB_ISNULL(index_schema)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KPC(create_index_arg),
KPC(data_table_schema), KPC(index_schema));
} else if (OB_FAIL(ObDDLTask::fetch_new_task_id(root_service_->get_sql_proxy(), data_table_schema->get_tenant_id(), task_id))) {
LOG_WARN("fetch new task id failed", K(ret));
} else if (OB_FAIL(index_task.init(data_table_schema->get_tenant_id(),
task_id,
data_table_schema,
index_schema,
data_table_schema->get_schema_version(),
parallelism,
consumer_group_id,
*create_index_arg,
parent_task_id))) {
LOG_WARN("init fts index task failed", K(ret), K(data_table_schema), K(index_schema));
} else if (OB_FAIL(index_task.set_trace_id(*ObCurTraceId::get_trace_id()))) {
LOG_WARN("set trace id failed", K(ret));
} else if (OB_FAIL(insert_task_record(proxy, index_task, allocator, task_record))) {
LOG_WARN("fail to insert task record", K(ret));
}
LOG_INFO("ddl_scheduler create build index task finished", K(ret), K(index_task));
}
return ret;
}
int ObDDLScheduler::create_build_index_task(
common::ObISQLClient &proxy,
const share::ObDDLType &ddl_type,
@ -1543,7 +1602,7 @@ int ObDDLScheduler::create_build_index_task(
} else if (OB_ISNULL(create_index_arg) || OB_ISNULL(data_table_schema) || OB_ISNULL(index_schema)
|| OB_UNLIKELY(tenant_data_version <= 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(create_index_arg), K(data_table_schema), K(index_schema), K(tenant_data_version));
LOG_WARN("invalid argument", K(ret), KPC(create_index_arg), KPC(data_table_schema), KPC(index_schema), K(tenant_data_version));
} else if (OB_FAIL(ObDDLTask::fetch_new_task_id(root_service_->get_sql_proxy(), data_table_schema->get_tenant_id(), task_id))) {
LOG_WARN("fetch new task id failed", K(ret));
} else if (OB_FAIL(index_task.init(data_table_schema->get_tenant_id(),
@ -1643,33 +1702,44 @@ int ObDDLScheduler::create_drop_fts_index_task(
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KP(index_schema));
} else if (FALSE_IT(is_fts_index = index_schema->is_fts_index_aux())) {
} else if (OB_ISNULL(rowkey_doc_schema)
|| OB_ISNULL(doc_rowkey_schema)
|| (is_fts_index && OB_ISNULL(doc_word_schema))
|| OB_UNLIKELY(schema_version <= 0)) {
} else if (OB_UNLIKELY(schema_version <= 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KP(index_schema), KP(rowkey_doc_schema), K(doc_rowkey_schema),
K(doc_word_schema), K(schema_version));
LOG_WARN("invalid argument", K(ret), KP(index_schema), K(schema_version));
} else if (OB_FAIL(ObDDLTask::fetch_new_task_id(root_service_->get_sql_proxy(), index_schema->get_tenant_id(),
task_id))) {
LOG_WARN("fetch new task id failed", K(ret));
} else if (OB_FAIL(index_schema->get_index_name(domain_index_name))) {
LOG_WARN("fail to get domain index name", K(ret), KPC(index_schema));
} else if (is_fts_index && OB_FAIL(doc_word_schema->get_index_name(fts_doc_word_name))) {
LOG_WARN("fail to get fts doc word name", K(ret), KPC(doc_word_schema));
} else if (OB_FAIL(rowkey_doc_schema->get_index_name(rowkey_doc_name))) {
LOG_WARN("fail to get rowkey doc name", K(ret), KPC(rowkey_doc_schema));
} else if (OB_FAIL(doc_rowkey_schema->get_index_name(doc_rowkey_name))) {
LOG_WARN("fail to get doc rowkey name", K(ret), KPC(doc_rowkey_schema));
} else {
if (is_fts_index) {
if (OB_FAIL(ret) || OB_ISNULL(doc_word_schema)) {
} else if (OB_FAIL(doc_word_schema->get_index_name(fts_doc_word_name))) {
LOG_WARN("fail to get fts doc word name", K(ret), KPC(doc_word_schema));
}
}
if (OB_FAIL(ret) || OB_ISNULL(rowkey_doc_schema)) {
} else if (OB_FAIL(rowkey_doc_schema->get_index_name(rowkey_doc_name))) {
LOG_WARN("fail to get rowkey doc name", K(ret), KPC(rowkey_doc_schema));
}
if (OB_FAIL(ret) || OB_ISNULL(doc_rowkey_schema)) {
} else if (OB_FAIL(doc_rowkey_schema->get_index_name(doc_rowkey_name))) {
LOG_WARN("fail to get doc rowkey name", K(ret), KPC(doc_rowkey_schema));
}
const uint64_t data_table_id = index_schema->get_data_table_id();
const ObFTSDDLChildTaskInfo domain_index(domain_index_name, index_schema->get_table_id(), 0/*task_id*/);
uint64_t rowkey_doc_table_id = OB_ISNULL(rowkey_doc_schema) ? OB_INVALID_ID :
rowkey_doc_schema->get_table_id();
uint64_t doc_rowkey_table_id = OB_ISNULL(doc_rowkey_schema) ? OB_INVALID_ID :
doc_rowkey_schema->get_table_id();
uint64_t doc_word_table_id = OB_ISNULL(doc_word_schema) ? OB_INVALID_ID :
doc_word_schema->get_table_id();
const ObFTSDDLChildTaskInfo fts_doc_word(fts_doc_word_name,
is_fts_index ? doc_word_schema->get_table_id() : OB_INVALID_ID, 0/*task_id*/);
const ObFTSDDLChildTaskInfo rowkey_doc(rowkey_doc_name, rowkey_doc_schema->get_table_id(), 0/*task_id*/);
const ObFTSDDLChildTaskInfo doc_rowkey(doc_rowkey_name, doc_rowkey_schema->get_table_id(), 0/*task_id*/);
is_fts_index ? doc_word_table_id : OB_INVALID_ID, 0/*task_id*/);
const ObFTSDDLChildTaskInfo rowkey_doc(rowkey_doc_name, rowkey_doc_table_id, 0/*task_id*/);
const ObFTSDDLChildTaskInfo doc_rowkey(doc_rowkey_name, doc_rowkey_table_id, 0/*task_id*/);
const ObDDLType ddl_type = is_fts_index ? DDL_DROP_FTS_INDEX : DDL_DROP_MULVALUE_INDEX;
if (OB_FAIL(index_task.init(index_schema->get_tenant_id(),
if (OB_FAIL(ret)) {
} else if (OB_FAIL(index_task.init(index_schema->get_tenant_id(),
task_id,
data_table_id,
ddl_type,
@ -2158,6 +2228,9 @@ int ObDDLScheduler::schedule_ddl_task(const ObDDLTaskRecord &record)
case ObDDLType::DDL_DROP_MLOG:
ret = schedule_drop_index_task(record);
break;
case ObDDLType::DDL_CREATE_FTS_INDEX:
ret = schedule_build_fts_index_task(record);
break;
case ObDDLType::DDL_DROP_FTS_INDEX:
case ObDDLType::DDL_DROP_MULVALUE_INDEX:
ret = schedule_drop_fts_index_task(record);
@ -2222,6 +2295,33 @@ int ObDDLScheduler::schedule_ddl_task(const ObDDLTaskRecord &record)
return ret;
}
int ObDDLScheduler::schedule_build_fts_index_task(
const ObDDLTaskRecord &task_record)
{
int ret = OB_SUCCESS;
ObFtsIndexBuildTask *build_index_task = nullptr;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_FAIL(alloc_ddl_task(build_index_task))) {
LOG_WARN("alloc ddl task failed", K(ret));
} else if (OB_FAIL(build_index_task->init(task_record))) {
LOG_WARN("init global_index_task failed", K(ret), K(task_record));
} else if (OB_FAIL(build_index_task->set_trace_id(task_record.trace_id_))) {
LOG_WARN("init build index task failed", K(ret));
} else if (OB_FAIL(inner_schedule_ddl_task(build_index_task, task_record))) {
if (OB_ENTRY_EXIST != ret) {
LOG_WARN("inner schedule task failed", K(ret), K(*build_index_task));
}
}
if (OB_FAIL(ret) && nullptr != build_index_task) {
build_index_task->~ObFtsIndexBuildTask();
allocator_.free(build_index_task);
build_index_task = nullptr;
}
return ret;
}
int ObDDLScheduler::schedule_build_index_task(
const ObDDLTaskRecord &task_record)
{

12
src/rootserver/ddl_task/ob_ddl_scheduler.h
View File

@ -367,6 +367,16 @@ private:
const uint64_t tenant_data_version,
ObIAllocator &allocator,
ObDDLTaskRecord &task_record);
int create_build_fts_index_task(
common::ObISQLClient &proxy,
const share::schema::ObTableSchema *data_table_schema,
const share::schema::ObTableSchema *index_schema,
const int64_t parallelism,
const int64_t parent_task_id,
const int64_t consumer_group_id,
const obrpc::ObCreateIndexArg *create_index_arg,
ObIAllocator &allocator,
ObDDLTaskRecord &task_record);
int create_constraint_task(
common::ObISQLClient &proxy,
const share::schema::ObTableSchema *table_schema,
@ -492,6 +502,8 @@ private:
ObIAllocator &allocator,
ObDDLTaskRecord &task_record);
int schedule_build_fts_index_task(
const ObDDLTaskRecord &task_record);
int schedule_build_index_task(
const ObDDLTaskRecord &task_record);
int schedule_build_mview_task(const ObDDLTaskRecord &task_record);

2
src/rootserver/ddl_task/ob_ddl_task.cpp
View File

@ -736,6 +736,8 @@ int ObDDLTask::get_ddl_type_str(const int64_t ddl_type, const char *&ddl_type_st
case DDL_CREATE_INDEX:
ddl_type_str = "create index";
break;
case DDL_CREATE_FTS_INDEX:
ddl_type_str = "create fts index";
case DDL_CREATE_PARTITIONED_LOCAL_INDEX:
ddl_type_str = "create partitioned local index";
break;

16
src/rootserver/ddl_task/ob_drop_fts_index_task.cpp
View File

@ -57,14 +57,11 @@ int ObDropFTSIndexTask::init(
if (OB_UNLIKELY(OB_INVALID_ID == tenant_id
|| task_id <= 0
|| OB_INVALID_ID == data_table_id
|| !rowkey_doc.is_valid()
|| !doc_rowkey.is_valid()
|| !domain_index.is_valid()
|| (is_fts_task && !fts_doc_word.is_valid())
|| schema_version <= 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K(tenant_id), K(task_id), K(data_table_id), K(rowkey_doc),
K(doc_rowkey), K(domain_index), K(fts_doc_word), K(schema_version));
LOG_WARN("invalid arguments", K(ret), K(tenant_id), K(task_id), K(data_table_id),
K(domain_index), K(schema_version));
} else if (OB_ISNULL(root_service_ = GCTX.root_service_)) {
ret = OB_ERR_SYS;
LOG_WARN("error sys, root service is null", K(ret));
@ -377,7 +374,7 @@ int ObDropFTSIndexTask::wait_child_task_finish(
for (int64_t i = 0; OB_SUCC(ret) && finished && i < child_task_ids.count(); ++i) {
const ObFTSDDLChildTaskInfo &task_info = child_task_ids.at(i);
finished = false;
if (-1 == task_info.task_id_) {
if (-1 == task_info.task_id_ || task_info.table_id_ == OB_INVALID_ID) {
finished = true;
} else if (OB_FAIL(check_drop_index_finish(tenant_id_, task_info.task_id_, task_info.table_id_, finished))) {
LOG_WARN("fail to check fts index child task finish", K(ret));
@ -434,9 +431,12 @@ int ObDropFTSIndexTask::create_drop_index_task(
if (OB_ISNULL(root_service_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected error, root service is nullptr", K(ret), KP(root_service_));
} else if (OB_UNLIKELY(OB_INVALID_ID == index_tid || index_name.empty())) {
} else if (OB_INVALID_ID == index_tid) {
// nothing to do, just by pass.
task_id = -1;
} else if (OB_UNLIKELY(index_name.empty())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K(index_tid), K(index_name));
LOG_WARN("invalid arguments", K(ret), K(index_name));
} else if (OB_FAIL(guard.check_table_exist(tenant_id_, index_tid, is_index_exist))) {
LOG_WARN("fail to check table exist", K(ret), K(tenant_id_), K(index_tid));
} else if (!is_index_exist) {

1667
src/rootserver/ddl_task/ob_fts_index_build_task.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

155
src/rootserver/ddl_task/ob_fts_index_build_task.h Normal file
View File

@ -0,0 +1,155 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_ROOTSERVER_OB_FTS_INDEX_BUILD_TASK_H_
#define OCEANBASE_ROOTSERVER_OB_FTS_INDEX_BUILD_TASK_H_
#include "rootserver/ddl_task/ob_ddl_task.h"
namespace oceanbase
{
namespace rootserver
{
class ObFtsIndexBuildTask : public ObDDLTask
{
public:
ObFtsIndexBuildTask();
virtual ~ObFtsIndexBuildTask();
int init(
const uint64_t tenant_id,
const int64_t task_id,
const ObTableSchema *data_table_schema,
const ObTableSchema *index_schema,
const int64_t schema_version,
const int64_t parallelism,
const int64_t consumer_group_id,
const obrpc::ObCreateIndexArg &create_index_arg,
const int64_t parent_task_id = 0,
const int64_t task_status = share::ObDDLTaskStatus::PREPARE,
const int64_t snapshot_version = 0);
int init(const ObDDLTaskRecord &task_record);
virtual int process() override;
virtual void flt_set_task_span_tag() const override;
virtual void flt_set_status_span_tag() const override;
virtual int cleanup_impl() override;
virtual bool is_valid() const override;
virtual int collect_longops_stat(share::ObLongopsValue &value) override;
virtual int serialize_params_to_message(
char *buf,
const int64_t buf_size,
int64_t &pos) const override;
virtual int deserialize_params_from_message(
const uint64_t tenant_id,
const char *buf,
const int64_t buf_size,
int64_t &pos) override;
virtual int64_t get_serialize_param_size() const override;
virtual bool support_longops_monitoring() const override { return false; }
virtual int on_child_task_finish(
const uint64_t child_task_key,
const int ret_code) override;
TO_STRING_KV(K(index_table_id_), K(rowkey_doc_aux_table_id_),
K(doc_rowkey_aux_table_id_), K(fts_index_aux_table_id_),
K(fts_doc_word_aux_table_id_), K(rowkey_doc_schema_generated_),
K(doc_rowkey_schema_generated_), K(fts_index_aux_schema_generated_),
K(fts_doc_word_schema_generated_), K(rowkey_doc_task_submitted_),
K(doc_rowkey_task_submitted_), K(fts_index_aux_task_submitted_),
K(fts_doc_word_task_submitted_), K(drop_index_task_id_),
K(drop_index_task_submitted_), K(schema_version_), K(execution_id_),
K(consumer_group_id_), K(trace_id_), K(parallelism_), K(create_index_arg_));
private:
int get_next_status(share::ObDDLTaskStatus &next_status);
int prepare_rowkey_doc_table();
int prepare_aux_index_tables();
int construct_rowkey_doc_arg(obrpc::ObCreateIndexArg &arg);
int construct_doc_rowkey_arg(obrpc::ObCreateIndexArg &arg);
int construct_fts_index_aux_arg(obrpc::ObCreateIndexArg &arg);
int construct_fts_doc_word_arg(obrpc::ObCreateIndexArg &arg);
int record_index_table_id(
const obrpc::ObCreateIndexArg *create_index_arg_,
uint64_t &aux_table_id);
int get_index_table_id(
const obrpc::ObCreateIndexArg *create_index_arg,
uint64_t &index_table_id);
int prepare();
int wait_aux_table_complement();
int submit_build_aux_index_task(
const obrpc::ObCreateIndexArg &create_index_arg,
ObDDLTaskRecord &task_record,
bool &task_submitted);
int validate_checksum();
int clean_on_failed();
int submit_drop_fts_index_task();
int wait_drop_index_finish(bool &is_finish);
int succ();
int update_index_status_in_schema(
const ObTableSchema &index_schema,
const ObIndexStatus new_status);
int check_health();
int check_aux_table_schemas_exist(bool &is_all_exist);
int deep_copy_index_arg(
common::ObIAllocator &allocator,
const obrpc::ObCreateIndexArg &source_arg,
obrpc::ObCreateIndexArg &dest_arg);
private:
struct DependTaskStatus final
{
public:
DependTaskStatus()
: ret_code_(INT64_MAX), task_id_(0)
{}
~DependTaskStatus() = default;
TO_STRING_KV(K_(task_id), K_(ret_code));
public:
int64_t ret_code_;
int64_t task_id_;
};
static const int64_t OB_FTS_INDEX_BUILD_TASK_VERSION = 1;
using ObDDLTask::tenant_id_;
using ObDDLTask::task_id_;
using ObDDLTask::schema_version_;
using ObDDLTask::parallelism_;
using ObDDLTask::consumer_group_id_;
using ObDDLTask::parent_task_id_;
using ObDDLTask::task_status_;
using ObDDLTask::snapshot_version_;
using ObDDLTask::object_id_;
using ObDDLTask::target_object_id_;
using ObDDLTask::is_inited_;
uint64_t &index_table_id_;
uint64_t rowkey_doc_aux_table_id_;
uint64_t doc_rowkey_aux_table_id_;
uint64_t fts_index_aux_table_id_;
uint64_t fts_doc_word_aux_table_id_;
bool rowkey_doc_schema_generated_;
bool doc_rowkey_schema_generated_;
bool fts_index_aux_schema_generated_;
bool fts_doc_word_schema_generated_;
bool rowkey_doc_task_submitted_;
bool doc_rowkey_task_submitted_;
bool fts_index_aux_task_submitted_;
bool fts_doc_word_task_submitted_;
int64_t drop_index_task_id_;
bool drop_index_task_submitted_;
ObRootService *root_service_;
ObDDLWaitTransEndCtx wait_trans_ctx_;
obrpc::ObCreateIndexArg create_index_arg_;
common::hash::ObHashMap<uint64_t, DependTaskStatus> dependent_task_result_map_;
};
} // end namespace rootserver
} // end namespace oceanbase
#endif /* OCEANBASE_ROOTSERVER_OB_FTS_INDEX_BUILD_TASK_H_*/

124
src/rootserver/ddl_task/ob_index_build_task.cpp
View File

@ -1051,67 +1051,73 @@ bool ObIndexBuildTask::is_create_partitioned_local_index()
int ObIndexBuildTask::wait_data_complement()
{
int ret = OB_SUCCESS;
bool state_finished = false;
bool is_request_end = false;
share::ObLSID ls_id;
common::ObAddr leader_addr;
ObArray<ObTabletID> index_partition_ids;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (ObDDLTaskStatus::REDEFINITION != task_status_) {
LOG_WARN("task status not match", K(ret), K(task_status_));
} else if (OB_UNLIKELY(snapshot_version_ <= 0)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected snapshot", K(ret), KPC(this));
}
// submit a job to complete sstable for the index table on snapshot_version
if (OB_SUCC(ret) && !state_finished && !is_sstable_complete_task_submitted()) {
bool need_exec_new_inner_sql = false;
if (OB_FAIL(reap_old_replica_build_task(need_exec_new_inner_sql))) {
if (OB_EAGAIN == ret) {
ret = OB_SUCCESS; // retry
} else {
LOG_WARN("failed to reap old task", K(ret));
}
} else if (!need_exec_new_inner_sql) {
state_finished = true;
} else if (OB_FAIL(send_build_single_replica_request(false, parallelism_, 0, ls_id, leader_addr, index_partition_ids))) {
LOG_WARN("fail to send build single replica request", K(ret), K(parallelism_), K(index_partition_ids));
}
}
DEBUG_SYNC(CREATE_INDEX_REPLICA_BUILD);
if (OB_SUCC(ret) && !state_finished && is_sstable_complete_task_submitted()) {
if (OB_FAIL(check_build_single_replica(is_request_end))) {
LOG_WARN("fail to check build single replica", K(ret));
} else if (OB_UNLIKELY(is_request_end)) {
ret = complete_sstable_job_ret_code_;
state_finished = true;
}
}
if (OB_SUCC(ret) && state_finished && !create_index_arg_.is_spatial_index()) {
bool dummy_equal = false;
bool need_verify_checksum = true;
#ifdef ERRSIM
// when the major compaction is delayed, skip verify column checksum
need_verify_checksum = 0 == GCONF.errsim_ddl_major_delay_time;
#endif
if (need_verify_checksum && OB_FAIL(ObDDLChecksumOperator::check_column_checksum(
tenant_id_, get_execution_id(), object_id_, index_table_id_, task_id_, false/*index build*/, dummy_equal, root_service_->get_sql_proxy()))) {
if (OB_ITER_END != ret) {
LOG_WARN("fail to check column checksum", K(ret), K(index_table_id_), K(object_id_), K(task_id_));
state_finished = true;
} else if (REACH_TIME_INTERVAL(1000L * 1000L)) {
LOG_INFO("index checksum has not been reported", K(ret), K(index_table_id_), K(object_id_), K(task_id_));
}
}
}
if (state_finished || OB_FAIL(ret)) {
// temporary bypass data complement for fts index
if (share::schema::is_fts_index(create_index_arg_.index_type_)) {
(void)switch_status(ObDDLTaskStatus::VALIDATE_CHECKSUM, true, ret);
LOG_INFO("wait data complement finished", K(ret), K(*this));
} else {
bool state_finished = false;
bool is_request_end = false;
share::ObLSID ls_id;
common::ObAddr leader_addr;
ObArray<ObTabletID> index_partition_ids;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (ObDDLTaskStatus::REDEFINITION != task_status_) {
LOG_WARN("task status not match", K(ret), K(task_status_));
} else if (OB_UNLIKELY(snapshot_version_ <= 0)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected snapshot", K(ret), KPC(this));
}
// submit a job to complete sstable for the index table on snapshot_version
if (OB_SUCC(ret) && !state_finished && !is_sstable_complete_task_submitted()) {
bool need_exec_new_inner_sql = false;
if (OB_FAIL(reap_old_replica_build_task(need_exec_new_inner_sql))) {
if (OB_EAGAIN == ret) {
ret = OB_SUCCESS; // retry
} else {
LOG_WARN("failed to reap old task", K(ret));
}
} else if (!need_exec_new_inner_sql) {
state_finished = true;
} else if (OB_FAIL(send_build_single_replica_request(false, parallelism_, 0, ls_id, leader_addr, index_partition_ids))) {
LOG_WARN("fail to send build single replica request", K(ret), K(parallelism_), K(index_partition_ids));
}
}
DEBUG_SYNC(CREATE_INDEX_REPLICA_BUILD);
if (OB_SUCC(ret) && !state_finished && is_sstable_complete_task_submitted()) {
if (OB_FAIL(check_build_single_replica(is_request_end))) {
LOG_WARN("fail to check build single replica", K(ret));
} else if (is_request_end) {
ret = complete_sstable_job_ret_code_;
state_finished = true;
}
}
if (OB_SUCC(ret) && state_finished && !create_index_arg_.is_spatial_index()) {
bool dummy_equal = false;
bool need_verify_checksum = true;
#ifdef ERRSIM
// when the major compaction is delayed, skip verify column checksum
need_verify_checksum = 0 == GCONF.errsim_ddl_major_delay_time;
#endif
if (need_verify_checksum && OB_FAIL(ObDDLChecksumOperator::check_column_checksum(
tenant_id_, get_execution_id(), object_id_, index_table_id_, task_id_, false/*index build*/, dummy_equal, root_service_->get_sql_proxy()))) {
if (OB_ITER_END != ret) {
LOG_WARN("fail to check column checksum", K(ret), K(index_table_id_), K(object_id_), K(task_id_));
state_finished = true;
} else if (REACH_TIME_INTERVAL(1000L * 1000L)) {
LOG_INFO("index checksum has not been reported", K(ret), K(index_table_id_), K(object_id_), K(task_id_));
}
}
}
if (state_finished || OB_FAIL(ret)) {
(void)switch_status(ObDDLTaskStatus::VALIDATE_CHECKSUM, true, ret);
LOG_INFO("wait data complement finished", K(ret), K(*this));
}
}
return ret;
}

155
src/rootserver/ob_ddl_service.cpp
View File

@ -6207,6 +6207,160 @@ int ObDDLService::lock_tables_of_database(const ObDatabaseSchema &database_schem
return ret;
}
int ObDDLService::check_schema_generated_for_aux_index_schema_(
const obrpc::ObGenerateAuxIndexSchemaArg &arg,
ObSchemaGetterGuard &schema_guard,
const ObTableSchema *data_schema,
bool &schema_generated,
uint64_t &aux_index_table_id)
{
int ret = OB_SUCCESS;
schema_generated = false;
aux_index_table_id = OB_INVALID_ID;
ObArenaAllocator allocator(ObModIds::OB_SCHEMA);
const uint64_t tenant_id = arg.tenant_id_;
ObIndexType index_type = arg.create_index_arg_.index_type_;
ObString index_table_name;
if (OB_ISNULL(data_schema)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KPC(data_schema));
} else if (OB_FAIL(ObTableSchema::build_index_table_name(allocator,
data_schema->get_table_id(),
arg.create_index_arg_.index_name_,
index_table_name))) {
LOG_WARN("failed to construct index table name", K(ret),
K(arg.create_index_arg_.index_name_));
} else if (share::schema::is_fts_index(index_type)) {
const ObTableSchema *index_schema = nullptr;
if (OB_FAIL(schema_guard.get_table_schema(tenant_id,
data_schema->get_database_id(),
index_table_name,
true/*is_index*/,
index_schema,
false/*with_hidden_flag*/,
share::schema::is_built_in_fts_index(index_type)))) {
LOG_WARN("failed to get index schema", K(ret), K(tenant_id), K(index_table_name));
} else if (OB_NOT_NULL(index_schema)) {
schema_generated = true;
aux_index_table_id = index_schema->get_table_id();
LOG_INFO("fts index aux table already exist, no need to generate",
K(index_table_name));
}
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected index type", K(ret), K(index_type));
}
return ret;
}
int ObDDLService::generate_aux_index_schema(
const obrpc::ObGenerateAuxIndexSchemaArg &arg,
obrpc::ObGenerateAuxIndexSchemaRes &result)
{
int ret = OB_SUCCESS;
const uint64_t tenant_id = arg.tenant_id_;
const uint64_t data_table_id = arg.data_table_id_;
SMART_VARS_2((obrpc::ObCreateIndexArg, create_index_arg),
(ObTableSchema, nonconst_data_schema)) {
ObSchemaGetterGuard schema_guard;
const ObTableSchema *data_schema = nullptr;
ObSEArray<ObColumnSchemaV2 *, 1> gen_columns;
ObArenaAllocator allocator(lib::ObLabel("DdlTaskTmp"));
if (!arg.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(arg));
} else if (OB_FAIL(get_tenant_schema_guard_with_version_in_inner_table(tenant_id,
schema_guard))) {
LOG_WARN("get schema guard failed", K(ret));
} else if (OB_FAIL(schema_guard.get_table_schema(tenant_id,
data_table_id,
data_schema))) {
LOG_WARN("get table schema failed", K(ret), K(tenant_id), K(data_table_id));
} else if (OB_ISNULL(data_schema)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error unexpected, table schema is nullptr", K(ret), K(data_table_id));
} else if (OB_FAIL(nonconst_data_schema.assign(*data_schema))) {
LOG_WARN("failed to assign to nonconst data schema", K(ret));
} else if (OB_FAIL(create_index_arg.assign(arg.create_index_arg_))) {
LOG_WARN("fail to assign create index arg", K(ret));
} else if (OB_FAIL(ObFtsIndexBuilderUtil::adjust_fts_args(create_index_arg,
nonconst_data_schema,
allocator,
gen_columns))) {
LOG_WARN("fail to adjust expr index args", K(ret));
} else {
if (OB_FAIL(check_schema_generated_for_aux_index_schema_(arg,
schema_guard,
data_schema,
result.schema_generated_,
result.aux_table_id_) )) {
LOG_WARN("failed to check if schema is generated for aux index table", K(ret), K(arg));
} else if (result.schema_generated_) {
// do nothing
} else {
ObIndexBuilder index_builder(*this);
ObDDLSQLTransaction trans(&get_schema_service());
const bool global_index_without_column_info = true;
int64_t refreshed_schema_version = 0;
uint64_t tenant_data_version = 0;
ObTableSchema index_schema;
if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) {
LOG_WARN("get min data version failed", K(ret), K(tenant_id));
} else if (OB_FAIL(schema_guard.get_schema_version(tenant_id,
refreshed_schema_version))) {
LOG_WARN("failed to get tenant schema version", KR(ret), K(tenant_id));
} else if (OB_FAIL(trans.start(&get_sql_proxy(),
tenant_id,
refreshed_schema_version))) {
LOG_WARN("start transaction failed", KR(ret), K(tenant_id),
K(refreshed_schema_version));
} else if (OB_FAIL(index_builder.generate_schema(create_index_arg,
nonconst_data_schema,
global_index_without_column_info,
true/*generate_id*/,
index_schema))) {
LOG_WARN("fail to generate schema", K(ret), K(create_index_arg));
} else if (OB_FAIL(nonconst_data_schema.check_create_index_on_hidden_primary_key(index_schema))) {
LOG_WARN("failed to check create index on table", K(ret), K(index_schema));
} else if (gen_columns.empty()) {
if (OB_FAIL(create_index_table(create_index_arg,
tenant_data_version,
index_schema,
trans))) {
LOG_WARN("fail to create index", K(ret), K(index_schema));
}
} else {
if (OB_FAIL(create_inner_expr_index(trans,
*data_schema,
tenant_data_version,
nonconst_data_schema,
gen_columns,
index_schema))) {
LOG_WARN("fail to create inner expr index", K(ret));
}
}
if (trans.is_started()) {
int temp_ret = OB_SUCCESS;
if (OB_SUCCESS != (temp_ret = trans.end(OB_SUCC(ret)))) {
LOG_WARN("trans end failed", "is_commit", OB_SUCCESS == ret, K(temp_ret));
ret = (OB_SUCC(ret)) ? temp_ret : ret;
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(publish_schema(tenant_id))) {
LOG_WARN("fail to publish schema", K(ret), K(tenant_id));
} else {
result.schema_generated_ = true;
result.aux_table_id_ = index_schema.get_table_id();
}
}
}
}
}
LOG_INFO("finish generate aux index schema", K(ret), K(arg), K(result), "ddl_event_info", ObDDLEventInfo());
return ret;
}
int ObDDLService::lock_tables_in_recyclebin(const ObDatabaseSchema &database_schema,
ObMySQLTransaction &trans)
{
@ -7642,7 +7796,6 @@ int ObDDLService::get_dropping_domain_index_invisiable_aux_table_schema(
const share::schema::ObTableSchema *doc_word_schema = nullptr;
const share::schema::ObTableSchema *rowkey_doc_schema = nullptr;
const share::schema::ObTableSchema *doc_rowkey_schema = nullptr;
for (int64_t i = 0; OB_SUCC(ret) && i < indexs.count(); ++i) {
const share::schema::ObAuxTableMetaInfo &info = indexs.at(i);
if (share::schema::is_rowkey_doc_aux(info.index_type_)) {

9
src/rootserver/ob_ddl_service.h
View File

@ -2127,6 +2127,9 @@ private:
ObDDLSQLTransaction &trans);
public:
int generate_aux_index_schema(
const obrpc::ObGenerateAuxIndexSchemaArg &arg,
obrpc::ObGenerateAuxIndexSchemaRes &result);
int check_parallel_ddl_conflict(
share::schema::ObSchemaGetterGuard &schema_guard,
const obrpc::ObDDLArg &arg);
@ -2186,6 +2189,12 @@ public:
common::ObIAllocator *allocator = NULL);
#endif
private:
int check_schema_generated_for_aux_index_schema_(
const obrpc::ObGenerateAuxIndexSchemaArg &arg,
ObSchemaGetterGuard &schema_guard,
const ObTableSchema *data_schema,
bool &schema_generated,
uint64_t &index_table_id);
int adjust_cg_for_offline(ObTableSchema &new_table_schema);
int add_column_group(const obrpc::ObAlterTableArg &alter_table_arg,
const share::schema::ObTableSchema &ori_table_schema,

42
src/rootserver/ob_index_builder.cpp
View File

@ -26,9 +26,11 @@
#include "share/schema/ob_multi_version_schema_service.h"
#include "share/schema/ob_schema_struct.h"
#include "share/schema/ob_part_mgr_util.h"
#include "share/schema/ob_table_schema.h"
#include "share/ob_common_rpc_proxy.h"
#include "share/config/ob_server_config.h"
#include "share/ob_index_builder_util.h"
#include "share/ob_fts_index_builder_util.h"
#include "observer/ob_server_struct.h"
#include "sql/resolver/ddl/ob_ddl_resolver.h"
#include "ob_zone_manager.h"
@ -455,14 +457,20 @@ int ObIndexBuilder::submit_build_index_task(
if (OB_UNLIKELY(nullptr == data_schema || nullptr == index_schema || tenant_data_version <= 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("schema is invalid", K(ret), KP(data_schema), KP(index_schema), K(tenant_data_version));
} else if (OB_FAIL(GCTX.root_service_->get_ddl_task_scheduler().create_ddl_task(param, trans, task_record))) {
LOG_WARN("submit create index ddl task failed", K(ret));
} else if (OB_FAIL(owner_id.convert_from_value(ObLockOwnerType::DEFAULT_OWNER_TYPE,
task_record.task_id_))) {
LOG_WARN("failed to get owner id", K(ret), K(task_record.task_id_));
} else if (OB_FAIL(ObDDLLock::lock_for_add_drop_index(
*data_schema, inc_data_tablet_ids, del_data_tablet_ids, *index_schema, owner_id, trans))) {
LOG_WARN("failed to lock online ddl lock", K(ret));
} else {
bool is_create_fts_index = share::schema::is_fts_index(create_index_arg.index_type_);
if (is_create_fts_index) {
param.type_ = ObDDLType::DDL_CREATE_FTS_INDEX;
}
if (OB_FAIL(GCTX.root_service_->get_ddl_task_scheduler().create_ddl_task(param, trans, task_record))) {
LOG_WARN("submit create index ddl task failed", K(ret));
} else if (OB_FAIL(owner_id.convert_from_value(ObLockOwnerType::DEFAULT_OWNER_TYPE,
task_record.task_id_))) {
LOG_WARN("failed to get owner id", K(ret), K(task_record.task_id_));
} else if (OB_FAIL(ObDDLLock::lock_for_add_drop_index(
*data_schema, inc_data_tablet_ids, del_data_tablet_ids, *index_schema, owner_id, trans))) {
LOG_WARN("failed to lock online ddl lock", K(ret));
}
}
return ret;
}
@ -637,11 +645,7 @@ int ObIndexBuilder::do_create_local_index(
int64_t refreshed_schema_version = 0;
const uint64_t tenant_id = table_schema.get_tenant_id();
uint64_t tenant_data_version = 0;
// TODO hanxuan support fulltext index after table created
if (share::schema::is_fts_index(create_index_arg.index_type_)) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("not supported", K(ret));
} else if (OB_FAIL(schema_guard.get_schema_version(tenant_id, refreshed_schema_version))) {
if (OB_FAIL(schema_guard.get_schema_version(tenant_id, refreshed_schema_version))) {
LOG_WARN("failed to get tenant schema version", KR(ret), K(tenant_id));
} else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) {
LOG_WARN("get tenant data version failed", K(ret));
@ -676,7 +680,11 @@ int ObIndexBuilder::do_create_local_index(
my_arg.index_schema_.set_index_type(INDEX_TYPE_SPATIAL_GLOBAL_LOCAL_STORAGE);
}
}
if (FAILEDx(ObIndexBuilderUtil::adjust_expr_index_args(
if (OB_FAIL(ret)) {
} else if (share::schema::is_fts_index(my_arg.index_type_) &&
OB_FAIL(ObFtsIndexBuilderUtil::generate_fts_aux_index_name(my_arg, &allocator))) {
LOG_WARN("failed to adjust fts index name", K(ret));
} else if (OB_FAIL(ObIndexBuilderUtil::adjust_expr_index_args(
my_arg, new_table_schema, allocator, gen_columns))) {
LOG_WARN("fail to adjust expr index args", K(ret));
} else if (OB_FAIL(generate_schema(
@ -887,10 +895,10 @@ int ObIndexBuilder::generate_schema(
if (OB_FAIL(GET_MIN_DATA_VERSION(data_schema.get_tenant_id(),
tenant_data_version))) {
LOG_WARN("failed to get tenant data version", K(ret));
} else if (tenant_data_version < DATA_VERSION_4_3_1_0) {
} else if (tenant_data_version < DATA_VERSION_4_3_2_0) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("tenant data version is less than 4.3.1, fulltext index is not supported", K(ret), K(tenant_data_version));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "tenant data version is less than 4.3.1, fulltext index");
LOG_WARN("tenant data version is less than 4.3.2, create fulltext index on existing table is not supported", K(ret), K(tenant_data_version));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "tenant data version is less than 4.3.2, fulltext index");
}
} else if (is_multivalue_index(arg.index_type_)) {
uint64_t tenant_data_version = 0;

19
src/rootserver/ob_root_service.cpp
View File

@ -51,6 +51,7 @@
#include "share/ob_time_zone_info_manager.h"
#include "share/ob_server_status.h"
#include "share/ob_index_builder_util.h"
#include "share/ob_fts_index_builder_util.h"
#include "observer/ob_server_struct.h"
#include "observer/omt/ob_tenant_config_mgr.h"
#include "observer/ob_server_event_history_table_operator.h"
@ -4647,6 +4648,21 @@ int ObRootService::exchange_partition(const obrpc::ObExchangePartitionArg &arg,
return ret;
}
int ObRootService::generate_aux_index_schema(
const ObGenerateAuxIndexSchemaArg &arg,
ObGenerateAuxIndexSchemaRes &result)
{
int ret = OB_SUCCESS;
if (!arg.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(arg));
} else if (OB_FAIL(ddl_service_.generate_aux_index_schema(arg, result))) {
LOG_WARN("failed to generate aux index schema", K(ret), K(arg), K(result));
}
LOG_INFO("finish generate aux index schema", K(ret), K(arg), K(result), "ddl_event_info", ObDDLEventInfo());
return ret;
}
int ObRootService::create_index(const ObCreateIndexArg &arg, obrpc::ObAlterTableRes &res)
{
int ret = OB_SUCCESS;
@ -4658,8 +4674,7 @@ int ObRootService::create_index(const ObCreateIndexArg &arg, obrpc::ObAlterTable
} else if (!arg.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arg", K(arg), K(ret));
} else if (is_fts_index(arg.index_type_) || is_multivalue_index(arg.index_type_)) {
// TODO hanxuan support create fulltext index
} else if (is_multivalue_index(arg.index_type_)) {
// todo yunyi not dynamic create multivlaue index
ret = OB_NOT_SUPPORTED;
LOG_WARN("not supported", K(ret));

3
src/rootserver/ob_root_service.h
View File

@ -502,6 +502,9 @@ public:
int truncate_table(const obrpc::ObTruncateTableArg &arg, obrpc::ObDDLRes &res);
int truncate_table_v2(const obrpc::ObTruncateTableArg &arg, obrpc::ObDDLRes &res);
int exchange_partition(const obrpc::ObExchangePartitionArg &arg, obrpc::ObAlterTableRes &res);
int generate_aux_index_schema(
const obrpc::ObGenerateAuxIndexSchemaArg &arg,
obrpc::ObGenerateAuxIndexSchemaRes &result);
int create_index(const obrpc::ObCreateIndexArg &arg, obrpc::ObAlterTableRes &res);
int drop_table(const obrpc::ObDropTableArg &arg, obrpc::ObDDLRes &res);
int drop_database(const obrpc::ObDropDatabaseArg &arg, obrpc::ObDropDatabaseRes &drop_database_res);

1
src/rootserver/ob_rs_rpc_processor.h
View File

@ -348,6 +348,7 @@ DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_DROP_TABLE, ObRpcDropTableP, drop_table(ar
DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_RENAME_TABLE, ObRpcRenameTableP, rename_table(arg_));
DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_TRUNCATE_TABLE, ObRpcTruncateTableP, truncate_table(arg_, result_));
DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_TRUNCATE_TABLE_V2, ObRpcTruncateTableV2P, truncate_table_v2(arg_, result_));
DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_GENERATE_AUX_INDEX_SCHEMA, ObRpcGenerateAuxIndexSchemaP, generate_aux_index_schema(arg_, result_));
DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_CREATE_INDEX, ObRpcCreateIndexP, create_index(arg_, result_));
DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_DROP_INDEX, ObRpcDropIndexP, drop_index(arg_, result_));
DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_CREATE_MLOG, ObRpcCreateMLogP, create_mlog(arg_, result_));

1
src/share/ob_common_rpc_proxy.h
View File

@ -76,6 +76,7 @@ public:
RPC_S(PRD rename_table, obrpc::OB_RENAME_TABLE, (ObRenameTableArg));
RPC_S(PRD truncate_table, obrpc::OB_TRUNCATE_TABLE, (ObTruncateTableArg), ObDDLRes);
RPC_S(PRD truncate_table_v2, obrpc::OB_TRUNCATE_TABLE_V2, (ObTruncateTableArg), ObDDLRes);
RPC_S(PRD generate_aux_index_schema, obrpc::OB_GENERATE_AUX_INDEX_SCHEMA, (obrpc::ObGenerateAuxIndexSchemaArg), obrpc::ObGenerateAuxIndexSchemaRes);
RPC_S(PRD create_index, obrpc::OB_CREATE_INDEX, (ObCreateIndexArg), ObAlterTableRes);
RPC_S(PRD drop_index, obrpc::OB_DROP_INDEX, (ObDropIndexArg), ObDropIndexRes);
RPC_S(PRD create_mlog, obrpc::OB_CREATE_MLOG, (ObCreateMLogArg), ObCreateMLogRes);

20
src/share/ob_ddl_common.h
View File

@ -164,10 +164,10 @@ enum ObDDLTaskStatus {
WAIT_LOCAL_INDEX_SPLIT_END = 22,
WAIT_LOB_TABLE_SPLIT_END = 23,
WAIT_PARTITION_SPLIT_RECOVERY_TASK_FINISH = 24,
//GENERATE_ROWKEY_DOC_SCHEMA = 25,
//WAIT_ROWKEY_DOC_TABLE_COMPLEMENT = 26,
//GENERATE_DOC_AUX_SCHEMA = 27,
//WAIT_AUX_TABLE_COMPLEMENT = 28,
GENERATE_ROWKEY_DOC_SCHEMA = 25,
WAIT_ROWKEY_DOC_TABLE_COMPLEMENT = 26,
GENERATE_DOC_AUX_SCHEMA = 27,
WAIT_AUX_TABLE_COMPLEMENT = 28,
FAIL = 99,
SUCCESS = 100
};
@ -275,6 +275,18 @@ static const char* ddl_task_status_to_str(const ObDDLTaskStatus &task_status) {
case ObDDLTaskStatus::WAIT_PARTITION_SPLIT_RECOVERY_TASK_FINISH:
str = "WAIT_PARTITION_SPLIT_RECOVERY_TASK_FINISH";
break;
case ObDDLTaskStatus::GENERATE_ROWKEY_DOC_SCHEMA:
str = "GENERATE_ROWKEY_DOC_SCHEMA";
break;
case ObDDLTaskStatus::GENERATE_DOC_AUX_SCHEMA:
str = "GENERATE_DOC_AUX_SCHEMA";
break;
case ObDDLTaskStatus::WAIT_ROWKEY_DOC_TABLE_COMPLEMENT:
str = "WAIT_ROWKEY_DOC_TABLE_COMPLEMENT";
break;
case ObDDLTaskStatus::WAIT_AUX_TABLE_COMPLEMENT:
str = "WAIT_AUX_TABLE_COMPLEMENT";
break;
case ObDDLTaskStatus::FAIL:
str = "FAIL";
break;

89
src/share/ob_fts_index_builder_util.cpp
View File

@ -155,6 +155,42 @@ int ObFtsIndexBuilderUtil::append_fts_doc_word_arg(
return ret;
}
int ObFtsIndexBuilderUtil::fts_doc_word_schema_exist(
uint64_t tenant_id,
uint64_t database_id,
ObSchemaGetterGuard &schema_guard,
const ObString &index_name,
bool &is_exist)
{
int ret = OB_SUCCESS;
is_exist = false;
const int64_t buf_size = OB_MAX_TABLE_NAME_BUF_LENGTH;
char buf[buf_size] = {0};
int64_t pos = 0;
ObString doc_word_index_name;
const ObTableSchema *fts_doc_word_schema = nullptr;
if (OB_FAIL(databuff_printf(buf,
buf_size,
pos,
"%.*s_fts_doc_word",
index_name.length(),
index_name.ptr()))) {
LOG_WARN("fail to printf fts doc word name str", K(ret), K(index_name));
} else if (OB_FALSE_IT(doc_word_index_name.assign_ptr(buf, pos))) {
} else if (OB_FAIL(schema_guard.get_table_schema(tenant_id,
database_id,
doc_word_index_name,
true/*is_index*/,
fts_doc_word_schema,
false/*with_hidden_flag*/,
true/*is_built_in_index*/))) {
LOG_WARN("failed to get index schema", K(ret), K(tenant_id));
} else if (OB_NOT_NULL(fts_doc_word_schema)) {
is_exist = true;
}
return ret;
}
int ObFtsIndexBuilderUtil::generate_fts_aux_index_name(
obrpc::ObCreateIndexArg &arg,
ObIAllocator *allocator)
@ -273,32 +309,22 @@ int ObFtsIndexBuilderUtil::adjust_fts_args(
ObColumnSchemaV2 *generated_word_col = nullptr;
ObColumnSchemaV2 *generated_doc_len_col = nullptr;
ObColumnSchemaV2 *generated_word_count_col = nullptr;
if (is_rowkey_doc || is_doc_rowkey) {
if (OB_ISNULL(existing_doc_id_col)) { // need to generate doc id col
doc_id_col_id = available_col_id++;
if (OB_FAIL(ret)) {
} else if (OB_FAIL(generate_doc_id_column(&index_arg,
doc_id_col_id,
data_schema,
generated_doc_id_col))) {
LOG_WARN("failed to generate doc id column", K(ret));
} else if (OB_FAIL(gen_columns.push_back(generated_doc_id_col))) {
LOG_WARN("failed to push back doc id col", K(ret));
}
}
if (OB_ISNULL(existing_doc_id_col)) { // need to generate doc id col
doc_id_col_id = available_col_id++;
if (OB_FAIL(ret)) {
} else if (OB_FAIL(push_back_gen_col(tmp_cols,
existing_doc_id_col,
generated_doc_id_col))) {
} else if (OB_FAIL(generate_doc_id_column(&index_arg,
doc_id_col_id,
data_schema,
generated_doc_id_col))) {
LOG_WARN("failed to generate doc id column", K(ret));
} else if (OB_FAIL(gen_columns.push_back(generated_doc_id_col))) {
LOG_WARN("failed to push back doc id col", K(ret));
} else if (OB_FAIL(adjust_fts_arg(&index_arg,
data_schema,
allocator,
tmp_cols))) {
LOG_WARN("failed to append fts_index arg", K(ret));
}
}
if (is_rowkey_doc || is_doc_rowkey) {
} else if (is_fts_index || is_doc_word) {
if (OB_ISNULL(existing_word_col)) {
if (OB_FAIL(ret)) {
} else if (OB_ISNULL(existing_word_col)) {
word_col_id = available_col_id++;
if (OB_FAIL(ret)) {
} else if (OB_FAIL(generate_word_segment_column(&index_arg,
@ -334,6 +360,20 @@ int ObFtsIndexBuilderUtil::adjust_fts_args(
LOG_WARN("fail to push back generated document length", K(ret));
}
}
}
if (is_rowkey_doc || is_doc_rowkey) {
if (OB_FAIL(ret)) {
} else if (OB_FAIL(push_back_gen_col(tmp_cols,
existing_doc_id_col,
generated_doc_id_col))) {
LOG_WARN("failed to push back doc id col", K(ret));
} else if (OB_FAIL(adjust_fts_arg(&index_arg,
data_schema,
allocator,
tmp_cols))) {
LOG_WARN("failed to append fts_index arg", K(ret));
}
} else if (is_fts_index || is_doc_word) {
if (OB_FAIL(ret)) {
} else if (is_fts_index) {
if (OB_FAIL(push_back_gen_col(tmp_cols,
@ -384,6 +424,7 @@ int ObFtsIndexBuilderUtil::adjust_fts_args(
}
}
}
FLOG_INFO("adjust fts arg finished", K(index_arg));
return ret;
}
@ -802,8 +843,8 @@ int ObFtsIndexBuilderUtil::inner_adjust_fts_arg(
!share::schema::is_fts_doc_word_aux(fts_arg->index_type_)) ||
fts_cols.count() != index_column_cnt + 2) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid argument", K(ret), KPC(fts_arg),
K(fts_cols.count()), K(index_column_cnt));
LOG_WARN("invalid argument", K(ret), KPC(fts_arg), K(fts_cols.count()),
K(index_column_cnt));
} else {
// 1. add doc id column, word column to arg->index_columns
for (int64_t i = 0; OB_SUCC(ret) && i < index_column_cnt; ++i) {

6
src/share/ob_fts_index_builder_util.h
View File

@ -46,6 +46,12 @@ public:
const obrpc::ObCreateIndexArg &index_arg,
ObIAllocator *allocator,
ObIArray<obrpc::ObCreateIndexArg> &index_arg_list);
static int fts_doc_word_schema_exist(
uint64_t tenant_id,
uint64_t database_id,
ObSchemaGetterGuard &schema_guard,
const ObString &index_name,
bool &is_exist);
static int generate_fts_aux_index_name(
obrpc::ObCreateIndexArg &arg,
ObIAllocator *allocator);

20
src/share/ob_rpc_struct.cpp
View File

@ -3175,6 +3175,26 @@ OB_SERIALIZE_MEMBER((ObCreateIndexArg, ObIndexArg),
exist_all_column_group_,
index_cgs_);
int ObGenerateAuxIndexSchemaArg::assign(const ObGenerateAuxIndexSchemaArg &other)
{
int ret = OB_SUCCESS;
if (OB_FAIL(create_index_arg_.assign(other.create_index_arg_))) {
SHARE_LOG(WARN, "fail to assign arg", K(ret));
} else {
tenant_id_ = other.tenant_id_;
data_table_id_ = other.data_table_id_;
}
return ret;
}
OB_SERIALIZE_MEMBER((ObGenerateAuxIndexSchemaArg, ObDDLArg),
tenant_id_,
data_table_id_,
create_index_arg_);
OB_SERIALIZE_MEMBER(ObGenerateAuxIndexSchemaRes,
aux_table_id_,
schema_generated_);
bool ObAlterIndexArg::is_valid() const
{
// store_columns_ can be empty

58
src/share/ob_rpc_struct.h
View File

@ -2700,6 +2700,64 @@ public:
common::ObSEArray<ObIndexColumnGroupItem, 1/*each*/> index_cgs_;
};
struct ObGenerateAuxIndexSchemaArg : public ObDDLArg
{
OB_UNIS_VERSION_V(1);
public:
ObGenerateAuxIndexSchemaArg()
: tenant_id_(OB_INVALID_TENANT_ID),
data_table_id_(OB_INVALID_ID)
{}
~ObGenerateAuxIndexSchemaArg() {}
bool is_valid() const
{
return tenant_id_ != OB_INVALID_TENANT_ID &&
data_table_id_ != OB_INVALID_ID&&
create_index_arg_.is_valid();
}
int assign(const ObGenerateAuxIndexSchemaArg &other);
void reset()
{
tenant_id_ = OB_INVALID_TENANT_ID;
data_table_id_ = OB_INVALID_ID;
create_index_arg_.reset();
}
TO_STRING_KV(K(tenant_id_), K(data_table_id_), K(create_index_arg_));
public:
uint64_t tenant_id_;
uint64_t data_table_id_;
ObCreateIndexArg create_index_arg_;
};
struct ObGenerateAuxIndexSchemaRes final
{
OB_UNIS_VERSION_V(1);
public:
ObGenerateAuxIndexSchemaRes()
: aux_table_id_(OB_INVALID_ID),
schema_generated_(false)
{}
~ObGenerateAuxIndexSchemaRes() {}
int assign(const ObGenerateAuxIndexSchemaRes &other)
{
int ret = OB_SUCCESS;
aux_table_id_ = other.aux_table_id_;
schema_generated_ = other.schema_generated_;
return ret;
}
void reset()
{
aux_table_id_ = OB_INVALID_ID;
schema_generated_ = false;
}
TO_STRING_KV(K(aux_table_id_), K(schema_generated_));
public:
uint64_t aux_table_id_;
bool schema_generated_;
};
typedef ObCreateIndexArg ObAlterPrimaryArg;
struct ObCreateMLogArg : public ObDDLArg

3
src/share/parameter/ob_parameter_seed.ipp
View File

@ -1399,6 +1399,9 @@ DEF_BOOL(_enable_px_batch_rescan, OB_TENANT_PARAMETER, "True",
DEF_BOOL(_enable_spf_batch_rescan, OB_TENANT_PARAMETER, "False",
"enable das batch rescan for subplan filter",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
DEF_BOOL(_enable_das_keep_order, OB_TENANT_PARAMETER, "True",
"enable das keep order optimization",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
DEF_INT(_parallel_max_active_sessions, OB_TENANT_PARAMETER, "0", "[0,]",
"max active parallel sessions allowed for tenant. Range: [0,+∞)",

8
src/sql/CMakeLists.txt
View File

@ -57,18 +57,24 @@ ob_set_subtarget(ob_sql das
das/ob_das_id_service.cpp
das/ob_das_id_rpc.cpp
das/ob_das_id_cache.cpp
das/ob_das_ir_define.cpp
das/ob_das_task_result.cpp
das/ob_das_spatial_index_lookup_op.cpp
das/ob_das_retry_ctrl.cpp
das/ob_das_simple_op.cpp
das/ob_das_domain_utils.cpp
das/ob_text_retrieval_op.cpp
das/ob_das_attach_define.cpp
das/ob_group_scan_iter.cpp
das/iter/ob_das_iter.cpp
das/iter/ob_das_merge_iter.cpp
das/iter/ob_das_lookup_iter.cpp
das/iter/ob_das_group_fold_iter.cpp
das/iter/ob_das_scan_iter.cpp
das/iter/ob_das_local_lookup_iter.cpp
das/iter/ob_das_global_lookup_iter.cpp
das/iter/ob_das_sort_iter.cpp
das/iter/ob_das_text_retrieval_iter.cpp
das/iter/ob_das_text_retrieval_merge_iter.cpp
das/iter/ob_das_iter_utils.cpp
)

96
src/sql/code_generator/ob_tsc_cg_service.cpp
View File

@ -105,6 +105,9 @@ int ObTscCgService::generate_tsc_ctdef(ObLogTableScan &op, ObTableScanCtDef &tsc
root_ctdef = &scan_ctdef;
}
}
if (OB_SUCC(ret) && op.das_need_keep_ordering()) {
tsc_ctdef.is_das_keep_order_ = true;
}
//to generate dynamic tsc partition pruning info
if (OB_SUCC(ret)) {
@ -166,7 +169,8 @@ int ObTscCgService::generate_tsc_ctdef(ObLogTableScan &op, ObTableScanCtDef &tsc
if (OB_SUCC(ret) && op.is_multivalue_index_scan()) {
ObDASIRAuxLookupCtDef *aux_lookup_ctdef = nullptr;
if (OB_FAIL(generate_doc_id_lookup_ctdef(op, tsc_ctdef, root_ctdef, aux_lookup_ctdef))) {
ObExpr *doc_id_col_expr = scan_ctdef.result_output_.at(scan_ctdef.result_output_.count() - 1);
if (OB_FAIL(generate_doc_id_lookup_ctdef(op, tsc_ctdef, root_ctdef, doc_id_col_expr, aux_lookup_ctdef))) {
LOG_WARN("failed to generate text ir ctdef", K(ret));
} else {
root_ctdef = aux_lookup_ctdef;
@ -194,14 +198,15 @@ int ObTscCgService::generate_tsc_ctdef(ObLogTableScan &op, ObTableScanCtDef &tsc
return ret;
}
int ObTscCgService::generate_table_param(const ObLogTableScan &op, ObDASScanCtDef &scan_ctdef)
int ObTscCgService::generate_table_param(const ObLogTableScan &op,
ObDASScanCtDef &scan_ctdef,
common::ObIArray<uint64_t> &tsc_out_cols)
{
int ret = OB_SUCCESS;
ObTableID index_id = scan_ctdef.ref_table_id_;
const ObTableSchema *table_schema = NULL;
const bool pd_agg = scan_ctdef.pd_expr_spec_.pd_storage_flag_.is_aggregate_pushdown();
const bool pd_group_by = scan_ctdef.pd_expr_spec_.pd_storage_flag_.is_group_by_pushdown();
ObArray<uint64_t> tsc_out_cols;
ObSqlSchemaGuard *schema_guard = cg_.opt_ctx_->get_sql_schema_guard();
CK(OB_NOT_NULL(schema_guard));
if (OB_UNLIKELY(pd_agg && 0 == scan_ctdef.aggregate_column_ids_.count()) ||
@ -242,8 +247,6 @@ int ObTscCgService::generate_table_param(const ObLogTableScan &op, ObDASScanCtDe
scan_ctdef.pd_expr_spec_.pd_storage_flag_))) {
LOG_WARN("convert group by failed", K(ret), K(*table_schema),
K(scan_ctdef.aggregate_column_ids_), K(scan_ctdef.group_by_column_ids_));
} else if (OB_FAIL(generate_das_result_output(tsc_out_cols, scan_ctdef, op.get_trans_info_expr(), pd_agg))) {
LOG_WARN("failed to init result outputs", K(ret));
}
return ret;
}
@ -297,6 +300,7 @@ int ObTscCgService::generate_das_result_output(const ObIArray<uint64_t> &output_
scan_ctdef.trans_info_expr_ = e;
}
}
return ret;
}
@ -594,15 +598,20 @@ int ObTscCgService::extract_das_access_exprs(const ObLogTableScan &op,
LOG_WARN("append the data table rowkey expr failed", K(ret), K(op.get_rowkey_exprs()));
} else if (OB_FAIL(append_array_no_dup(access_exprs, op.get_part_exprs()))) {
LOG_WARN("append the data table part expr failed", K(ret), K(op.get_part_exprs()));
} else if (NULL != op.get_group_id_expr() && op.use_batch()
&& OB_FAIL(add_var_to_array_no_dup(access_exprs,
const_cast<ObRawExpr *>(op.get_group_id_expr())))) {
LOG_WARN("fail to add group id", K(ret));
}
}
} else if (OB_FAIL(access_exprs.assign(op.get_access_exprs()))) {
LOG_WARN("assign access exprs failed", K(ret));
}
// store group_id_expr when use group id
if (OB_SUCC(ret) && op.use_group_id()) {
const ObRawExpr* group_id_expr = op.get_group_id_expr();
if (group_id_expr != nullptr &&
OB_FAIL(add_var_to_array_no_dup(access_exprs, const_cast<ObRawExpr*>(group_id_expr)))) {
LOG_WARN("failed to push back group id expr", K(ret));
}
}
if (OB_SUCC(ret) && is_oracle_mapping_real_virtual_table(op.get_ref_table_id())) {
//the access exprs are the agent virtual table columns, but das need the real table columns
//now to replace the real table column
@ -897,7 +906,7 @@ int ObTscCgService::generate_das_scan_ctdef(const ObLogTableScan &op,
}
//4. generate batch scan ctdef
if (OB_SUCC(ret) && op.use_batch()) {
if (OB_SUCC(ret) && op.use_group_id()) {
if (OB_FAIL(cg_.generate_rt_expr(*op.get_group_id_expr(), scan_ctdef.group_id_expr_))) {
LOG_WARN("generate group id expr failed", K(ret));
}
@ -911,11 +920,19 @@ int ObTscCgService::generate_das_scan_ctdef(const ObLogTableScan &op,
}
}
//6. generate table param
ObArray<uint64_t> tsc_out_cols;
if (OB_SUCC(ret)) {
if (OB_FAIL(generate_table_param(op, scan_ctdef))) {
if (OB_FAIL(generate_table_param(op, scan_ctdef, tsc_out_cols))) {
LOG_WARN("generate table param failed", K(ret));
}
}
//7. generate das result output
if (OB_SUCC(ret)) {
const bool pd_agg = scan_ctdef.pd_expr_spec_.pd_storage_flag_.is_aggregate_pushdown();
if (OB_FAIL(generate_das_result_output(tsc_out_cols, scan_ctdef, op.get_trans_info_expr(), pd_agg))) {
LOG_WARN("failed to init result outputs", K(ret));
}
}
return ret;
}
@ -945,7 +962,7 @@ int ObTscCgService::extract_das_output_column_ids(const ObLogTableScan &op,
LOG_WARN("unexpected nullptr to table schema", K(ret));
} else if (OB_FAIL(table_schema->get_rowkey_column_ids(output_cids))) {
LOG_WARN("get rowkey column ids failed", K(ret));
} else if (nullptr != op.get_group_id_expr() && op.use_batch()) {
} else if (nullptr != op.get_group_id_expr() && op.use_group_id()) {
if (OB_FAIL(output_cids.push_back(OB_HIDDEN_GROUP_IDX_COLUMN_ID))) {
LOG_WARN("store group column id failed", K(ret));
}
@ -986,6 +1003,7 @@ int ObTscCgService::extract_das_output_column_ids(const ObLogTableScan &op,
LOG_WARN("store colum id failed", K(ret));
}
}
//column expr in non-pushdown filter need to be output,
//because filter_row will use it in TSC operator
} else if (OB_FAIL(extract_tsc_access_columns(op, das_output_cols))) {
@ -1003,7 +1021,7 @@ int ObTscCgService::extract_das_output_column_ids(const ObLogTableScan &op,
LOG_WARN("store real output colum id failed", K(ret), KPC(mapping_expr));
}
}
} else if (nullptr != op.get_group_id_expr() && op.use_batch() &&
} else if (nullptr != op.get_group_id_expr() && op.use_group_id() &&
OB_FAIL(das_output_cols.push_back(const_cast<ObRawExpr*>(op.get_group_id_expr())))) {
LOG_WARN("store group id expr failed", K(ret));
} else if (OB_FAIL(extract_das_column_ids(das_output_cols, output_cids))) {
@ -1114,6 +1132,7 @@ int ObTscCgService::generate_text_ir_ctdef(const ObLogTableScan &op,
ObSqlSchemaGuard *schema_guard = cg_.opt_ctx_->get_sql_schema_guard();
ObDASIRScanCtDef *ir_scan_ctdef = nullptr;
ObDASSortCtDef *sort_ctdef = nullptr;
ObExpr *index_back_doc_id_column = nullptr;
const bool use_approx_pre_agg = true; // TODO: support differentiate use approx agg or not
if (OB_ISNULL(match_against) || OB_ISNULL(schema_guard)) {
ret = OB_ERR_UNEXPECTED;
@ -1221,6 +1240,7 @@ int ObTscCgService::generate_text_ir_ctdef(const ObLogTableScan &op,
partition_row_cnt = est_cost_info->table_meta_info_->table_row_count_ / est_cost_info->table_meta_info_->part_count_;
}
ir_scan_ctdef->estimated_total_doc_cnt_ = partition_row_cnt;
index_back_doc_id_column = ir_scan_ctdef->inv_scan_doc_id_col_;
}
}
@ -1245,7 +1265,8 @@ int ObTscCgService::generate_text_ir_ctdef(const ObLogTableScan &op,
ObDASIRAuxLookupCtDef *aux_lookup_ctdef = nullptr;
ObDASBaseCtDef *ir_output_ctdef = nullptr == sort_ctdef ?
static_cast<ObDASBaseCtDef *>(ir_scan_ctdef) : static_cast<ObDASBaseCtDef *>(sort_ctdef);
if (OB_FAIL(generate_doc_id_lookup_ctdef(op, tsc_ctdef, ir_output_ctdef, aux_lookup_ctdef))) {
if (OB_FAIL(generate_doc_id_lookup_ctdef(
op, tsc_ctdef, ir_output_ctdef, index_back_doc_id_column, aux_lookup_ctdef))) {
LOG_WARN("generate doc id lookup ctdef failed", K(ret));
} else if (OB_FAIL(append_fts_relavence_project_col(aux_lookup_ctdef, ir_scan_ctdef))) {
LOG_WARN("failed to append fts relavence project col", K(ret));
@ -1511,6 +1532,7 @@ int ObTscCgService::generate_text_ir_spec_exprs(const ObLogTableScan &op,
int ObTscCgService::generate_doc_id_lookup_ctdef(const ObLogTableScan &op,
ObTableScanCtDef &tsc_ctdef,
ObDASBaseCtDef *ir_scan_ctdef,
ObExpr *doc_id_expr,
ObDASIRAuxLookupCtDef *&aux_lookup_ctdef)
{
int ret = OB_SUCCESS;
@ -1560,6 +1582,13 @@ int ObTscCgService::generate_doc_id_lookup_ctdef(const ObLogTableScan &op,
LOG_WARN("generate das lookup scan ctdef failed", K(ret));
} else if (OB_FAIL(result_outputs.assign(scan_ctdef->result_output_))) {
LOG_WARN("construct aux lookup ctdef failed", K(ret));
} else if (OB_ISNULL(doc_id_expr) || OB_UNLIKELY(!scan_ctdef->rowkey_exprs_.empty())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected doc id expr status", K(ret), KPC(doc_id_expr), KPC(scan_ctdef));
} else if (OB_FAIL(scan_ctdef->rowkey_exprs_.reserve(1))) {
LOG_WARN("failed to reserve doc id lookup rowkey exprs", K(ret));
} else if (OB_FAIL(scan_ctdef->rowkey_exprs_.push_back(doc_id_expr))) {
LOG_WARN("failed to append doc id rowkey expr", K(ret));
} else if (OB_FAIL(generate_table_loc_meta(op.get_table_id(),
*op.get_stmt(),
*index_schema,
@ -1637,6 +1666,17 @@ int ObTscCgService::generate_table_lookup_ctdef(const ObLogTableScan &op,
LOG_WARN("fail to generate rowkey exprs", K(ret));
}
}
if (OB_SUCC(ret) && op.get_index_back()) {
ObArray<ObRawExpr*> rowkey_exprs;
if (OB_FAIL(rowkey_exprs.assign(op.get_rowkey_exprs()))) {
LOG_WARN("failed to assign rowkey exprs", K(ret));
} else if (!op.get_is_index_global() && OB_FAIL(mapping_oracle_real_agent_virtual_exprs(op, rowkey_exprs))) {
LOG_WARN("failed to mapping oracle real virtual exprs", K(ret));
} else if (OB_FAIL(cg_.generate_rt_exprs(rowkey_exprs, tsc_ctdef.lookup_ctdef_->rowkey_exprs_))) {
LOG_WARN("failed to generate main table rowkey exprs", K(ret));
}
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(ObDASTaskFactory::alloc_das_ctdef(DAS_OP_TABLE_LOOKUP, allocator, lookup_ctdef))) {
@ -1828,5 +1868,33 @@ int ObTscCgService::generate_das_sort_ctdef(
return ret;
}
int ObTscCgService::mapping_oracle_real_agent_virtual_exprs(const ObLogTableScan &op,
common::ObIArray<ObRawExpr*> &access_exprs)
{
int ret = OB_SUCCESS;
if (is_oracle_mapping_real_virtual_table(op.get_ref_table_id())) {
//the access exprs are the agent virtual table columns, but das need the real table columns
//now to replace the real table column
for (int64_t i = 0; OB_SUCC(ret) && i < access_exprs.count(); ++i) {
ObRawExpr *expr = access_exprs.at(i);
ObRawExpr *mapping_expr = nullptr;
uint64_t column_id = UINT64_MAX;
if (OB_ISNULL(expr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr expr", K(expr), K(ret));
} else if (T_ORA_ROWSCN == expr->get_expr_type()) {
// keep orign expr as access expr
} else if (OB_ISNULL(mapping_expr = op.get_real_expr(expr))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("mapping expr is null", K(ret), KPC(expr));
} else {
//replace the agent virtual table column expr
access_exprs.at(i) = mapping_expr;
}
}
}
return ret;
}
} // namespace sql
} // namespace oceanbase

6
src/sql/code_generator/ob_tsc_cg_service.h
View File

@ -53,7 +53,7 @@ private:
int generate_access_ctdef(const ObLogTableScan &op, ObDASScanCtDef &scan_ctdef, bool &has_rowscn);
int generate_pushdown_aggr_ctdef(const ObLogTableScan &op, ObDASScanCtDef &scan_ctdef);
int generate_das_scan_ctdef(const ObLogTableScan &op, ObDASScanCtDef &scan_ctdef, bool &has_rowscn);
int generate_table_param(const ObLogTableScan &op, ObDASScanCtDef &scan_ctdef);
int generate_table_param(const ObLogTableScan &op, ObDASScanCtDef &scan_ctdef, common::ObIArray<uint64_t> &tsc_out_cols);
int extract_das_output_column_ids(const ObLogTableScan &op,
ObDASScanCtDef &scan_ctdef,
const ObTableSchema &index_schema,
@ -79,6 +79,7 @@ private:
int generate_doc_id_lookup_ctdef(const ObLogTableScan &op,
ObTableScanCtDef &tsc_ctdef,
ObDASBaseCtDef *ir_scan_ctdef,
ObExpr *doc_id_expr,
ObDASIRAuxLookupCtDef *&aux_lookup_ctdef);
int generate_table_lookup_ctdef(const ObLogTableScan &op,
ObTableScanCtDef &tsc_ctdef,
@ -98,6 +99,9 @@ private:
ObRawExpr *topk_offset_expr,
ObDASBaseCtDef *child_ctdef,
ObDASSortCtDef *&sort_ctdef);
int mapping_oracle_real_agent_virtual_exprs(const ObLogTableScan &op,
common::ObIArray<ObRawExpr*> &access_exprs);
private:
ObStaticEngineCG &cg_;
};

241
src/sql/das/iter/ob_das_global_lookup_iter.cpp Normal file
View File

@ -0,0 +1,241 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_DAS
#include "sql/das/iter/ob_das_global_lookup_iter.h"
#include "sql/das/iter/ob_das_merge_iter.h"
#include "sql/engine/ob_exec_context.h"
namespace oceanbase
{
using namespace common;
namespace sql
{
int ObDASGlobalLookupIter::inner_init(ObDASIterParam &param)
{
int ret = OB_SUCCESS;
if (OB_FAIL(ObDASLookupIter::inner_init(param))) {
LOG_WARN("failed to init das lookup iter", K(ret));
} else if (param.type_ != ObDASIterType::DAS_ITER_GLOBAL_LOOKUP) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("inner init das iter with bad param type", K(param), K(ret));
} else {
ObDASGlobalLookupIterParam &lookup_param = static_cast<ObDASGlobalLookupIterParam&>(param);
can_retry_ = lookup_param.can_retry_;
calc_part_id_ = lookup_param.calc_part_id_;
// use lookup_memctx for lookup to avoid memory expansion during global index lookup
lookup_rtdef_->scan_allocator_.set_alloc(&get_arena_allocator());
lookup_rtdef_->stmt_allocator_.set_alloc(&get_arena_allocator());
if (lookup_param.rowkey_exprs_->empty()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected empty global rowkey exprs", K(ret));
} else if (OB_FAIL(rowkey_exprs_.assign(*lookup_param.rowkey_exprs_))) {
LOG_WARN("failed to assign rowkey exprs", K(ret));
}
}
return ret;
}
int ObDASGlobalLookupIter::inner_reuse()
{
int ret = OB_SUCCESS;
// for global lookup, the reuse of index table will handled in TSC.
if (OB_FAIL(data_table_iter_->reuse())) {
LOG_WARN("failed to reuse data table iter", K(ret));
} else if (OB_FAIL(ObDASLookupIter::inner_reuse())) {
LOG_WARN("failed to reuse das lookup iter", K(ret));
}
index_ordered_idx_ = 0;
return ret;
}
int ObDASGlobalLookupIter::inner_release()
{
int ret = OB_SUCCESS;
if (OB_FAIL(ObDASLookupIter::inner_release())) {
LOG_WARN("failed to release das lookup iter", K(ret));
}
return ret;
}
void ObDASGlobalLookupIter::reset_lookup_state()
{
index_ordered_idx_ = 0;
ObDASLookupIter::reset_lookup_state();
}
int ObDASGlobalLookupIter::add_rowkey()
{
int ret = OB_SUCCESS;
ObObjectID partition_id = OB_INVALID_ID;
ObTabletID tablet_id(OB_INVALID_ID);
ObDASScanOp *das_scan_op = nullptr;
ObDASTabletLoc *tablet_loc = nullptr;
ObDASCtx *das_ctx = nullptr;
bool reuse_das_op = false;
OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_MERGE);
if (OB_ISNULL(exec_ctx_) || OB_ISNULL(das_ctx = &exec_ctx_->get_das_ctx())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get das ctx", KPC_(exec_ctx));
} else {
ObDASMergeIter *merge_iter = static_cast<ObDASMergeIter*>(data_table_iter_);
if (OB_FAIL(ObExprCalcPartitionBase::calc_part_and_tablet_id(calc_part_id_,
*eval_ctx_,
partition_id,
tablet_id))) {
LOG_WARN("failed to calc part id", K(ret), KPC(calc_part_id_));
} else if (OB_FAIL(das_ctx->extended_tablet_loc(*lookup_rtdef_->table_loc_,
tablet_id,
tablet_loc))) {
LOG_WARN("failed to get tablet loc by tablet_id", K(ret));
} else if (OB_FAIL(merge_iter->create_das_task(tablet_loc, das_scan_op, reuse_das_op))) {
LOG_WARN("failed to create das task", K(ret));
} else if (!reuse_das_op) {
das_scan_op->set_scan_ctdef(lookup_ctdef_);
das_scan_op->set_scan_rtdef(lookup_rtdef_);
das_scan_op->set_can_part_retry(can_retry_);
}
}
if (OB_SUCC(ret)) {
storage::ObTableScanParam &scan_param = das_scan_op->get_scan_param();
ObNewRange lookup_range;
int64_t group_id = 0;
OB_ASSERT(DAS_OP_TABLE_SCAN == index_ctdef_->op_type_);
const ObDASScanCtDef *index_ctdef = static_cast<const ObDASScanCtDef*>(index_ctdef_);
if (nullptr != index_ctdef->group_id_expr_) {
group_id = index_ctdef->group_id_expr_->locate_expr_datum(*eval_ctx_).get_int();
}
if (nullptr != index_ctdef->trans_info_expr_) {
ObDatum *datum_ptr = nullptr;
if (OB_FAIL(build_trans_info_datum(index_ctdef->trans_info_expr_, datum_ptr))) {
LOG_WARN("failed to build trans info datum", K(ret));
} else if (OB_ISNULL(datum_ptr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K(ret));
} else if (OB_FAIL(das_scan_op->trans_info_array_.push_back(datum_ptr))) {
LOG_WARN("failed to push back trans info array", K(ret), KPC(datum_ptr));
}
}
int64_t group_idx = ObNewRange::get_group_idx(group_id);
index_ordered_idx_++;
if (OB_FAIL(ret)) {
} else if (OB_FAIL(build_lookup_range(lookup_range))) {
LOG_WARN("failed to build lookup range", K(ret));
} else if (FALSE_IT(lookup_range.group_idx_ = group_idx)) {
} else if (FALSE_IT(lookup_range.index_ordered_idx_ = index_ordered_idx_)) {
} else if (OB_FAIL(scan_param.key_ranges_.push_back(lookup_range))) {
LOG_WARN("failed to push back lookup range", K(ret));
} else {
scan_param.is_get_ = true;
}
LOG_DEBUG("build global lookup range", K(lookup_range), K(ret));
}
return ret;
}
int ObDASGlobalLookupIter::add_rowkeys(int64_t count)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(eval_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K_(eval_ctx));
} else {
ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*eval_ctx_);
batch_info_guard.set_batch_size(count);
for(int i = 0; OB_SUCC(ret) && i < count; i++) {
batch_info_guard.set_batch_idx(i);
if(OB_FAIL(add_rowkey())) {
LOG_WARN("failed to add rowkey", K(ret), K(i));
}
}
}
return ret;
}
int ObDASGlobalLookupIter::do_index_lookup()
{
int ret = OB_SUCCESS;
OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_MERGE);
ObDASMergeIter *merge_iter = static_cast<ObDASMergeIter*>(data_table_iter_);
if (OB_FAIL(merge_iter->do_table_scan())) {
LOG_WARN("failed to do global index lookup", K(ret));
} else {
merge_iter->set_merge_status(merge_iter->get_merge_type());
}
return ret;
}
int ObDASGlobalLookupIter::check_index_lookup()
{
int ret = OB_SUCCESS;
OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_MERGE);
ObDASMergeIter *merge_iter = static_cast<ObDASMergeIter*>(data_table_iter_);
if (GCONF.enable_defensive_check() &&
lookup_ctdef_->pd_expr_spec_.pushdown_filters_.empty()) {
if (OB_UNLIKELY(lookup_rowkey_cnt_ != lookup_row_cnt_)) {
ret = OB_ERR_DEFENSIVE_CHECK;
ObSQLSessionInfo *my_session = exec_ctx_->get_my_session();
ObString func_name = ObString::make_string("check_lookup_row_cnt");
LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr());
LOG_ERROR("Fatal Error!!! Catch a defensive error!",
K(ret), K(lookup_rowkey_cnt_), K(lookup_row_cnt_),
"main table id", lookup_ctdef_->ref_table_id_,
KPC(my_session->get_tx_desc()));
int64_t row_num = 0;
for (DASTaskIter task_iter = merge_iter->begin_task_iter(); !task_iter.is_end(); ++task_iter) {
ObDASScanOp *das_op = static_cast<ObDASScanOp*>(*task_iter);
if (das_op->trans_info_array_.count() == das_op->get_scan_param().key_ranges_.count()) {
for (int64_t i = 0; i < das_op->trans_info_array_.count(); i++) {
row_num++;
ObDatum *datum = das_op->trans_info_array_.at(i);
LOG_ERROR("dump lookup range and trans info of global lookup das task",
K(row_num), KPC(datum),
K(das_op->get_scan_param().key_ranges_.at(i)),
K(das_op->get_tablet_id()));
}
} else {
for (int64_t i = 0; i < das_op->get_scan_param().key_ranges_.count(); i++) {
row_num++;
LOG_ERROR("dump lookup range of global lookup das task",
K(row_num),
K(das_op->get_scan_param().key_ranges_.at(i)),
K(das_op->get_tablet_id()));
}
}
}
}
}
int simulate_error = EVENT_CALL(EventTable::EN_DAS_SIMULATE_DUMP_WRITE_BUFFER);
if (0 != simulate_error) {
for (DASTaskIter task_iter = merge_iter->begin_task_iter(); !task_iter.is_end(); ++task_iter) {
ObDASScanOp *das_op = static_cast<ObDASScanOp*>(*task_iter);
for (int64_t i = 0; i < das_op->trans_info_array_.count(); i++) {
ObDatum *datum = das_op->trans_info_array_.at(i);
LOG_INFO("dump trans info of global lookup das task", K(i),
KPC(das_op->trans_info_array_.at(i)),
K(das_op->get_scan_param().key_ranges_.at(i)),
K(das_op->get_tablet_id()));
}
}
}
return ret;
}
} // namespace sql
} // namespace oceanbase

72
src/sql/das/iter/ob_das_global_lookup_iter.h Normal file
View File

@ -0,0 +1,72 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OBDEV_SRC_SQL_DAS_ITER_OB_DAS_GLOBAL_LOOKUP_ITER_H_
#define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_GLOBAL_LOOKUP_ITER_H_
#include "sql/das/iter/ob_das_lookup_iter.h"
namespace oceanbase
{
using namespace common;
namespace sql
{
struct ObDASGlobalLookupIterParam : public ObDASLookupIterParam
{
public:
ObDASGlobalLookupIterParam()
: ObDASLookupIterParam(true /*global lookup*/),
can_retry_(false),
calc_part_id_(nullptr)
{}
bool can_retry_;
const ObExpr *calc_part_id_;
virtual bool is_valid() const override
{
return ObDASLookupIterParam::is_valid() && calc_part_id_ != nullptr;
}
};
class ObDASGlobalLookupIter : public ObDASLookupIter
{
public:
ObDASGlobalLookupIter()
: ObDASLookupIter(ObDASIterType::DAS_ITER_GLOBAL_LOOKUP),
can_retry_(false),
calc_part_id_(nullptr),
index_ordered_idx_(0)
{}
virtual ~ObDASGlobalLookupIter() {}
protected:
virtual int inner_init(ObDASIterParam &param) override;
virtual int inner_reuse() override;
virtual int inner_release() override;
virtual int add_rowkey() override;
virtual int add_rowkeys(int64_t count) override;
virtual int do_index_lookup() override;
virtual int check_index_lookup() override;
virtual void reset_lookup_state();
private:
bool can_retry_;
const ObExpr *calc_part_id_;
int64_t index_ordered_idx_; // used for keep order of global lookup
};
} // namespace sql
} // namespace oceanbase
#endif /* OBDEV_SRC_SQL_DAS_ITER_OB_DAS_GLOBAL_LOOKUP_ITER_H_ */

10
src/sql/das/iter/ob_das_group_fold_iter.cpp
View File

@ -101,8 +101,8 @@ int ObGroupResultSaveRows::to_expr(bool is_vectorized, int64_t start_pos, int64_
int64_t ObGroupResultSaveRows::cur_group_idx()
{
return start_pos_ >= saved_size_ ?
OB_INVALID_INDEX : store_rows_[start_pos_].store_row_->cells()[group_id_idx_].get_int();
return start_pos_ >= saved_size_ ? OB_INVALID_INDEX :
ObNewRange::get_group_idx(store_rows_[start_pos_].store_row_->cells()[group_id_idx_].get_int());
}
@ -140,7 +140,7 @@ int ObDASGroupFoldIter::set_scan_group(int64_t group_id)
cur_group_idx_ = group_id;
}
if (group_save_rows_.need_check_output_datum_) {
reset_expr_datum_ptr();
reset_expr_datum_ptr();
}
if (cur_group_idx_ >= group_size_) {
ret = OB_ITER_END;
@ -277,7 +277,7 @@ int ObDASGroupFoldIter::inner_get_next_rows(int64_t &count, int64_t capacity)
PRINT_VECTORIZED_ROWS(SQL, DEBUG, *eval_ctx_, *output_, storage_count, skip);
ObDatum *group_idx_batch = group_id_expr_->locate_batch_datums(*group_save_rows_.eval_ctx_);
for (int64_t i = 0; OB_SUCC(ret) && i < storage_count; i++) {
group_idx = group_idx_batch[i].get_int();
group_idx = ObNewRange::get_group_idx(group_idx_batch[i].get_int());
if (group_idx >= cur_group_idx_) {
if (OB_FAIL(group_save_rows_.save(true, i, storage_count - i))) {
LOG_WARN("das group fold iter failed to save batch result", K(ret));
@ -352,7 +352,7 @@ int ObDASGroupFoldIter::inner_get_next_row()
} else if (OB_FAIL(group_id_expr_->eval(*group_save_rows_.eval_ctx_, group_idx))) {
LOG_WARN("failed to eval group id", K(ret));
} else {
available_group_idx_ = group_idx->get_int();
available_group_idx_ = ObNewRange::get_group_idx(group_idx->get_int());
}
} // while end

7
src/sql/das/iter/ob_das_group_fold_iter.h
View File

@ -75,6 +75,10 @@ public:
struct ObDASGroupFoldIterParam : public ObDASIterParam
{
public:
ObDASGroupFoldIterParam()
: ObDASIterParam(ObDASIterType::DAS_ITER_GROUP_FOLD)
{}
bool need_check_output_datum_;
ObDASIter *iter_tree_;
@ -88,7 +92,8 @@ class ObDASGroupFoldIter : public ObDASIter
{
public:
ObDASGroupFoldIter()
: cur_group_idx_(0),
: ObDASIter(ObDASIterType::DAS_ITER_GROUP_FOLD),
cur_group_idx_(0),
available_group_idx_(MIN_GROUP_INDEX),
group_size_(0),
need_check_output_datum_(false),

29
src/sql/das/iter/ob_das_iter.cpp
View File

@ -24,9 +24,11 @@ namespace sql
int ObDASIter::set_merge_status(MergeType merge_type)
{
int ret = OB_SUCCESS;
ObDASIter *child = child_;
for (; child != nullptr && OB_SUCC(ret); child = child->right_) {
if (OB_FAIL(child->set_merge_status(merge_type))) {
for (uint32_t i = 0; i < children_cnt_; i++) {
if (OB_ISNULL(children_[i])) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr das iter child", K(i), K_(children_cnt), K(ret));
} else if (OB_FAIL(children_[i]->set_merge_status(merge_type))) {
LOG_WARN("failed to set merge status", K(ret));
}
}
@ -50,8 +52,6 @@ int ObDASIter::init(ObDASIterParam &param)
exec_ctx_ = param.exec_ctx_;
output_ = param.output_;
group_id_expr_ = param.group_id_expr_;
child_ = param.child_;
right_ = param.right_;
if (OB_FAIL(inner_init(param))) {
LOG_WARN("failed to inner init das iter", K(param), K(ret));
}
@ -60,6 +60,7 @@ int ObDASIter::init(ObDASIterParam &param)
return ret;
}
// NOTE: unlike release(), reuse() does not recursively call the reuse() of its children.
int ObDASIter::reuse()
{
int ret = OB_SUCCESS;
@ -76,15 +77,15 @@ int ObDASIter::release()
{
int ret = OB_SUCCESS;
int child_ret = OB_SUCCESS;
ObDASIter *child = child_;
int tmp_ret = OB_SUCCESS;
while (child != nullptr) {
ObDASIter *right = child->right_;
if (OB_TMP_FAIL(child->release())) {
LOG_WARN("failed to release child iter", K(tmp_ret), KPC(child));
child_ret = tmp_ret;
for (uint32_t i = 0; i < children_cnt_; i++) {
if (OB_ISNULL(children_[i])) {
tmp_ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr das iter child", K(i), K_(children_cnt), K(tmp_ret));
} else if (OB_TMP_FAIL(children_[i]->release())) {
LOG_WARN("failed to release child iter", K(tmp_ret), KPC(children_[i]));
}
child = right;
child_ret = tmp_ret;
}
if (OB_FAIL(inner_release())) {
LOG_WARN("failed to inner release das iter", K(ret), KPC(this));
@ -92,8 +93,8 @@ int ObDASIter::release()
ret = child_ret;
}
inited_ = false;
right_ = nullptr;
child_ = nullptr;
children_cnt_ = 0;
children_ = nullptr;
group_id_expr_ = nullptr;
output_ = nullptr;
exec_ctx_ = nullptr;

71
src/sql/das/iter/ob_das_iter.h
View File

@ -13,9 +13,9 @@
#ifndef OBDEV_SRC_SQL_DAS_ITER_OB_DAS_ITER_H_
#define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_ITER_H_
#include "sql/engine/expr/ob_expr.h"
#include "sql/engine/ob_exec_context.h"
#include "lib/container/ob_fixed_array.h"
#include "sql/das/ob_das_context.h"
#include "common/row/ob_row_iterator.h"
#include "sql/das/iter/ob_das_iter_define.h"
namespace oceanbase
{
@ -23,36 +23,18 @@ using namespace common;
namespace sql
{
class ObDASIter;
enum ObDASIterType : uint32_t
{
DAS_ITER_INVALID = 0,
DAS_ITER_SCAN,
DAS_ITER_MERGE,
DAS_ITER_GROUP_FOLD,
DAS_ITER_LOOKUP,
// append DASIterType before me
DAS_ITER_MAX
};
enum MergeType : uint32_t {
SEQUENTIAL_MERGE = 0,
SORT_MERGE
};
class ObEvalCtx;
class ObExecContext;
struct ObDASIterParam
{
public:
ObDASIterParam()
: type_(ObDASIterType::DAS_ITER_INVALID),
ObDASIterParam(ObDASIterType type=ObDASIterType::DAS_ITER_INVALID)
: type_(type),
max_size_(0),
eval_ctx_(nullptr),
exec_ctx_(nullptr),
output_(nullptr),
group_id_expr_(nullptr),
child_(nullptr),
right_(nullptr)
group_id_expr_(nullptr)
{}
virtual ~ObDASIterParam() {}
@ -65,8 +47,6 @@ public:
exec_ctx_ = param.exec_ctx_;
output_ = param.output_;
group_id_expr_ = param.group_id_expr_;
child_ = param.child_;
right_ = param.right_;
}
virtual bool is_valid() const
@ -80,33 +60,34 @@ public:
ObExecContext *exec_ctx_;
const ObIArray<ObExpr*> *output_;
const ObExpr *group_id_expr_;
ObDASIter *child_;
ObDASIter *right_;
TO_STRING_KV(K_(type), K_(max_size), K_(eval_ctx), K_(exec_ctx), KPC_(output), K_(group_id_expr),
K_(child), K_(right));
TO_STRING_KV(K_(type), K_(max_size), K_(eval_ctx), K_(exec_ctx), KPC_(output), K_(group_id_expr));
};
class ObDASIter
class ObDASIter : public common::ObNewRowIterator
{
public:
ObDASIter()
: type_(ObDASIterType::DAS_ITER_INVALID),
max_size_(0),
ObDASIter(const ObDASIterType type = ObDASIterType::DAS_ITER_INVALID)
: type_(type),
max_size_(1),
eval_ctx_(nullptr),
exec_ctx_(nullptr),
output_(nullptr),
group_id_expr_(nullptr),
child_(nullptr),
right_(nullptr),
children_(nullptr),
children_cnt_(0),
inited_(false)
{}
virtual ~ObDASIter() { release(); }
VIRTUAL_TO_STRING_KV(K_(type), K_(max_size), K_(eval_ctx), K_(exec_ctx), K_(output),
K_(group_id_expr), K_(child), K_(right), K_(inited));
K_(group_id_expr), K_(children_cnt), K_(inited));
void set_type(ObDASIterType type) { type_ = type; }
ObDASIterType get_type() const { return type_; }
ObDASIter **&get_children() { return children_; }
void set_children_cnt(uint32_t children_cnt) { children_cnt_ = children_cnt; }
int64_t get_children_cnt() const { return children_cnt_; }
const ObIArray<ObExpr*> *get_output() { return output_; }
// The state of ObDASMergeIter may change many times during execution, e.g., the merge_type
// changing from SEQUENTIAL_MERGE to SORT_MERGE, or the creation of a new batch of DAS tasks.
@ -123,7 +104,17 @@ public:
// get_next_row(s) should be called after init().
int get_next_row();
int get_next_rows(int64_t &count, int64_t capacity);
virtual void clear_evaluated_flag() {}
// required by iters related to DAS SCAN OP
virtual int do_table_scan() { return OB_NOT_IMPLEMENT; }
virtual int rescan() { return OB_NOT_IMPLEMENT; }
// required by iters related to DAS SCAN OP
// for compatibility with ObNewRowIterator
virtual int get_next_row(ObNewRow *&row) override { return OB_NOT_IMPLEMENT; }
virtual void reset() override {}
// for compatibility with ObNewRowIterator
protected:
virtual int inner_init(ObDASIterParam &param) = 0;
@ -138,8 +129,8 @@ protected:
ObExecContext *exec_ctx_;
const ObIArray<ObExpr*> *output_;
const ObExpr *group_id_expr_;
ObDASIter *child_;
ObDASIter *right_;
ObDASIter **children_;
uint32_t children_cnt_;
private:
bool inited_;

89
src/sql/das/iter/ob_das_iter_define.h Normal file
View File

@ -0,0 +1,89 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OBDEV_SRC_SQL_DAS_ITER_OB_DAS_ITER_DEFINE_H
#define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_ITER_DEFINE_H
#include "common/ob_tablet_id.h"
namespace oceanbase
{
namespace sql
{
enum ObDASIterType : uint32_t
{
DAS_ITER_INVALID = 0,
DAS_ITER_SCAN,
DAS_ITER_MERGE,
DAS_ITER_GROUP_FOLD,
DAS_ITER_LOCAL_LOOKUP,
DAS_ITER_GLOBAL_LOOKUP,
DAS_ITER_TEXT_RETRIEVAL,
DAS_ITER_SORT,
DAS_ITER_TEXT_RETRIEVAL_MERGE,
// append DASIterType before me
DAS_ITER_MAX
};
#define IS_LOOKUP_ITER(_iter_type) \
({ \
DAS_ITER_LOCAL_LOOKUP == (_iter_type) || \
DAS_ITER_GLOBAL_LOOKUP == (_iter_type); \
})
enum MergeType : uint32_t {
SEQUENTIAL_MERGE = 0,
SORT_MERGE
};
// group fold iter is used on demand and not considered as a part of iter tree,
// it is placed above iter tree when in use.
enum ObDASIterTreeType : uint32_t
{
ITER_TREE_INVALID = 0,
ITER_TREE_TABLE_SCAN,
ITER_TREE_GLOBAL_LOOKUP,
ITER_TREE_PARTITION_SCAN,
ITER_TREE_LOCAL_LOOKUP,
ITER_TREE_GIS_LOOKUP,
ITER_TREE_DOMAIN_LOOKUP,
ITER_TREE_TEXT_RETRIEVAL,
// append iter tree type before me
ITER_TREE_MAX
};
struct ObDASRelatedTabletID
{
public:
common::ObTabletID lookup_tablet_id_;
common::ObTabletID aux_lookup_tablet_id_;
/* used by fulltext index */
common::ObTabletID inv_idx_tablet_id_;
common::ObTabletID fwd_idx_tablet_id_;
common::ObTabletID doc_id_idx_tablet_id_;
/* used by fulltext index */
void reset()
{
lookup_tablet_id_.reset();
aux_lookup_tablet_id_.reset();
inv_idx_tablet_id_.reset();
fwd_idx_tablet_id_.reset();
doc_id_idx_tablet_id_.reset();
}
};
} // namespace sql
} // namespace oceanbase
#endif /* OBDEV_SRC_SQL_DAS_ITER_OB_DAS_ITER_DEFINE_H */

956
src/sql/das/iter/ob_das_iter_utils.cpp
View File

File diff suppressed because it is too large Load Diff

171
src/sql/das/iter/ob_das_iter_utils.h
View File

@ -13,20 +13,124 @@
#ifndef OBDEV_SRC_SQL_DAS_ITER_OB_DAS_ITER_UTILS_H_
#define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_ITER_UTILS_H_
#include "sql/das/iter/ob_das_iter_define.h"
#include "sql/das/iter/ob_das_scan_iter.h"
#include "sql/das/iter/ob_das_merge_iter.h"
#include "sql/das/iter/ob_das_lookup_iter.h"
#include "sql/das/iter/ob_das_local_lookup_iter.h"
#include "sql/das/iter/ob_das_global_lookup_iter.h"
#include "sql/das/iter/ob_das_group_fold_iter.h"
#include "sql/das/iter/ob_das_sort_iter.h"
#include "sql/das/iter/ob_das_text_retrieval_iter.h"
#include "sql/das/iter/ob_das_text_retrieval_merge_iter.h"
#include "sql/engine/table/ob_table_scan_op.h"
namespace oceanbase
{
namespace sql
{
class ObDASIterUtils
{
public:
static int create_das_scan_iter_tree(ObDASIterTreeType tree_type,
storage::ObTableScanParam &scan_param,
const ObDASScanCtDef *scan_ctdef,
ObDASScanRtDef *scan_rtdef,
const ObDASScanCtDef *lookup_ctdef,
ObDASScanRtDef *lookup_rtdef,
const ObDASBaseCtDef *attach_ctdef,
ObDASBaseRtDef *attach_rtdef,
const ObDASRelatedTabletID &related_tablet_ids,
transaction::ObTxDesc *trans_desc,
transaction::ObTxReadSnapshot *snapshot,
common::ObIAllocator &alloc,
ObDASIter *&iter_tree);
static int create_tsc_iter_tree(ObDASIterTreeType tree_type,
const ObTableScanCtDef &tsc_ctdef,
ObTableScanRtDef &tsc_rtdef,
ObEvalCtx &eval_ctx,
ObExecContext &exec_ctx,
ObFixedArray<ObEvalInfo *, ObIAllocator> &eval_infos,
const ObTableScanSpec &spec,
bool can_retry,
ObDASMergeIter *&scan_iter,
ObDASIter *&iter_tree);
static int create_group_fold_iter(const ObTableScanCtDef &tsc_ctdef,
ObTableScanRtDef &tsc_rtdef,
ObEvalCtx &eval_ctx,
ObExecContext &exec_ctx,
ObFixedArray<ObEvalInfo *, ObIAllocator> &eval_infos,
const ObTableScanSpec &spec,
ObDASIter *iter_tree,
ObDASGroupFoldIter *&fold_iter);
static int set_text_retrieval_related_ids(const ObDASBaseCtDef *attach_ctdef,
const ObDASRelatedTabletID &related_tablet_ids,
const ObLSID &ls_id,
ObDASIter *root_iter);
private:
static int create_partition_scan_tree(ObTableScanParam &scan_param,
common::ObIAllocator &alloc,
const ObDASScanCtDef *scan_ctdef,
ObDASScanRtDef *scan_rtdef,
ObDASIter *&iter_tree);
static int create_local_lookup_tree(ObTableScanParam &scan_param,
common::ObIAllocator &alloc,
const ObDASScanCtDef *scan_ctdef,
ObDASScanRtDef *scan_rtdef,
const ObDASScanCtDef *lookup_ctdef,
ObDASScanRtDef *lookup_rtdef,
const ObDASRelatedTabletID &related_tablet_ids,
transaction::ObTxDesc *trans_desc,
transaction::ObTxReadSnapshot *snapshot,
ObDASIter *&iter_tree);
static int create_domain_lookup_tree(ObTableScanParam &scan_param,
common::ObIAllocator &alloc,
const ObDASBaseCtDef *attach_ctdef,
ObDASBaseRtDef *attach_rtdef,
ObDASIter *&iter_tree);
static int create_text_retrieval_tree(ObTableScanParam &scan_param,
common::ObIAllocator &alloc,
const ObDASBaseCtDef *attach_ctdef,
ObDASBaseRtDef *attach_rtdef,
const ObDASRelatedTabletID &related_tablet_ids,
transaction::ObTxDesc *trans_desc,
transaction::ObTxReadSnapshot *snapshot,
ObDASIter *&iter_tree);
static int create_domain_lookup_sub_tree(const ObLSID &ls_id,
common::ObIAllocator &alloc,
const ObDASTableLookupCtDef *table_lookup_ctdef,
ObDASTableLookupRtDef *table_lookup_rtdef,
const ObDASRelatedTabletID &related_tablet_ids,
transaction::ObTxDesc *trans_desc,
transaction::ObTxReadSnapshot *snapshot,
ObDASIter *doc_id_iter,
ObDASIter *&domain_lookup_result);
static int create_text_retrieval_sub_tree(const ObLSID &ls_id,
common::ObIAllocator &alloc,
const ObDASIRScanCtDef *ir_scan_ctdef,
ObDASIRScanRtDef *ir_scan_rtdef,
const ObDASRelatedTabletID &related_tablet_ids,
transaction::ObTxDesc *trans_desc,
transaction::ObTxReadSnapshot *snapshot,
ObDASIter *&retrieval_result);
static int create_sort_sub_tree(common::ObIAllocator &alloc,
const ObDASSortCtDef *sort_ctdef,
ObDASSortRtDef *sort_rtdef,
ObDASIter *sort_input,
ObDASIter *&sort_result);
static int create_table_scan_iter_tree(const ObTableScanCtDef &tsc_ctdef,
ObTableScanRtDef &tsc_rtdef,
ObEvalCtx &eval_ctx,
ObExecContext &exec_ctx,
ObFixedArray<ObEvalInfo *, ObIAllocator> &eval_infos,
@ -34,15 +138,6 @@ public:
ObDASMergeIter *&scan_iter,
ObDASIter *&iter_tree);
static int create_local_lookup_iter_tree(const ObTableScanCtDef &tsc_ctdef,
ObTableScanRtDef &tsc_rtdef,
ObEvalCtx &eval_ctx,
ObExecContext &exec_ctx,
ObFixedArray<ObEvalInfo *, ObIAllocator> &eval_infos,
const ObTableScanSpec &spec,
ObDASMergeIter *&scan_iter,
ObDASIter *&iter_tree);
static int create_global_lookup_iter_tree(const ObTableScanCtDef &tsc_ctdef,
ObTableScanRtDef &tsc_rtdef,
ObEvalCtx &eval_ctx,
@ -53,38 +148,32 @@ public:
ObDASMergeIter *&scan_iter,
ObDASIter *&iter_tree);
static int create_group_fold_iter(const ObTableScanCtDef &tsc_ctdef,
ObTableScanRtDef &tsc_rtdef,
ObEvalCtx &eval_ctx,
ObExecContext &exec_ctx,
ObFixedArray<ObEvalInfo *, ObIAllocator> &eval_infos,
const ObTableScanSpec &spec,
ObDASIter *iter_tree,
ObDASGroupFoldIter *&fold_iter);
private:
static int create_das_merge_iter_help(ObDASIterParam &param,
static int create_iter_children_array(const int64_t children_cnt,
common::ObIAllocator &alloc,
bool is_global_lookup,
ObFixedArray<ObEvalInfo *, ObIAllocator> &eval_infos,
const ObTableScanSpec &spec,
ObDASMergeIter *&result);
ObDASIter *iter);
static int create_das_global_lookup_iter_help(ObDASIterParam &param,
common::ObIAllocator &alloc,
int64_t default_batch_row_count,
ObDASMergeIter *index_table_iter,
ObDASMergeIter *data_table_iter,
bool can_retry,
const ObTableScanCtDef &tsc_ctdef,
ObTableScanRtDef &tsc_rtdef,
ObDASGlobalLookupIter *&result);
static int create_das_group_fold_iter_help(ObDASIterParam &param,
common::ObIAllocator &alloc,
bool need_check_output_datum,
ObDASIter *iter_tree,
ObDASGroupFoldIter *&result);
template<class IterType, class IterParamType>
static int create_das_iter(common::ObIAllocator &alloc, IterParamType &param, IterType *&result)
{
int ret = OB_SUCCESS;
IterType *iter = nullptr;
if (OB_ISNULL(iter = OB_NEWx(IterType, &alloc))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to new a das iter", K(ret));
} else if (OB_FAIL(iter->init(param))) {
LOG_WARN("failed to init das iter", K(param), K(ret));
}
if (OB_SUCC(ret)) {
result = iter;
} else {
if (OB_NOT_NULL(iter)) {
iter->release();
alloc.free(iter);
iter = nullptr;
}
}
return ret;
}
ObDASIterUtils() = delete;
~ObDASIterUtils() = delete;

366
src/sql/das/iter/ob_das_local_lookup_iter.cpp Normal file
View File

@ -0,0 +1,366 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_DAS
#include "sql/das/iter/ob_das_local_lookup_iter.h"
#include "sql/das/iter/ob_das_scan_iter.h"
#include "sql/das/ob_das_scan_op.h"
#include "sql/das/ob_das_ir_define.h"
#include "storage/concurrency_control/ob_data_validation_service.h"
namespace oceanbase
{
using namespace common;
namespace sql
{
int ObDASLocalLookupIter::inner_init(ObDASIterParam &param)
{
int ret = OB_SUCCESS;
if (OB_FAIL(ObDASLookupIter::inner_init(param))) {
LOG_WARN("failed to init das lookup iter", K(ret));
} else if (param.type_ != ObDASIterType::DAS_ITER_LOCAL_LOOKUP) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("inner init das iter with bad param type", K(param), K(ret));
} else {
ObDASLocalLookupIterParam &lookup_param = static_cast<ObDASLocalLookupIterParam&>(param);
trans_desc_ = lookup_param.trans_desc_;
snapshot_ = lookup_param.snapshot_;
if (lookup_param.rowkey_exprs_->empty()) {
// for compatibility
if (OB_FAIL(init_rowkey_exprs_for_compat())) {
LOG_WARN("failed eto init rowkeys exprs for compat", K(ret));
}
} else if (OB_FAIL(rowkey_exprs_.assign(*lookup_param.rowkey_exprs_))) {
LOG_WARN("failed to assign rowkey exprs", K(ret));
}
}
return ret;
}
int ObDASLocalLookupIter::inner_reuse()
{
int ret = OB_SUCCESS;
// index_scan_param is maintained by das scan op.
if (OB_FAIL(index_table_iter_->reuse())) {
LOG_WARN("failed to reuse index table iter", K(ret));
} else {
const ObTabletID &old_tablet_id = lookup_param_.tablet_id_;
lookup_param_.need_switch_param_ = lookup_param_.need_switch_param_ ||
((old_tablet_id.is_valid() && old_tablet_id != lookup_tablet_id_) ? true : false);
if (OB_FAIL(data_table_iter_->reuse())) {
LOG_WARN("failed to reuse data table iter", K(ret));
}
}
if (OB_SUCC(ret) && OB_FAIL(ObDASLookupIter::inner_reuse())) {
LOG_WARN("failed to reuse das lookup iter", K(ret));
}
trans_info_array_.reuse();
return ret;
}
int ObDASLocalLookupIter::inner_release()
{
int ret = OB_SUCCESS;
if (OB_FAIL(ObDASLookupIter::inner_release())) {
LOG_WARN("failed to release lookup iter", K(ret));
}
lookup_param_.destroy_schema_guard();
lookup_param_.snapshot_.reset();
lookup_param_.destroy();
trans_info_array_.reset();
return ret;
}
int ObDASLocalLookupIter::rescan()
{
int ret = OB_SUCCESS;
// only rescan index table, data table will be rescan in do_lookup.
if (OB_FAIL(index_table_iter_->rescan())) {
LOG_WARN("failed to rescan index table iter", K(ret));
}
return ret;
}
int ObDASLocalLookupIter::init_scan_param(ObTableScanParam &param, const ObDASScanCtDef *ctdef, ObDASScanRtDef *rtdef)
{
int ret = OB_SUCCESS;
uint64_t tenant_id = MTL_ID();
param.tenant_id_ = tenant_id;
param.key_ranges_.set_attr(ObMemAttr(tenant_id, "ScanParamKR"));
param.ss_key_ranges_.set_attr(ObMemAttr(tenant_id, "ScanParamSSKR"));
if (OB_ISNULL(ctdef) || OB_ISNULL(rtdef)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr ctdef or rtdef", K(ctdef), K(rtdef));
} else {
param.tablet_id_ = lookup_tablet_id_;
param.ls_id_ = lookup_ls_id_;
param.scan_allocator_ = &get_arena_allocator();
param.allocator_ = &rtdef->stmt_allocator_;
param.tx_lock_timeout_ = rtdef->tx_lock_timeout_;
param.index_id_ = ctdef->ref_table_id_;
param.is_get_ = ctdef->is_get_;
param.is_for_foreign_check_ = rtdef->is_for_foreign_check_;
param.timeout_ = rtdef->timeout_ts_;
param.scan_flag_ = rtdef->scan_flag_;
param.reserved_cell_count_ = ctdef->access_column_ids_.count();
param.sql_mode_ = rtdef->sql_mode_;
param.frozen_version_ = rtdef->frozen_version_;
param.force_refresh_lc_ = rtdef->force_refresh_lc_;
param.output_exprs_ = &(ctdef->pd_expr_spec_.access_exprs_);
param.aggregate_exprs_ = &(ctdef->pd_expr_spec_.pd_storage_aggregate_output_);
param.ext_file_column_exprs_ = &(ctdef->pd_expr_spec_.ext_file_column_exprs_);
param.ext_column_convert_exprs_ = &(ctdef->pd_expr_spec_.ext_column_convert_exprs_);
param.calc_exprs_ = &(ctdef->pd_expr_spec_.calc_exprs_);
param.table_param_ = &(ctdef->table_param_);
param.op_ = rtdef->p_pd_expr_op_;
param.row2exprs_projector_ = rtdef->p_row2exprs_projector_;
param.schema_version_ = ctdef->schema_version_;
param.tenant_schema_version_ = rtdef->tenant_schema_version_;
param.limit_param_ = rtdef->limit_param_;
param.need_scn_ = rtdef->need_scn_;
param.pd_storage_flag_ = ctdef->pd_expr_spec_.pd_storage_flag_.pd_flag_;
param.fb_snapshot_ = rtdef->fb_snapshot_;
param.fb_read_tx_uncommitted_ = rtdef->fb_read_tx_uncommitted_;
if (rtdef->is_for_foreign_check_) {
param.trans_desc_ = trans_desc_;
}
if (OB_NOT_NULL(snapshot_)) {
param.snapshot_ = *snapshot_;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null snapshot", K(ret), KPC(this));
}
if (OB_NOT_NULL(trans_desc_)) {
param.tx_id_ = trans_desc_->get_tx_id();
} else {
param.tx_id_.reset();
}
if (!ctdef->pd_expr_spec_.pushdown_filters_.empty()) {
param.op_filters_ = &ctdef->pd_expr_spec_.pushdown_filters_;
}
param.pd_storage_filters_ = rtdef->p_pd_expr_op_->pd_storage_filters_;
if (OB_FAIL(param.column_ids_.assign(ctdef->access_column_ids_))) {
LOG_WARN("failed to assign column ids", K(ret));
}
if (rtdef->sample_info_ != nullptr) {
param.sample_info_ = *rtdef->sample_info_;
}
}
LOG_DEBUG("init local index lookup param finished", K(param), K(ret));
return ret;
}
void ObDASLocalLookupIter::reset_lookup_state()
{
ObDASLookupIter::reset_lookup_state();
trans_info_array_.reuse();
}
int ObDASLocalLookupIter::add_rowkey()
{
int ret = OB_SUCCESS;
OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_SCAN);
ObDASScanIter *scan_iter = static_cast<ObDASScanIter *>(data_table_iter_);
storage::ObTableScanParam &scan_param = scan_iter->get_scan_param();
ObNewRange lookup_range;
int64 group_id = 0;
if (DAS_OP_TABLE_SCAN == index_ctdef_->op_type_) {
const ObDASScanCtDef *index_ctdef = static_cast<const ObDASScanCtDef*>(index_ctdef_);
if (nullptr != index_ctdef->group_id_expr_) {
group_id = index_ctdef->group_id_expr_->locate_expr_datum(*eval_ctx_).get_int();
}
if (nullptr != index_ctdef->trans_info_expr_) {
ObDatum *datum_ptr = nullptr;
if (OB_FAIL(build_trans_info_datum(index_ctdef->trans_info_expr_, datum_ptr))) {
LOG_WARN("failed to build trans info datum", K(ret));
} else if (OB_ISNULL(datum_ptr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K(ret));
} else if (OB_FAIL(trans_info_array_.push_back(datum_ptr))) {
LOG_WARN("failed to push back trans info array", K(ret), KPC(datum_ptr));
}
}
}
int64_t group_idx = ObNewRange::get_group_idx(group_id);
if (OB_FAIL(ret)) {
} else if (OB_FAIL(build_lookup_range(lookup_range))) {
LOG_WARN("failed to build lookup range", K(ret));
} else if (FALSE_IT(lookup_range.group_idx_ = group_idx)) {
} else if (OB_FAIL(scan_param.key_ranges_.push_back(lookup_range))) {
LOG_WARN("failed to push back lookup range", K(ret));
} else {
scan_param.is_get_ = true;
}
LOG_DEBUG("build local lookup range", K(lookup_range), K(ret));
return ret;
}
int ObDASLocalLookupIter::do_table_scan()
{
int ret = OB_SUCCESS;
if (OB_FAIL(index_table_iter_->do_table_scan())) {
LOG_WARN("failed to scan index table", K(ret));
}
return ret;
}
int ObDASLocalLookupIter::add_rowkeys(int64_t count)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(eval_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K_(eval_ctx));
} else {
ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*eval_ctx_);
batch_info_guard.set_batch_size(count);
for(int i = 0; OB_SUCC(ret) && i < count; i++) {
batch_info_guard.set_batch_idx(i);
if(OB_FAIL(add_rowkey())) {
LOG_WARN("failed to add rowkey", K(ret), K(i));
}
}
}
return ret;
}
int ObDASLocalLookupIter::do_index_lookup()
{
int ret = OB_SUCCESS;
OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_SCAN);
if (is_first_lookup_) {
is_first_lookup_ = false;
if (OB_FAIL(init_scan_param(lookup_param_, lookup_ctdef_, lookup_rtdef_))) {
LOG_WARN("failed to init scan param", K(ret));
} else if (OB_FAIL(data_table_iter_->do_table_scan())) {
if (OB_SNAPSHOT_DISCARDED == ret && lookup_param_.fb_snapshot_.is_valid()) {
ret = OB_INVALID_QUERY_TIMESTAMP;
} else if (OB_TRY_LOCK_ROW_CONFLICT != ret) {
LOG_WARN("failed to do partition scan", K(lookup_param_), K(ret));
}
}
} else {
// reuse -> real rescan
// reuse: store next tablet_id, ls_id and reuse storage iter;
// rescan: bind tablet_id, ls_id to scan_param and rescan;
lookup_param_.tablet_id_ = lookup_tablet_id_;
lookup_param_.ls_id_ = lookup_ls_id_;
if (OB_FAIL(data_table_iter_->rescan())) {
LOG_WARN("failed to rescan data table", K(ret));
}
}
return ret;
}
int ObDASLocalLookupIter::check_index_lookup()
{
int ret = OB_SUCCESS;
OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_SCAN);
ObDASScanIter *scan_iter = static_cast<ObDASScanIter*>(data_table_iter_);
if (GCONF.enable_defensive_check() &&
lookup_ctdef_->pd_expr_spec_.pushdown_filters_.empty()) {
if (OB_UNLIKELY(lookup_rowkey_cnt_ != lookup_row_cnt_)) {
ret = OB_ERR_DEFENSIVE_CHECK;
ObString func_name = ObString::make_string("check_lookup_row_cnt");
LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr());
LOG_ERROR("Fatal Error!!! Catch a defensive error!",
K(ret), K_(lookup_rowkey_cnt), K_(lookup_row_cnt),
"main table id", lookup_ctdef_->ref_table_id_,
"main tablet id", lookup_tablet_id_,
KPC_(trans_desc), KPC_(snapshot));
concurrency_control::ObDataValidationService::set_delay_resource_recycle(lookup_ls_id_);
const ObTableScanParam &scan_param = scan_iter->get_scan_param();
if (trans_info_array_.count() == scan_param.key_ranges_.count()) {
for (int64_t i = 0; i < trans_info_array_.count(); i++) {
ObDatum *datum = trans_info_array_.at(i);
LOG_ERROR("dump lookup range and trans info of local lookup das task",
K(i), KPC(trans_info_array_.at(i)), K(scan_param.key_ranges_.at(i)));
}
} else {
for (int64_t i = 0; i < scan_param.key_ranges_.count(); i++) {
LOG_ERROR("dump lookup range of local lookup das task",
K(i), K(scan_param.key_ranges_.at(i)));
}
}
}
}
int simulate_error = EVENT_CALL(EventTable::EN_DAS_SIMULATE_DUMP_WRITE_BUFFER);
if (0 != simulate_error) {
for (int64_t i = 0; i < trans_info_array_.count(); i++) {
LOG_INFO("dump trans info of local lookup das task", K(i), KPC(trans_info_array_.at(i)));
}
}
return ret;
}
int ObDASLocalLookupIter::init_rowkey_exprs_for_compat()
{
int ret = OB_SUCCESS;
if (ObDASOpType::DAS_OP_TABLE_SCAN == index_ctdef_->op_type_
|| ObDASOpType::DAS_OP_IR_AUX_LOOKUP == index_ctdef_->op_type_) {
const ObDASScanCtDef *scan_ctdef = static_cast<const ObDASScanCtDef*>(index_ctdef_);
int64_t rowkey_cnt = scan_ctdef->result_output_.count();
if (nullptr != scan_ctdef->group_id_expr_) {
rowkey_cnt -= 1;
}
if (nullptr != scan_ctdef->trans_info_expr_) {
rowkey_cnt -= 1;
}
if (OB_FAIL(rowkey_exprs_.reserve(rowkey_cnt))) {
LOG_WARN("failed to reserve rowkey exprs cnt", K(rowkey_cnt), K(ret));
} else {
for (int64_t i = 0; i < scan_ctdef->result_output_.count(); i++) {
ObExpr* const expr = scan_ctdef->result_output_.at(i);
if (T_PSEUDO_GROUP_ID == expr->type_ || T_PSEUDO_ROW_TRANS_INFO_COLUMN == expr->type_) {
// skip
} else if (OB_FAIL(rowkey_exprs_.push_back(expr))) {
LOG_WARN("failed to push back expr", K(ret));
}
}
}
} else if (ObDASOpType::DAS_OP_IR_SCAN == index_ctdef_->op_type_
|| ObDASOpType::DAS_OP_SORT == index_ctdef_->op_type_) {
// only doc_id as rowkey for text retrieval index back
const ObDASIRScanCtDef *ir_ctdef = nullptr;
if (ObDASOpType::DAS_OP_SORT == index_ctdef_->op_type_) {
if (OB_UNLIKELY(ObDASOpType::DAS_OP_IR_SCAN != index_ctdef_->children_[0]->op_type_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected child of sort iter is not an ir scan iter for compatible scan", K(ret));
} else {
ir_ctdef = static_cast<const ObDASIRScanCtDef *>(index_ctdef_->children_[0]);
}
} else {
ir_ctdef = static_cast<const ObDASIRScanCtDef *>(index_ctdef_);
}
if (OB_SUCC(ret)) {
if (OB_FAIL(rowkey_exprs_.push_back(ir_ctdef->inv_scan_doc_id_col_))) {
LOG_WARN("gailed to add rowkey exprs", K(ret));
}
}
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected compatible das scan op type", K(ret), K(index_ctdef_->op_type_));
}
return ret;
}
} // namespace sql
} // namespace oceanbase

94
src/sql/das/iter/ob_das_local_lookup_iter.h Normal file
View File

@ -0,0 +1,94 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OBDEV_SRC_SQL_DAS_ITER_OB_DAS_LOCAL_LOOKUP_ITER_H_
#define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_LOCAL_LOOKUP_ITER_H_
#include "sql/das/iter/ob_das_lookup_iter.h"
#include "storage/access/ob_dml_param.h"
namespace oceanbase
{
using namespace common;
namespace sql
{
struct ObDASLocalLookupIterParam : public ObDASLookupIterParam
{
public:
ObDASLocalLookupIterParam()
: ObDASLookupIterParam(false /*local lookup*/),
trans_desc_(nullptr),
snapshot_(nullptr)
{}
transaction::ObTxDesc *trans_desc_;
transaction::ObTxReadSnapshot *snapshot_;
virtual bool is_valid() const override
{
return true;
}
};
class ObDASScanCtDef;
class ObDASScanRtDef;
class ObDASLocalLookupIter : public ObDASLookupIter
{
public:
ObDASLocalLookupIter()
: ObDASLookupIter(ObDASIterType::DAS_ITER_LOCAL_LOOKUP),
trans_info_array_(),
lookup_param_(),
lookup_tablet_id_(),
lookup_ls_id_(),
trans_desc_(nullptr),
snapshot_(nullptr),
is_first_lookup_(true)
{}
virtual ~ObDASLocalLookupIter() {}
storage::ObTableScanParam &get_lookup_param() { return lookup_param_; }
void set_tablet_id(const ObTabletID &tablet_id) { lookup_tablet_id_ = tablet_id; }
void set_ls_id(const share::ObLSID &ls_id) { lookup_ls_id_ = ls_id; }
int init_scan_param(storage::ObTableScanParam &param, const ObDASScanCtDef *ctdef, ObDASScanRtDef *rtdef);
protected:
virtual int inner_init(ObDASIterParam &param) override;
virtual int inner_reuse() override;
virtual int inner_release() override;
virtual int do_table_scan() override;
virtual int rescan() override;
virtual void reset_lookup_state();
virtual int add_rowkey() override;
virtual int add_rowkeys(int64_t count) override;
virtual int do_index_lookup() override;
virtual int check_index_lookup() override;
private:
int init_rowkey_exprs_for_compat();
private:
ObSEArray<ObDatum *, 4> trans_info_array_;
// Local lookup das task could rescan multiple times during execution, lookup_tablet_id_ and
// lookup_ls_id_ store the lookup parameter for this time.
storage::ObTableScanParam lookup_param_;
ObTabletID lookup_tablet_id_;
share::ObLSID lookup_ls_id_;
transaction::ObTxDesc *trans_desc_;
transaction::ObTxReadSnapshot *snapshot_;
bool is_first_lookup_;
};
} // namespace sql
} // namespace oceanbase
#endif /* OBDEV_SRC_SQL_DAS_ITER_OB_DAS_LOOKUP_ITER_H_ */

241
src/sql/das/iter/ob_das_lookup_iter.cpp
View File

@ -23,9 +23,9 @@ namespace sql
int ObDASLookupIter::inner_init(ObDASIterParam &param)
{
int ret = OB_SUCCESS;
if (param.type_ != ObDASIterType::DAS_ITER_LOOKUP) {
if (!IS_LOOKUP_ITER(param.type_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("specific_init with bad param type", K(param.type_));
LOG_WARN("inner init das iter with bad param type", K(param), K(ret));
} else {
ObDASLookupIterParam &lookup_param = static_cast<ObDASLookupIterParam&>(param);
state_ = LookupState::INDEX_SCAN;
@ -35,15 +35,16 @@ int ObDASLookupIter::inner_init(ObDASIterParam &param)
lookup_row_cnt_ = 0;
index_table_iter_ = lookup_param.index_table_iter_;
data_table_iter_ = lookup_param.data_table_iter_;
can_retry_ = lookup_param.can_retry_;
calc_part_id_ = lookup_param.calc_part_id_;
index_ctdef_ = lookup_param.index_ctdef_;
index_rtdef_ = lookup_param.index_rtdef_;
lookup_ctdef_ = lookup_param.lookup_ctdef_;
lookup_rtdef_ = lookup_param.lookup_rtdef_;
rowkey_exprs_ = lookup_param.rowkey_exprs_;
iter_alloc_ = new (iter_alloc_buf_) common::ObArenaAllocator();
iter_alloc_->set_attr(ObMemAttr(MTL_ID(), "TableLookup"));
lookup_rtdef_->scan_allocator_.set_alloc(iter_alloc_);
lookup_rtdef_->stmt_allocator_.set_alloc(iter_alloc_);
lib::ContextParam param;
param.set_mem_attr(MTL_ID(), ObModIds::OB_SQL_TABLE_LOOKUP, ObCtxIds::DEFAULT_CTX_ID)
.set_properties(lib::USE_TL_PAGE_OPTIONAL);
if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(lookup_memctx_, param))) {
LOG_WARN("failed to create lookup memctx", K(ret));
}
}
return ret;
}
@ -51,12 +52,8 @@ int ObDASLookupIter::inner_init(ObDASIterParam &param)
int ObDASLookupIter::inner_reuse()
{
int ret = OB_SUCCESS;
// the reuse() of index table iter will be handled in TSC.
if (OB_FAIL(data_table_iter_->reuse())) {
LOG_WARN("failed to reuse data table iter", K(ret));
}
if (OB_NOT_NULL(iter_alloc_)) {
iter_alloc_->reset_remain_one_page();
if (OB_NOT_NULL(lookup_memctx_)) {
lookup_memctx_->reset_remain_one_page();
}
lookup_row_cnt_ = 0;
lookup_rowkey_cnt_ = 0;
@ -68,25 +65,27 @@ int ObDASLookupIter::inner_reuse()
int ObDASLookupIter::inner_release()
{
int ret = OB_SUCCESS;
if (OB_NOT_NULL(iter_alloc_)) {
iter_alloc_->reset();
iter_alloc_->~ObArenaAllocator();
iter_alloc_ = nullptr;
if (OB_NOT_NULL(lookup_memctx_)) {
DESTROY_CONTEXT(lookup_memctx_);
lookup_memctx_ = nullptr;
}
index_table_iter_ = nullptr;
data_table_iter_ = nullptr;
rowkey_exprs_.reset();
return ret;
}
void ObDASLookupIter::reset_lookup_state()
{
lookup_rowkey_cnt_ = 0;
lookup_row_cnt_ = 0;
lookup_rowkey_cnt_ = 0;
index_end_ = false;
state_ = LookupState::INDEX_SCAN;
if (OB_NOT_NULL(data_table_iter_)) {
data_table_iter_->reuse();
}
if (OB_NOT_NULL(iter_alloc_)) {
iter_alloc_->reset_remain_one_page();
if (OB_NOT_NULL(lookup_memctx_)) {
lookup_memctx_->reset_remain_one_page();
}
}
@ -95,13 +94,15 @@ int ObDASLookupIter::inner_get_next_row()
int ret = OB_SUCCESS;
bool got_next_row = false;
int64_t simulate_batch_row_cnt = - EVENT_CALL(EventTable::EN_TABLE_LOOKUP_BATCH_ROW_COUNT);
int64_t default_row_batch_cnt = simulate_batch_row_cnt > 0 ? simulate_batch_row_cnt : default_batch_row_count_;
const bool use_simulate_batch_row_cnt = simulate_batch_row_cnt > 0 && simulate_batch_row_cnt < default_batch_row_count_;
int64_t default_row_batch_cnt = use_simulate_batch_row_cnt ? simulate_batch_row_cnt : default_batch_row_count_;
LOG_DEBUG("simulate lookup row batch count", K(simulate_batch_row_cnt), K(default_row_batch_cnt));
do {
switch (state_) {
case INDEX_SCAN: {
reset_lookup_state();
while (OB_SUCC(ret) && !index_end_ && lookup_rowkey_cnt_ < default_row_batch_cnt) {
index_table_iter_->clear_evaluated_flag();
if (OB_FAIL(index_table_iter_->get_next_row())) {
if(OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next row from index table", K(ret));
@ -136,6 +137,7 @@ int ObDASLookupIter::inner_get_next_row()
}
case OUTPUT_ROWS: {
data_table_iter_->clear_evaluated_flag();
if (OB_FAIL(data_table_iter_->get_next_row())) {
if (OB_LIKELY(OB_ITER_END == ret)) {
ret = OB_SUCCESS;
@ -173,7 +175,8 @@ int ObDASLookupIter::inner_get_next_rows(int64_t &count, int64_t capacity)
int ret = OB_SUCCESS;
bool get_next_rows = false;
int64_t simulate_batch_row_cnt = - EVENT_CALL(EventTable::EN_TABLE_LOOKUP_BATCH_ROW_COUNT);
int64_t default_row_batch_cnt = simulate_batch_row_cnt > 0 ? simulate_batch_row_cnt : default_batch_row_count_;
const bool use_simulate_batch_row_cnt = simulate_batch_row_cnt > 0 && simulate_batch_row_cnt < default_batch_row_count_;
int64_t default_row_batch_cnt = use_simulate_batch_row_cnt ? simulate_batch_row_cnt : default_batch_row_count_;
LOG_DEBUG("simulate lookup row batch count", K(simulate_batch_row_cnt), K(default_row_batch_cnt));
do {
switch (state_) {
@ -184,6 +187,7 @@ int ObDASLookupIter::inner_get_next_rows(int64_t &count, int64_t capacity)
while (OB_SUCC(ret) && !index_end_ && lookup_rowkey_cnt_ < default_row_batch_cnt) {
storage_count = 0;
index_capacity = std::min(max_size_, default_row_batch_cnt - lookup_rowkey_cnt_);
index_table_iter_->clear_evaluated_flag();
if (OB_FAIL(index_table_iter_->get_next_rows(storage_count, index_capacity))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next rows from index table", K(ret));
@ -224,6 +228,7 @@ int ObDASLookupIter::inner_get_next_rows(int64_t &count, int64_t capacity)
case OUTPUT_ROWS: {
count = 0;
data_table_iter_->clear_evaluated_flag();
if (OB_FAIL(data_table_iter_->get_next_rows(count, capacity))) {
if (OB_LIKELY(OB_ITER_END == ret)) {
ret = OB_SUCCESS;
@ -260,46 +265,39 @@ int ObDASLookupIter::inner_get_next_rows(int64_t &count, int64_t capacity)
int ObDASLookupIter::build_lookup_range(ObNewRange &range)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(rowkey_exprs_) || OB_ISNULL(eval_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K(ret), K(rowkey_exprs_), K(eval_ctx_));
if (OB_ISNULL(eval_ctx_) || OB_UNLIKELY(rowkey_exprs_.empty())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid eval ctx or rowkey exprs", K_(eval_ctx), K_(rowkey_exprs), K(ret));
} else {
int64_t rowkey_cnt = rowkey_exprs_->count();
ObObj *obj_ptr = nullptr;
void *buf = nullptr;
if (OB_ISNULL(buf = iter_alloc_->alloc(sizeof(ObObj) * rowkey_cnt))) {
int64_t rowkey_cnt = rowkey_exprs_.count();
common::ObArenaAllocator& lookup_alloc = lookup_memctx_->get_arena_allocator();
if (OB_ISNULL(buf = lookup_alloc.alloc(sizeof(ObObj) * rowkey_cnt))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate enough memory", K(ret), K(rowkey_cnt));
LOG_WARN("failed to allocate enough memory", K(rowkey_cnt), K(ret));
} else {
obj_ptr = new (buf) ObObj[rowkey_cnt];
}
for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt; ++i) {
for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt; i++) {
ObObj tmp_obj;
ObExpr *expr = rowkey_exprs_->at(i);
const ObExpr *expr = rowkey_exprs_.at(i);
ObDatum &col_datum = expr->locate_expr_datum(*eval_ctx_);
if (OB_FAIL(col_datum.to_obj(tmp_obj, expr->obj_meta_, expr->obj_datum_map_))) {
if (OB_UNLIKELY(T_PSEUDO_GROUP_ID == expr->type_ || T_PSEUDO_ROW_TRANS_INFO_COLUMN == expr->type_)) {
// skip.
} else if (OB_FAIL(col_datum.to_obj(tmp_obj, expr->obj_meta_, expr->obj_datum_map_))) {
LOG_WARN("failed to convert datum to obj", K(ret));
} else if (OB_FAIL(ob_write_obj(*iter_alloc_, tmp_obj, obj_ptr[i]))) {
} else if (OB_FAIL(ob_write_obj(lookup_alloc, tmp_obj, obj_ptr[i]))) {
LOG_WARN("failed to deep copy rowkey", K(ret), K(tmp_obj));
}
}
int64_t group_id = 0;
if (OB_NOT_NULL(group_id_expr_)) {
ObDatum &group_datum = group_id_expr_->locate_expr_datum(*eval_ctx_);
OB_ASSERT(T_PSEUDO_GROUP_ID == group_id_expr_->type_);
group_id = group_datum.get_int();
}
if (OB_SUCC(ret)) {
ObRowkey row_key(obj_ptr, rowkey_cnt);
if (OB_FAIL(range.build_range(lookup_ctdef_->ref_table_id_, row_key))) {
LOG_WARN("failed to build lookup range", K(ret), K(lookup_ctdef_->ref_table_id_), K(row_key));
} else {
range.group_idx_ = group_id;
}
LOG_DEBUG("build lookup range", K(ret), K(row_key), K(range));
}
}
@ -318,7 +316,7 @@ int ObDASLookupIter::build_trans_info_datum(const ObExpr *trans_info_expr, ObDat
ObDatum &col_datum = trans_info_expr->locate_expr_datum(*eval_ctx_);
int64_t pos = sizeof(ObDatum);
int64_t len = sizeof(ObDatum) + col_datum.len_;
if (OB_ISNULL(buf = iter_alloc_->alloc(len))) {
if (OB_ISNULL(buf = lookup_memctx_->get_arena_allocator().alloc(len))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate enough memory", K(ret));
} else if (FALSE_IT(datum_ptr = new (buf) ObDatum)) {
@ -330,158 +328,5 @@ int ObDASLookupIter::build_trans_info_datum(const ObExpr *trans_info_expr, ObDat
return ret;
}
int ObDASGlobalLookupIter::add_rowkey()
{
int ret = OB_SUCCESS;
ObObjectID partition_id = OB_INVALID_ID;
ObTabletID tablet_id(OB_INVALID_ID);
ObDASScanOp *das_scan_op = nullptr;
ObDASTabletLoc *tablet_loc = nullptr;
ObDASCtx *das_ctx = nullptr;
bool reuse_das_op = false;
OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_MERGE);
if (OB_ISNULL(exec_ctx_) || OB_ISNULL(das_ctx = &exec_ctx_->get_das_ctx())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get das ctx", KPC_(exec_ctx));
} else {
ObDASMergeIter *merge_iter = static_cast<ObDASMergeIter*>(data_table_iter_);
if (OB_FAIL(ObExprCalcPartitionBase::calc_part_and_tablet_id(calc_part_id_,
*eval_ctx_,
partition_id,
tablet_id))) {
LOG_WARN("failed to calc part id", K(ret), KPC(calc_part_id_));
} else if (OB_FAIL(das_ctx->extended_tablet_loc(*lookup_rtdef_->table_loc_,
tablet_id,
tablet_loc))) {
LOG_WARN("failed to get tablet loc by tablet_id", K(ret));
} else if (OB_FAIL(merge_iter->create_das_task(tablet_loc, das_scan_op, reuse_das_op))) {
LOG_WARN("failed to create das task", K(ret));
} else if (!reuse_das_op) {
das_scan_op->set_scan_ctdef(lookup_ctdef_);
das_scan_op->set_scan_rtdef(lookup_rtdef_);
das_scan_op->set_can_part_retry(can_retry_);
}
}
if (OB_SUCC(ret)) {
storage::ObTableScanParam &scan_param = das_scan_op->get_scan_param();
ObNewRange lookup_range;
if (OB_FAIL(build_lookup_range(lookup_range))) {
LOG_WARN("failed to build lookup range", K(ret));
} else if (OB_FAIL(scan_param.key_ranges_.push_back(lookup_range))) {
LOG_WARN("failed to push back lookup range", K(ret));
} else {
scan_param.is_get_ = true;
}
}
const ObExpr *trans_info_expr = lookup_ctdef_->trans_info_expr_;
if (OB_SUCC(ret) && OB_NOT_NULL(trans_info_expr)) {
void *buf = nullptr;
ObDatum *datum_ptr = nullptr;
if (OB_FAIL(build_trans_info_datum(trans_info_expr, datum_ptr))) {
LOG_WARN("failed to build trans info datum", K(ret));
} else if (OB_ISNULL(datum_ptr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K(ret));
} else if (OB_FAIL(das_scan_op->trans_info_array_.push_back(datum_ptr))) {
LOG_WARN("failed to push back trans info array", K(ret), KPC(datum_ptr));
}
}
return ret;
}
int ObDASGlobalLookupIter::add_rowkeys(int64_t count)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(eval_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K_(eval_ctx));
} else {
ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*eval_ctx_);
batch_info_guard.set_batch_size(count);
for(int i = 0; OB_SUCC(ret) && i < count; i++) {
batch_info_guard.set_batch_idx(i);
if(OB_FAIL(add_rowkey())) {
LOG_WARN("failed to add rowkey", K(ret), K(i));
}
}
}
return ret;
}
int ObDASGlobalLookupIter::do_index_lookup()
{
int ret = OB_SUCCESS;
OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_MERGE);
ObDASMergeIter *merge_iter = static_cast<ObDASMergeIter*>(data_table_iter_);
if (OB_FAIL(merge_iter->do_table_scan())) {
LOG_WARN("failed to do global index lookup", K(ret));
} else if (OB_FAIL(merge_iter->set_merge_status(merge_iter->get_merge_type()))) {
LOG_WARN("failed to set merge status for das iter", K(ret));
}
return ret;
}
int ObDASGlobalLookupIter::check_index_lookup()
{
int ret = OB_SUCCESS;
OB_ASSERT(data_table_iter_->get_type() == DAS_ITER_MERGE);
ObDASMergeIter *merge_iter = static_cast<ObDASMergeIter*>(data_table_iter_);
if (GCONF.enable_defensive_check() &&
lookup_ctdef_->pd_expr_spec_.pushdown_filters_.empty()) {
if (OB_UNLIKELY(lookup_rowkey_cnt_ != lookup_row_cnt_)) {
ret = OB_ERR_DEFENSIVE_CHECK;
ObSQLSessionInfo *my_session = exec_ctx_->get_my_session();
ObString func_name = ObString::make_string("check_lookup_row_cnt");
LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr());
LOG_ERROR("Fatal Error!!! Catch a defensive error!",
K(ret), K(lookup_rowkey_cnt_), K(lookup_row_cnt_),
"main table id", lookup_ctdef_->ref_table_id_,
KPC(my_session->get_tx_desc()));
int64_t row_num = 0;
for (DASTaskIter task_iter = merge_iter->begin_task_iter(); !task_iter.is_end(); ++task_iter) {
ObDASScanOp *das_op = static_cast<ObDASScanOp*>(*task_iter);
if (das_op->trans_info_array_.count() == das_op->get_scan_param().key_ranges_.count()) {
for (int64_t i = 0; i < das_op->trans_info_array_.count(); i++) {
row_num++;
ObDatum *datum = das_op->trans_info_array_.at(i);
LOG_ERROR("dump GLobalIndexBack das task lookup range and trans info",
K(row_num), KPC(datum),
K(das_op->get_scan_param().key_ranges_.at(i)),
K(das_op->get_tablet_id()));
}
} else {
for (int64_t i = 0; i < das_op->get_scan_param().key_ranges_.count(); i++) {
row_num++;
LOG_ERROR("dump GLobalIndexBack das task lookup range",
K(row_num),
K(das_op->get_scan_param().key_ranges_.at(i)),
K(das_op->get_tablet_id()));
}
}
}
}
}
int simulate_error = EVENT_CALL(EventTable::EN_DAS_SIMULATE_DUMP_WRITE_BUFFER);
if (0 != simulate_error) {
for (DASTaskIter task_iter = merge_iter->begin_task_iter(); !task_iter.is_end(); ++task_iter) {
ObDASScanOp *das_op = static_cast<ObDASScanOp*>(*task_iter);
for (int64_t i = 0; i < das_op->trans_info_array_.count(); i++) {
ObDatum *datum = das_op->trans_info_array_.at(i);
LOG_INFO("dump GLobalIndexBack das task trans info", K(i),
KPC(das_op->trans_info_array_.at(i)),
K(das_op->get_scan_param().key_ranges_.at(i)),
K(das_op->get_tablet_id()));
}
}
}
return ret;
}
} // namespace sql
} // namespace oceanbase

71
src/sql/das/iter/ob_das_lookup_iter.h
View File

@ -21,47 +21,64 @@ using namespace common;
namespace sql
{
class ObDASBaseCtDef;
class ObDASBaseRtDef;
class ObDASScanCtDef;
class ObDASScanRtDef;
struct ObDASLookupIterParam : public ObDASIterParam
{
public:
ObDASLookupIterParam(bool is_global_index)
: ObDASIterParam(is_global_index ? DAS_ITER_GLOBAL_LOOKUP : DAS_ITER_LOCAL_LOOKUP),
default_batch_row_count_(0),
index_ctdef_(nullptr),
index_rtdef_(nullptr),
lookup_ctdef_(nullptr),
lookup_rtdef_(nullptr),
index_table_iter_(nullptr),
data_table_iter_(nullptr),
rowkey_exprs_(nullptr)
{}
int64_t default_batch_row_count_;
bool can_retry_;
const ObExpr *calc_part_id_;
const ObDASBaseCtDef *index_ctdef_;
ObDASBaseRtDef *index_rtdef_;
const ObDASScanCtDef *lookup_ctdef_;
ObDASScanRtDef *lookup_rtdef_;
const ExprFixedArray *rowkey_exprs_;
ObTableID ref_table_id_;
ObDASIter *index_table_iter_;
ObDASIter *data_table_iter_;
const ExprFixedArray *rowkey_exprs_;
virtual bool is_valid() const override
{
return ObDASIterParam::is_valid() &&
index_table_iter_ != nullptr && data_table_iter_ != nullptr && calc_part_id_ != nullptr &&
lookup_ctdef_ != nullptr && lookup_rtdef_ != nullptr && rowkey_exprs_ != nullptr;
return ObDASIterParam::is_valid()
&& index_table_iter_ != nullptr && data_table_iter_ != nullptr
&& index_ctdef_ != nullptr && index_rtdef_ != nullptr
&& lookup_ctdef_ != nullptr && lookup_rtdef_ != nullptr && rowkey_exprs_ != nullptr;
}
};
class ObDASLookupIter : public ObDASIter
{
public:
ObDASLookupIter()
: calc_part_id_(nullptr),
ObDASLookupIter(const ObDASIterType type = ObDASIterType::DAS_ITER_INVALID)
: ObDASIter(type),
index_ctdef_(nullptr),
index_rtdef_(nullptr),
lookup_ctdef_(nullptr),
lookup_rtdef_(nullptr),
rowkey_exprs_(nullptr),
rowkey_exprs_(),
index_table_iter_(nullptr),
data_table_iter_(nullptr),
lookup_rowkey_cnt_(0),
lookup_row_cnt_(0),
can_retry_(false),
state_(INDEX_SCAN),
index_end_(false),
default_batch_row_count_(0),
iter_alloc_(nullptr)
lookup_memctx_()
{}
virtual ~ObDASLookupIter() {}
INHERIT_TO_STRING_KV("ObDASIter", ObDASIter, K_(state), K_(index_end), K(lookup_ctdef_->ref_table_id_));
INHERIT_TO_STRING_KV("ObDASIter", ObDASIter, K_(state), K_(index_end), K_(default_batch_row_count));
protected:
virtual int inner_init(ObDASIterParam &param) override;
@ -70,24 +87,24 @@ protected:
virtual int inner_get_next_row() override;
virtual int inner_get_next_rows(int64_t &count, int64_t capacity) override;
virtual void reset_lookup_state();
virtual int add_rowkey() = 0;
virtual int add_rowkeys(int64_t count) = 0;
virtual int do_index_lookup() = 0;
virtual int check_index_lookup() = 0;
protected:
const ObExpr *calc_part_id_;
const ObDASBaseCtDef *index_ctdef_;
ObDASBaseRtDef *index_rtdef_;
const ObDASScanCtDef *lookup_ctdef_;
ObDASScanRtDef *lookup_rtdef_;
const ExprFixedArray *rowkey_exprs_;
common::ObSEArray<ObExpr*, 2> rowkey_exprs_;
ObDASIter *index_table_iter_;
ObDASIter *data_table_iter_;
int64_t lookup_rowkey_cnt_;
int64_t lookup_row_cnt_;
bool can_retry_;
int build_lookup_range(ObNewRange &range);
int build_trans_info_datum(const ObExpr *trans_info_expr, ObDatum *&datum_ptr);
common::ObArenaAllocator &get_arena_allocator() { return lookup_memctx_->get_arena_allocator(); }
private:
enum LookupState : uint32_t
@ -101,28 +118,10 @@ private:
LookupState state_;
bool index_end_;
int64_t default_batch_row_count_;
common::ObArenaAllocator *iter_alloc_;
char iter_alloc_buf_[sizeof(common::ObArenaAllocator)];
};
class ObDASGlobalLookupIter : public ObDASLookupIter
{
public:
ObDASGlobalLookupIter()
: ObDASLookupIter()
{}
virtual ~ObDASGlobalLookupIter() {}
protected:
virtual int add_rowkey() override;
virtual int add_rowkeys(int64_t count) override;
virtual int do_index_lookup() override;
virtual int check_index_lookup() override;
lib::MemoryContext lookup_memctx_;
};
} // namespace sql
} // namespace oceanbase
#endif /* OBDEV_SRC_SQL_DAS_ITER_OB_DAS_LOOKUP_ITER_H_ */

61
src/sql/das/iter/ob_das_merge_iter.cpp
View File

@ -91,7 +91,7 @@ int MergeStoreRows::to_expr(bool is_vectorized, int64_t size)
int64_t MergeStoreRows::get_group_idx(int64_t idx)
{
OB_ASSERT(idx < saved_size_);
return store_rows_[idx].store_row_->cells()[group_id_idx_].get_int();
return ObNewRange::get_group_idx(store_rows_[idx].store_row_->cells()[group_id_idx_].get_int());
}
int64_t MergeStoreRows::cur_group_idx()
@ -111,6 +111,12 @@ int64_t MergeStoreRows::row_cnt_with_cur_group_idx()
return end_idx - cur_idx_;
}
const ObDatum *MergeStoreRows::cur_datums()
{
OB_ASSERT(cur_idx_ < saved_size_);
return store_rows_[cur_idx_].store_row_->cells();
}
void MergeStoreRows::reuse()
{
cur_idx_ = OB_INVALID_INDEX;
@ -136,7 +142,7 @@ void MergeStoreRows::reset()
int ObDASMergeIter::set_merge_status(MergeType merge_type)
{
int ret = OB_SUCCESS;
merge_type_ = merge_type;
merge_type_ = used_for_keep_order_ ? MergeType::SORT_MERGE : merge_type;
if (merge_type == MergeType::SEQUENTIAL_MERGE) {
get_next_row_ = &ObDASMergeIter::get_next_seq_row;
get_next_rows_ = &ObDASMergeIter::get_next_seq_rows;
@ -270,7 +276,7 @@ int ObDASMergeIter::inner_init(ObDASIterParam &param)
int ret = OB_SUCCESS;
if (param.type_ != ObDASIterType::DAS_ITER_MERGE) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("inner init das iter with bad param type", K(param));
LOG_WARN("inner init das iter with bad param type", K(param), K(ret));
} else {
ObDASMergeIterParam &merge_param = static_cast<ObDASMergeIterParam&>(param);
eval_infos_ = merge_param.eval_infos_;
@ -287,6 +293,8 @@ int ObDASMergeIter::inner_init(ObDASIterParam &param)
das_ref_->set_expr_frame_info(merge_param.frame_info_);
das_ref_->set_execute_directly(merge_param.execute_das_directly_);
das_ref_->set_enable_rich_format(merge_param.enable_rich_format_);
used_for_keep_order_ = merge_param.used_for_keep_order_;
merge_type_ = used_for_keep_order_ ? SORT_MERGE : SEQUENTIAL_MERGE;
if (group_id_expr_ != nullptr) {
for (int64_t i = 0; i < output_->count(); i++) {
@ -589,7 +597,9 @@ int ObDASMergeIter::get_next_sorted_row()
LOG_WARN("failed to save store row", K(ret));
} else {
merge_state_arr_[i].row_store_have_data_ = true;
compare(i, output_idx);
if (OB_FAIL(compare(i, output_idx))) {
LOG_WARN("failed to compare two rows", K(ret));
}
}
} else if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
@ -597,8 +607,8 @@ int ObDASMergeIter::get_next_sorted_row()
} else {
LOG_WARN("das iter failed to get next row", K(ret));
}
} else {
compare(i, output_idx);
} else if (OB_FAIL(compare(i, output_idx))) {
LOG_WARN("failed to compare two rows", K(ret));
}
}
} // for end
@ -679,7 +689,9 @@ int ObDASMergeIter::get_next_sorted_rows(int64_t &count, int64_t capacity)
LOG_WARN("failed to save store row", K(ret));
} else {
merge_state_arr_[i].row_store_have_data_ = true;
compare(i, output_idx);
if (OB_FAIL(compare(i, output_idx))) {
LOG_WARN("failed to compare two rows", K(ret));
}
}
} else if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
@ -689,7 +701,9 @@ int ObDASMergeIter::get_next_sorted_rows(int64_t &count, int64_t capacity)
}
}
} else {
compare(i, output_idx);
if (OB_FAIL(compare(i, output_idx))) {
LOG_WARN("failed to compare two rows", K(ret));
}
}
}
}
@ -708,7 +722,7 @@ int ObDASMergeIter::get_next_sorted_rows(int64_t &count, int64_t capacity)
}
}
MergeStoreRows &store_rows = merge_store_rows_arr_.at(output_idx);
int64_t ret_count = store_rows.row_cnt_with_cur_group_idx();
int64_t ret_count = used_for_keep_order_ ? 1 : store_rows.row_cnt_with_cur_group_idx();
ret = store_rows.to_expr(true, ret_count);
if (OB_SUCC(ret)) {
count = ret_count;
@ -719,7 +733,6 @@ int ObDASMergeIter::get_next_sorted_rows(int64_t &count, int64_t capacity)
}
}
}
return ret;
}
@ -770,18 +783,34 @@ int ObDASMergeIter::prepare_sort_merge_info()
return ret;
}
void ObDASMergeIter::compare(int64_t cur_idx, int64_t &output_idx)
// [GROUP_ID] is composed of group_idx and index_ordered_idx now,
// we should compare group_idx first and then index_ordered_idx,
// group_idx and index_ordered_idx should always be sorted in ascending order.
int ObDASMergeIter::compare(int64_t cur_idx, int64_t &output_idx)
{
int ret = OB_SUCCESS;
if (OB_INVALID_INDEX == output_idx) {
output_idx = cur_idx;
} else {
// compare the values of group_idx.
int64_t output_group_idx = merge_store_rows_arr_[output_idx].cur_group_idx();
int64_t cur_group_idx = merge_store_rows_arr_[cur_idx].cur_group_idx();
if (output_group_idx > cur_group_idx) {
output_idx = cur_idx;
const ObDatum *cur_datums = merge_store_rows_arr_[cur_idx].cur_datums();
const ObDatum *output_datums = merge_store_rows_arr_[output_idx].cur_datums();
if (nullptr != group_id_expr_) {
int64_t cur_group_idx = ObNewRange::get_group_idx(cur_datums[group_id_idx_].get_int());
int64_t output_group_idx = ObNewRange::get_group_idx(output_datums[group_id_idx_].get_int());
if (cur_group_idx != output_group_idx) {
output_idx = cur_group_idx < output_group_idx ? cur_idx : output_idx;
} else {
int64_t cur_order_idx = ObNewRange::get_index_ordered_idx(cur_datums[group_id_idx_].get_int());
int64_t output_order_idx = ObNewRange::get_index_ordered_idx(output_datums[group_id_idx_].get_int());
if (cur_order_idx != output_order_idx) {
output_idx = cur_order_idx < output_order_idx ? cur_idx : output_idx;
}
}
}
}
LOG_DEBUG("das merge iter compare finished", K(cur_idx), K(output_idx), K(used_for_keep_order_));
return ret;
}
}//end namespace sql

22
src/sql/das/iter/ob_das_merge_iter.h
View File

@ -14,6 +14,8 @@
#define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_MERGE_ITER_H_
#include "sql/das/ob_das_utils.h"
#include "sql/das/iter/ob_das_iter.h"
#include "sql/das/ob_das_ref.h"
#include "sql/das/ob_das_scan_op.h"
namespace oceanbase
{
@ -21,9 +23,12 @@ using namespace common;
namespace sql
{
class ObDASMergeIterParam : public ObDASIterParam
struct ObDASMergeIterParam : public ObDASIterParam
{
public:
ObDASMergeIterParam()
: ObDASIterParam(ObDASIterType::DAS_ITER_MERGE)
{}
ObFixedArray<ObEvalInfo*, ObIAllocator> *eval_infos_;
bool need_update_partition_id_;
ObExpr *pdml_partition_id_;
@ -34,6 +39,7 @@ public:
const ObExprFrameInfo *frame_info_;
bool execute_das_directly_;
bool enable_rich_format_;
bool used_for_keep_order_;
virtual bool is_valid() const override
{
@ -73,6 +79,8 @@ public:
int64_t get_group_idx(int64_t idx);
int64_t cur_group_idx();
int64_t row_cnt_with_cur_group_idx();
const ObDatum *cur_datums();
void reuse();
void reset();
TO_STRING_KV(K_(saved_size),
@ -93,7 +101,8 @@ class ObDASMergeIter : public ObDASIter
{
public:
ObDASMergeIter()
: wild_datum_info_(),
: ObDASIter(ObDASIterType::DAS_ITER_MERGE),
wild_datum_info_(),
merge_type_(SEQUENTIAL_MERGE),
eval_infos_(nullptr),
need_update_partition_id_(false),
@ -111,11 +120,13 @@ public:
group_id_idx_(OB_INVALID_INDEX),
need_prepare_sort_merge_info_(false),
merge_state_arr_(),
merge_store_rows_arr_()
merge_store_rows_arr_(),
used_for_keep_order_(false)
{}
virtual ~ObDASMergeIter() {}
virtual int set_merge_status(MergeType merge_type) override;
virtual int do_table_scan() override;
MergeType get_merge_type() const { return merge_type_; }
void set_global_lookup_iter(ObDASMergeIter *global_lookup_iter);
INHERIT_TO_STRING_KV("ObDASIter", ObDASIter, K_(merge_type), K_(ref_table_id));
@ -128,8 +139,6 @@ public:
DASTaskIter begin_task_iter();
bool is_all_local_task() const;
int rescan_das_task(ObDASScanOp *scan_op);
// do_table_scan() need be called before get_next_row(s).
int do_table_scan();
/********* DAS REF END *********/
protected:
@ -151,7 +160,7 @@ private:
int get_next_sorted_row();
int get_next_sorted_rows(int64_t &count, int64_t capacity);
int prepare_sort_merge_info();
void compare(int64_t cur_idx, int64_t &output_idx);
int compare(int64_t cur_idx, int64_t &output_idx);
private:
@ -218,6 +227,7 @@ private:
typedef common::ObSEArray<MergeStoreRows, 8> MergeStoreRowsArray;
MergeStateArray merge_state_arr_;
MergeStoreRowsArray merge_store_rows_arr_;
bool used_for_keep_order_;
/********* SORT MERGE END *********/
};

141
src/sql/das/iter/ob_das_scan_iter.cpp Normal file
View File

@ -0,0 +1,141 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_DAS
#include "sql/das/iter/ob_das_scan_iter.h"
#include "sql/das/ob_das_scan_op.h"
#include "storage/tx_storage/ob_access_service.h"
namespace oceanbase
{
using namespace common;
namespace sql
{
int ObDASScanIter::inner_init(ObDASIterParam &param)
{
int ret = OB_SUCCESS;
if (param.type_ != ObDASIterType::DAS_ITER_SCAN) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("inner init das iter with bad param type", K(param), K(ret));
} else {
const ObDASScanCtDef *scan_ctdef = (static_cast<ObDASScanIterParam&>(param)).scan_ctdef_;
output_ = &scan_ctdef->result_output_;
tsc_service_ = is_virtual_table(scan_ctdef->ref_table_id_) ? GCTX.vt_par_ser_
: scan_ctdef->is_external_table_ ? GCTX.et_access_service_
: MTL(ObAccessService *);
}
return ret;
}
int ObDASScanIter::inner_reuse()
{
int ret = OB_SUCCESS;
// NOTE: need_switch_param_ should have been set before call reuse().
if (OB_ISNULL(scan_param_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr scan param", K(ret));
} else if (OB_FAIL(tsc_service_->reuse_scan_iter(scan_param_->need_switch_param_, result_))) {
LOG_WARN("failed to reuse storage scan iter", K(ret));
} else {
scan_param_->key_ranges_.reuse();
scan_param_->ss_key_ranges_.reuse();
scan_param_->mbr_filters_.reuse();
}
return ret;
}
int ObDASScanIter::inner_release()
{
int ret = OB_SUCCESS;
if (OB_NOT_NULL(result_)) {
if (OB_FAIL(tsc_service_->revert_scan_iter(result_))) {
LOG_WARN("failed to revert storage scan iter", K(ret));
}
result_ = nullptr;
}
return ret;
}
int ObDASScanIter::do_table_scan()
{
int ret = OB_SUCCESS;
result_ = nullptr;
if (OB_ISNULL(scan_param_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr scan param", K(ret));
} else if (OB_FAIL(tsc_service_->table_scan(*scan_param_, result_))) {
if (OB_SNAPSHOT_DISCARDED == ret && scan_param_->fb_snapshot_.is_valid()) {
ret = OB_INVALID_QUERY_TIMESTAMP;
} else if (OB_TRY_LOCK_ROW_CONFLICT != ret) {
LOG_WARN("fail to scan table", KPC_(scan_param), K(ret));
}
}
LOG_DEBUG("das scan iter do table scan", KPC_(scan_param), K(ret));
return ret;
}
int ObDASScanIter::rescan()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(scan_param_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr scan param", K(ret));
} else if (OB_FAIL(tsc_service_->table_rescan(*scan_param_, result_))) {
LOG_WARN("failed to rescan tablet", K(scan_param_->tablet_id_), K(ret));
} else {
// reset need_switch_param_ after real rescan.
scan_param_->need_switch_param_ = false;
}
LOG_DEBUG("das scan iter rescan", KPC_(scan_param), K(ret));
return ret;
}
int ObDASScanIter::inner_get_next_row()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(result_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr scan iter", K(ret));
} else if (OB_FAIL(result_->get_next_row())) {
if (ret != OB_ITER_END) {
LOG_WARN("failed to get next row", K(ret));
}
}
return ret;
}
int ObDASScanIter::inner_get_next_rows(int64_t &count, int64_t capacity)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(result_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr scan iter", K(ret));
} else if (OB_FAIL(result_->get_next_rows(count, capacity))) {
if (ret != OB_ITER_END) {
LOG_WARN("failed to get next row", K(ret));
}
}
return ret;
}
void ObDASScanIter::clear_evaluated_flag()
{
OB_ASSERT(nullptr != scan_param_);
scan_param_->op_->clear_evaluated_flag();
}
} // namespace sql
} // namespace oceanbase

77
src/sql/das/iter/ob_das_scan_iter.h Normal file
View File

@ -0,0 +1,77 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OBDEV_SRC_SQL_DAS_ITER_OB_DAS_SCAN_ITER_H_
#define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_SCAN_ITER_H_
#include "sql/das/iter/ob_das_iter.h"
namespace oceanbase
{
using namespace common;
namespace sql
{
// DASScanIter is a wrapper class for storage iter, it doesn't require eval_ctx or exprs like other iters.
struct ObDASScanIterParam : public ObDASIterParam
{
public:
ObDASScanIterParam()
: ObDASIterParam(ObDASIterType::DAS_ITER_SCAN),
scan_ctdef_(nullptr)
{}
const ObDASScanCtDef *scan_ctdef_;
virtual bool is_valid() const override
{
return nullptr != scan_ctdef_;
}
};
class ObDASScanIter : public ObDASIter
{
public:
ObDASScanIter()
: ObDASIter(ObDASIterType::DAS_ITER_SCAN),
tsc_service_(nullptr),
result_(nullptr),
scan_param_(nullptr)
{}
virtual ~ObDASScanIter() {}
common::ObNewRowIterator *&get_output_result_iter() { return result_; }
void set_scan_param(storage::ObTableScanParam &scan_param) { scan_param_ = &scan_param; }
storage::ObTableScanParam &get_scan_param() { return *scan_param_; }
virtual int do_table_scan() override;
virtual int rescan() override;
virtual void clear_evaluated_flag() override;
protected:
virtual int inner_init(ObDASIterParam &param) override;
virtual int inner_reuse() override;
virtual int inner_release() override;
virtual int inner_get_next_row() override;
virtual int inner_get_next_rows(int64_t &count, int64_t capacity) override;
private:
common::ObITabletScan *tsc_service_;
common::ObNewRowIterator *result_;
// must ensure the lifecycle of scan param is longer than scan iter.
storage::ObTableScanParam *scan_param_;
};
} // namespace sql
} // namespace oceanbase
#endif /* OBDEV_SRC_SQL_DAS_ITER_OB_DAS_SCAN_ITER_H_ */

272
src/sql/das/iter/ob_das_sort_iter.cpp Normal file
View File

@ -0,0 +1,272 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_DAS
#include "sql/das/iter/ob_das_sort_iter.h"
#include "sql/das/ob_das_attach_define.h"
#include "sql/engine/ob_bit_vector.h"
namespace oceanbase
{
using namespace common;
namespace sql
{
int ObDASSortIter::inner_init(ObDASIterParam &param)
{
int ret = OB_SUCCESS;
if (param.type_ != ObDASIterType::DAS_ITER_SORT) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("inner init das iter with bad param type", K(param), K(ret));
} else {
lib::ContextParam context_param;
context_param.set_mem_attr(MTL_ID(), "DASSortIter", ObCtxIds::DEFAULT_CTX_ID)
.set_properties(lib::USE_TL_PAGE_OPTIONAL);
if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(sort_memctx_, context_param))) {
LOG_WARN("failed to create lookup memctx", K(ret));
} else {
ObDASSortIterParam &sort_param = static_cast<ObDASSortIterParam&>(param);
sort_ctdef_ = sort_param.sort_ctdef_;
child_ = sort_param.child_;
// init top-n parameter
if ((nullptr != sort_ctdef_->limit_expr_ || nullptr != sort_ctdef_->offset_expr_)
&& sort_param.limit_param_.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected both limit offset expr and limit param", K(ret));
} else if (sort_param.limit_param_.is_valid()) {
limit_param_ = sort_param.limit_param_;
} else {
if (nullptr != sort_ctdef_->limit_expr_) {
ObDatum *limit_datum = nullptr;
if (OB_FAIL(sort_ctdef_->limit_expr_->eval(*eval_ctx_, limit_datum))) {
LOG_WARN("failed to eval limit expr", K(ret));
} else {
limit_param_.limit_ = (limit_datum->is_null() || limit_datum->get_int() < 0) ? 0 : limit_datum->get_int();
}
}
if (nullptr != sort_ctdef_->offset_expr_) {
ObDatum *offset_datum = nullptr;
if (OB_FAIL(sort_ctdef_->offset_expr_->eval(*eval_ctx_, offset_datum))) {
LOG_WARN("failed to eval limit expr", K(ret));
} else if (offset_datum->is_null()) {
limit_param_.limit_ = 0;
limit_param_.offset_ = 0;
} else {
limit_param_.offset_ = offset_datum->get_int() < 0 ? 0 : offset_datum->get_int();
}
}
}
if (OB_SUCC(ret)) {
const bool top_k_overflow = INT64_MAX - limit_param_.offset_ < limit_param_.limit_;
const int64_t top_k = (limit_param_.is_valid() && !top_k_overflow)
? (limit_param_.limit_ + limit_param_.offset_) : INT64_MAX;
if (OB_FAIL(sort_impl_.init(MTL_ID(),
&sort_ctdef_->sort_collations_,
&sort_ctdef_->sort_cmp_funcs_,
eval_ctx_,
exec_ctx_,
false, // enable encode sort key
false, // is local order
false, // need rewind
0, // part cnt
top_k,
sort_ctdef_->fetch_with_ties_))) {
LOG_WARN("failed to init sort impl", K(ret));
} else if (OB_FAIL(append(sort_row_, sort_ctdef_->sort_exprs_))) {
LOG_WARN("failed to append sort exprs", K(ret));
} else if (OB_FAIL(append_array_no_dup(sort_row_, *child_->get_output()))) {
LOG_WARN("failed to append sort rows", K(ret));
}
}
}
}
return ret;
}
int ObDASSortIter::inner_reuse()
{
int ret = OB_SUCCESS;
if (OB_NOT_NULL(child_)) {
if (OB_FAIL(child_->reuse())) {
LOG_WARN("failed to reuse child iter", K(ret));
}
}
if (OB_NOT_NULL(sort_memctx_)) {
sort_memctx_->reset_remain_one_page();
}
sort_finished_ = false;
output_row_cnt_ = 0;
input_row_cnt_ = 0;
// TODO: check if we can reuse sort impl here
// reset sort impl here since ObSortOpImpl::outputted_rows_cnt_ is not reset in reuse()
sort_impl_.reset();
fake_skip_ = nullptr;
return ret;
}
int ObDASSortIter::inner_release()
{
int ret = OB_SUCCESS;
sort_impl_.reset();
if (OB_NOT_NULL(sort_memctx_)) {
sort_memctx_->reset_remain_one_page();
DESTROY_CONTEXT(sort_memctx_);
sort_memctx_ = nullptr;
}
sort_row_.reset();
return ret;
}
int ObDASSortIter::do_table_scan()
{
int ret = OB_SUCCESS;
if (OB_FAIL(child_->do_table_scan())) {
LOG_WARN("failed to do table scan", K(ret));
}
return ret;
}
int ObDASSortIter::rescan()
{
int ret = OB_SUCCESS;
const bool top_k_overflow = INT64_MAX - limit_param_.offset_ < limit_param_.limit_;
const int64_t top_k = (limit_param_.is_valid() && !top_k_overflow)
? (limit_param_.limit_ + limit_param_.offset_) : INT64_MAX;
if (OB_FAIL(child_->rescan())) {
LOG_WARN("failed to rescan child", K(ret));
} else if (OB_FAIL(sort_impl_.init(MTL_ID(),
&sort_ctdef_->sort_collations_,
&sort_ctdef_->sort_cmp_funcs_,
eval_ctx_,
exec_ctx_,
false, // enable encode sort key
false, // is local order
false, // need rewind
0, // part cnt
top_k,
sort_ctdef_->fetch_with_ties_))) {
LOG_WARN("failed to init sort impl", K(ret));
}
return ret;
}
int ObDASSortIter::inner_get_next_row()
{
int ret = OB_SUCCESS;
if (limit_param_.limit_ > 0 && output_row_cnt_ >= limit_param_.limit_) {
ret = OB_ITER_END;
LOG_DEBUG("das sort iter got enough rows", K_(limit_param), K_(output_row_cnt), K_(input_row_cnt), K(ret));
} else if (!sort_finished_ && OB_FAIL(do_sort(false))) {
LOG_WARN("failed to do sort", K(ret));
} else {
bool got_row = false;
while (OB_SUCC(ret) && !got_row) {
if (OB_FAIL(sort_impl_.get_next_row(sort_row_))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next row from sort impl", K(ret));
}
} else {
++input_row_cnt_;
if (input_row_cnt_ > limit_param_.offset_) {
got_row = true;
++output_row_cnt_;
}
}
}
}
return ret;
}
int ObDASSortIter::inner_get_next_rows(int64_t &count, int64_t capacity)
{
int ret = OB_SUCCESS;
if (limit_param_.limit_ > 0 && output_row_cnt_ >= limit_param_.limit_) {
ret = OB_ITER_END;
LOG_DEBUG("das sort iter got enough rows", K_(limit_param), K_(output_row_cnt), K_(input_row_cnt), K(ret));
} else if (!sort_finished_ && OB_FAIL(do_sort(true))) {
LOG_WARN("failed to do sort", K(ret));
} else {
bool got_rows = false;
// TODO: @bingfan use vectorized interface instead.
while (OB_SUCC(ret) && !got_rows) {
if (OB_FAIL(sort_impl_.get_next_row(sort_row_))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next row from sort impl", K(ret));
}
} else {
++input_row_cnt_;
if (input_row_cnt_ > limit_param_.offset_) {
got_rows = true;
count = 1;
++output_row_cnt_;
}
}
}
}
return ret;
}
int ObDASSortIter::do_sort(bool is_vectorized)
{
int ret = OB_SUCCESS;
if (OB_LIKELY(is_vectorized)) {
int64_t read_size = 0;
fake_skip_ = to_bit_vector(sort_memctx_->get_arena_allocator().alloc(ObBitVector::memory_size(max_size_)));
fake_skip_->init(max_size_);
while (OB_SUCC(ret)) {
read_size = 0;
if (OB_FAIL(child_->get_next_rows(read_size, max_size_))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed ro get next rows from child iter", K(ret));
} else if (OB_FAIL(sort_impl_.add_batch(sort_row_, *fake_skip_, read_size, 0, nullptr))) {
LOG_WARN("failed to add batch to sort impl", K(ret));
} else {
ret = OB_ITER_END;
}
} else if (OB_FAIL(sort_impl_.add_batch(sort_row_, *fake_skip_, read_size, 0, nullptr))) {
LOG_WARN("failed to add batch to sort impl", K(ret));
}
}
} else {
while (OB_SUCC(ret)) {
if (OB_FAIL(child_->get_next_row())) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed ro get next rows from child iter", K(ret));
}
} else if (OB_FAIL(sort_impl_.add_row(sort_row_))) {
LOG_WARN("failed to add row to sort impl", K(ret));
}
}
}
if (OB_LIKELY(OB_ITER_END == ret)) {
ret = OB_SUCCESS;
if (OB_FAIL(sort_impl_.sort())) {
LOG_WARN("failed to do sort", K(ret));
} else {
sort_finished_ = true;
}
}
return ret;
}
void ObDASSortIter::clear_evaluated_flag()
{
OB_ASSERT(nullptr != child_);
child_->clear_evaluated_flag();
}
} // namespace sql
} // namespace oceanbase

95
src/sql/das/iter/ob_das_sort_iter.h Normal file
View File

@ -0,0 +1,95 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OBDEV_SRC_SQL_DAS_ITER_OB_DAS_SORT_ITER_H_
#define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_SORT_ITER_H_
#include "sql/das/iter/ob_das_iter.h"
#include "sql/engine/sort/ob_sort_op_impl.h"
namespace oceanbase
{
using namespace common;
namespace sql
{
struct ObDASSortIterParam : public ObDASIterParam
{
public:
ObDASSortIterParam()
: ObDASIterParam(ObDASIterType::DAS_ITER_SORT),
sort_ctdef_(nullptr),
child_(nullptr),
limit_param_() {}
virtual ~ObDASSortIterParam() {}
const ObDASSortCtDef *sort_ctdef_;
ObDASIter *child_;
common::ObLimitParam limit_param_;
virtual bool is_valid() const override
{
return ObDASIterParam::is_valid() && nullptr != sort_ctdef_ && nullptr != child_;
}
};
class ObDASSortIter : public ObDASIter
{
public:
ObDASSortIter()
: ObDASIter(ObDASIterType::DAS_ITER_SORT),
sort_impl_(),
sort_memctx_(),
sort_ctdef_(nullptr),
sort_row_(),
child_(nullptr),
sort_finished_(false),
limit_param_(),
input_row_cnt_(0),
output_row_cnt_(0),
fake_skip_(nullptr)
{}
virtual ~ObDASSortIter() {}
virtual int do_table_scan() override;
virtual int rescan() override;
virtual void clear_evaluated_flag() override;
protected:
virtual int inner_init(ObDASIterParam &param) override;
virtual int inner_reuse() override;
virtual int inner_release() override;
virtual int inner_get_next_row() override;
virtual int inner_get_next_rows(int64_t &count, int64_t capacity) override;
private:
int do_sort(bool is_vectorized);
private:
ObSortOpImpl sort_impl_;
lib::MemoryContext sort_memctx_;
const ObDASSortCtDef *sort_ctdef_;
ObSEArray<ObExpr *, 2> sort_row_;
ObDASIter *child_;
bool sort_finished_;
// limit param was set only once at do_table_scan of TSC, which means it should not be reset at reuse,
// input row cnt and output row cnt are the same as well.
common::ObLimitParam limit_param_;
int64_t input_row_cnt_;
int64_t output_row_cnt_;
ObBitVector *fake_skip_;
};
} // namespace sql
} // namespace oceanbase
#endif /* OBDEV_SRC_SQL_DAS_ITER_OB_DAS_SORT_ITER_H_ */

506
src/storage/fts/ob_text_retrieval_iterator.cpp → src/sql/das/iter/ob_das_text_retrieval_iter.cpp
View File

@ -10,86 +10,33 @@
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE_FTS
#define USING_LOG_PREFIX SQL_DAS
#include "ob_das_text_retrieval_iter.h"
#include "ob_das_scan_iter.h"
#include "sql/das/ob_das_ir_define.h"
#include "sql/engine/expr/ob_expr_bm25.h"
#include "sql/das/ob_text_retrieval_op.h"
#include "storage/fts/ob_text_retrieval_iterator.h"
#include "storage/tx_storage/ob_access_service.h"
namespace oceanbase
{
namespace storage
namespace sql
{
bool ObTokenRetrievalParam::need_relevance() const
{
OB_ASSERT(nullptr != ir_ctdef_);
return ir_ctdef_->need_calc_relevance();
}
const share::ObLSID &ObTokenRetrievalParam::get_ls_id() const
{
return ls_id_;
}
const sql::ObDASIRScanCtDef *ObTokenRetrievalParam::get_ir_ctdef() const
{
return ir_ctdef_;
}
sql::ObDASIRScanRtDef *ObTokenRetrievalParam::get_ir_rtdef()
{
return ir_rtdef_;
}
const sql::ObDASScanCtDef *ObTokenRetrievalParam::get_inv_idx_scan_ctdef() const
{
OB_ASSERT(nullptr != ir_ctdef_);
return ir_ctdef_->get_inv_idx_scan_ctdef();
}
const sql::ObDASScanCtDef *ObTokenRetrievalParam::get_inv_idx_agg_ctdef() const
{
OB_ASSERT(nullptr != ir_ctdef_);
return ir_ctdef_->get_inv_idx_agg_ctdef();
}
const sql::ObDASScanCtDef *ObTokenRetrievalParam::get_fwd_idx_agg_ctdef() const
{
OB_ASSERT(nullptr != ir_ctdef_);
return ir_ctdef_->get_fwd_idx_agg_ctdef();
}
const sql::ObDASScanCtDef *ObTokenRetrievalParam::get_doc_id_idx_agg_ctdef() const
{
OB_ASSERT(nullptr != ir_ctdef_);
return ir_ctdef_->get_doc_id_idx_agg_ctdef();
}
const common::ObTabletID &ObTokenRetrievalParam::get_inv_idx_tablet_id() const
{
return inv_idx_tablet_id_;
}
const common::ObTabletID &ObTokenRetrievalParam::get_fwd_idx_tablet_id() const
{
return fwd_idx_tablet_id_;
}
const common::ObTabletID &ObTokenRetrievalParam::get_doc_id_idx_tablet_id() const
{
return doc_id_idx_tablet_id_;
}
ObTextRetrievalIterator::ObTextRetrievalIterator()
: ObNewRowIterator(),
ObDASTextRetrievalIter::ObDASTextRetrievalIter()
: ObDASIter(ObDASIterType::DAS_ITER_TEXT_RETRIEVAL),
mem_context_(nullptr),
retrieval_param_(nullptr),
ir_ctdef_(nullptr),
ir_rtdef_(nullptr),
tx_desc_(nullptr),
snapshot_(nullptr),
ls_id_(),
inv_idx_tablet_id_(),
fwd_idx_tablet_id_(),
inv_idx_scan_param_(),
inv_idx_agg_param_(),
fwd_idx_scan_param_(),
calc_exprs_(),
inverted_idx_iter_(nullptr),
inverted_idx_scan_iter_(nullptr),
inverted_idx_agg_iter_(nullptr),
forward_idx_iter_(nullptr),
fwd_range_objs_(nullptr),
doc_token_cnt_expr_(nullptr),
@ -97,34 +44,59 @@ ObTextRetrievalIterator::ObTextRetrievalIterator()
need_fwd_idx_agg_(false),
need_inv_idx_agg_(false),
inv_idx_agg_evaluated_(false),
not_first_fwd_agg_(false),
is_inited_(false)
{
}
ObTextRetrievalIterator::~ObTextRetrievalIterator()
int ObDASTextRetrievalIter::set_query_token(const ObString &query_token)
{
reset();
int ret = OB_SUCCESS;
ObNewRange inv_idx_scan_range;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("text retrieval iter not inited", K(ret));
} else if (OB_UNLIKELY(!inv_idx_scan_param_.key_ranges_.empty() ||
(need_inv_idx_agg_ && !inv_idx_agg_param_.key_ranges_.empty()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected set query token with not null query range", K(ret), K(query_token),
K(inv_idx_scan_param_.key_ranges_), K_(need_inv_idx_agg), K(inv_idx_agg_param_.key_ranges_));
} else if (OB_FAIL(gen_inv_idx_scan_range(query_token, inv_idx_scan_range))) {
LOG_WARN("failed to generate inverted index scan range", K(ret), K(query_token));
} else if (OB_FAIL(inv_idx_scan_param_.key_ranges_.push_back(inv_idx_scan_range))) {
LOG_WARN("failed to add scan range for inv idx scan", K(ret));
} else if (need_inv_idx_agg_ && OB_FAIL(inv_idx_agg_param_.key_ranges_.push_back(inv_idx_scan_range))) {
LOG_WARN("failed to add scan range for inv idx agg", K(ret));
}
return ret;
}
int ObTextRetrievalIterator::init(
ObTokenRetrievalParam &retrieval_param,
const ObString &query_token,
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot)
int ObDASTextRetrievalIter::inner_init(ObDASIterParam &param)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("double initialization", K(ret), KPC(this));
} else if (OB_UNLIKELY(nullptr == tx_desc || nullptr == snapshot || !tx_desc->is_valid())) {
LOG_WARN("double initialization", K(ret));
} else if (OB_UNLIKELY(ObDASIterType::DAS_ITER_TEXT_RETRIEVAL != param.type_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KPC(tx_desc), KPC(snapshot));
LOG_WARN("invalid das iter param type for text retrieval iter", K(ret), K(param));
} else {
retrieval_param_ = &retrieval_param;
tx_desc_ = tx_desc;
snapshot_ = snapshot;
need_fwd_idx_agg_ = retrieval_param.get_ir_ctdef()->has_fwd_agg_ && retrieval_param.need_relevance();
need_inv_idx_agg_ = retrieval_param.need_relevance();
ObDASTextRetrievalIterParam &retrieval_param = static_cast<ObDASTextRetrievalIterParam &>(param);
inverted_idx_scan_iter_ = static_cast<ObDASScanIter *>(retrieval_param.inv_idx_scan_iter_);
ir_ctdef_ = retrieval_param.ir_ctdef_;
ir_rtdef_ = retrieval_param.ir_rtdef_;
tx_desc_ = retrieval_param.tx_desc_;
snapshot_ = retrieval_param.snapshot_;
need_fwd_idx_agg_ = ir_ctdef_->need_fwd_idx_agg();
need_inv_idx_agg_ = ir_ctdef_->need_inv_idx_agg();
if (need_inv_idx_agg_) {
inverted_idx_agg_iter_ = static_cast<ObDASScanIter *>(retrieval_param.inv_idx_agg_iter_);
}
if (need_fwd_idx_agg_) {
forward_idx_iter_ = static_cast<ObDASScanIter *>(retrieval_param.fwd_idx_iter_);
}
if (OB_ISNULL(mem_context_)) {
lib::ContextParam param;
@ -134,10 +106,7 @@ int ObTextRetrievalIterator::init(
}
}
if (FAILEDx(init_inv_idx_scan_param(query_token))) {
LOG_WARN("failed to init inverted index scan param", K(ret), K_(inv_idx_scan_param), K_(inv_idx_agg_param));
} else if (need_fwd_idx_agg_ && OB_FAIL(init_fwd_idx_scan_param())) {
LOG_WARN("failed to init forward index scan param", K(ret), K_(fwd_idx_scan_param));
if (OB_FAIL(ret)) {
} else if (OB_FAIL(init_calc_exprs())) {
LOG_WARN("failed to init row-wise calc exprs", K(ret));
} else {
@ -147,94 +116,148 @@ int ObTextRetrievalIterator::init(
return ret;
}
void ObTextRetrievalIterator::reset()
int ObDASTextRetrievalIter::inner_reuse()
{
int ret = OB_SUCCESS;
ObAccessService *tsc_service = MTL(ObAccessService *);
if (nullptr == tsc_service) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("failed to get access service when reset text retrieval iterator", K(ret));
if (nullptr != mem_context_) {
mem_context_->reset_remain_one_page();
}
inv_idx_agg_evaluated_ = false;
const ObTabletID &old_inv_scan_id = inv_idx_scan_param_.tablet_id_;
inverted_idx_scan_iter_->set_scan_param(inv_idx_scan_param_);
inv_idx_scan_param_.need_switch_param_ = inv_idx_scan_param_.need_switch_param_ ||
((old_inv_scan_id.is_valid() && old_inv_scan_id != inv_idx_tablet_id_) ? true : false);
if (OB_FAIL(inverted_idx_scan_iter_->reuse())) {
LOG_WARN("failed to reuse inverted index iter", K(ret));
} else {
if (nullptr != inverted_idx_iter_) {
if (OB_FAIL(tsc_service->revert_scan_iter(inverted_idx_iter_))) {
LOG_ERROR("failed to revert inverted index iter", K(ret));
if (need_inv_idx_agg_) {
const ObTabletID &old_inv_agg_id = inv_idx_agg_param_.tablet_id_;
inverted_idx_agg_iter_->set_scan_param(inv_idx_agg_param_);
inv_idx_agg_param_.need_switch_param_ = inv_idx_agg_param_.need_switch_param_ ||
((old_inv_agg_id.is_valid() && old_inv_agg_id != inv_idx_tablet_id_) ? true : false);
if (OB_FAIL(inverted_idx_agg_iter_->reuse())) {
LOG_WARN("failed to reuse inverted index agg iter", K(ret));
}
inverted_idx_iter_ = nullptr;
}
if (nullptr != forward_idx_iter_) {
if (OB_FAIL(tsc_service->revert_scan_iter(forward_idx_iter_))) {
LOG_ERROR("failed to revert forward index iter", K(ret));
if (OB_SUCC(ret) && need_fwd_idx_agg_) {
const ObTabletID &old_fwd_agg_id = fwd_idx_scan_param_.tablet_id_;
forward_idx_iter_->set_scan_param(fwd_idx_scan_param_);
fwd_idx_scan_param_.need_switch_param_ = fwd_idx_scan_param_.need_switch_param_ ||
((old_fwd_agg_id.is_valid() && old_fwd_agg_id != fwd_idx_tablet_id_) ? true : false);
if (OB_FAIL(forward_idx_iter_->reuse())) {
LOG_WARN("failed to reuse forward index iter", K(ret));
}
forward_idx_iter_ = nullptr;
}
}
inv_idx_scan_param_.need_switch_param_ = false;
inv_idx_scan_param_.destroy_schema_guard();
inv_idx_agg_param_.need_switch_param_ = false;
inv_idx_agg_param_.destroy_schema_guard();
fwd_idx_scan_param_.need_switch_param_ = false;
fwd_idx_scan_param_.destroy_schema_guard();
calc_exprs_.reset();
if (OB_SUCC(ret)) {
inv_idx_agg_evaluated_ = false;
}
return ret;
}
int ObDASTextRetrievalIter::inner_release()
{
int ret = OB_SUCCESS;
inv_idx_scan_param_.destroy_schema_guard();
inv_idx_scan_param_.snapshot_.reset();
inv_idx_scan_param_.destroy();
inv_idx_agg_param_.destroy_schema_guard();
inv_idx_agg_param_.snapshot_.reset();
inv_idx_agg_param_.destroy();
fwd_idx_scan_param_.destroy_schema_guard();
fwd_idx_scan_param_.snapshot_.reset();
fwd_idx_scan_param_.destroy();
calc_exprs_.reset();
if (nullptr != mem_context_) {
mem_context_->reset_remain_one_page();
DESTROY_CONTEXT(mem_context_);
mem_context_ = nullptr;
}
ir_ctdef_ = nullptr;
ir_rtdef_ = nullptr;
inverted_idx_scan_iter_ = nullptr;
inverted_idx_agg_iter_ = nullptr;
forward_idx_iter_ = nullptr;
fwd_range_objs_ = nullptr;
doc_token_cnt_expr_ = nullptr;
retrieval_param_ = nullptr;
tx_desc_ = nullptr;
snapshot_ = nullptr;
token_doc_cnt_ = 0;
need_fwd_idx_agg_ = false;
need_inv_idx_agg_ = false;
inv_idx_agg_evaluated_ = false;
not_first_fwd_agg_ = false;
is_inited_ = false;
return ret;
}
int ObTextRetrievalIterator::get_next_row(ObNewRow *&row)
{
UNUSED(row);
return OB_NOT_IMPLEMENT;
}
int ObTextRetrievalIterator::get_next_row()
int ObDASTextRetrievalIter::do_table_scan()
{
int ret = OB_SUCCESS;
inverted_idx_scan_iter_->set_scan_param(inv_idx_scan_param_);
if (need_inv_idx_agg_) {
inverted_idx_agg_iter_->set_scan_param(inv_idx_agg_param_);
}
if (OB_FAIL(init_inv_idx_scan_param())) {
LOG_WARN("failed to init inv idx scan param", K(ret));
} else if (OB_FAIL(inverted_idx_scan_iter_->do_table_scan())) {
LOG_WARN("failed to do inverted index table scan", K(ret));
} else if (need_inv_idx_agg_ && OB_FAIL(inverted_idx_agg_iter_->do_table_scan())) {
LOG_WARN("failed to do inverted index agg", K(ret));
}
return ret;
}
int ObDASTextRetrievalIter::rescan()
{
int ret = OB_SUCCESS;
inv_idx_scan_param_.tablet_id_ = inv_idx_tablet_id_;
inv_idx_scan_param_.ls_id_ = ls_id_;
if (need_inv_idx_agg_) {
inv_idx_agg_param_.tablet_id_ = inv_idx_tablet_id_;
inv_idx_agg_param_.ls_id_ = ls_id_;
}
if (OB_FAIL(inverted_idx_scan_iter_->rescan())) {
LOG_WARN("failed to rescan inverted scan iter", K(ret));
} else if (need_inv_idx_agg_ && OB_FAIL(inverted_idx_agg_iter_->rescan())) {
LOG_WARN("failed to rescan inverted index agg iter", K(ret));
}
return ret;
}
int ObDASTextRetrievalIter::inner_get_next_row()
{
int ret = OB_SUCCESS;
ObAccessService *tsc_service = MTL(ObAccessService *);
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("retrieval iterator not inited", K(ret));
} else if (!inv_idx_agg_evaluated_ && retrieval_param_->need_relevance()) {
} else if (!inv_idx_agg_evaluated_ && need_inv_idx_agg_) {
if (OB_FAIL(do_doc_cnt_agg())) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("Fail to do document count aggregation", K(ret), K_(inv_idx_agg_param));
}
} else if (OB_FAIL(tsc_service->revert_scan_iter(inverted_idx_iter_))) {
LOG_WARN("Fail to revert inverted index scan iterator after count aggregation", K(ret));
} else {
inverted_idx_iter_ = nullptr;
inv_idx_agg_evaluated_ = true;
}
}
if (OB_SUCC(ret) && nullptr == inverted_idx_iter_) {
if (OB_FAIL(tsc_service->table_scan(inv_idx_scan_param_, inverted_idx_iter_))) {
LOG_WARN("failed to init inverted index scan iterator", K(ret));
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(get_next_single_row(inv_idx_scan_param_.op_->is_vectorized(), inverted_idx_iter_))) {
} else if (OB_FAIL(get_next_single_row(inv_idx_scan_param_.op_->is_vectorized(), inverted_idx_scan_iter_))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next row from inverted index", K(ret), K_(inv_idx_scan_param), KPC_(inverted_idx_iter));
LOG_WARN("failed to get next row from inverted index", K(ret), K_(inv_idx_scan_param), KPC_(inverted_idx_scan_iter));
}
} else {
LOG_DEBUG("get one invert index scan row", "row",
ROWEXPR2STR(*retrieval_param_->get_ir_rtdef()->get_inv_idx_scan_rtdef()->eval_ctx_,
ROWEXPR2STR(*ir_rtdef_->get_inv_idx_scan_rtdef()->eval_ctx_,
*inv_idx_scan_param_.output_exprs_));
if (retrieval_param_->need_relevance()) {
if (ir_ctdef_->need_calc_relevance()) {
clear_row_wise_evaluated_flag();
if (OB_FAIL(get_next_doc_token_cnt(need_fwd_idx_agg_))) {
LOG_WARN("failed to get next doc token count", K(ret));
@ -248,67 +271,39 @@ int ObTextRetrievalIterator::get_next_row()
return ret;
}
int ObTextRetrievalIterator::get_next_rows(int64_t &count, int64_t capacity)
int ObDASTextRetrievalIter::inner_get_next_rows(int64_t &count, int64_t capacity)
{
UNUSEDx(count, capacity);
return OB_NOT_IMPLEMENT;
}
int ObTextRetrievalIterator::get_curr_iter_row(
const sql::ExprFixedArray *&curr_row,
sql::ObEvalCtx *&curr_eval_ctx)
{
UNUSEDx(curr_row, curr_eval_ctx);
return OB_NOT_IMPLEMENT;
}
int ObTextRetrievalIterator::get_curr_doc_id()
{
return OB_NOT_IMPLEMENT;
}
int ObTextRetrievalIterator::forward_to_doc(const ObDocId &doc_id)
{
UNUSED(doc_id);
return OB_NOT_IMPLEMENT;
}
int ObTextRetrievalIterator::init_inv_idx_scan_param(const ObString &query_token)
int ObDASTextRetrievalIter::init_inv_idx_scan_param()
{
int ret = OB_SUCCESS;
ObNewRange inv_idx_scan_range;
if (OB_FAIL(gen_inv_idx_scan_range(query_token, inv_idx_scan_range))) {
LOG_WARN("failed to generate inverted index scan range", K(ret), K(query_token));
} else if (OB_FAIL(init_base_idx_scan_param(
retrieval_param_->get_ls_id(),
retrieval_param_->get_inv_idx_tablet_id(),
retrieval_param_->get_inv_idx_scan_ctdef(),
retrieval_param_->get_ir_rtdef()->get_inv_idx_scan_rtdef(),
if (OB_FAIL(init_base_idx_scan_param(
ls_id_,
inv_idx_tablet_id_,
ir_ctdef_->get_inv_idx_scan_ctdef(),
ir_rtdef_->get_inv_idx_scan_rtdef(),
tx_desc_,
snapshot_,
inv_idx_scan_param_))) {
LOG_WARN("fail to init inverted index scan param", K(ret), KPC_(retrieval_param));
} else if (OB_FAIL(inv_idx_scan_param_.key_ranges_.push_back(inv_idx_scan_range))) {
LOG_WARN("failed to append scan range", K(ret));
}
if (OB_SUCC(ret) && need_inv_idx_agg_) {
LOG_WARN("fail to init inverted index scan param", K(ret), KPC_(ir_ctdef));
} else if (need_inv_idx_agg_) {
if (OB_FAIL(init_base_idx_scan_param(
retrieval_param_->get_ls_id(),
retrieval_param_->get_inv_idx_tablet_id(),
retrieval_param_->get_inv_idx_agg_ctdef(),
retrieval_param_->get_ir_rtdef()->get_inv_idx_agg_rtdef(),
ls_id_,
inv_idx_tablet_id_,
ir_ctdef_->get_inv_idx_agg_ctdef(),
ir_rtdef_->get_inv_idx_agg_rtdef(),
tx_desc_,
snapshot_,
inv_idx_agg_param_))) {
LOG_WARN("fail to init inverted index count aggregate param", K(ret), KPC_(retrieval_param));
} else if (OB_FAIL(inv_idx_agg_param_.key_ranges_.push_back(inv_idx_scan_range))) {
LOG_WARN("failed to append scan range", K(ret));
LOG_WARN("fail to init inverted index count aggregate param", K(ret), KPC_(ir_ctdef));
} else {
if (OB_UNLIKELY(!static_cast<sql::ObStoragePushdownFlag>(
retrieval_param_->get_inv_idx_agg_ctdef()->pd_expr_spec_.pd_storage_flag_).is_aggregate_pushdown())) {
ir_ctdef_->get_inv_idx_agg_ctdef()->pd_expr_spec_.pd_storage_flag_).is_aggregate_pushdown())) {
ret = OB_NOT_IMPLEMENT;
LOG_ERROR("not pushdown aggregate not supported", K(ret), K_(retrieval_param));
LOG_ERROR("not pushdown aggregate not supported", K(ret), KPC(ir_ctdef_->get_inv_idx_agg_ctdef()));
}
}
}
@ -316,25 +311,25 @@ int ObTextRetrievalIterator::init_inv_idx_scan_param(const ObString &query_token
return ret;
}
int ObTextRetrievalIterator::init_fwd_idx_scan_param()
int ObDASTextRetrievalIter::init_fwd_idx_scan_param()
{
int ret = OB_SUCCESS;
if (!retrieval_param_->need_relevance()) {
if (!ir_ctdef_->need_calc_relevance()) {
} else if (OB_FAIL(init_base_idx_scan_param(
retrieval_param_->get_ls_id(),
retrieval_param_->get_fwd_idx_tablet_id(),
retrieval_param_->get_fwd_idx_agg_ctdef(),
retrieval_param_->get_ir_rtdef()->get_fwd_idx_agg_rtdef(),
ls_id_,
fwd_idx_tablet_id_,
ir_ctdef_->get_fwd_idx_agg_ctdef(),
ir_rtdef_->get_fwd_idx_agg_rtdef(),
tx_desc_,
snapshot_,
fwd_idx_scan_param_))) {
LOG_WARN("Fail to init foward index scan param", K(ret), KPC_(retrieval_param));
LOG_WARN("Fail to init foward index scan param", K(ret), KPC_(ir_ctdef));
}
return ret;
}
int ObTextRetrievalIterator::init_base_idx_scan_param(
int ObDASTextRetrievalIter::init_base_idx_scan_param(
const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id,
const sql::ObDASScanCtDef *ctdef,
@ -403,42 +398,30 @@ int ObTextRetrievalIterator::init_base_idx_scan_param(
return ret;
}
int ObTextRetrievalIterator::do_doc_cnt_agg()
int ObDASTextRetrievalIter::do_doc_cnt_agg()
{
int ret = OB_SUCCESS;
ObAccessService *tsc_service = MTL(ObAccessService *);
if (OB_ISNULL(tsc_service)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get table access service", K(ret));
} else if (OB_FAIL(tsc_service->table_scan(inv_idx_agg_param_, inverted_idx_iter_))) {
if (OB_SNAPSHOT_DISCARDED == ret && inv_idx_agg_param_.fb_snapshot_.is_valid()) {
ret = OB_INVALID_QUERY_TIMESTAMP;
} else if (OB_TRY_LOCK_ROW_CONFLICT != ret) {
LOG_WARN("failed to do table scan for document count aggregation", K(ret));
if (OB_UNLIKELY(!static_cast<sql::ObStoragePushdownFlag>(inv_idx_agg_param_.pd_storage_flag_).is_aggregate_pushdown())) {
ret = OB_NOT_SUPPORTED;
LOG_ERROR("aggregate without pushdown not supported", K(ret));
} else if (OB_FAIL(get_next_single_row(inv_idx_agg_param_.op_->is_vectorized(), inverted_idx_agg_iter_))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get aggregated row from iter", K(ret));
}
} else {
if (OB_UNLIKELY(!static_cast<sql::ObStoragePushdownFlag>(inv_idx_agg_param_.pd_storage_flag_).is_aggregate_pushdown())) {
ret = OB_NOT_SUPPORTED;
LOG_ERROR("aggregate without pushdown not supported", K(ret));
} else if (OB_FAIL(get_next_single_row(inv_idx_agg_param_.op_->is_vectorized(), inverted_idx_iter_))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get aggregated row from iter", K(ret));
}
const sql::ObExpr *inv_idx_agg_expr = inv_idx_agg_param_.aggregate_exprs_->at(0);
sql::ObEvalCtx *eval_ctx = ir_rtdef_->get_inv_idx_agg_rtdef()->eval_ctx_;
ObDatum *doc_cnt_datum = nullptr;
if (OB_FAIL(inv_idx_agg_expr->eval(*eval_ctx, doc_cnt_datum))) {
LOG_WARN("failed to evaluate aggregated expr", K(ret));
} else {
const sql::ObExpr *inv_idx_agg_expr = inv_idx_agg_param_.aggregate_exprs_->at(0);
sql::ObEvalCtx *eval_ctx = retrieval_param_->get_ir_rtdef()->get_inv_idx_agg_rtdef()->eval_ctx_;
ObDatum *doc_cnt_datum = nullptr;
if (OB_FAIL(inv_idx_agg_expr->eval(*eval_ctx, doc_cnt_datum))) {
LOG_WARN("failed to evaluate aggregated expr", K(ret));
} else {
token_doc_cnt_ = doc_cnt_datum->get_int();
}
token_doc_cnt_ = doc_cnt_datum->get_int();
}
}
return ret;
}
int ObTextRetrievalIterator::get_next_doc_token_cnt(const bool use_fwd_idx_agg)
int ObDASTextRetrievalIter::get_next_doc_token_cnt(const bool use_fwd_idx_agg)
{
int ret = OB_SUCCESS;
if (use_fwd_idx_agg) {
@ -457,11 +440,11 @@ int ObTextRetrievalIterator::get_next_doc_token_cnt(const bool use_fwd_idx_agg)
return ret;
}
int ObTextRetrievalIterator::get_inv_idx_scan_doc_id(ObDocId &doc_id)
int ObDASTextRetrievalIter::get_inv_idx_scan_doc_id(ObDocId &doc_id)
{
int ret = OB_SUCCESS;
sql::ObExpr *doc_id_expr = retrieval_param_->get_ir_ctdef()->inv_scan_doc_id_col_;
sql::ObEvalCtx *eval_ctx = retrieval_param_->get_ir_rtdef()->get_inv_idx_scan_rtdef()->eval_ctx_;
sql::ObExpr *doc_id_expr = ir_ctdef_->inv_scan_doc_id_col_;
sql::ObEvalCtx *eval_ctx = ir_rtdef_->get_inv_idx_scan_rtdef()->eval_ctx_;
ObDatum &doc_id_datum = doc_id_expr->locate_expr_datum(*eval_ctx);
if (OB_FAIL(doc_id.from_string(doc_id_datum.get_string()))) {
LOG_WARN("failed to get ObDocId from datum", K(ret));
@ -470,59 +453,51 @@ int ObTextRetrievalIterator::get_inv_idx_scan_doc_id(ObDocId &doc_id)
return ret;
}
int ObTextRetrievalIterator::do_token_cnt_agg(const ObDocId &doc_id, int64_t &token_count)
int ObDASTextRetrievalIter::do_token_cnt_agg(const ObDocId &doc_id, int64_t &token_count)
{
int ret = OB_SUCCESS;
token_count = 0;
ObNewRange scan_range;
if (OB_FAIL(reuse_fwd_idx_iter())) {
LOG_WARN("failed to reuse forward index iterator", K(ret));
} else if (OB_UNLIKELY(!fwd_idx_scan_param_.key_ranges_.empty())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected non empty forward index scan range", K(ret));
} else if (OB_FAIL(gen_fwd_idx_scan_range(doc_id, scan_range))) {
if (OB_FAIL(gen_fwd_idx_scan_range(doc_id, scan_range))) {
LOG_WARN("failed to generate forward index scan range", K(ret));
} else if (OB_FAIL(fwd_idx_scan_param_.key_ranges_.push_back(scan_range))) {
LOG_WARN("failed to add forward index scan range", K(ret), K(scan_range));
}
if (OB_SUCC(ret)) {
ObAccessService *tsc_service = MTL(ObAccessService *);
if (OB_ISNULL(tsc_service)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get table access service", K(ret));
} else if (nullptr == forward_idx_iter_) {
if (OB_FAIL(tsc_service->table_scan(fwd_idx_scan_param_, forward_idx_iter_))) {
if (OB_SNAPSHOT_DISCARDED == ret && fwd_idx_scan_param_.fb_snapshot_.is_valid()) {
ret = OB_INVALID_QUERY_TIMESTAMP;
} else if (OB_TRY_LOCK_ROW_CONFLICT != ret) {
LOG_WARN("failed to init forward index scan iterator", K(ret), K_(fwd_idx_scan_param));
}
if (not_first_fwd_agg_) {
fwd_idx_scan_param_.tablet_id_ = fwd_idx_tablet_id_;
fwd_idx_scan_param_.ls_id_ = ls_id_;
if (OB_FAIL(reuse_fwd_idx_iter())) {
LOG_WARN("failed to reuse forward index iterator", K(ret));
} else if (OB_FAIL(fwd_idx_scan_param_.key_ranges_.push_back(scan_range))) {
LOG_WARN("failed to add forward index scan range", K(ret), K(scan_range));
} else if (OB_FAIL(forward_idx_iter_->rescan())) {
LOG_WARN("failed to rescan forward index", K(ret));
}
} else {
const ObTabletID &storage_tablet_id = fwd_idx_scan_param_.tablet_id_;
const bool need_switch_param =
storage_tablet_id.is_valid() && storage_tablet_id != retrieval_param_->get_fwd_idx_tablet_id();
fwd_idx_scan_param_.need_switch_param_ = need_switch_param;
if (OB_FAIL(tsc_service->reuse_scan_iter(need_switch_param, forward_idx_iter_))) {
LOG_WARN("failed to reuse scan iter", K(ret));
} else if (OB_FAIL(tsc_service->table_rescan(fwd_idx_scan_param_, forward_idx_iter_))) {
LOG_WARN("failed to rescan forward index table", K(ret), K_(fwd_idx_scan_param));
if (OB_FAIL(init_fwd_idx_scan_param())) {
LOG_WARN("failed to init forward index scan param", K(ret));
} else if (OB_FAIL(fwd_idx_scan_param_.key_ranges_.push_back(scan_range))) {
LOG_WARN("failed to add forward index scan range", K(ret), K(scan_range));
} else if (FALSE_IT(forward_idx_iter_->set_scan_param(fwd_idx_scan_param_))) {
} else if (OB_FAIL(forward_idx_iter_->do_table_scan())) {
LOG_WARN("failed to do forward index scan", K(ret));
} else {
not_first_fwd_agg_ = true;
}
}
if (OB_SUCC(ret)) {
if (!static_cast<sql::ObStoragePushdownFlag>(
retrieval_param_->get_fwd_idx_agg_ctdef()->pd_expr_spec_.pd_storage_flag_).is_aggregate_pushdown()) {
ir_ctdef_->get_fwd_idx_agg_ctdef()->pd_expr_spec_.pd_storage_flag_).is_aggregate_pushdown()) {
ret = OB_NOT_IMPLEMENT;
LOG_ERROR("aggregate without pushdown not implemented", K(ret));
} else {
if (OB_FAIL(forward_idx_iter_->get_next_row())) {
LOG_WARN("failed to get next row from forward index iterator", K(ret));
} else {
const sql::ObExpr *agg_expr = retrieval_param_->get_fwd_idx_agg_ctdef()->pd_expr_spec_.pd_storage_aggregate_output_.at(0);
sql::ObEvalCtx *eval_ctx = retrieval_param_->get_ir_rtdef()->get_fwd_idx_agg_rtdef()->eval_ctx_;
const sql::ObExpr *agg_expr = ir_ctdef_->get_fwd_idx_agg_ctdef()->pd_expr_spec_.pd_storage_aggregate_output_.at(0);
sql::ObEvalCtx *eval_ctx = ir_rtdef_->get_fwd_idx_agg_rtdef()->eval_ctx_;
const ObDatum &word_cnt_datum = agg_expr->locate_expr_datum(*eval_ctx);
token_count = word_cnt_datum.get_int();
LOG_DEBUG("retrieval iterator get token cnt for doc", K(ret), K(doc_id), K(token_count));
@ -534,12 +509,12 @@ int ObTextRetrievalIterator::do_token_cnt_agg(const ObDocId &doc_id, int64_t &to
return ret;
}
int ObTextRetrievalIterator::fill_token_cnt_with_doc_len()
int ObDASTextRetrievalIter::fill_token_cnt_with_doc_len()
{
int ret = OB_SUCCESS;
const sql::ObExpr *agg_expr = doc_token_cnt_expr_;
const sql::ObExpr *doc_length_expr = retrieval_param_->get_ir_ctdef()->inv_scan_doc_length_col_;
sql::ObEvalCtx *eval_ctx = retrieval_param_->get_ir_rtdef()->eval_ctx_;
const sql::ObExpr *doc_length_expr = ir_ctdef_->inv_scan_doc_length_col_;
sql::ObEvalCtx *eval_ctx = ir_rtdef_->eval_ctx_;
ObDatum *doc_length_datum = nullptr;
if (OB_ISNULL(agg_expr) || OB_ISNULL(doc_length_expr) || OB_ISNULL(eval_ctx)
|| OB_UNLIKELY(agg_expr->datum_meta_.get_type() != ObDecimalIntType)) {
@ -554,11 +529,11 @@ int ObTextRetrievalIterator::fill_token_cnt_with_doc_len()
return ret;
}
int ObTextRetrievalIterator::fill_token_doc_cnt()
int ObDASTextRetrievalIter::fill_token_doc_cnt()
{
int ret = OB_SUCCESS;
const sql::ObExpr *inv_idx_agg_expr = inv_idx_agg_param_.aggregate_exprs_->at(0);
sql::ObEvalCtx *eval_ctx = retrieval_param_->get_ir_rtdef()->get_inv_idx_agg_rtdef()->eval_ctx_;
sql::ObEvalCtx *eval_ctx = ir_rtdef_->get_inv_idx_agg_rtdef()->eval_ctx_;
if (OB_ISNULL(inv_idx_agg_expr) || OB_ISNULL(eval_ctx)
|| OB_UNLIKELY(inv_idx_agg_expr->datum_meta_.get_type() != ObIntType)) {
ret = OB_ERR_UNEXPECTED;
@ -570,31 +545,32 @@ int ObTextRetrievalIterator::fill_token_doc_cnt()
return ret;
}
int ObTextRetrievalIterator::project_relevance_expr()
int ObDASTextRetrievalIter::project_relevance_expr()
{
int ret = OB_SUCCESS;
const sql::ObDASIRScanRtDef *ir_rtdef = retrieval_param_->get_ir_rtdef();
sql::ObExpr *relevance_expr = retrieval_param_->get_ir_ctdef()->relevance_expr_;
sql::ObExpr *relevance_expr = ir_ctdef_->relevance_expr_;
ObDatum *relevance_datum = nullptr;
if (OB_ISNULL(relevance_expr)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid relevance expr", K(ret));
} else if (OB_FAIL(relevance_expr->eval(*ir_rtdef->eval_ctx_, relevance_datum))) {
} else if (OB_FAIL(relevance_expr->eval(*ir_rtdef_->eval_ctx_, relevance_datum))) {
LOG_WARN("failed to evaluate relevance", K(ret));
}
return ret;
}
int ObTextRetrievalIterator::reuse_fwd_idx_iter()
int ObDASTextRetrievalIter::reuse_fwd_idx_iter()
{
int ret = OB_SUCCESS;
if (nullptr != forward_idx_iter_) {
fwd_idx_scan_param_.key_ranges_.reuse();
if (OB_FAIL(forward_idx_iter_->reuse())) {
LOG_WARN("failed to reuse forward index iter", K(ret));
}
}
return ret;
}
int ObTextRetrievalIterator::gen_inv_idx_scan_range(const ObString &query_token, ObNewRange &scan_range)
int ObDASTextRetrievalIter::gen_inv_idx_scan_range(const ObString &query_token, ObNewRange &scan_range)
{
int ret = OB_SUCCESS;
void *buf = nullptr;
@ -604,7 +580,7 @@ int ObTextRetrievalIterator::gen_inv_idx_scan_range(const ObString &query_token,
ObObj tmp_obj;
tmp_obj.set_string(ObVarcharType, query_token);
// We need to ensure collation type / level between query text and token column is compatible
tmp_obj.set_meta_type(retrieval_param_->get_ir_ctdef()->search_text_->obj_meta_);
tmp_obj.set_meta_type(ir_ctdef_->search_text_->obj_meta_);
if (OB_ISNULL(buf = ctx_alloc.alloc(sizeof(ObObj) * obj_cnt))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
@ -621,7 +597,7 @@ int ObTextRetrievalIterator::gen_inv_idx_scan_range(const ObString &query_token,
obj_ptr[3].set_max_value();
ObRowkey start_key(obj_ptr, INV_IDX_ROWKEY_COL_CNT);
ObRowkey end_key(&obj_ptr[2], INV_IDX_ROWKEY_COL_CNT);
common::ObTableID inv_table_id = retrieval_param_->get_inv_idx_scan_ctdef()->ref_table_id_;
common::ObTableID inv_table_id = ir_ctdef_->get_inv_idx_scan_ctdef()->ref_table_id_;
scan_range.table_id_ = inv_table_id;
scan_range.start_key_.assign(obj_ptr, INV_IDX_ROWKEY_COL_CNT);
scan_range.end_key_.assign(&obj_ptr[2], INV_IDX_ROWKEY_COL_CNT);
@ -631,7 +607,7 @@ int ObTextRetrievalIterator::gen_inv_idx_scan_range(const ObString &query_token,
return ret;
}
int ObTextRetrievalIterator::gen_fwd_idx_scan_range(const ObDocId &doc_id, ObNewRange &scan_range)
int ObDASTextRetrievalIter::gen_fwd_idx_scan_range(const ObDocId &doc_id, ObNewRange &scan_range)
{
int ret = OB_SUCCESS;
if (nullptr == fwd_range_objs_) {
@ -651,7 +627,7 @@ int ObTextRetrievalIterator::gen_fwd_idx_scan_range(const ObDocId &doc_id, ObNew
fwd_range_objs_[1].set_min_value();
fwd_range_objs_[2].set_varbinary(doc_id.get_string());
fwd_range_objs_[3].set_max_value();
scan_range.table_id_ = retrieval_param_->get_fwd_idx_agg_ctdef()->ref_table_id_;
scan_range.table_id_ = ir_ctdef_->get_fwd_idx_agg_ctdef()->ref_table_id_;
scan_range.start_key_.assign(fwd_range_objs_, FWD_IDX_ROWKEY_COL_CNT);
scan_range.end_key_.assign(&fwd_range_objs_[2], FWD_IDX_ROWKEY_COL_CNT);
scan_range.border_flag_.set_inclusive_start();
@ -660,12 +636,12 @@ int ObTextRetrievalIterator::gen_fwd_idx_scan_range(const ObDocId &doc_id, ObNew
return ret;
}
int ObTextRetrievalIterator::init_calc_exprs()
int ObDASTextRetrievalIter::init_calc_exprs()
{
int ret = OB_SUCCESS;
if (retrieval_param_->get_ir_ctdef()->need_calc_relevance()) {
sql::ObExpr *relevance_expr = retrieval_param_->get_ir_ctdef()->relevance_expr_;
sql::ObEvalCtx *eval_ctx = retrieval_param_->get_ir_rtdef()->eval_ctx_;
if (ir_ctdef_->need_calc_relevance()) {
sql::ObExpr *relevance_expr = ir_ctdef_->relevance_expr_;
sql::ObEvalCtx *eval_ctx = ir_rtdef_->eval_ctx_;
if (OB_ISNULL(relevance_expr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null relevance expr", K(ret));
@ -701,9 +677,9 @@ int ObTextRetrievalIterator::init_calc_exprs()
return ret;
}
void ObTextRetrievalIterator::clear_row_wise_evaluated_flag()
void ObDASTextRetrievalIter::clear_row_wise_evaluated_flag()
{
sql::ObEvalCtx *eval_ctx = retrieval_param_->get_ir_rtdef()->eval_ctx_;
sql::ObEvalCtx *eval_ctx = ir_rtdef_->eval_ctx_;
for (int64_t i = 0; i < calc_exprs_.count(); ++i) {
sql::ObExpr *expr = calc_exprs_.at(i);
if (expr->is_batch_result()) {
@ -714,5 +690,5 @@ void ObTextRetrievalIterator::clear_row_wise_evaluated_flag()
}
}
} // end storage
} // end oceanbase
} // namespace sql
} // namespace oceanbase

151
src/sql/das/iter/ob_das_text_retrieval_iter.h Normal file
View File

@ -0,0 +1,151 @@
/**
* Copyright (c) 2024 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OB_DAS_TEXT_RETRIEVAL_ITER_H_
#define OB_DAS_TEXT_RETRIEVAL_ITER_H_
#include "ob_das_iter.h"
#include "lib/container/ob_se_array.h"
namespace oceanbase
{
namespace sql
{
struct ObDASIRScanCtDef;
struct ObDASIRScanRtDef;
struct ObDASTextRetrievalIterParam : public ObDASIterParam
{
public:
ObDASTextRetrievalIterParam()
: ObDASIterParam(DAS_ITER_TEXT_RETRIEVAL),
ir_ctdef_(nullptr),
ir_rtdef_(nullptr),
inv_idx_scan_iter_(nullptr),
inv_idx_agg_iter_(nullptr),
fwd_idx_iter_(nullptr),
tx_desc_(nullptr),
snapshot_(nullptr)
{}
virtual bool is_valid() const override
{
return nullptr != ir_ctdef_ && nullptr != ir_rtdef_ && nullptr != inv_idx_scan_iter_;
}
const ObDASIRScanCtDef *ir_ctdef_;
ObDASIRScanRtDef *ir_rtdef_;
ObDASIter *inv_idx_scan_iter_;
ObDASIter *inv_idx_agg_iter_;
ObDASIter *fwd_idx_iter_;
transaction::ObTxDesc *tx_desc_;
transaction::ObTxReadSnapshot *snapshot_;
};
// single token
class ObDASTextRetrievalIter : public ObDASIter
{
public:
ObDASTextRetrievalIter();
virtual ~ObDASTextRetrievalIter() {}
virtual int do_table_scan() override;
virtual int rescan() override;
int set_query_token(const ObString &query_token);
void set_ls_tablet_ids(
const share::ObLSID &ls_id,
const ObTabletID &inv_tablet_id,
const ObTabletID &fwd_tablet_id)
{
ls_id_ = ls_id;
inv_idx_tablet_id_ = inv_tablet_id;
fwd_idx_tablet_id_ = fwd_tablet_id;
}
INHERIT_TO_STRING_KV("ObDASIter", ObDASIter, K_(calc_exprs), K_(need_fwd_idx_agg),
K_(need_inv_idx_agg), K_(inv_idx_agg_evaluated), K_(not_first_fwd_agg), K_(is_inited));
protected:
virtual int inner_init(ObDASIterParam &param) override;
virtual int inner_reuse() override;
virtual int inner_release() override;
virtual int inner_get_next_row() override;
virtual int inner_get_next_rows(int64_t &count, int64_t capacity) override;
private:
int init_inv_idx_scan_param();
int init_fwd_idx_scan_param();
static int init_base_idx_scan_param(
const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id,
const sql::ObDASScanCtDef *ctdef,
sql::ObDASScanRtDef *rtdef,
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot,
ObTableScanParam &scan_param);
int get_next_doc_token_cnt(const bool use_fwd_idx_agg);
int do_doc_cnt_agg();
int do_token_cnt_agg(const ObDocId &doc_id, int64_t &token_count);
int get_inv_idx_scan_doc_id(ObDocId &doc_id);
int fill_token_cnt_with_doc_len();
int fill_token_doc_cnt();
int project_relevance_expr();
int reuse_fwd_idx_iter();
int gen_inv_idx_scan_range(const ObString &query_token, ObNewRange &scan_range);
int gen_fwd_idx_scan_range(const ObDocId &doc_id, ObNewRange &scan_range);
inline bool need_calc_relevance() { return true; } // TODO: reduce tsc ops if no need to calc relevance
int init_calc_exprs();
void clear_row_wise_evaluated_flag();
// TODO: delete this after enable standard vectorized execution
inline int get_next_single_row(const bool is_vectorized, ObNewRowIterator *iter)
{
int ret = OB_SUCCESS;
if (is_vectorized) {
int64_t scan_row_cnt = 0;
ret = iter->get_next_rows(scan_row_cnt, 1);
} else {
ret = iter->get_next_row();
}
return ret;
}
private:
static const int64_t FWD_IDX_ROWKEY_COL_CNT = 2;
static const int64_t INV_IDX_ROWKEY_COL_CNT = 2;
private:
lib::MemoryContext mem_context_;
const ObDASIRScanCtDef *ir_ctdef_;
ObDASIRScanRtDef *ir_rtdef_;
transaction::ObTxDesc *tx_desc_;
transaction::ObTxReadSnapshot *snapshot_;
share::ObLSID ls_id_;
common::ObTabletID inv_idx_tablet_id_;
common::ObTabletID fwd_idx_tablet_id_;
ObTableScanParam inv_idx_scan_param_;
ObTableScanParam inv_idx_agg_param_;
ObTableScanParam fwd_idx_scan_param_;
common::ObSEArray<sql::ObExpr *, 2> calc_exprs_;
ObDASScanIter *inverted_idx_scan_iter_;
ObDASScanIter *inverted_idx_agg_iter_;
ObDASScanIter *forward_idx_iter_;
ObObj *fwd_range_objs_;
sql::ObExpr *doc_token_cnt_expr_;
int64_t token_doc_cnt_;
bool need_fwd_idx_agg_;
bool need_inv_idx_agg_;
bool inv_idx_agg_evaluated_;
bool not_first_fwd_agg_;
bool is_inited_;
};
} // namespace sql
} // namespace oceanbase
#endif // OB_DAS_TEXT_RETRIEVAL_ITER_H_

708
src/sql/das/iter/ob_das_text_retrieval_merge_iter.cpp Normal file
View File

@ -0,0 +1,708 @@
/**
* Copyright (c) 2024 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_DAS
#include "ob_das_scan_iter.h"
#include "ob_das_text_retrieval_merge_iter.h"
#include "ob_das_text_retrieval_iter.h"
#include "sql/das/ob_das_ir_define.h"
#include "share/text_analysis/ob_text_analyzer.h"
#include "storage/fts/ob_fts_plugin_helper.h"
namespace oceanbase
{
namespace sql
{
ObIRIterLoserTreeItem::ObIRIterLoserTreeItem()
: relevance_(0), doc_id_(), iter_idx_(-1)
{
}
ObIRIterLoserTreeCmp::ObIRIterLoserTreeCmp()
: cmp_func_(), is_inited_(false)
{
}
ObIRIterLoserTreeCmp::~ObIRIterLoserTreeCmp()
{
}
int ObIRIterLoserTreeCmp::init()
{
int ret = OB_SUCCESS;
sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(ObVarcharType, CS_TYPE_BINARY);
cmp_func_ = lib::is_oracle_mode() ? basic_funcs->null_last_cmp_ : basic_funcs->null_first_cmp_;
if (OB_ISNULL(cmp_func_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to init IRIterLoserTreeCmp", K(ret));
} else {
is_inited_ = true;
}
return ret;
}
int ObIRIterLoserTreeCmp::cmp(
const ObIRIterLoserTreeItem &l,
const ObIRIterLoserTreeItem &r,
int64_t &cmp_ret)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret));
} else {
ObDatum l_datum;
ObDatum r_datum;
l_datum.set_string(l.doc_id_.get_string());
r_datum.set_string(r.doc_id_.get_string());
int tmp_ret = 0;
if (OB_FAIL(cmp_func_(l_datum, r_datum, tmp_ret))) {
LOG_WARN("failed to compare doc id by datum", K(ret));
} else {
cmp_ret = tmp_ret;
}
}
return ret;
}
ObDASTextRetrievalMergeIter::ObDASTextRetrievalMergeIter()
: ObDASIter(ObDASIterType::DAS_ITER_TEXT_RETRIEVAL_MERGE),
mem_context_(nullptr),
relation_type_(MAX_RELATION_TYPE),
processing_type_(MAX_PROC_TYPE),
ir_ctdef_(nullptr),
ir_rtdef_(nullptr),
tx_desc_(nullptr),
snapshot_(nullptr),
ls_id_(),
doc_id_idx_tablet_id_(),
query_tokens_(),
loser_tree_cmp_(),
iter_row_heap_(nullptr),
next_batch_iter_idxes_(),
next_batch_cnt_(0),
whole_doc_cnt_iter_(nullptr),
whole_doc_agg_param_(),
limit_param_(),
input_row_cnt_(0),
output_row_cnt_(0),
doc_cnt_calculated_(false),
is_inited_(false)
{
}
int ObDASTextRetrievalMergeIter::inner_init(ObDASIterParam &param)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("double initialization", K(ret));
} else if (OB_UNLIKELY(ObDASIterType::DAS_ITER_TEXT_RETRIEVAL_MERGE != param.type_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid das iter param type for text retrieval iter", K(ret), K(param));
} else {
ObDASTextRetrievalMergeIterParam &retrieval_param = static_cast<ObDASTextRetrievalMergeIterParam &>(param);
ir_ctdef_ = retrieval_param.ir_ctdef_;
ir_rtdef_ = retrieval_param.ir_rtdef_;
whole_doc_cnt_iter_ = static_cast<ObDASScanIter *>(retrieval_param.doc_cnt_iter_);
tx_desc_ = retrieval_param.tx_desc_;
snapshot_ = retrieval_param.snapshot_;
relation_type_ = TokenRelationType::DISJUNCTIVE;
processing_type_ = RetrievalProcType::DAAT;
if (OB_ISNULL(mem_context_)) {
lib::ContextParam param;
param.set_mem_attr(MTL_ID(), "TextIRIter", ObCtxIds::DEFAULT_CTX_ID);
if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(mem_context_, param))) {
LOG_WARN("failed to create text retrieval iterator memory context", K(ret));
}
}
if (FAILEDx(loser_tree_cmp_.init())) {
LOG_WARN("failed to init loser tree comparator", K(ret));
} else if (OB_ISNULL(iter_row_heap_ = OB_NEWx(ObIRIterLoserTree, &mem_context_->get_arena_allocator(), loser_tree_cmp_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate loser tree", K(ret));
} else if (OB_FAIL(init_query_tokens(ir_ctdef_, ir_rtdef_))) {
LOG_WARN("failed to init query tokens", K(ret));
} else if (0 == query_tokens_.count()) {
// empty token set
LOG_DEBUG("empty query token set after tokenization", K(ret), KPC_(ir_ctdef));
is_inited_ = true;
} else if (OB_UNLIKELY(query_tokens_.count() > OB_MAX_TEXT_RETRIEVAL_TOKEN_CNT)) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("too many query tokens in a single query not supported", K(ret), K_(query_tokens));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "text retrieval query with token count exceed limit");
} else if (OB_FAIL(iter_row_heap_->init(query_tokens_.count(), mem_context_->get_arena_allocator()))) {
LOG_WARN("failed to init iter loser tree", K(ret));
} else {
limit_param_ = ir_rtdef_->get_inv_idx_scan_rtdef()->limit_param_;
is_inited_ = true;
}
LOG_DEBUG("init text retrieval op", K(ret), KPC_(ir_ctdef), KPC_(ir_rtdef));
}
return ret;
}
int ObDASTextRetrievalMergeIter::rescan()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(query_tokens_.count() != token_iters_.count())) {
ret= OB_ERR_UNEXPECTED;
LOG_WARN("unexpected iter count mismatch with query tokens",
K(ret), K_(query_tokens), K_(token_iters));
} else if (0 == query_tokens_.count()) {
} else if (nullptr != whole_doc_cnt_iter_ && OB_FAIL(whole_doc_cnt_iter_->rescan())) {
LOG_WARN("failed to rescan doc count iter", K(ret));
} else if (OB_FAIL(next_batch_iter_idxes_.init(query_tokens_.count()))) {
LOG_WARN("failed to init next batch iter idxes array", K(ret));
} else if (OB_FAIL(next_batch_iter_idxes_.prepare_allocate(query_tokens_.count()))) {
LOG_WARN("failed to prepare allocate next batch iter idxes array", K(ret));
} else if (OB_FAIL(iter_row_heap_->open(query_tokens_.count()))) {
LOG_WARN("failed to open iter row heap", K(ret), K_(query_tokens));
} else {
limit_param_ = ir_rtdef_->get_inv_idx_scan_rtdef()->limit_param_;
next_batch_cnt_ = token_iters_.count();
for (int64_t i = 0; OB_SUCC(ret) && i < token_iters_.count(); ++i) {
ObDASTextRetrievalIter *iter = token_iters_.at(i);
if (OB_FAIL(token_iters_.at(i)->set_query_token(query_tokens_.at(i)))) {
LOG_WARN("failed to set query token before rescan", K(ret), K(query_tokens_.at(i)));
} else if (OB_FAIL(iter->rescan())) {
LOG_WARN("failed to append token iter to array", K(ret));
} else {
next_batch_iter_idxes_[i] = i;
}
}
}
return ret;
}
int ObDASTextRetrievalMergeIter::do_table_scan()
{
int ret = OB_SUCCESS;
for (int64_t i = 0; i < token_iters_.count(); ++i) {
if (OB_FAIL(token_iters_.at(i)->do_table_scan())) {
LOG_WARN("failed to do table scan for token iter", K(ret));
}
}
return ret;
}
int ObDASTextRetrievalMergeIter::set_related_tablet_ids(
const ObLSID &ls_id,
const ObDASRelatedTabletID &related_tablet_ids)
{
int ret = OB_SUCCESS;
ls_id_ = ls_id;
doc_id_idx_tablet_id_ = related_tablet_ids.doc_id_idx_tablet_id_;
for (int64_t i = 0; i < token_iters_.count(); ++i) {
token_iters_.at(i)->set_ls_tablet_ids(
ls_id,
related_tablet_ids.inv_idx_tablet_id_,
related_tablet_ids.fwd_idx_tablet_id_);
}
return ret;
}
int ObDASTextRetrievalMergeIter::inner_reuse()
{
int ret = OB_SUCCESS;
next_batch_iter_idxes_.reuse();
iter_row_heap_->reuse();
next_batch_cnt_ = 0;
doc_cnt_calculated_ = false;
input_row_cnt_ = 0;
output_row_cnt_ = 0;
const ObTabletID &old_doc_id_tablet_id = whole_doc_agg_param_.tablet_id_;
whole_doc_agg_param_.need_switch_param_ = whole_doc_agg_param_.need_switch_param_ ||
((old_doc_id_tablet_id.is_valid() && old_doc_id_tablet_id != doc_id_idx_tablet_id_) ? true : false);
if (nullptr != whole_doc_cnt_iter_) {
whole_doc_cnt_iter_->set_scan_param(whole_doc_agg_param_);
if (OB_FAIL(whole_doc_cnt_iter_->reuse())) {
LOG_WARN("failed to reuse whole doc cnt iter", K(ret));
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < token_iters_.count(); ++i) {
if (OB_FAIL(token_iters_.at(i)->reuse())) {
LOG_WARN("failed to reuse token iters", K(ret));
}
}
return ret;
}
int ObDASTextRetrievalMergeIter::inner_release()
{
int ret = OB_SUCCESS;
query_tokens_.reset();
if (nullptr != iter_row_heap_) {
iter_row_heap_->~ObIRIterLoserTree();
iter_row_heap_ = nullptr;
}
whole_doc_agg_param_.destroy_schema_guard();
whole_doc_agg_param_.snapshot_.reset();
whole_doc_agg_param_.destroy();
whole_doc_cnt_iter_ = nullptr;
token_iters_.reset();
next_batch_iter_idxes_.reset();
if (nullptr != mem_context_) {
mem_context_->reset_remain_one_page();
DESTROY_CONTEXT(mem_context_);
mem_context_ = nullptr;
}
next_batch_cnt_ = 0;
input_row_cnt_ = 0;
output_row_cnt_ = 0;
limit_param_.offset_ = 0;
limit_param_.limit_ = -1;
doc_cnt_calculated_ = false;
is_inited_ = false;
return ret;
}
int ObDASTextRetrievalMergeIter::inner_get_next_row()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret));
} else if (0 == query_tokens_.count()) {
ret = OB_ITER_END;
} else if (limit_param_.limit_ > 0 && output_row_cnt_ >= limit_param_.limit_) {
ret = OB_ITER_END;
LOG_DEBUG("get row with limit finished",
K(ret), K_(limit_param), K_(output_row_cnt), K_(input_row_cnt));
} else if (!doc_cnt_calculated_) {
if (OB_FAIL(do_total_doc_cnt())) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to do total document count", K(ret), KPC_(ir_ctdef));
}
} else {
doc_cnt_calculated_ = true;
}
}
bool filter_valid = false;
bool got_valid_document = false;
ObExpr *match_filter = ir_ctdef_->need_calc_relevance() ? ir_ctdef_->match_filter_ : nullptr;
ObDatum *filter_res = nullptr;
while (OB_SUCC(ret) && !got_valid_document) {
clear_evaluated_infos();
filter_valid = false;
if (OB_FAIL(pull_next_batch_rows())) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to pull next batch rows from iterator", K(ret));
}
} else if (OB_FAIL(next_disjunctive_document())) {
LOG_WARN("failed to get next document with disjunctive tokens", K(ret));
} else if (OB_ISNULL(match_filter)) {
filter_valid = true;
} else if (OB_FAIL(match_filter->eval(*ir_rtdef_->eval_ctx_, filter_res))) {
LOG_WARN("failed to evaluate match filter", K(ret));
} else {
filter_valid = !(filter_res->is_null() || 0 == filter_res->get_int());
}
if (OB_SUCC(ret)) {
if (filter_valid) {
++input_row_cnt_;
if (input_row_cnt_ > limit_param_.offset_) {
got_valid_document = true;
++output_row_cnt_;
}
}
}
}
return ret;
}
int ObDASTextRetrievalMergeIter::inner_get_next_rows(int64_t &count, int64_t capacity)
{
// only one row at a time
// TODO: support batch vectorized execution later
int ret = OB_SUCCESS;
if (OB_FAIL(inner_get_next_row())) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next row", K(ret));
}
} else {
count += 1;
}
return ret;
}
int ObDASTextRetrievalMergeIter::init_query_tokens(const ObDASIRScanCtDef *ir_ctdef, ObDASIRScanRtDef *ir_rtdef)
{
int ret = OB_SUCCESS;
ObExpr *search_text = ir_ctdef->search_text_;
ObEvalCtx *eval_ctx = ir_rtdef->eval_ctx_;
ObDatum *search_text_datum = nullptr;
if (OB_ISNULL(search_text) || OB_ISNULL(eval_ctx)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K(ret), KP(search_text), KP(eval_ctx));
} else if (OB_FAIL(search_text->eval(*eval_ctx, search_text_datum))) {
LOG_WARN("expr evaluation failed", K(ret));
} else if (0 == search_text_datum->len_) {
// empty query text
} else {
// TODO: FTParseHelper currently does not support deduplicate tokens
// We should abstract such universal analyse functors into utility structs
const ObString &search_text_string = search_text_datum->get_string();
const ObString &parser_name = ir_ctdef->get_inv_idx_scan_ctdef()->table_param_.get_parser_name();
const ObCollationType &cs_type = search_text->datum_meta_.cs_type_;
int64_t doc_length = 0;
storage::ObFTParseHelper tokenize_helper;
common::ObSEArray<ObFTWord, 16> tokens;
hash::ObHashMap<ObFTWord, int64_t> token_map;
const int64_t ft_word_bkt_cnt = MAX(search_text_string.length() / 10, 2);
if (OB_FAIL(tokenize_helper.init(&mem_context_->get_arena_allocator(), parser_name))) {
LOG_WARN("failed to init tokenize helper", K(ret));
} else if (OB_FAIL(token_map.create(ft_word_bkt_cnt, common::ObMemAttr(MTL_ID(), "FTWordMap")))) {
LOG_WARN("failed to create token map", K(ret));
} else if (OB_FAIL(tokenize_helper.segment(
cs_type, search_text_string.ptr(), search_text_string.length(), doc_length, token_map))) {
LOG_WARN("failed to segment");
} else {
for (hash::ObHashMap<ObFTWord, int64_t>::const_iterator iter = token_map.begin();
OB_SUCC(ret) && iter != token_map.end();
++iter) {
const ObFTWord &token = iter->first;
ObString token_string;
if (OB_FAIL(ob_write_string(mem_context_->get_arena_allocator(), token.get_word(), token_string))) {
LOG_WARN("failed to deep copy query token", K(ret));
} else if (OB_FAIL(query_tokens_.push_back(token_string))) {
LOG_WARN("failed to append query token", K(ret));
}
}
}
// TODO: try use this interface instead
/*
share::ObITokenStream *token_stream = nullptr;
share::ObTextAnalysisCtx query_analysis_ctx;
query_analysis_ctx.need_grouping_ = true;
query_analysis_ctx.filter_stopword_ = true;
query_analysis_ctx.cs_ = common::ObCharset::get_charset(search_text->obj_meta_.get_collation_type());
share::ObEnglishTextAnalyzer query_analyzer;
if (OB_FAIL(query_analyzer.init(query_analysis_ctx, token_analyze_alloc))) {
LOG_WARN("failed to init query text analyzer", K(ret));
} else if (OB_FAIL(query_analyzer.analyze(*search_text_datum, token_stream))) {
LOG_WARN("failed to analyze search text", K(ret), K(query_analysis_ctx), KPC(search_text_datum));
}
while (OB_SUCC(ret)) {
ObDatum token;
ObString token_string;
if (OB_FAIL(token_stream->get_next(token))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next query token", K(ret));
}
} else if (OB_FAIL(ob_write_string(token_analyze_alloc, token.get_string(), token_string))) {
LOG_WARN("failed to deep copy query token", K(ret));
} else if (OB_FAIL(query_tokens_.push_back(token_string))) {
LOG_WARN("failed to append query token", K(ret));
}
}
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to init query tokens", K(ret));
} else {
ret = OB_SUCCESS;
}
*/
LOG_DEBUG("tokenized text query:", K(ret), KPC(search_text_datum), K_(query_tokens));
}
return ret;
}
int ObDASTextRetrievalMergeIter::set_merge_iters(const ObIArray<ObDASIter *> &retrieval_iters)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret));
} else if (OB_UNLIKELY(retrieval_iters.count() != query_tokens_.count())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid retrieval iter count mismatch with query tokens", K(ret), K_(query_tokens), K(retrieval_iters));
} else if (token_iters_.count() != 0) {
ret = OB_INIT_TWICE;
LOG_WARN("set retrieval iters to a non-empty merge iter", K(ret));
} else if (0 == query_tokens_.count()) {
// no valid tokens
} else if (FALSE_IT(next_batch_iter_idxes_.set_allocator(&mem_context_->get_arena_allocator()))) {
} else if (OB_FAIL(next_batch_iter_idxes_.init(query_tokens_.count()))) {
LOG_WARN("failed to init next batch iter idxes array", K(ret));
} else if (OB_FAIL(next_batch_iter_idxes_.prepare_allocate(query_tokens_.count()))) {
LOG_WARN("failed to prepare allocate next batch iter idxes array", K(ret));
} else {
next_batch_cnt_ = query_tokens_.count();
for (int64_t i = 0; OB_SUCC(ret) && i < query_tokens_.count(); ++i) {
const ObString &query_token = query_tokens_.at(i);
ObDASTextRetrievalIter *iter = static_cast<ObDASTextRetrievalIter *>(retrieval_iters.at(i));
if (OB_FAIL(token_iters_.push_back(iter))) {
LOG_WARN("failed to append token iter to array", K(ret));
} else {
next_batch_iter_idxes_[i] = i;
}
}
}
return ret;
}
int ObDASTextRetrievalMergeIter::pull_next_batch_rows()
{
int ret = OB_SUCCESS;
ObIRIterLoserTreeItem item;
for (int64_t i = 0; OB_SUCC(ret) && i < next_batch_cnt_; ++i) {
const int64_t iter_idx = next_batch_iter_idxes_[i];
ObDASTextRetrievalIter *iter = nullptr;
if (OB_ISNULL(iter = token_iters_.at(iter_idx))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null token iter ptr", K(ret), K(iter_idx), K(token_iters_.count()));
} else if (OB_FAIL(iter->get_next_row())) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get pull next batch rows from iterator", K(ret));
} else {
ret = OB_SUCCESS;
}
} else if (OB_FAIL(fill_loser_tree_item(*iter, iter_idx, item))) {
LOG_WARN("fail to fill loser tree item", K(ret));
} else if (OB_FAIL(iter_row_heap_->push(item))) {
LOG_WARN("fail to push item to loser tree", K(ret));
}
}
if (OB_SUCC(ret)) {
if (iter_row_heap_->empty()) {
ret = OB_ITER_END;
} else if (OB_FAIL(iter_row_heap_->rebuild())) {
LOG_WARN("fail to rebuild loser tree", K(ret), K_(next_batch_cnt));
} else {
next_batch_cnt_ = 0;
}
}
return ret;
}
int ObDASTextRetrievalMergeIter::next_disjunctive_document()
{
int ret = OB_SUCCESS;
int64_t doc_cnt = 0;
bool curr_doc_end = false;
const ObIRIterLoserTreeItem *top_item = nullptr;
// Do we need to use ObExpr to collect relevance?
double cur_doc_relevance = 0.0;
while (OB_SUCC(ret) && !iter_row_heap_->empty() && !curr_doc_end) {
if (iter_row_heap_->is_unique_champion()) {
curr_doc_end = true;
}
if (OB_FAIL(iter_row_heap_->top(top_item))) {
LOG_WARN("failed to get top item from heap", K(ret));
} else {
// consider to add an expr for collectiong conjunction result between query tokens here?
cur_doc_relevance += top_item->relevance_;
next_batch_iter_idxes_[next_batch_cnt_++] = top_item->iter_idx_;
if (OB_FAIL(iter_row_heap_->pop())) {
LOG_WARN("failed to pop top item in heap", K(ret));
}
}
}
if (OB_SUCC(ret)) {
const double project_relevance = ir_ctdef_->need_calc_relevance() ? cur_doc_relevance : 1;
if (OB_FAIL(project_result(*top_item, project_relevance))) {
LOG_WARN("failed to project relevance", K(ret));
}
}
return ret;
}
int ObDASTextRetrievalMergeIter::project_result(const ObIRIterLoserTreeItem &item, const double relevance)
{
int ret = OB_SUCCESS;
// TODO: usage of doc id column is somehow weird here, since in single token retrieval iterators,
// we use doc id expr to scan doc_id column for scan document. But here after DaaT processing, we use this expr
// to record current disjunctive documents. Though current implementation can make sure lifetime is
// safe, but it's tricky and indirect to read.
// P.S we cannot allocate multiple doc id expr at cg for every query token since tokenization now is an runtime operation
ObExpr *doc_id_col = ir_ctdef_->inv_scan_doc_id_col_;
ObEvalCtx *eval_ctx = ir_rtdef_->eval_ctx_;
if (OB_ISNULL(doc_id_col) || OB_ISNULL(eval_ctx)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr to relevance proejction column",
K(ret), KP(doc_id_col), KP(eval_ctx));
} else {
ObDatum &doc_id_proj_datum = doc_id_col->locate_datum_for_write(*eval_ctx);
doc_id_proj_datum.set_string(item.doc_id_.get_string());
if (ir_ctdef_->need_proj_relevance_score()) {
ObExpr *relevance_proj_col = ir_ctdef_->relevance_proj_col_;
if (OB_ISNULL(relevance_proj_col)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null relevance proj col", K(ret));
} else {
ObDatum &relevance_proj_datum = relevance_proj_col->locate_datum_for_write(*eval_ctx);
relevance_proj_datum.set_double(relevance);
}
}
LOG_DEBUG("project one fulltext search result", K(ret), K(item));
}
return ret;
}
int ObDASTextRetrievalMergeIter::fill_loser_tree_item(
ObDASTextRetrievalIter &iter,
const int64_t iter_idx,
ObIRIterLoserTreeItem &item)
{
int ret = OB_SUCCESS;
item.iter_idx_ = iter_idx;
ObExpr *doc_id_expr = ir_ctdef_->inv_scan_doc_id_col_;
const ObDatum &doc_id_datum = doc_id_expr->locate_expr_datum(*ir_rtdef_->eval_ctx_);
if (OB_FAIL(item.doc_id_.from_string(doc_id_datum.get_string()))) {
LOG_WARN("failed to get ObDocId from string", K(ret), K(doc_id_datum), KPC(doc_id_expr));
} else if (ir_ctdef_->need_calc_relevance()) {
ObExpr *relevance_expr = ir_ctdef_->relevance_expr_;
const ObDatum &relevance_datum = relevance_expr->locate_expr_datum(*ir_rtdef_->eval_ctx_);
item.relevance_ = relevance_datum.get_double();
}
return ret;
}
int ObDASTextRetrievalMergeIter::init_total_doc_cnt_param(
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot)
{
int ret = OB_SUCCESS;
const ObDASScanCtDef *ctdef = ir_ctdef_->get_doc_id_idx_agg_ctdef();
ObDASScanRtDef *rtdef = ir_rtdef_->get_doc_id_idx_agg_rtdef();
if (!ir_ctdef_->need_calc_relevance()) {
// no need to do total doc cnt
} else if (OB_ISNULL(ctdef) || OB_ISNULL(rtdef)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected scan descriptor", K(ret));
} else {
ObTableScanParam &scan_param = whole_doc_agg_param_;
scan_param.tenant_id_ = MTL_ID();
scan_param.tx_lock_timeout_ = rtdef->tx_lock_timeout_;
scan_param.index_id_ = ctdef->ref_table_id_;
scan_param.is_get_ = false; // scan
scan_param.is_for_foreign_check_ = false;
scan_param.timeout_ = rtdef->timeout_ts_;
scan_param.scan_flag_ = rtdef->scan_flag_;
scan_param.reserved_cell_count_ = ctdef->access_column_ids_.count();
scan_param.allocator_ = &rtdef->stmt_allocator_;
scan_param.scan_allocator_ = &rtdef->scan_allocator_;
scan_param.sql_mode_ = rtdef->sql_mode_;
scan_param.frozen_version_ = rtdef->frozen_version_;
scan_param.force_refresh_lc_ = rtdef->force_refresh_lc_;
scan_param.output_exprs_ = &(ctdef->pd_expr_spec_.access_exprs_);
scan_param.calc_exprs_ = &(ctdef->pd_expr_spec_.calc_exprs_);
scan_param.aggregate_exprs_ = &(ctdef->pd_expr_spec_.pd_storage_aggregate_output_);
scan_param.table_param_ = &(ctdef->table_param_);
scan_param.op_ = rtdef->p_pd_expr_op_;
scan_param.row2exprs_projector_ = rtdef->p_row2exprs_projector_;
scan_param.schema_version_ = ctdef->schema_version_;
scan_param.tenant_schema_version_ = rtdef->tenant_schema_version_;
scan_param.limit_param_ = rtdef->limit_param_;
scan_param.need_scn_ = rtdef->need_scn_;
scan_param.pd_storage_flag_ = ctdef->pd_expr_spec_.pd_storage_flag_.pd_flag_;
scan_param.fb_snapshot_ = rtdef->fb_snapshot_;
scan_param.fb_read_tx_uncommitted_ = rtdef->fb_read_tx_uncommitted_;
scan_param.ls_id_ = ls_id_;
scan_param.tablet_id_ = doc_id_idx_tablet_id_;
if (ctdef->pd_expr_spec_.pushdown_filters_.empty()) {
scan_param.op_filters_ = &ctdef->pd_expr_spec_.pushdown_filters_;
}
scan_param.pd_storage_filters_ = rtdef->p_pd_expr_op_->pd_storage_filters_;
if (OB_NOT_NULL(tx_desc)) {
scan_param.tx_id_ = tx_desc->get_tx_id();
} else {
scan_param.tx_id_.reset();
}
if (OB_NOT_NULL(snapshot)) {
scan_param.snapshot_ = *snapshot;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("null snapshot", K(ret), KP(snapshot));
}
if (FAILEDx(scan_param.column_ids_.assign(ctdef->access_column_ids_))) {
LOG_WARN("failed to init column ids", K(ret));
}
}
return ret;
}
int ObDASTextRetrievalMergeIter::do_total_doc_cnt()
{
int ret = OB_SUCCESS;
if (!ir_ctdef_->need_calc_relevance()) {
// skip
} else if (ir_ctdef_->need_do_total_doc_cnt()) {
// When estimation info not exist, or we found estimation info not accurate, calculate document count by scan
if (OB_FAIL(init_total_doc_cnt_param(tx_desc_, snapshot_))) {
LOG_WARN("failed to do total doc cnt", K(ret));
} else if (FALSE_IT(whole_doc_cnt_iter_->set_scan_param(whole_doc_agg_param_))) {
} else if (OB_FAIL(whole_doc_cnt_iter_->do_table_scan())) {
LOG_WARN("failed to do table scan for document count aggregation", K(ret));
} else {
if (OB_UNLIKELY(!static_cast<sql::ObStoragePushdownFlag>(whole_doc_agg_param_.pd_storage_flag_).is_aggregate_pushdown())) {
ret = OB_NOT_IMPLEMENT;
LOG_ERROR("aggregate without pushdown not implemented", K(ret));
} else if (OB_FAIL(whole_doc_cnt_iter_->get_next_row())) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get aggregated row from iter", K(ret));
}
}
}
} else {
// use estimated document count for relevance estimation
// Need to note that when total doc count is under estimated too much, the IDF component in BM25
// would be invalidate and result to token frequence have major influence on final relevance score
ObExpr *total_doc_cnt_expr = ir_ctdef_->get_doc_id_idx_agg_ctdef()->pd_expr_spec_.pd_storage_aggregate_output_.at(0);
if (OB_ISNULL(total_doc_cnt_expr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null total doc cnt expr", K(ret));
} else {
ObDatum &total_doc_cnt = total_doc_cnt_expr->locate_datum_for_write(*ir_rtdef_->eval_ctx_);
total_doc_cnt.set_int(ir_ctdef_->estimated_total_doc_cnt_);
LOG_TRACE("use estimated row count as partition document count", K(ret), K(total_doc_cnt));
}
}
return ret;
}
void ObDASTextRetrievalMergeIter::clear_evaluated_infos()
{
ObExpr *match_filter = ir_ctdef_->match_filter_;
ObEvalCtx *eval_ctx = ir_rtdef_->eval_ctx_;
if (nullptr != match_filter) {
if (match_filter->is_batch_result()) {
match_filter->get_evaluated_flags(*eval_ctx).unset(eval_ctx->get_batch_idx());
} else {
match_filter->get_eval_info(*eval_ctx).clear_evaluated_flag();
}
}
}
} // namespace sql
} // namespace oceanbase

165
src/sql/das/iter/ob_das_text_retrieval_merge_iter.h Normal file
View File

@ -0,0 +1,165 @@
/**
* Copyright (c) 2024 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OB_DAS_TEXT_RETRIEVAL_MERGE_ITER_H_
#define OB_DAS_TEXT_RETRIEVAL_MERGE_ITER_H_
#include "ob_das_iter.h"
#include "lib/container/ob_loser_tree.h"
namespace oceanbase
{
namespace sql
{
struct ObDASIRScanCtDef;
struct ObDASIRScanRtDef;
class ObDASTextRetrievalIterator;
static const int64_t OB_MAX_TEXT_RETRIEVAL_TOKEN_CNT = 256;
struct ObIRIterLoserTreeItem
{
ObIRIterLoserTreeItem();
~ObIRIterLoserTreeItem() = default;
TO_STRING_KV(K_(iter_idx), K_(relevance), K_(doc_id), K(doc_id_.get_string()));
double relevance_;
ObDocId doc_id_;
int64_t iter_idx_;
};
struct ObIRIterLoserTreeCmp
{
ObIRIterLoserTreeCmp();
virtual ~ObIRIterLoserTreeCmp();
int init();
int cmp(const ObIRIterLoserTreeItem &l, const ObIRIterLoserTreeItem &r, int64_t &cmp_ret);
private:
common::ObDatumCmpFuncType cmp_func_;
bool is_inited_;
};
typedef common::ObLoserTree<ObIRIterLoserTreeItem, ObIRIterLoserTreeCmp, OB_MAX_TEXT_RETRIEVAL_TOKEN_CNT> ObIRIterLoserTree;
struct ObDASTextRetrievalMergeIterParam : public ObDASIterParam
{
public:
ObDASTextRetrievalMergeIterParam()
: ObDASIterParam(DAS_ITER_TEXT_RETRIEVAL_MERGE),
ir_ctdef_(nullptr),
ir_rtdef_(nullptr),
doc_cnt_iter_(nullptr),
tx_desc_(nullptr),
snapshot_(nullptr)
{}
virtual bool is_valid() const override
{
return nullptr != ir_ctdef_ && nullptr != ir_rtdef_;
}
const ObDASIRScanCtDef *ir_ctdef_;
ObDASIRScanRtDef *ir_rtdef_;
ObDASIter *doc_cnt_iter_;
transaction::ObTxDesc *tx_desc_;
transaction::ObTxReadSnapshot *snapshot_;
};
class ObDASTextRetrievalMergeIter : public ObDASIter
{
public:
enum TokenRelationType
{
DISJUNCTIVE = 0,
// CONJUNCTIVE = 1,
// BOOLEAN = 2,
MAX_RELATION_TYPE
};
enum RetrievalProcType
{
DAAT = 0,
// TAAT = 1,
// VAAT = 2,
MAX_PROC_TYPE
};
public:
ObDASTextRetrievalMergeIter();
virtual ~ObDASTextRetrievalMergeIter() {}
virtual int do_table_scan() override;
virtual int rescan() override;
void set_doc_id_idx_tablet_id(const ObTabletID &tablet_id) { doc_id_idx_tablet_id_ = tablet_id; }
void set_ls_id(const ObLSID &ls_id) { ls_id_ = ls_id; }
storage::ObTableScanParam &get_doc_agg_param() { return whole_doc_agg_param_; }
int set_related_tablet_ids(const ObLSID &ls_id, const ObDASRelatedTabletID &related_tablet_ids);
int set_merge_iters(const ObIArray<ObDASIter *> &retrieval_iters);
const ObIArray<ObString> &get_query_tokens() { return query_tokens_; }
protected:
virtual int inner_init(ObDASIterParam &param) override;
virtual int inner_reuse() override;
virtual int inner_release() override;
virtual int inner_get_next_row() override;
virtual int inner_get_next_rows(int64_t &count, int64_t capacity) override;
private:
int init_iters(
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot,
const ObIArray<ObString> &query_tokens);
int init_query_tokens(const ObDASIRScanCtDef *ir_ctdef, ObDASIRScanRtDef *ir_rtdef);
void release_iters();
int pull_next_batch_rows();
int fill_loser_tree_item(
ObDASTextRetrievalIter &iter,
const int64_t iter_idx,
ObIRIterLoserTreeItem &item);
int next_disjunctive_document();
int init_total_doc_cnt_param(transaction::ObTxDesc *tx_desc, transaction::ObTxReadSnapshot *snapshot);
int do_total_doc_cnt();
int project_result(const ObIRIterLoserTreeItem &item, const double relevance);
void clear_evaluated_infos();
private:
static const int64_t OB_DEFAULT_QUERY_TOKEN_ITER_CNT = 4;
typedef ObSEArray<ObDASTextRetrievalIter *, OB_DEFAULT_QUERY_TOKEN_ITER_CNT> ObDASTokenRetrievalIterArray;
lib::MemoryContext mem_context_;
TokenRelationType relation_type_;
RetrievalProcType processing_type_;
ObIAllocator *allocator_;
const ObDASIRScanCtDef *ir_ctdef_;
ObDASIRScanRtDef *ir_rtdef_;
transaction::ObTxDesc *tx_desc_;
transaction::ObTxReadSnapshot *snapshot_;
share::ObLSID ls_id_;
common::ObTabletID doc_id_idx_tablet_id_;
ObArray<ObString> query_tokens_;
ObDASTokenRetrievalIterArray token_iters_;
ObIRIterLoserTreeCmp loser_tree_cmp_;
ObIRIterLoserTree *iter_row_heap_;
ObFixedArray<int64_t, ObIAllocator> next_batch_iter_idxes_;
int64_t next_batch_cnt_;
ObDASScanIter *whole_doc_cnt_iter_;
ObTableScanParam whole_doc_agg_param_;
common::ObLimitParam limit_param_;
int64_t input_row_cnt_;
int64_t output_row_cnt_;
bool doc_cnt_calculated_;
bool is_inited_;
};
} // namespace sql
} // namespace oceanbase
#endif // OB_DAS_TEXT_RETRIEVAL_MERGE_ITER_H_

2
src/sql/das/ob_das_factory.cpp
View File

@ -23,7 +23,7 @@
#include "sql/das/ob_das_rpc_processor.h"
#include "sql/das/ob_das_ref.h"
#include "sql/das/ob_das_attach_define.h"
#include "sql/das/ob_text_retrieval_op.h"
#include "sql/das/ob_das_ir_define.h"
#include "share/datum/ob_datum_util.h"
#define STORE_DAS_OBJ(obj_store, das_obj, class_name) \

39
src/sql/das/ob_das_ir_define.cpp Normal file
View File

@ -0,0 +1,39 @@
/**
* Copyright (c) 2024 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_DAS
#include "ob_das_ir_define.h"
namespace oceanbase
{
namespace sql
{
OB_SERIALIZE_MEMBER((ObDASIRScanCtDef, ObDASAttachCtDef),
flags_,
search_text_,
inv_scan_doc_id_col_,
inv_scan_doc_length_col_,
match_filter_,
relevance_expr_,
relevance_proj_col_,
estimated_total_doc_cnt_);
OB_SERIALIZE_MEMBER(ObDASIRScanRtDef);
OB_SERIALIZE_MEMBER((ObDASIRAuxLookupCtDef, ObDASAttachCtDef),
relevance_proj_col_);
OB_SERIALIZE_MEMBER((ObDASIRAuxLookupRtDef, ObDASAttachRtDef));
} // sql
} // oceanbase

190
src/sql/das/ob_text_retrieval_op.h → src/sql/das/ob_das_ir_define.h
View File

@ -10,21 +10,15 @@
* See the Mulan PubL v2 for more details.
*/
#ifndef OBDEV_SRC_SQL_DAS_OB_TEXT_RETRIEVAL_OP_H_
#define OBDEV_SRC_SQL_DAS_OB_TEXT_RETRIEVAL_OP_H_
#ifndef OB_DAS_IR_DEFINE_H_
#define OB_DAS_IR_DEFINE_H_
#include "lib/container/ob_loser_tree.h"
#include "sql/das/ob_das_task.h"
#include "sql/das/ob_das_scan_op.h"
#include "sql/das/ob_das_attach_define.h"
#include "sql/engine/sort/ob_sort_op_impl.h"
#include "storage/fts/ob_text_retrieval_iterator.h"
#include "ob_das_attach_define.h"
namespace oceanbase
{
namespace sql
{
static const int64_t OB_MAX_TEXT_RETRIEVAL_TOKEN_CNT = 256;
struct ObDASIRScanCtDef : ObDASAttachCtDef
{
@ -44,6 +38,8 @@ public:
}
bool need_calc_relevance() const { return nullptr != relevance_expr_; }
bool need_proj_relevance_score() const { return nullptr != relevance_proj_col_; }
bool need_fwd_idx_agg() const { return has_fwd_agg_ && need_calc_relevance(); }
bool need_inv_idx_agg() const { return has_inv_agg_ && need_calc_relevance(); }
const ObDASScanCtDef *get_inv_idx_scan_ctdef() const
{
const ObDASScanCtDef *idx_scan_ctdef = nullptr;
@ -92,7 +88,7 @@ public:
int64_t get_inv_agg_idx() const { return has_inv_agg_ ? 1 : -1; }
int64_t get_doc_agg_idx() const { return has_doc_id_agg_ ? (1 + has_inv_agg_) : -1; }
int64_t get_fwd_agg_idx() const { return has_fwd_agg_ ? (1 + has_inv_agg_ + has_doc_id_agg_) : -1; }
bool need_do_total_doc_cnt() const { return 0 == estimated_total_doc_cnt_; }
bool need_do_total_doc_cnt() const { return need_calc_relevance() && 0 == estimated_total_doc_cnt_; }
INHERIT_TO_STRING_KV("ObDASBaseCtDef", ObDASBaseCtDef,
K_(flags),
@ -220,180 +216,6 @@ public:
}
};
struct ObIRIterLoserTreeItem
{
ObIRIterLoserTreeItem();
~ObIRIterLoserTreeItem() = default;
TO_STRING_KV(K_(iter_idx), K_(relevance), K_(doc_id), K(doc_id_.get_string()));
double relevance_;
ObDocId doc_id_;
int64_t iter_idx_;
};
struct ObIRIterLoserTreeCmp
{
ObIRIterLoserTreeCmp();
virtual ~ObIRIterLoserTreeCmp();
int init();
int cmp(const ObIRIterLoserTreeItem &l, const ObIRIterLoserTreeItem &r, int64_t &cmp_ret);
private:
common::ObDatumCmpFuncType cmp_func_;
bool is_inited_;
};
typedef common::ObLoserTree<ObIRIterLoserTreeItem, ObIRIterLoserTreeCmp, OB_MAX_TEXT_RETRIEVAL_TOKEN_CNT> ObIRIterLoserTree;
class ObTextRetrievalMerge : public common::ObNewRowIterator
{
public:
enum TokenRelationType
{
DISJUNCTIVE = 0,
// CONJUNCTIVE = 1,
// BOOLEAN = 2,
MAX_RELATION_TYPE
};
enum RetrievalProcType
{
DAAT = 0,
// TAAT = 1,
// VAAT = 2,
MAX_PROC_TYPE
};
public:
ObTextRetrievalMerge();
virtual ~ObTextRetrievalMerge();
int init(
const share::ObLSID &ls_id,
const ObTabletID &inv_idx_tablet_id,
const ObTabletID &fwd_idx_tablet_id,
const ObTabletID &doc_id_idx_tablet_id,
const ObDASIRScanCtDef *ir_ctdef,
ObDASIRScanRtDef *ir_rtdef,
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot,
ObIAllocator &allocator);
int rescan(
const share::ObLSID &ls_id,
const ObTabletID &inv_idx_tablet_id,
const ObTabletID &fwd_idx_tablet_id,
const ObTabletID &doc_id_idx_tablet_id,
const ObDASIRScanCtDef *ir_ctdef,
ObDASIRScanRtDef *ir_rtdef,
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot,
ObIAllocator &allocator);
virtual int get_next_row(ObNewRow *&row) override;
virtual int get_next_row() override { ObNewRow *r = nullptr; return get_next_row(r); }
virtual int get_next_rows(int64_t &count, int64_t capacity) override;
virtual void reset() override;
private:
int init_iter_params(
const share::ObLSID &ls_id,
const ObTabletID &inv_idx_tablet_id,
const ObTabletID &fwd_idx_tablet_id,
const ObTabletID &doc_id_idx_tablet_id,
const ObDASIRScanCtDef *ir_ctdef,
ObDASIRScanRtDef *ir_rtdef);
int init_iters(
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot,
const ObIArray<ObString> &query_tokens);
int init_query_tokens(const ObDASIRScanCtDef *ir_ctdef, ObDASIRScanRtDef *ir_rtdef);
void release_iters();
int pull_next_batch_rows();
int fill_loser_tree_item(
storage::ObTextRetrievalIterator &iter,
const int64_t iter_idx,
ObIRIterLoserTreeItem &item);
int next_disjunctive_document();
int init_total_doc_cnt_param(transaction::ObTxDesc *tx_desc, transaction::ObTxReadSnapshot *snapshot);
int do_total_doc_cnt();
int project_result(const ObIRIterLoserTreeItem &item, const double relevance);
void clear_evaluated_infos();
private:
static const int64_t OB_DEFAULT_QUERY_TOKEN_ITER_CNT = 4;
typedef ObSEArray<storage::ObTextRetrievalIterator *, OB_DEFAULT_QUERY_TOKEN_ITER_CNT> ObTokenRetrievalIterArray;
TokenRelationType relation_type_;
RetrievalProcType processing_type_;
ObIAllocator *allocator_;
ObTokenRetrievalParam retrieval_param_;
ObArray<ObString> query_tokens_;
ObTokenRetrievalIterArray token_iters_;
ObIRIterLoserTreeCmp loser_tree_cmp_;
ObIRIterLoserTree *iter_row_heap_;
ObFixedArray<int64_t, ObIAllocator> next_batch_iter_idxes_;
int64_t next_batch_cnt_;
common::ObNewRowIterator *whole_doc_cnt_iter_;
ObTableScanParam whole_doc_agg_param_;
bool doc_cnt_calculated_;
bool is_inited_;
};
class ObTextRetrievalOp : public common::ObNewRowIterator
{
public:
ObTextRetrievalOp();
virtual ~ObTextRetrievalOp();
int init(
const share::ObLSID &ls_id,
const ObTabletID &inv_idx_tablet_id,
const ObTabletID &fwd_idx_tablet_id,
const ObTabletID &doc_id_idx_tablet_id,
const ObDASIRScanCtDef *ir_ctdef,
ObDASIRScanRtDef *ir_rtdef,
const ObDASSortCtDef *sort_ctdef,
ObDASSortRtDef *sort_rtdef,
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot);
int rescan(
const share::ObLSID &ls_id,
const ObTabletID &inv_idx_tablet_id,
const ObTabletID &fwd_idx_tablet_id,
const ObTabletID &doc_id_idx_tablet_id,
const ObDASIRScanCtDef *ir_ctdef,
ObDASIRScanRtDef *ir_rtdef,
const ObDASSortCtDef *sort_ctdef,
ObDASSortRtDef *sort_rtdef,
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot);
virtual int get_next_row(ObNewRow *&row) override;
virtual int get_next_row() override { ObNewRow *r = nullptr; return get_next_row(r); }
virtual int get_next_rows(int64_t &count, int64_t capacity) override;
virtual void reset() override;
private:
int inner_get_next_row_for_output();
int init_sort(
const ObDASIRScanCtDef *ir_ctdef,
const ObDASSortCtDef *sort_ctdef,
ObDASSortRtDef *sort_rtdef);
int init_limit(
const ObDASIRScanCtDef *ir_ctdef,
ObDASIRScanRtDef *ir_rtdef,
const ObDASSortCtDef *sort_ctdef,
ObDASSortRtDef *sort_rtdef);
int do_sort();
private:
lib::MemoryContext mem_context_;
ObTextRetrievalMerge token_merge_;
common::ObLimitParam limit_param_;
int64_t input_row_cnt_;
int64_t output_row_cnt_;
ObSortOpImpl *sort_impl_;
ObSEArray<ObExpr *, 2> sort_row_;
bool sort_finished_;
bool is_inited_;
};
} // namespace sql
} // namespace oceanbase

448
src/sql/das/ob_das_scan_op.cpp
View File

@ -21,6 +21,7 @@
#include "sql/engine/ob_des_exec_context.h"
#include "storage/access/ob_table_scan_iterator.h"
#include "storage/concurrency_control/ob_data_validation_service.h"
#include "sql/das/iter/ob_das_iter_utils.h"
namespace oceanbase
{
@ -320,6 +321,42 @@ ObITabletScan &ObDASScanOp::get_tsc_service()
: *(MTL(ObAccessService *));
}
ObDASIterTreeType ObDASScanOp::get_iter_tree_type() const
{
ObDASIterTreeType tree_type = ObDASIterTreeType::ITER_TREE_INVALID;
bool is_fts_index = scan_param_.table_param_->is_fts_index() && attach_ctdef_ != nullptr;
bool is_spatial_index = scan_param_.table_param_->is_spatial_index();
bool is_multivalue_index = scan_param_.table_param_->is_multivalue_index();
if (is_fts_index) {
tree_type = ObDASIterTreeType::ITER_TREE_TEXT_RETRIEVAL;
} else if (is_spatial_index) {
tree_type = ObDASIterTreeType::ITER_TREE_GIS_LOOKUP;
} else if (is_multivalue_index) {
tree_type = ObDASIterTreeType::ITER_TREE_DOMAIN_LOOKUP;
} else {
tree_type = OB_ISNULL(get_lookup_ctdef()) ? ObDASIterTreeType::ITER_TREE_PARTITION_SCAN
: ObDASIterTreeType::ITER_TREE_LOCAL_LOOKUP;
}
return tree_type;
}
int ObDASScanOp::init_related_tablet_ids(ObDASRelatedTabletID &related_tablet_ids)
{
int ret = OB_SUCCESS;
related_tablet_ids.reset();
if (OB_FAIL(get_table_lookup_tablet_id(related_tablet_ids.lookup_tablet_id_))) {
LOG_WARN("failed to get table lookup tablet id", K(ret));
} else if (OB_ISNULL(attach_ctdef_) || OB_ISNULL(attach_rtdef_)) { // no attached task.
} else if (OB_FAIL(get_aux_lookup_tablet_id(related_tablet_ids.aux_lookup_tablet_id_))) {
LOG_WARN("failed to get aux lookup tablet id", K(ret));
} else if (OB_FAIL(get_text_ir_tablet_ids(related_tablet_ids.inv_idx_tablet_id_,
related_tablet_ids.fwd_idx_tablet_id_,
related_tablet_ids.doc_id_idx_tablet_id_))) {
LOG_WARN("failed to get text ir tablet ids", K(ret));
}
return ret;
}
int ObDASScanOp::open_op()
{
int ret = OB_SUCCESS;
@ -330,8 +367,35 @@ int ObDASScanOp::open_op()
init_retry_alloc();
}
reset_access_datums_ptr();
ObDASIterTreeType tree_type = ITER_TREE_INVALID;
if (OB_FAIL(init_scan_param())) {
LOG_WARN("init scan param failed", K(ret));
} else if (FALSE_IT(tree_type = get_iter_tree_type())) {
} else if (ITER_TREE_PARTITION_SCAN == tree_type || ITER_TREE_LOCAL_LOOKUP == tree_type
|| ITER_TREE_TEXT_RETRIEVAL == tree_type) {
ObDASIter *result = nullptr;
if (OB_FAIL(init_related_tablet_ids(tablet_ids_))) {
LOG_WARN("failed to init related tablet ids", K(ret));
} else if (OB_FAIL(ObDASIterUtils::create_das_scan_iter_tree(tree_type,
scan_param_,
scan_ctdef_,
scan_rtdef_,
get_lookup_ctdef(),
get_lookup_rtdef(),
attach_ctdef_,
attach_rtdef_,
tablet_ids_,
trans_desc_,
snapshot_,
op_alloc_,
result))) {
LOG_WARN("failed to create das scan iter tree", K(get_iter_tree_type()), K(ret));
} else {
result_ = result;
if (OB_FAIL(result->do_table_scan())) {
LOG_WARN("iter tree failed to do table scan", K(ret));
}
}
} else if (OB_FAIL(do_table_scan())) {
if (OB_SNAPSHOT_DISCARDED == ret && scan_param_.fb_snapshot_.is_valid()) {
ret = OB_INVALID_QUERY_TIMESTAMP;
@ -343,46 +407,58 @@ int ObDASScanOp::open_op()
LOG_WARN("do local index lookup failed", K(ret));
}
}
int simulate_error = EVENT_CALL(EventTable::EN_DAS_SIMULATE_LOOKUPOP_INIT_ERROR);
if (OB_UNLIKELY(OB_SUCCESS != simulate_error)) {
ret = simulate_error;
}
return ret;
}
int ObDASScanOp::release_op()
{
int ret = OB_SUCCESS;
int lookup_ret = OB_SUCCESS;
ObITabletScan &tsc_service = get_tsc_service();
if (result_ != nullptr) {
if (ObNewRowIterator::IterType::ObLocalIndexLookupIterator == result_->get_type() ||
ObNewRowIterator::IterType::ObGroupLookupOp == result_->get_type()) {
ObLocalIndexLookupOp *lookup_op = static_cast<ObLocalIndexLookupOp*>(result_);
ObDASIterTreeType tree_type = get_iter_tree_type();
if (ITER_TREE_PARTITION_SCAN == tree_type || ITER_TREE_LOCAL_LOOKUP == tree_type
|| ITER_TREE_TEXT_RETRIEVAL == tree_type) {
if (OB_NOT_NULL(result_)) {
ObDASIter *result = static_cast<ObDASIter*>(result_);
if (OB_FAIL(result->release())) {
LOG_WARN("failed to release das iter tree", K(ret));
}
}
} else {
int lookup_ret = OB_SUCCESS;
ObITabletScan &tsc_service = get_tsc_service();
if (result_ != nullptr) {
if (ObNewRowIterator::IterType::ObLocalIndexLookupIterator == result_->get_type() ||
ObNewRowIterator::IterType::ObGroupLookupOp == result_->get_type()) {
ObLocalIndexLookupOp *lookup_op = static_cast<ObLocalIndexLookupOp*>(result_);
ret = tsc_service.revert_scan_iter(lookup_op->get_rowkey_iter());
if (OB_SUCCESS != ret) {
LOG_WARN("revert scan iterator failed", K(ret));
}
ret = tsc_service.revert_scan_iter(lookup_op->get_rowkey_iter());
if (OB_SUCCESS != ret) {
LOG_WARN("revert scan iterator failed", K(ret));
}
lookup_ret = lookup_op->revert_iter();
if (OB_SUCCESS != lookup_ret) {
LOG_WARN("revert lookup iterator failed", K(lookup_ret));
}
lookup_ret = lookup_op->revert_iter();
if (OB_SUCCESS != lookup_ret) {
LOG_WARN("revert lookup iterator failed", K(lookup_ret));
}
result_ = nullptr;
//if row_key revert is succ return look_up iter ret code.
//if row_key revert is fail return row_key iter ret code.
//In short if row_key and look_up iter all revert fail.
//We just ignore lookup_iter ret code.
if (OB_SUCCESS == ret) {
ret = lookup_ret;
result_ = nullptr;
//if row_key revert is succ return look_up iter ret code.
//if row_key revert is fail return row_key iter ret code.
//In short if row_key and look_up iter all revert fail.
//We just ignore lookup_iter ret code.
if (OB_SUCCESS == ret) {
ret = lookup_ret;
}
} else {
if (OB_FAIL(tsc_service.revert_scan_iter(result_))) {
LOG_WARN("revert scan iterator failed", K(ret));
}
result_ = nullptr;
}
} else if (ObNewRowIterator::IterType::ObTextRetrievalOp == result_->get_type()) {
ObTextRetrievalOp *text_retrieval_op = static_cast<ObTextRetrievalOp *>(result_);
text_retrieval_op->reset();
result_ = nullptr;
} else {
if (OB_FAIL(tsc_service.revert_scan_iter(result_))) {
LOG_WARN("revert scan iterator failed", K(ret));
}
result_ = nullptr;
}
}
//need to clear the flag:need_switch_param_
@ -440,12 +516,7 @@ ObLocalIndexLookupOp *ObDASScanOp::get_lookup_op()
int ObDASScanOp::do_table_scan()
{
int ret = OB_SUCCESS;
if (scan_param_.table_param_->is_fts_index() && attach_ctdef_ != nullptr) {
// full text index retrieval scan
if (OB_FAIL(do_text_retrieve(result_))) {
LOG_WARN("fail to retrieve token from full text index", K(ret));
}
} else if (OB_FAIL(get_tsc_service().table_scan(scan_param_, result_))) {
if (OB_FAIL(get_tsc_service().table_scan(scan_param_, result_))) {
if (OB_SNAPSHOT_DISCARDED == ret && scan_param_.fb_snapshot_.is_valid()) {
ret = OB_INVALID_QUERY_TIMESTAMP;
} else if (OB_TRY_LOCK_ROW_CONFLICT != ret) {
@ -458,8 +529,8 @@ int ObDASScanOp::do_table_scan()
int ObDASScanOp::do_local_index_lookup()
{
int ret = OB_SUCCESS;
if (scan_param_.table_param_->is_fts_index() ||
scan_param_.table_param_->is_multivalue_index()) {
ObTabletID lookup_tablet_id;
if (scan_param_.table_param_->is_multivalue_index()) {
if (OB_FAIL(do_domain_index_lookup())) {
LOG_WARN("failed to do domain index lookup", K(ret));
}
@ -475,8 +546,9 @@ int ObDASScanOp::do_local_index_lookup()
if (OB_FAIL(op->init(get_lookup_ctdef(), get_lookup_rtdef(), scan_ctdef_, scan_rtdef_,
trans_desc_, snapshot_, scan_param_))) {
LOG_WARN("init spatial lookup op failed", K(ret));
} else if (FALSE_IT(get_table_lookup_tablet_id(lookup_tablet_id))) {
} else {
op->set_tablet_id(get_table_lookup_tablet_id());
op->set_tablet_id(lookup_tablet_id);
op->set_ls_id(ls_id_);
}
}
@ -496,8 +568,9 @@ int ObDASScanOp::do_local_index_lookup()
trans_desc_,
snapshot_))) {
LOG_WARN("init lookup op failed", K(ret));
} else if (FALSE_IT(get_table_lookup_tablet_id(lookup_tablet_id))) {
} else {
op->set_tablet_id(get_table_lookup_tablet_id());
op->set_tablet_id(lookup_tablet_id);
op->set_ls_id(ls_id_);
}
}
@ -509,27 +582,8 @@ int ObDASScanOp::do_domain_index_lookup()
{
int ret = OB_SUCCESS;
ObTabletID doc_id_idx_tablet_id;
if (scan_param_.table_param_->is_fts_index()) {
ObFullTextIndexLookupOp *op = nullptr;
ObTabletID doc_id_idx_tablet_id;
const ObDASTableLookupCtDef *table_lookup_ctdef = nullptr;
ObDASTableLookupRtDef *table_lookup_rtdef = nullptr;
if (OB_FAIL(get_aux_lookup_tablet_id(doc_id_idx_tablet_id))) {
LOG_WARN("failed to get doc id idx tablet id", K(ret), K_(related_tablet_ids));
} else if (OB_ISNULL(op = OB_NEWx(ObFullTextIndexLookupOp, &op_alloc_, op_alloc_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate full text index lookup op", K(ret));
} else if (FALSE_IT(op->set_text_retrieval_iter(result_))) {
} else if (FALSE_IT(result_ = op)) {
} else if (OB_FAIL(op->init(attach_ctdef_, attach_rtdef_, trans_desc_, snapshot_, scan_param_))) {
LOG_WARN("failed to init full text index lookup op", K(ret));
} else {
op->set_tablet_id(get_table_lookup_tablet_id());
op->set_doc_id_idx_tablet_id(doc_id_idx_tablet_id);
op->set_ls_id(ls_id_);
}
} else if (scan_param_.table_param_->is_multivalue_index()) {
ObTabletID lookup_tablet_id;
if (scan_param_.table_param_->is_multivalue_index()) {
ObMulValueIndexLookupOp* op = nullptr;
if (OB_FAIL(get_aux_lookup_tablet_id(doc_id_idx_tablet_id))) {
LOG_WARN("failed to get doc id idx tablet id", K(ret), K_(related_tablet_ids));
@ -540,8 +594,9 @@ int ObDASScanOp::do_domain_index_lookup()
} else if (FALSE_IT(result_ = op)) {
} else if (OB_FAIL(op->init(attach_ctdef_, attach_rtdef_, trans_desc_, snapshot_, scan_param_))) {
LOG_WARN("failed to init multivalue index lookup op", K(ret));
} else if (FALSE_IT(get_table_lookup_tablet_id(lookup_tablet_id))) {
} else {
op->set_tablet_id(get_table_lookup_tablet_id());
op->set_tablet_id(lookup_tablet_id);
op->set_doc_id_idx_tablet_id(doc_id_idx_tablet_id);
op->set_ls_id(ls_id_);
}
@ -769,19 +824,30 @@ int ObDASScanOp::rescan()
"tablet_id", scan_param_.tablet_id_,
"scan_range", scan_param_.key_ranges_,
"range_pos", scan_param_.range_array_pos_);
ObLocalIndexLookupOp *lookup_op = get_lookup_op();
if (scan_param_.table_param_->is_fts_index() && attach_ctdef_ != nullptr) {
if (OB_FAIL(do_text_retrieve_rescan())) {
LOG_WARN("failed to do text retrieval rescan", K(ret));
ObDASIterTreeType tree_type = get_iter_tree_type();
if (ITER_TREE_PARTITION_SCAN == tree_type || ITER_TREE_LOCAL_LOOKUP == tree_type
|| ITER_TREE_TEXT_RETRIEVAL == tree_type) {
if (OB_ISNULL(result_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr das iter tree", K(ret));
} else {
ObDASIter *result = static_cast<ObDASIter*>(result_);
if (OB_FAIL(result->rescan())) {
LOG_WARN("failed to rescan das iter", K(ret));
}
}
} else {
ObLocalIndexLookupOp *lookup_op = get_lookup_op();
ObTabletID lookup_tablet_id;
if (OB_FAIL(tsc_service.table_rescan(scan_param_, get_storage_scan_iter()))) {
LOG_WARN("rescan the table iterator failed", K(ret));
} else if (lookup_op != nullptr) {
OZ(get_table_lookup_tablet_id(lookup_tablet_id));
lookup_op->set_tablet_id(lookup_tablet_id);
lookup_op->set_ls_id(ls_id_);
}
} else if (OB_FAIL(tsc_service.table_rescan(scan_param_, get_storage_scan_iter()))) {
LOG_WARN("rescan the table iterator failed", K(ret));
} else if (lookup_op != nullptr) {
lookup_op->set_tablet_id(get_table_lookup_tablet_id());
lookup_op->set_ls_id(ls_id_);
//lookup op's table_rescan will be drive by its get_next_row()
//so will can not call it here
}
return ret;
}
@ -791,15 +857,54 @@ int ObDASScanOp::reuse_iter()
//May be retry change to retry alloc.
//Change back.
scan_param_.scan_allocator_ = &scan_rtdef_->scan_allocator_;
ObITabletScan &tsc_service = get_tsc_service();
ObLocalIndexLookupOp *lookup_op = get_lookup_op();
const ObTabletID &storage_tablet_id = scan_param_.tablet_id_;
scan_param_.need_switch_param_ = (storage_tablet_id.is_valid() && storage_tablet_id != tablet_id_ ? true : false);
if (scan_param_.table_param_->is_fts_index() && attach_ctdef_ != nullptr) {
if (nullptr != lookup_op
&& OB_FAIL(static_cast<ObFullTextIndexLookupOp *>(lookup_op)->reuse_scan_iter())) {
LOG_WARN("failed to reuse text lookup iters", K(ret));
scan_param_.key_ranges_.reuse();
scan_param_.ss_key_ranges_.reuse();
scan_param_.mbr_filters_.reuse();
ObDASIterTreeType tree_type = get_iter_tree_type();
ObITabletScan &tsc_service = get_tsc_service();
ObLocalIndexLookupOp *lookup_op = get_lookup_op();
if (ITER_TREE_PARTITION_SCAN == tree_type || ITER_TREE_LOCAL_LOOKUP == tree_type
|| ITER_TREE_TEXT_RETRIEVAL == tree_type) {
if (OB_ISNULL(result_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr das iter tree", K(ret));
} else {
ObDASIter *result = static_cast<ObDASIter*>(result_);
if (OB_FAIL(init_related_tablet_ids(tablet_ids_))) {
LOG_WARN("fail to init related tablet ids", K(ret));
} else {
switch (get_iter_tree_type()) {
case ITER_TREE_PARTITION_SCAN: {
break;
}
case ITER_TREE_LOCAL_LOOKUP: {
ObDASLocalLookupIter *lookup_iter = static_cast<ObDASLocalLookupIter*>(result_);
lookup_iter->set_tablet_id(tablet_ids_.lookup_tablet_id_);
lookup_iter->set_ls_id(ls_id_);
break;
}
case ITER_TREE_DOMAIN_LOOKUP: {
break;
}
case ITER_TREE_TEXT_RETRIEVAL: {
ObDASIter *result_iter = static_cast<ObDASIter *>(result_);
if (OB_FAIL(ObDASIterUtils::set_text_retrieval_related_ids(
attach_ctdef_, tablet_ids_, ls_id_, result_iter))) {
LOG_WARN("failed to set text retrieval related ids", K(ret));
}
break;
}
default: {
ret = OB_ERR_UNEXPECTED;
}
}
}
if (OB_SUCC(ret) && OB_FAIL(result->reuse())) {
LOG_WARN("failed to reuse das iter tree", K(ret));
}
}
} else if (scan_param_.table_param_->is_multivalue_index() && attach_ctdef_ != nullptr) {
if (nullptr != lookup_op
@ -811,11 +916,8 @@ int ObDASScanOp::reuse_iter()
} else if (lookup_op != nullptr
&& OB_FAIL(lookup_op->reset_lookup_state())) {
LOG_WARN("reuse lookup iterator failed", K(ret));
} else {
scan_param_.key_ranges_.reuse();
scan_param_.ss_key_ranges_.reuse();
scan_param_.mbr_filters_.reuse();
}
return ret;
}
@ -899,14 +1001,13 @@ int ObDASScanOp::get_aux_lookup_tablet_id(common::ObTabletID &tablet_id) const
int ret = OB_SUCCESS;
tablet_id.reset();
const ObDASIRAuxLookupCtDef *aux_lookup_ctdef = nullptr;
ObDASIRAuxLookupRtDef *aux_lookup_rtdef = nullptr;
if (OB_FAIL(ObDASUtils::find_target_das_def(attach_ctdef_,
attach_rtdef_,
DAS_OP_IR_AUX_LOOKUP,
aux_lookup_ctdef,
aux_lookup_rtdef))) {
LOG_WARN("find aux lookup definition failed", K(ret));
} else {
if (nullptr != attach_ctdef_ && ObDASOpType::DAS_OP_TABLE_LOOKUP == attach_ctdef_->op_type_) {
const ObDASTableLookupCtDef *table_lookup_ctdef = static_cast<const ObDASTableLookupCtDef*>(attach_ctdef_);
if (ObDASOpType::DAS_OP_IR_AUX_LOOKUP == table_lookup_ctdef->get_rowkey_scan_ctdef()->op_type_) {
aux_lookup_ctdef = static_cast<const ObDASIRAuxLookupCtDef *>(table_lookup_ctdef->get_rowkey_scan_ctdef());
} else if (ObDASOpType::DAS_OP_SORT == table_lookup_ctdef->get_rowkey_scan_ctdef()->op_type_) {
aux_lookup_ctdef = static_cast<const ObDASIRAuxLookupCtDef *>(table_lookup_ctdef->get_rowkey_scan_ctdef()->children_[0]);
}
for (int i = 0; !tablet_id.is_valid() && i < related_ctdefs_.count(); ++i) {
if (aux_lookup_ctdef->get_lookup_scan_ctdef() == related_ctdefs_.at(i)) {
tablet_id = related_tablet_ids_.at(i);
@ -916,9 +1017,10 @@ int ObDASScanOp::get_aux_lookup_tablet_id(common::ObTabletID &tablet_id) const
return ret;
}
ObTabletID ObDASScanOp::get_table_lookup_tablet_id() const
int ObDASScanOp::get_table_lookup_tablet_id(common::ObTabletID &tablet_id) const
{
ObTabletID tablet_id;
int ret = OB_SUCCESS;
tablet_id.reset();
if (get_lookup_ctdef() != nullptr) {
for (int i = 0; !tablet_id.is_valid() && i < related_ctdefs_.count(); ++i) {
if (get_lookup_ctdef() == related_ctdefs_.at(i)) {
@ -926,158 +1028,6 @@ ObTabletID ObDASScanOp::get_table_lookup_tablet_id() const
}
}
}
return tablet_id;
}
int ObDASScanOp::do_text_retrieve(common::ObNewRowIterator *&retrieval_iter)
{
int ret = OB_SUCCESS;
retrieval_iter = nullptr;
ObTextRetrievalOp *retrieval_op = nullptr;
ObTabletID inv_idx_tablet_id;
ObTabletID fwd_idx_tablet_id;
ObTabletID doc_id_idx_tablet_id;
const ObDASIRScanCtDef *ir_scan_ctdef = nullptr;
ObDASIRScanRtDef *ir_scan_rtdef = nullptr;
const ObDASIRAuxLookupCtDef *aux_lookup_ctdef = nullptr;
ObDASIRAuxLookupRtDef *aux_lookup_rtdef = nullptr;
const ObDASSortCtDef *sort_ctdef = nullptr;
ObDASSortRtDef *sort_rtdef = nullptr;
const bool has_lookup = nullptr != get_lookup_ctdef();
if (OB_ISNULL(retrieval_op = OB_NEWx(ObTextRetrievalOp, &op_alloc_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate text retrieval op", K(ret));
} else if (FALSE_IT(retrieval_iter = retrieval_op)) {
} else if (OB_FAIL(get_text_ir_tablet_ids(inv_idx_tablet_id, fwd_idx_tablet_id, doc_id_idx_tablet_id))) {
LOG_WARN("failed to get text ir tablet ids", K(ret));
} else if (OB_FAIL(ObDASUtils::find_target_das_def(attach_ctdef_,
attach_rtdef_,
DAS_OP_IR_SCAN,
ir_scan_ctdef,
ir_scan_rtdef))) {
LOG_WARN("find ir scan definition failed", K(ret));
} else if (has_lookup && OB_FAIL(ObDASUtils::find_target_das_def(attach_ctdef_,
attach_rtdef_,
DAS_OP_IR_AUX_LOOKUP,
aux_lookup_ctdef,
aux_lookup_rtdef))) {
LOG_WARN("find aux lookup definition failed", K(ret));
}
if (OB_SUCC(ret)) {
if (has_lookup) {
// relevance sort would be child of aux lookup if aux lookup exists
if (DAS_OP_SORT == aux_lookup_ctdef->get_doc_id_scan_ctdef()->op_type_) {
sort_ctdef = static_cast<const ObDASSortCtDef *>(aux_lookup_ctdef->get_doc_id_scan_ctdef());
sort_rtdef = static_cast<ObDASSortRtDef *>(aux_lookup_rtdef->get_doc_id_scan_rtdef());
}
} else {
// relevance sort would be the root attach ctdef if no aux/table lookup
if (DAS_OP_SORT == attach_ctdef_->op_type_) {
sort_ctdef = static_cast<const ObDASSortCtDef *>(attach_ctdef_);
sort_rtdef = static_cast<ObDASSortRtDef *>(attach_rtdef_);
}
}
}
if (FAILEDx(retrieval_op->init(ls_id_,
inv_idx_tablet_id,
fwd_idx_tablet_id,
doc_id_idx_tablet_id,
ir_scan_ctdef,
ir_scan_rtdef,
sort_ctdef,
sort_rtdef,
trans_desc_,
snapshot_))) {
LOG_WARN("failed to init text retrieval op", K(ret));
}
return ret;
}
int ObDASScanOp::do_text_retrieve_rescan()
{
int ret = OB_SUCCESS;
if (nullptr == result_ || (result_->get_type() != ObNewRowIterator::IterType::ObLocalIndexLookupIterator
&& result_->get_type() != ObNewRowIterator::IterType::ObTextRetrievalOp)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected text retrieve rescan status", K(ret), KP_(result));
} else {
ObTabletID inv_idx_tablet_id;
ObTabletID fwd_idx_tablet_id;
ObTabletID doc_id_idx_tablet_id;
ObTabletID aux_lookup_tablet_id;
const ObDASIRScanCtDef *ir_scan_ctdef = nullptr;
ObDASIRScanRtDef *ir_scan_rtdef = nullptr;
const ObDASIRAuxLookupCtDef *aux_lookup_ctdef = nullptr;
ObDASIRAuxLookupRtDef *aux_lookup_rtdef = nullptr;
const ObDASSortCtDef *sort_ctdef = nullptr;
ObDASSortRtDef *sort_rtdef = nullptr;
const bool has_lookup = nullptr != get_lookup_ctdef();
ObFullTextIndexLookupOp *text_lookup_op = has_lookup
? static_cast<ObFullTextIndexLookupOp *>(result_)
: nullptr;
ObTextRetrievalOp * text_retrieval_op = has_lookup
? static_cast<ObTextRetrievalOp *>(text_lookup_op->get_text_retrieval_iter())
: static_cast<ObTextRetrievalOp *>(result_);
if (OB_FAIL(get_text_ir_tablet_ids(inv_idx_tablet_id, fwd_idx_tablet_id, doc_id_idx_tablet_id))) {
LOG_WARN("failed to get text ir tablet ids", K(ret));
} else if (OB_FAIL(ObDASUtils::find_target_das_def(attach_ctdef_,
attach_rtdef_,
DAS_OP_IR_SCAN,
ir_scan_ctdef,
ir_scan_rtdef))) {
LOG_WARN("find ir scan definition failed", K(ret));
} else if (!has_lookup) {
// skip
} else if (OB_ISNULL(text_lookup_op)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr to text lookup op", K(ret), KPC(result_));
} else if (OB_FAIL(get_aux_lookup_tablet_id(aux_lookup_tablet_id))) {
LOG_WARN("failed to get doc id idx tablet id", K(ret), K_(related_tablet_ids));
} else if (OB_FAIL(ObDASUtils::find_target_das_def(attach_ctdef_,
attach_rtdef_,
DAS_OP_IR_AUX_LOOKUP,
aux_lookup_ctdef,
aux_lookup_rtdef))) {
LOG_WARN("find aux lookup definition failed", K(ret));
} else {
text_lookup_op->set_tablet_id(get_table_lookup_tablet_id());
text_lookup_op->set_ls_id(ls_id_);
text_lookup_op->set_doc_id_idx_tablet_id(aux_lookup_tablet_id);
}
if (OB_SUCC(ret)) {
if (has_lookup) {
// relevance sort would be child of aux lookup if aux lookup exists
if (DAS_OP_SORT == aux_lookup_ctdef->get_doc_id_scan_ctdef()->op_type_) {
sort_ctdef = static_cast<const ObDASSortCtDef *>(aux_lookup_ctdef->get_doc_id_scan_ctdef());
sort_rtdef = static_cast<ObDASSortRtDef *>(aux_lookup_rtdef->get_doc_id_scan_rtdef());
}
} else {
// relevance sort would be the root attach ctdef if no aux/table lookup
if (DAS_OP_SORT == attach_ctdef_->op_type_) {
sort_ctdef = static_cast<const ObDASSortCtDef *>(attach_ctdef_);
sort_rtdef = static_cast<ObDASSortRtDef *>(attach_rtdef_);
}
}
}
if (OB_FAIL(ret)) {
} else if (nullptr != text_retrieval_op
&& OB_FAIL(text_retrieval_op->rescan(ls_id_,
inv_idx_tablet_id,
fwd_idx_tablet_id,
doc_id_idx_tablet_id,
ir_scan_ctdef,
ir_scan_rtdef,
sort_ctdef,
sort_rtdef,
trans_desc_,
snapshot_))) {
LOG_WARN("failed to do text retrieval rescan", K(ret));
}
}
return ret;
}
@ -1350,10 +1300,6 @@ int ObLocalIndexLookupOp::init(const ObDASScanCtDef *lookup_ctdef,
LOG_WARN("create lookup mem context entity failed", K(ret));
}
}
int simulate_error = EVENT_CALL(EventTable::EN_DAS_SIMULATE_LOOKUPOP_INIT_ERROR);
if (OB_UNLIKELY(OB_SUCCESS != simulate_error)) {
ret = simulate_error;
}
return ret;
}
@ -1458,7 +1404,7 @@ int ObLocalIndexLookupOp::process_data_table_rowkey()
ObObj tmp_obj;
ObExpr *expr = index_ctdef_->result_output_.at(i);
if (T_PSEUDO_GROUP_ID == expr->type_) {
group_idx = expr->locate_expr_datum(*lookup_rtdef_->eval_ctx_).get_int();
group_idx = ObNewRange::get_group_idx(expr->locate_expr_datum(*lookup_rtdef_->eval_ctx_).get_int());
} else if (T_PSEUDO_ROW_TRANS_INFO_COLUMN == expr->type_) {
// do nothing
} else {

11
src/sql/das/ob_das_scan_op.h
View File

@ -17,6 +17,7 @@
#include "sql/engine/basic/ob_chunk_datum_store.h"
#include "sql/engine/table/ob_index_lookup_op_impl.h"
#include "sql/das/ob_group_scan_iter.h"
#include "sql/das/iter/ob_das_iter.h"
namespace oceanbase
{
@ -202,6 +203,8 @@ public:
storage::ObTableScanParam &get_scan_param() { return scan_param_; }
const storage::ObTableScanParam &get_scan_param() const { return scan_param_; }
int init_related_tablet_ids(ObDASRelatedTabletID &related_tablet_ids);
virtual int decode_task_result(ObIDASTaskResult *task_result) override;
virtual int fill_task_result(ObIDASTaskResult &task_result, bool &has_more, int64_t &memory_limit) override;
virtual int fill_extra_result() override;
@ -221,7 +224,7 @@ public:
const ObDASScanCtDef *get_lookup_ctdef() const;
ObDASScanRtDef *get_lookup_rtdef();
int get_aux_lookup_tablet_id(common::ObTabletID &tablet_id) const;
common::ObTabletID get_table_lookup_tablet_id() const;
int get_table_lookup_tablet_id(common::ObTabletID &tablet_id) const;
int init_scan_param();
int rescan();
int reuse_iter();
@ -230,8 +233,6 @@ public:
bool is_contain_trans_info() {return NULL != scan_ctdef_->trans_info_expr_; }
int do_table_scan();
int do_domain_index_lookup();
int do_text_retrieve(common::ObNewRowIterator *&retrieval_iter);
int do_text_retrieve_rescan();
int get_text_ir_tablet_ids(
common::ObTabletID &inv_idx_tablet_id,
common::ObTabletID &fwd_idx_tablet_id,
@ -248,6 +249,7 @@ protected:
int do_local_index_lookup();
common::ObNewRowIterator *get_storage_scan_iter();
common::ObNewRowIterator *get_output_result_iter() { return result_; }
ObDASIterTreeType get_iter_tree_type() const;
public:
ObSEArray<ObDatum *, 4> trans_info_array_;
protected:
@ -266,9 +268,12 @@ protected:
storage::ObTableScanParam scan_param_;
const ObDASScanCtDef *scan_ctdef_;
ObDASScanRtDef *scan_rtdef_;
// result_ is actually a ObDASIter during execution
common::ObNewRowIterator *result_;
//Indicates the number of remaining rows currently that need to be sent through DTL
int64_t remain_row_cnt_;
// only can be used in runner server
ObDASRelatedTabletID tablet_ids_;
common::ObArenaAllocator *retry_alloc_;
union {

2
src/sql/das/ob_das_spatial_index_lookup_op.cpp
View File

@ -132,7 +132,7 @@ int ObSpatialIndexLookupOp::save_rowkeys()
ObObj tmp_obj;
ObExpr *expr = index_ctdef_->result_output_.at(i);
if (T_PSEUDO_GROUP_ID == expr->type_) {
group_idx = expr->locate_expr_datum(*lookup_rtdef_->eval_ctx_).get_int();
group_idx = ObNewRange::get_group_idx(expr->locate_expr_datum(*lookup_rtdef_->eval_ctx_).get_int());
}
}
if (OB_FAIL(lookup_range.build_range(ref_table_id, *idx_row))) {

293
src/sql/das/ob_domain_index_lookup_op.cpp
View File

@ -11,7 +11,7 @@
*/
#define USING_LOG_PREFIX SQL_DAS
#include "sql/das/ob_das_scan_op.h"
#include "sql/das/ob_text_retrieval_op.h"
#include "sql/das/ob_das_ir_define.h"
#include "sql/das/ob_domain_index_lookup_op.h"
#include "sql/das/ob_das_utils.h"
#include "sql/engine/ob_exec_context.h"
@ -385,233 +385,6 @@ int ObDomainIndexLookupOp::reuse_scan_iter()
return OB_SUCCESS;
}
int ObFullTextIndexLookupOp::reuse_scan_iter()
{
int ret = OB_SUCCESS;
if (OB_FAIL(ObDomainIndexLookupOp::reuse_scan_iter())) {
LOG_WARN("failed to reuse scan iter", K(ret));
} else {
ObITabletScan &tsc_service = get_tsc_service();
const ObTabletID &scan_tablet_id = doc_id_scan_param_.tablet_id_;
doc_id_scan_param_.need_switch_param_ = scan_tablet_id.is_valid() && scan_tablet_id != doc_id_idx_tablet_id_;
if (OB_FAIL(tsc_service.reuse_scan_iter(doc_id_scan_param_.need_switch_param_, rowkey_iter_))) {
LOG_WARN("failed to reuse scan iter", K(ret));
} else if (nullptr != lookup_iter_) {
doc_id_scan_param_.key_ranges_.reuse();
doc_id_scan_param_.ss_key_ranges_.reuse();
}
}
return ret;
}
int ObFullTextIndexLookupOp::init(const ObDASBaseCtDef *table_lookup_ctdef,
ObDASBaseRtDef *table_lookup_rtdef,
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot,
storage::ObTableScanParam &scan_param)
{
int ret = OB_SUCCESS;
const ObDASTableLookupCtDef *tbl_lookup_ctdef = nullptr;
ObDASTableLookupRtDef *tbl_lookup_rtdef = nullptr;
const ObDASIRAuxLookupCtDef *aux_lookup_ctdef = nullptr;
ObDASIRAuxLookupRtDef *aux_lookup_rtdef = nullptr;
const ObDASIRScanCtDef *ir_scan_ctdef = nullptr;
ObDASIRScanRtDef *ir_scan_rtdef = nullptr;
if (OB_ISNULL(table_lookup_ctdef) || OB_ISNULL(table_lookup_rtdef)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table lookup param is nullptr", KP(table_lookup_ctdef), KP(table_lookup_rtdef));
} else if (OB_FAIL(ObDASUtils::find_target_das_def(table_lookup_ctdef,
table_lookup_rtdef,
DAS_OP_TABLE_LOOKUP,
tbl_lookup_ctdef,
tbl_lookup_rtdef))) {
LOG_WARN("find data table lookup def failed", K(ret));
} else if (OB_FAIL(ObDASUtils::find_target_das_def(tbl_lookup_ctdef,
tbl_lookup_rtdef,
DAS_OP_IR_AUX_LOOKUP,
aux_lookup_ctdef,
aux_lookup_rtdef))) {
LOG_WARN("find ir aux lookup def failed", K(ret));
} else if (OB_FAIL(ObDASUtils::find_target_das_def(aux_lookup_ctdef,
aux_lookup_rtdef,
DAS_OP_IR_SCAN,
ir_scan_ctdef,
ir_scan_rtdef))) {
LOG_WARN("find ir scan def failed", K(ret));
} else {
if (OB_FAIL(ObDomainIndexLookupOp::init(tbl_lookup_ctdef->get_lookup_scan_ctdef(),
tbl_lookup_rtdef->get_lookup_scan_rtdef(),
ir_scan_ctdef->get_inv_idx_scan_ctdef(),
ir_scan_rtdef->get_inv_idx_scan_rtdef(),
aux_lookup_ctdef->get_lookup_scan_ctdef(),
aux_lookup_rtdef->get_lookup_scan_rtdef(),
tx_desc,
snapshot,
scan_param))) {
LOG_WARN("failed to init domain index lookup op", K(ret));
} else {
need_scan_aux_ = true;
doc_id_lookup_ctdef_ = aux_lookup_ctdef->get_lookup_scan_ctdef();
doc_id_lookup_rtdef_ = aux_lookup_rtdef->get_lookup_scan_rtdef();
doc_id_expr_ = ir_scan_ctdef->inv_scan_doc_id_col_;
retrieval_ctx_ = ir_scan_rtdef->eval_ctx_;
}
}
return ret;
}
int ObFullTextIndexLookupOp::reset_lookup_state()
{
int ret = OB_SUCCESS;
if (OB_FAIL(ObDomainIndexLookupOp::reset_lookup_state())) {
LOG_WARN("failed to reset lookup state for domain index lookup op", K(ret));
} else {
if (nullptr != lookup_iter_) {
doc_id_scan_param_.key_ranges_.reuse();
doc_id_scan_param_.ss_key_ranges_.reuse();
}
}
return ret;
}
void ObFullTextIndexLookupOp::do_clear_evaluated_flag()
{
return ObDomainIndexLookupOp::do_clear_evaluated_flag();
}
int ObFullTextIndexLookupOp::revert_iter()
{
int ret = OB_SUCCESS;
if (nullptr != text_retrieval_iter_) {
text_retrieval_iter_->reset();
text_retrieval_iter_->~ObNewRowIterator();
if (nullptr != allocator_) {
allocator_->free(text_retrieval_iter_);
}
text_retrieval_iter_ = nullptr;
}
if (OB_FAIL(ObDomainIndexLookupOp::revert_iter())) {
LOG_WARN("failed to revert local index lookup op iter", K(ret));
}
return ret;
}
int ObFullTextIndexLookupOp::fetch_index_table_rowkey()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(text_retrieval_iter_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null text retrieval iterator for index lookup", K(ret), KP(text_retrieval_iter_));
} else if (OB_FAIL(text_retrieval_iter_->get_next_row())) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next next row from text retrieval iter", K(ret));
}
} else if (OB_FAIL(set_lookup_doc_id_key(doc_id_expr_, retrieval_ctx_))) {
LOG_WARN("failed to set lookup doc id query key", K(ret));
}
return ret;
}
int ObFullTextIndexLookupOp::fetch_index_table_rowkeys(int64_t &count, const int64_t capacity)
{
int ret = OB_SUCCESS;
int64_t index_scan_row_cnt = 0;
if (OB_ISNULL(text_retrieval_iter_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null text retrieval iterator for index lookup", K(ret), KP(text_retrieval_iter_));
} else if (OB_FAIL(text_retrieval_iter_->get_next_rows(index_scan_row_cnt, capacity))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next next row from text retrieval iter", K(ret));
} else {
ret = OB_SUCCESS;
}
}
if (OB_SUCC(ret) && index_scan_row_cnt > 0) {
if (OB_FAIL(set_lookup_doc_id_keys(index_scan_row_cnt))) {
LOG_WARN("failed to set lookup doc id query key", K(ret));
} else {
count += index_scan_row_cnt;
}
}
return ret;
}
int ObFullTextIndexLookupOp::do_aux_table_lookup()
{
int ret = OB_SUCCESS;
ObITabletScan &tsc_service = get_tsc_service();
if (nullptr == rowkey_iter_) {
// init doc_id -> rowkey table iterator as rowkey iter
if (OB_FAIL(set_doc_id_idx_lookup_param(
doc_id_lookup_ctdef_, doc_id_lookup_rtdef_, doc_id_scan_param_, doc_id_idx_tablet_id_, ls_id_))) {
LOG_WARN("failed to init doc id lookup scan param", K(ret));
} else if (tsc_service.table_scan(doc_id_scan_param_, rowkey_iter_)) {
if (OB_SNAPSHOT_DISCARDED == ret && scan_param_.fb_snapshot_.is_valid()) {
ret = OB_INVALID_QUERY_TIMESTAMP;
} else if (OB_TRY_LOCK_ROW_CONFLICT != ret) {
LOG_WARN("fail to scan table", K(scan_param_), K(ret));
}
}
} else {
const ObTabletID &scan_tablet_id = doc_id_scan_param_.tablet_id_;
doc_id_scan_param_.need_switch_param_ = scan_tablet_id.is_valid() && (doc_id_idx_tablet_id_ != scan_tablet_id);
doc_id_scan_param_.tablet_id_ = doc_id_idx_tablet_id_;
doc_id_scan_param_.ls_id_ = ls_id_;
if (OB_FAIL(tsc_service.reuse_scan_iter(doc_id_scan_param_.need_switch_param_, rowkey_iter_))) {
LOG_WARN("failed to reuse doc id iterator", K(ret));
} else if (OB_FAIL(tsc_service.table_rescan(doc_id_scan_param_, rowkey_iter_))) {
LOG_WARN("failed to rescan doc id rowkey table", K(ret), K_(doc_id_idx_tablet_id), K(scan_tablet_id));
}
}
return ret;
}
int ObFullTextIndexLookupOp::get_aux_table_rowkey()
{
int ret = OB_SUCCESS;
doc_id_lookup_rtdef_->p_pd_expr_op_->clear_evaluated_flag();
if (index_end_ && doc_id_scan_param_.key_ranges_.empty()) {
ret = OB_ITER_END;
} else if (OB_FAIL(do_aux_table_lookup())) {
LOG_WARN("failed to do aux table lookup", K(ret));
} else if (OB_FAIL(rowkey_iter_->get_next_row())) {
LOG_WARN("failed to get rowkey by doc id", K(ret));
} else if (OB_FAIL(set_main_table_lookup_key())) {
LOG_WARN("failed to set main table lookup key", K(ret));
}
return ret;
}
int ObFullTextIndexLookupOp::get_aux_table_rowkeys(const int64_t lookup_row_cnt)
{
int ret = OB_SUCCESS;
doc_id_lookup_rtdef_->p_pd_expr_op_->clear_evaluated_flag();
int64_t rowkey_cnt = 0;
if (index_end_ && doc_id_scan_param_.key_ranges_.empty()) {
ret = OB_ITER_END;
} else if (OB_FAIL(do_aux_table_lookup())) {
LOG_WARN("failed to do aux table lookup", K(ret));
} else if (OB_FAIL(rowkey_iter_->get_next_rows(rowkey_cnt, lookup_row_cnt))) {
LOG_WARN("failed to get rowkey by doc id", K(ret), K(doc_id_scan_param_.key_ranges_));
} else if (OB_UNLIKELY(lookup_row_cnt != rowkey_cnt)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected aux lookup row count not match", K(ret), K(rowkey_cnt), K(lookup_row_cnt));
} else {
ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*doc_id_lookup_rtdef_->eval_ctx_);
batch_info_guard.set_batch_size(lookup_row_cnt);
for (int64_t i = 0; OB_SUCC(ret) && i < lookup_row_cnt; ++i) {
batch_info_guard.set_batch_idx(i);
if (OB_FAIL(set_main_table_lookup_key())) {
LOG_WARN("failed to set main table lookup key", K(ret));
}
}
}
return ret;
}
int ObMulValueIndexLookupOp::revert_iter()
{
int ret = OB_SUCCESS;
@ -979,70 +752,6 @@ int ObMulValueIndexLookupOp::save_rowkeys()
return ret;
}
int ObFullTextIndexLookupOp::set_lookup_doc_id_keys(const int64_t size)
{
int ret = OB_SUCCESS;
ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*retrieval_ctx_);
batch_info_guard.set_batch_size(size);
for (int64_t i = 0; OB_SUCC(ret) && i < size; ++i) {
batch_info_guard.set_batch_idx(i);
if (OB_FAIL(set_lookup_doc_id_key(doc_id_expr_, retrieval_ctx_))) {
LOG_WARN("failed to set lookup doc id key", K(ret));
}
}
return ret;
}
int ObFullTextIndexLookupOp::set_main_table_lookup_key()
{
int ret = OB_SUCCESS;
int64_t rowkey_cnt = doc_id_lookup_ctdef_->result_output_.count();
void *buf = nullptr;
ObObj *obj_ptr = nullptr;
common::ObArenaAllocator &lookup_alloc = lookup_memctx_->get_arena_allocator();
ObNewRange lookup_range;
if (nullptr != doc_id_lookup_ctdef_->trans_info_expr_) {
rowkey_cnt = rowkey_cnt - 1;
}
if (OB_UNLIKELY(rowkey_cnt <= 0)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid rowkey cnt", K(ret), KPC(doc_id_lookup_ctdef_));
} else if (OB_ISNULL(buf = lookup_alloc.alloc(sizeof(ObObj) * rowkey_cnt))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate memory failed", K(ret), K(rowkey_cnt));
} else {
obj_ptr = new (buf) ObObj[rowkey_cnt];
}
for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt; ++i) {
ObObj tmp_obj;
ObExpr *expr = doc_id_lookup_ctdef_->result_output_.at(i);
if (T_PSEUDO_GROUP_ID == expr->type_) {
// do nothing
} else {
ObDatum &col_datum = expr->locate_expr_datum(*doc_id_lookup_rtdef_->eval_ctx_);
if (OB_FAIL(col_datum.to_obj(tmp_obj, expr->obj_meta_, expr->obj_datum_map_))) {
LOG_WARN("convert datum to obj failed", K(ret));
} else if (OB_FAIL(ob_write_obj(lookup_alloc, tmp_obj, obj_ptr[i]))) {
LOG_WARN("deep copy rowkey value failed", K(ret), K(tmp_obj));
}
}
}
if (OB_SUCC(ret)) {
ObRowkey table_rowkey(obj_ptr, rowkey_cnt);
if (OB_FAIL(lookup_range.build_range(lookup_ctdef_->ref_table_id_, table_rowkey))) {
LOG_WARN("failed to build lookup range", K(ret), K(table_rowkey));
} else if (OB_FAIL(scan_param_.key_ranges_.push_back(lookup_range))) {
LOG_WARN("store lookup key range failed", K(ret), K(scan_param_));
} else {
LOG_DEBUG("get rowkey from docid rowkey table", K(ret), K(table_rowkey), K(lookup_range));
}
}
return ret;
}
int ObMulValueIndexLookupOp::get_aux_table_rowkey()
{
INIT_SUCC(ret);

38
src/sql/das/ob_domain_index_lookup_op.h
View File

@ -116,44 +116,6 @@ protected:
static const int64_t MAX_NUM_PER_BATCH = 1000;
};
class ObFullTextIndexLookupOp : public ObDomainIndexLookupOp
{
public:
explicit ObFullTextIndexLookupOp(ObIAllocator &allocator)
: ObDomainIndexLookupOp(allocator),
text_retrieval_iter_(nullptr),
retrieval_ctx_(nullptr) {}
virtual ~ObFullTextIndexLookupOp() {}
int init(const ObDASBaseCtDef *table_lookup_ctdef,
ObDASBaseRtDef *table_lookup_rtdef,
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot,
storage::ObTableScanParam &scan_param);
void set_text_retrieval_iter(common::ObNewRowIterator *text_retrieval_iter)
{
text_retrieval_iter_ = text_retrieval_iter;
}
common::ObNewRowIterator *get_text_retrieval_iter() { return text_retrieval_iter_; }
virtual int reset_lookup_state() override;
virtual int do_aux_table_lookup();
virtual int revert_iter() override;
virtual int reuse_scan_iter();
virtual void do_clear_evaluated_flag() override;
protected:
virtual int fetch_index_table_rowkey() override;
virtual int fetch_index_table_rowkeys(int64_t &count, const int64_t capacity) override;
virtual int get_aux_table_rowkey() override;
virtual int get_aux_table_rowkeys(const int64_t lookup_row_cnt) override;
private:
int set_lookup_doc_id_keys(const int64_t size);
int set_main_table_lookup_key();
private:
ObNewRowIterator *text_retrieval_iter_;
ObEvalCtx *retrieval_ctx_;
};
class ObMulValueIndexLookupOp : public ObDomainIndexLookupOp
{
public:

153
src/sql/engine/table/ob_table_scan_op.cpp
View File

@ -136,6 +136,7 @@ OB_DEF_SERIALIZE(ObTableScanCtDef)
OB_UNIS_ENCODE(abandoned_always_false_aux_lookup);
OB_UNIS_ENCODE(abandoned_always_false_text_ir);
OB_UNIS_ENCODE(attach_spec_);
OB_UNIS_ENCODE(flags_);
return ret;
}
@ -169,6 +170,7 @@ OB_DEF_SERIALIZE_SIZE(ObTableScanCtDef)
OB_UNIS_ADD_LEN(abandoned_always_false_aux_lookup);
OB_UNIS_ADD_LEN(abandoned_always_false_text_ir);
OB_UNIS_ADD_LEN(attach_spec_);
OB_UNIS_ADD_LEN(flags_);
return len;
}
@ -219,6 +221,7 @@ OB_DEF_DESERIALIZE(ObTableScanCtDef)
OB_UNIS_DECODE(abandoned_always_false_aux_lookup);
OB_UNIS_DECODE(abandoned_always_false_text_ir);
OB_UNIS_DECODE(attach_spec_);
OB_UNIS_DECODE(flags_);
return ret;
}
@ -801,6 +804,23 @@ int ObTableScanOp::prepare_pushdown_limit_param()
need_final_limit_ = true;
tsc_rtdef_.prepare_multi_part_limit_param();
}
// NOTICE: TSC operator can not apply final limit when das need keep ordering for multi partitions,
// consider following:
// TSC (limit 10), need_keep_order
// / \
// / \
// / \
// DAS SCAN DAS SCAN
// p0 p1
// (limit 10) (limit 10)
// when das need keep ordering, TSC should get 10 rows from each partition, with the upper operator
// applying merge sort and final limit.
// However, if we apply final limit on TSC operator, it will exit after got 10 rows from p0.
// TODO: @bingfan remove need_final_limit_ from TSC operator
if (MY_CTDEF.is_das_keep_order_ && OB_NOT_NULL(scan_iter_) && scan_iter_->get_das_task_cnt() > 1) {
need_final_limit_ = false;
}
return ret;
}
@ -1102,60 +1122,6 @@ OB_INLINE int ObTableScanOp::init_das_scan_rtdef(const ObDASScanCtDef &das_ctdef
return ret;
}
int ObTableScanOp::update_output_tablet_id()
{
int ret = OB_SUCCESS;
if (NULL != MY_SPEC.pdml_partition_id_) {
const ObDASTabletLoc *data_tablet_loc = nullptr;
int64_t output_id = OB_INVALID_ID;
if (MY_SPEC.partition_id_calc_type_ > 0) {
// partition id for gather statistics, index scan should output index partition id
data_tablet_loc = scan_result_.get_tablet_loc();
} else if (MY_SPEC.should_scan_index()) {
data_tablet_loc = ObDASUtils::get_related_tablet_loc(*scan_result_.get_tablet_loc(), MY_SPEC.ref_table_id_);
} else {
data_tablet_loc = scan_result_.get_tablet_loc();
}
if (OB_ISNULL(data_tablet_loc)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("data tablet loc is null, value of pdml partition id will not be set", K(ret),
K(MY_SPEC.should_scan_index()), K(MY_SPEC.ref_table_id_));
} else {
if (MY_SPEC.partition_id_calc_type_ == 0) {
output_id = data_tablet_loc->tablet_id_.id();
} else if (MY_SPEC.partition_id_calc_type_ == 1) {
output_id = data_tablet_loc->first_level_part_id_ != OB_INVALID_ID ?
data_tablet_loc->first_level_part_id_ : data_tablet_loc->partition_id_;
} else if (MY_SPEC.partition_id_calc_type_ == 2) {
output_id = data_tablet_loc->partition_id_;
} else {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid partition id cacl type", K(ret));
}
if (OB_FAIL(ret)) {
} else if (is_vectorized()) {
const int64_t batch_size = MY_SPEC.max_batch_size_;
ObExpr *expr = MY_SPEC.pdml_partition_id_;
ObDatum *datums = expr->locate_datums_for_update(eval_ctx_, batch_size);
for (int64_t i = 0; i < batch_size; i++) {
datums[i].set_int(output_id);
}
expr->set_evaluated_projected(eval_ctx_);
LOG_TRACE("find the partition id expr in pdml table scan", K(ret), KPC(expr), KPC(data_tablet_loc), K(output_id));
} else {
// handle PDML partition id:
// if partition id expr in TSC output_exprs,
// set the TSC partition id to the corresponding expr frame
ObExpr *expr = MY_SPEC.pdml_partition_id_;
expr->locate_datum_for_write(eval_ctx_).set_int(output_id);
expr->set_evaluated_projected(eval_ctx_);
LOG_TRACE("find the partition id expr in pdml table scan", K(ret), KPC(data_tablet_loc), K(output_id));
}
}
}
return ret;
}
int ObTableScanOp::prepare_scan_range()
{
int ret = OB_SUCCESS;
@ -1196,6 +1162,7 @@ int ObTableScanOp::prepare_batch_scan_range()
LOG_DEBUG("after prepare batch scan range", K(MY_INPUT.key_ranges_), K(MY_INPUT.ss_key_ranges_));
return ret;
}
int ObTableScanOp::build_bnlj_params()
{
int ret = OB_SUCCESS;
@ -1492,10 +1459,9 @@ int ObTableScanOp::inner_open()
// create and init iter_tree_.
const ObTableScanSpec &spec = MY_SPEC;
if (OB_SUCC(ret)) {
if (spec.should_scan_index()) {
if (spec.is_global_index_back()) {
if (OB_FAIL(ObDASIterUtils::create_global_lookup_iter_tree(MY_CTDEF,
ObDASIterTreeType tree_type = spec.is_global_index_back() ? ITER_TREE_GLOBAL_LOOKUP : ITER_TREE_TABLE_SCAN;
if (OB_SUCC(ret) && OB_FAIL(ObDASIterUtils::create_tsc_iter_tree(tree_type,
spec.tsc_ctdef_,
tsc_rtdef_,
eval_ctx_,
ctx_,
@ -1504,28 +1470,7 @@ int ObTableScanOp::inner_open()
can_partition_retry(),
scan_iter_,
iter_tree_))) {
LOG_WARN("failed to create global lookup iter tree", K(ret));
}
} else if (OB_FAIL(ObDASIterUtils::create_local_lookup_iter_tree(MY_CTDEF,
tsc_rtdef_,
eval_ctx_,
ctx_,
eval_infos_,
spec,
scan_iter_,
iter_tree_))) {
LOG_WARN("failed to create local lookup iter tree", K(ret));
}
} else if (OB_FAIL(ObDASIterUtils::create_table_scan_iter_tree(MY_CTDEF,
tsc_rtdef_,
eval_ctx_,
ctx_,
eval_infos_,
spec,
scan_iter_,
iter_tree_))) {
LOG_WARN("failed to create table scan iter tree", K(ret));
}
LOG_WARN("failed to create table scan iter tree", K(tree_type), K(ret));
}
output_ = iter_tree_;
return ret;
@ -1823,28 +1768,10 @@ int ObTableScanOp::local_iter_rescan()
DASTaskIter task_iter = scan_iter_->begin_task_iter();
for (; OB_SUCC(ret) && !task_iter.is_end(); ++task_iter) {
ObDASScanOp *scan_op = DAS_SCAN_OP(*task_iter);
if (OB_ISNULL(scan_op)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr das scan op", K(ret));
} else if (MY_SPEC.gi_above_) {
if (!MY_SPEC.is_index_global_ && MY_CTDEF.lookup_ctdef_ != nullptr) {
//is local index lookup, need to set the lookup ctdef to the das scan op
if (OB_FAIL(pushdown_normal_lookup_to_das(*scan_op))) {
LOG_WARN("pushdown normal lookup to das failed", K(ret));
}
}
if (OB_SUCC(ret) && MY_CTDEF.attach_spec_.attach_ctdef_ != nullptr) {
if (OB_FAIL(pushdown_attach_task_to_das(*scan_op))) {
LOG_WARN("pushdown attach task to das failed", K(ret));
}
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(cherry_pick_range_by_tablet_id(scan_op))) {
LOG_WARN("prune query range by partition id failed", K(ret));
} else if (OB_FAIL(scan_iter_->rescan_das_task(scan_op))) {
LOG_WARN("rescan das task failed", K(ret));
}
if (OB_FAIL(cherry_pick_range_by_tablet_id(scan_op))) {
LOG_WARN("prune query range by partition id failed", K(ret));
} else if (OB_FAIL(scan_iter_->rescan_das_task(scan_op))) {
LOG_WARN("rescan das task failed", K(ret));
}
}
}
@ -1882,9 +1809,23 @@ int ObTableScanOp::local_iter_reuse()
scan_op->set_ls_id(MY_INPUT.tablet_loc_->ls_id_);
scan_op->set_tablet_loc(MY_INPUT.tablet_loc_);
}
if (MY_SPEC.gi_above_) {
if (!MY_SPEC.is_index_global_ && MY_CTDEF.lookup_ctdef_ != nullptr) {
//is local index lookup, need to set the lookup ctdef to the das scan op
if (OB_FAIL(pushdown_normal_lookup_to_das(*scan_op))) {
LOG_WARN("pushdown normal lookup to das failed", K(ret));
}
}
if (OB_SUCC(ret) && MY_CTDEF.attach_spec_.attach_ctdef_ != nullptr) {
if (OB_FAIL(pushdown_attach_task_to_das(*scan_op))) {
LOG_WARN("pushdown attach task to das failed", K(ret));
}
}
}
scan_op->reuse_iter();
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(reuse_table_rescan_allocator())) {
LOG_WARN("get table allocator", K(ret));
@ -1975,7 +1916,9 @@ void ObTableScanOp::reset_iter_tree_for_rescan()
if (OB_NOT_NULL(fold_iter_)) {
fold_iter_->reuse();
}
if (iter_tree_->get_type() == DAS_ITER_LOOKUP) {
// we cannot simply reuse iter tree due to local iter rescan optimization.
if (iter_tree_->get_type() == DAS_ITER_GLOBAL_LOOKUP) {
iter_tree_->reuse();
}
}
@ -1992,9 +1935,6 @@ int ObTableScanOp::set_batch_iter(int64_t group_id)
return ret;
}
int ObTableScanOp::get_next_row_with_das()
{
int ret = OB_SUCCESS;
@ -2464,6 +2404,7 @@ int ObTableScanOp::cherry_pick_range_by_tablet_id(ObDASScanOp *scan_op)
ObNewRange whole_range;
false_range.set_false_range();
false_range.group_idx_ = input_ranges.at(0).group_idx_;
false_range.index_ordered_idx_ = input_ranges.at(0).index_ordered_idx_;
whole_range.set_whole_range();
if (OB_FAIL(scan_ranges.push_back(false_range))) {
LOG_WARN("store false range to scan ranges failed", K(ret));

14
src/sql/engine/table/ob_table_scan_op.h
View File

@ -24,9 +24,8 @@
#include "sql/das/ob_das_ref.h"
#include "sql/das/ob_data_access_service.h"
#include "sql/das/ob_das_scan_op.h"
#include "sql/das/ob_text_retrieval_op.h"
#include "sql/das/ob_das_attach_define.h"
#include "sql/das/ob_text_retrieval_op.h"
#include "sql/das/ob_das_ir_define.h"
#include "sql/engine/basic/ob_pushdown_filter.h"
#include "sql/engine/table/ob_index_lookup_op_impl.h"
#include "sql/das/iter/ob_das_iter.h"
@ -147,7 +146,8 @@ public:
allocator_(allocator),
calc_part_id_expr_(NULL),
global_index_rowkey_exprs_(allocator),
attach_spec_(allocator_, &scan_ctdef_)
attach_spec_(allocator_, &scan_ctdef_),
flags_(0)
{ }
const ExprFixedArray &get_das_output_exprs() const
{
@ -199,6 +199,13 @@ public:
ExprFixedArray global_index_rowkey_exprs_;
// end for Global Index Lookup
ObDASAttachSpec attach_spec_;
union {
uint64_t flags_;
struct {
uint64_t is_das_keep_order_ : 1; // whether das need keep ordering
uint64_t reserved_ : 63;
};
};
};
struct ObTableScanRtDef
@ -470,7 +477,6 @@ protected:
int local_iter_rescan();
int close_and_reopen();
int update_output_tablet_id();
int cherry_pick_range_by_tablet_id(ObDASScanOp *scan_op);
int can_prune_by_tablet_id(const common::ObTabletID &tablet_id,

1
src/sql/ob_optimizer_trace_impl.cpp
View File

@ -937,6 +937,7 @@ int ObOptimizerTraceImpl::trace_parameters()
TRACE_PARAMETER(_optimizer_sortmerge_join_enabled, bool);
TRACE_PARAMETER(_nested_loop_join_enabled, bool);
TRACE_PARAMETER(_enable_var_assign_use_das, bool);
TRACE_PARAMETER(_enable_das_keep_order, bool);
//for system variables
TRACE_SYS_VAR(_PX_SHARED_HASH_JOIN, int64_t);
TRACE_SYS_VAR(_ENABLE_PARALLEL_DML, int64_t);

26
src/sql/optimizer/ob_join_order.cpp
View File

@ -618,8 +618,15 @@ int ObJoinOrder::compute_base_table_path_ordering(AccessPath *path)
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(path), K(ret));
} else if (path->use_das_ &&
!path->ordering_.empty() &&
path->table_partition_info_->get_phy_tbl_location_info().get_partition_cnt() > 1) {
path->ordering_.reset();
if (get_plan()->get_optimizer_context().is_das_keep_order_enabled()) {
// when enable das keep order optimization, DAS layer can provide a guarantee of local order,
// otherwise the order is totally not guaranteed.
path->is_local_order_ = true;
} else {
path->ordering_.reset();
}
} else if (path->ordering_.empty() || is_at_most_one_row_ || !path->strong_sharding_->is_distributed()) {
path->is_local_order_ = false;
} else if (get_plan()->get_optimizer_context().is_online_ddl()) {
@ -1782,10 +1789,13 @@ int ObJoinOrder::create_one_access_path(const uint64_t table_id,
LOG_WARN("failed to add access filters", K(*ap), K(ordering_info.get_index_keys()), K(ret));
} else if (OB_FAIL(get_plan()->get_stmt()->get_column_items(table_id, ap->est_cost_info_.access_column_items_))) {
LOG_WARN("failed to get column items", K(ret));
} else if ((!ap->is_global_index_ || !index_info_entry->is_index_back()) &&
OB_FAIL(ObOptimizerUtil::make_sort_keys(ordering_info.get_ordering(),
ordering_info.get_scan_direction(),
ap->ordering_))) {
} else if ((!ap->is_global_index_ ||
!index_info_entry->is_index_back() ||
get_plan()->get_optimizer_context().is_das_keep_order_enabled())
// for global index lookup without keep order, the ordering is wrong.
&& OB_FAIL(ObOptimizerUtil::make_sort_keys(ordering_info.get_ordering(),
ordering_info.get_scan_direction(),
ap->ordering_))) {
LOG_WARN("failed to create index keys expression array", K(index_id), K(ret));
} else if (ordering_info.get_index_keys().count() > 0) {
ap->pre_query_range_ = const_cast<ObQueryRange *>(range_info.get_query_range());
@ -2153,8 +2163,8 @@ int ObJoinOrder::get_access_path_ordering(const uint64_t table_id,
LOG_WARN("NULL pointer error", K(ret), K(table_id), K(ref_table_id), K(index_id));
} else if (!index_schema->is_ordered()) {
// for virtual table, we have HASH index which offers no ordering on index keys
} else if (index_schema->is_global_index_table() && is_index_back) {
// for global index lookup, the order is wrong.
} else if (index_schema->is_global_index_table() && is_index_back && !opt_ctx->is_das_keep_order_enabled()) {
// for global index lookup without keep order, the ordering is wrong.
} else if (OB_FAIL(append(ordering, index_keys))) {
LOG_WARN("failed to append index ordering expr", K(ret));
} else if (OB_FAIL(get_index_scan_direction(ordering, stmt,
@ -2616,7 +2626,7 @@ int ObJoinOrder::fill_index_info_entry(const uint64_t table_id,
entry->get_ordering_info().get_ordering(),
direction,
is_index_back))) {
LOG_WARN("get_access_path_ordering ", K(ret));
LOG_WARN("get access path ordering ", K(ret));
} else {
entry->set_is_index_global(is_index_global);
entry->set_is_index_geo(is_index_geo);

38
src/sql/optimizer/ob_log_plan.cpp
View File

@ -7455,7 +7455,18 @@ int ObLogPlan::try_push_limit_into_table_scan(ObLogicalOperator *top,
(NULL == table_scan->get_limit_expr() ||
ObOptimizerUtil::is_point_based_sub_expr(limit_expr, table_scan->get_limit_expr())) &&
table_scan->get_text_retrieval_info().topk_limit_expr_ == NULL) {
if (!top->is_distributed()) {
bool das_multi_partition = false;
if (table_scan->use_das() && NULL != table_scan->get_table_partition_info()) {
int64_t partition_count = table_scan->get_table_partition_info()->
get_phy_tbl_location_info().get_phy_part_loc_info_list().count();
if (1 != partition_count) {
das_multi_partition = true;
}
}
if (das_multi_partition) {
new_limit_expr = pushed_expr;
} else if (!top->is_distributed()) {
new_limit_expr = limit_expr;
new_offset_expr = offset_expr;
} else {
@ -7469,6 +7480,9 @@ int ObLogPlan::try_push_limit_into_table_scan(ObLogicalOperator *top,
} else {
is_pushed = true;
}
if (das_multi_partition) {
is_pushed = false;
}
} else if (OB_NOT_NULL(table_scan->get_text_retrieval_info().topk_limit_expr_)) {
is_pushed = true;
}
@ -10664,6 +10678,8 @@ int ObLogPlan::do_post_plan_processing()
LOG_WARN("failed to collect table location", K(ret));
} else if (OB_FAIL(build_location_related_tablet_ids())) {
LOG_WARN("build location related tablet ids failed", K(ret));
} else if (OB_FAIL(check_das_need_keep_ordering(root))) {
LOG_WARN("failed to check das need keep ordering", K(ret));
} else { /*do nothing*/ }
return ret;
}
@ -11355,6 +11371,26 @@ int ObLogPlan::build_location_related_tablet_ids()
return ret;
}
int ObLogPlan::check_das_need_keep_ordering(ObLogicalOperator *op)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(op)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null param", K(ret));
} else if (log_op_def::LOG_TABLE_SCAN == op->get_type()) {
ObLogTableScan *scan = static_cast<ObLogTableScan*>(op);
if (OB_FAIL(scan->check_das_need_keep_ordering())) {
LOG_WARN("failed to check das need keep ordering", K(ret));
}
}
for (int i = 0; OB_SUCC(ret) && i < op->get_num_of_child(); ++i) {
if (OB_FAIL(SMART_CALL(check_das_need_keep_ordering(op->get_child(i))))) {
LOG_WARN("failed to check das need keep ordering", K(ret));
}
}
return ret;
}
int ObLogPlan::calc_plan_resource()
{
int ret = OB_SUCCESS;

1
src/sql/optimizer/ob_log_plan.h
View File

@ -262,6 +262,7 @@ public:
int collect_location_related_info(ObLogicalOperator &op);
int build_location_related_tablet_ids();
int check_das_need_keep_ordering(ObLogicalOperator *op);
int gen_das_table_location_info(ObLogTableScan *table_scan,
ObTablePartitionInfo *&table_partition_info);

51
src/sql/optimizer/ob_log_table_scan.cpp
View File

@ -196,6 +196,8 @@ int ObLogTableScan::get_op_exprs(ObIArray<ObRawExpr*> &all_exprs)
LOG_WARN("failed to add lookup trans expr", K(ret));
} else if (NULL != trans_info_expr_ && OB_FAIL(all_exprs.push_back(trans_info_expr_))) {
LOG_WARN("failed to push back expr", K(ret));
} else if (NULL != group_id_expr_ && OB_FAIL(all_exprs.push_back(group_id_expr_))) {
LOG_WARN("failed to push back expr", K(ret));
} else if (is_text_retrieval_scan() && OB_FAIL(get_text_retrieval_calc_exprs(all_exprs))) {
LOG_WARN("failed to get text retrieval exprs", K(ret));
} else if (OB_FAIL(append(all_exprs, access_exprs_))) {
@ -304,7 +306,7 @@ int ObLogTableScan::check_output_dependance(common::ObIArray<ObRawExpr *> &child
LOG_WARN("failed to append exprs", K(ret));
} else if (OB_FAIL(append_array_no_dup(exprs, spatial_exprs_))) {
LOG_WARN("failed to append exprs", K(ret));
} else if (use_batch() && nullptr != group_id_expr_
} else if (use_group_id() && nullptr != group_id_expr_
&& OB_FAIL(add_var_to_array_no_dup(exprs, group_id_expr_))) {
LOG_WARN("failed to push back group id expr", K(ret));
} else if (index_back_ &&
@ -435,10 +437,9 @@ int ObLogTableScan::generate_access_exprs()
LOG_WARN("failed to copy text retrieval aggr exprs", K(ret));
} else if (OB_FAIL(generate_necessary_rowkey_and_partkey_exprs())) {
LOG_WARN("failed to generate rowkey and part exprs", K(ret));
} else if (use_batch()
&& OB_FAIL(ObOptimizerUtil::allocate_group_id_expr(get_plan(), group_id_expr_))) {
LOG_WARN("allocate group id expr failed", K(ret));
} else if (nullptr != group_id_expr_ && OB_FAIL(access_exprs_.push_back(group_id_expr_))) {
} else if (OB_FAIL(allocate_group_id_expr())) {
LOG_WARN("failed to allocate group id expr", K(ret));
} else if (NULL != group_id_expr_ && use_batch_ && OB_FAIL(access_exprs_.push_back(group_id_expr_))) {
LOG_WARN("failed to push back expr", K(ret));
} else if (OB_FAIL(append_array_no_dup(access_exprs_, rowkey_exprs_))) {
LOG_WARN("failed to push back exprs", K(ret));
@ -923,6 +924,23 @@ int ObLogTableScan::allocate_lookup_trans_info_expr()
return ret;
}
int ObLogTableScan::allocate_group_id_expr()
{
int ret = OB_SUCCESS;
// [GROUP_ID] expr is now used for group rescan and global lookup keep order, it is handled
// by DAS layer and transparent to TSC operator.
ObRawExpr *group_id_expr = nullptr;
if (OB_ISNULL(get_plan())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K(ret));
} else if (use_group_id() && OB_FAIL(ObOptimizerUtil::allocate_group_id_expr(get_plan(), group_id_expr))) {
LOG_WARN("failed to allocate group id expr", K(ret));
} else {
group_id_expr_ = group_id_expr;
}
return ret;
}
int ObLogTableScan::generate_necessary_rowkey_and_partkey_exprs()
{
int ret = OB_SUCCESS;
@ -1370,6 +1388,12 @@ int ObLogTableScan::get_plan_item_info(PlanText &plan_text,
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(BUF_PRINTF("is_global_index=%s", is_index_global_? "true" : "false"))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (!das_keep_ordering_) {
//do nothing
} else if (OB_FAIL(BUF_PRINTF(", "))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(BUF_PRINTF("keep_ordering=%s", das_keep_ordering_ ? "true" : "false"))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else { /* Do nothing */ }
if (OB_SUCC(ret) && (0 != filter_before_index_back_.count())) {
@ -1379,7 +1403,6 @@ int ObLogTableScan::get_plan_item_info(PlanText &plan_text,
LOG_WARN("BUF_PRINTF fails", K(ret));
} else { /* Do nothing */ }
}
//Print ranges
if (OB_FAIL(ret) || is_text_retrieval_scan()) {
} else if (OB_FAIL(BUF_PRINTF(", "))) {
@ -2598,6 +2621,20 @@ int ObLogTableScan::get_card_without_filter(double &card)
return ret;
}
int ObLogTableScan::check_das_need_keep_ordering()
{
int ret = OB_SUCCESS;
das_keep_ordering_ = true;
bool ordering_be_used = true;
if (!use_das_ && !(is_index_global_ && index_back_)) {
das_keep_ordering_ = false;
} else if (OB_FAIL(check_op_orderding_used_by_parent(ordering_be_used))) {
LOG_WARN("failed to check op ordering used by parent", K(ret));
} else if (!ordering_be_used) {
das_keep_ordering_ = false;
}
return ret;
}
int ObLogTableScan::generate_filter_monotonicity()
{
int ret = OB_SUCCESS;
@ -2803,4 +2840,4 @@ int ObLogTableScan::get_filter_assist_exprs(ObIArray<ObRawExpr *> &assist_exprs)
}
}
return ret;
}
}

10
src/sql/optimizer/ob_log_table_scan.h
View File

@ -155,6 +155,7 @@ public:
use_column_store_(false),
doc_id_table_id_(common::OB_INVALID_ID),
text_retrieval_info_(),
das_keep_ordering_(false),
filter_monotonicity_()
{
}
@ -520,7 +521,8 @@ public:
int copy_filter_before_index_back();
void set_use_batch(bool use_batch) { use_batch_ = use_batch; }
bool use_batch() const { return use_batch_; }
// only use group_id_expr_ when use_batch() is true.
// use group_id_expr_ when batch rescan or keep order for global lookup.
bool use_group_id() const { return use_batch_ || (is_index_global_ && index_back_ && das_keep_ordering_); }
inline const ObRawExpr *get_group_id_expr() const { return group_id_expr_; }
int extract_bnlj_param_idxs(common::ObIArray<int64_t> &bnlj_params);
@ -582,6 +584,10 @@ public:
}
void set_identify_seq_expr(ObRawExpr *expr) { identify_seq_expr_ = expr; }
inline bool das_need_keep_ordering() const { return das_keep_ordering_; }
int check_das_need_keep_ordering();
const ObIArray<ObRawFilterMonotonicity>& get_filter_monotonicity() const
{ return filter_monotonicity_; }
int get_filter_monotonicity(const ObRawExpr *filter,
@ -603,6 +609,7 @@ private: // member functions
int add_mapping_columns_for_vt(ObIArray<ObRawExpr*> &access_exprs);
int get_mbr_column_exprs(const uint64_t table_id, ObIArray<ObRawExpr *> &mbr_exprs);
int allocate_lookup_trans_info_expr();
int allocate_group_id_expr();
int extract_doc_id_index_back_expr(ObIArray<ObRawExpr *> &exprs);
int extract_text_retrieval_access_expr(ObIArray<ObRawExpr *> &exprs);
int get_text_retrieval_calc_exprs(ObIArray<ObRawExpr *> &all_exprs);
@ -725,6 +732,7 @@ protected: // memeber variables
ObTextRetrievalInfo text_retrieval_info_;
ObPxRFStaticInfo px_rf_info_;
bool das_keep_ordering_;
typedef common::ObSEArray<ObRawFilterMonotonicity, 4, common::ModulePageAllocator, true> FilterMonotonicity;
FilterMonotonicity filter_monotonicity_;
// disallow copy and assign

56
src/sql/optimizer/ob_logical_operator.cpp
View File

@ -6288,7 +6288,11 @@ int ObLogicalOperator::check_use_child_ordering(bool &used, int64_t &inherit_chi
} else {
used = true;
}
inherit_child_ordering_index = first_child;
if (LOG_TEMP_TABLE_TRANSFORMATION == get_type()) {
inherit_child_ordering_index = get_num_of_child()-1;
} else {
inherit_child_ordering_index = first_child;
}
} else {
used = false;
inherit_child_ordering_index = -1;
@ -6488,36 +6492,38 @@ int ObLogicalOperator::check_op_orderding_used_by_parent(bool &used)
{
int ret = OB_SUCCESS;
used = true;
bool is_first_child = true;
bool inherit_child_ordering = true;
int64_t inherit_child_ordering_index = -1;
ObLogicalOperator *parent = get_parent();
ObLogicalOperator *child = this;
while (OB_SUCC(ret) && NULL != parent) {
if (OB_FAIL(parent->check_use_child_ordering(used, inherit_child_ordering_index))) {
LOG_WARN("failed to check use child ordering", K(ret));
} else {
inherit_child_ordering = child == parent->get_child(inherit_child_ordering_index);
if (!used && inherit_child_ordering && child->is_plan_root()) {
ObLogPlan *plan = child->get_plan();
const ObDMLStmt *stmt = NULL;
if (OB_ISNULL(plan) || OB_ISNULL(stmt=plan->get_stmt())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpect null param", K(ret));
} else if (0 == stmt->get_order_item_size()) {
//do nothing
} else {
used = true;
}
while (OB_SUCC(ret) && NULL != child) {
if (child->is_plan_root()) {
ObLogPlan *plan = child->get_plan();
const ObDMLStmt *stmt = NULL;
if (OB_ISNULL(plan) || OB_ISNULL(stmt=plan->get_stmt())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpect null param", K(ret));
} else if (0 == stmt->get_order_item_size()) {
//do nothing
} else {
used = true;
break;
}
if (NULL == parent) {
break;
}
}
if (OB_FAIL(ret)) {
} else if (used || !inherit_child_ordering) {
break;
} else {
child = parent;
parent = parent->get_parent();
if (OB_SUCC(ret) && NULL != parent) {
if (OB_FAIL(parent->check_use_child_ordering(used, inherit_child_ordering_index))) {
LOG_WARN("failed to check use child ordering", K(ret));
} else if (OB_FALSE_IT(inherit_child_ordering = child == parent->get_child(inherit_child_ordering_index))) {
} else if (used || !inherit_child_ordering) {
break;
} else {
child = parent;
parent = parent->get_parent();
}
}
}
return ret;
}
}

5
src/sql/optimizer/ob_optimizer.cpp
View File

@ -560,6 +560,8 @@ int ObOptimizer::extract_opt_ctx_basic_flags(const ObDMLStmt &stmt, ObSQLSession
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(session.get_effective_tenant_id()));
bool rowsets_enabled = tenant_config.is_valid() && tenant_config->_rowsets_enabled;
ctx_.set_is_online_ddl(session.get_ddl_info().is_ddl()); // set is online ddl first, is used by other extract operations
bool das_keep_order_enabled = tenant_config.is_valid() && tenant_config->_enable_das_keep_order;
const ObOptParamHint &opt_params = ctx_.get_global_hint().opt_params_;
if (OB_FAIL(check_whether_contain_nested_sql(stmt))) {
LOG_WARN("check whether contain nested sql failed", K(ret));
} else if (OB_FAIL(stmt.check_has_subquery_in_function_table(has_subquery_in_function_table))) {
@ -582,6 +584,8 @@ int ObOptimizer::extract_opt_ctx_basic_flags(const ObDMLStmt &stmt, ObSQLSession
LOG_WARN("fail to check cursor expression info", K(ret));
} else if (OB_FAIL(session.is_storage_estimation_enabled(storage_estimation_enabled))) {
LOG_WARN("fail to get storage_estimation_enabled", K(ret));
} else if (OB_FAIL(opt_params.get_bool_opt_param(ObOptParamHint::ENABLE_DAS_KEEP_ORDER, das_keep_order_enabled))) {
LOG_WARN("failed to check das keep order enabled", K(ret));
} else {
ctx_.set_storage_estimation_enabled(storage_estimation_enabled);
ctx_.set_serial_set_order(force_serial_set_order);
@ -592,6 +596,7 @@ int ObOptimizer::extract_opt_ctx_basic_flags(const ObDMLStmt &stmt, ObSQLSession
ctx_.set_has_dblink(has_dblink);
ctx_.set_cost_model_type(rowsets_enabled ? ObOptEstCost::VECTOR_MODEL : ObOptEstCost::NORMAL_MODEL);
ctx_.set_has_cursor_expression(has_cursor_expr);
ctx_.set_das_keep_order_enabled(GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_3_2_0 ? false : das_keep_order_enabled);
if (!tenant_config.is_valid() ||
(!tenant_config->_hash_join_enabled &&
!tenant_config->_optimizer_sortmerge_join_enabled &&

4
src/sql/optimizer/ob_optimizer_context.h
View File

@ -234,6 +234,7 @@ ObOptimizerContext(ObSQLSessionInfo *session_info,
nested_loop_join_enabled_(true),
system_stat_(),
storage_estimation_enabled_(false),
das_keep_order_enabled_(true),
generate_random_plan_(false)
{ }
inline common::ObOptStatManager *get_opt_stat_manager() { return opt_stat_manager_; }
@ -311,6 +312,8 @@ ObOptimizerContext(ObSQLSessionInfo *session_info,
inline bool is_storage_estimation_enabled() const { return storage_estimation_enabled_; }
void set_storage_estimation_enabled(bool storage_estimation_enabled) { storage_estimation_enabled_ = storage_estimation_enabled; }
inline bool is_das_keep_order_enabled() const { return das_keep_order_enabled_; }
void set_das_keep_order_enabled(bool das_keep_order_enabled) { das_keep_order_enabled_ = das_keep_order_enabled; }
inline int64_t get_parallel() const { return parallel_; }
inline int64_t get_max_parallel() const { return max_parallel_; }
inline int64_t get_parallel_degree_limit(const int64_t server_cnt) const { return auto_dop_params_.get_parallel_degree_limit(server_cnt); }
@ -690,6 +693,7 @@ private:
bool nested_loop_join_enabled_;
OptSystemStat system_stat_;
bool storage_estimation_enabled_;
bool das_keep_order_enabled_;
bool generate_random_plan_;
};

12
src/sql/plan_cache/ob_plan_cache_util.cpp
View File

@ -537,6 +537,7 @@ int ObConfigInfoInPC::load_influence_plan_config()
min_cluster_version_ = GET_MIN_CLUSTER_VERSION();
enable_spf_batch_rescan_ = tenant_config->_enable_spf_batch_rescan;
enable_var_assign_use_das_ = tenant_config->_enable_var_assign_use_das;
enable_das_keep_order_ = tenant_config->_enable_das_keep_order;
}
return ret;
@ -581,16 +582,19 @@ int ObConfigInfoInPC::serialize_configs(char *buf, int buf_len, int64_t &pos)
"%lu,", min_cluster_version_))) {
SQL_PC_LOG(WARN, "failed to databuff_printf", K(ret), K(min_cluster_version_));
} else if (OB_FAIL(databuff_printf(buf, buf_len, pos,
"%d", is_enable_px_fast_reclaim_))) {
"%d,", is_enable_px_fast_reclaim_))) {
SQL_PC_LOG(WARN, "failed to databuff_printf", K(ret), K(is_enable_px_fast_reclaim_));
} else if (OB_FAIL(databuff_printf(buf, buf_len, pos,
"%d", enable_spf_batch_rescan_))) {
"%d,", enable_spf_batch_rescan_))) {
SQL_PC_LOG(WARN, "failed to databuff_printf", K(ret), K(enable_spf_batch_rescan_));
} else if (OB_FAIL(databuff_printf(buf, buf_len, pos,
"%d", enable_var_assign_use_das_))) {
"%d,", enable_var_assign_use_das_))) {
SQL_PC_LOG(WARN, "failed to databuff_printf", K(ret), K(enable_var_assign_use_das_));
} else if (OB_FAIL(databuff_printf(buf, buf_len, pos,
"%d", bloom_filter_ratio_))) {
"%d,", enable_das_keep_order_))) {
SQL_PC_LOG(WARN, "failed to databuff_printf", K(ret), K(enable_das_keep_order_));
} else if (OB_FAIL(databuff_printf(buf, buf_len, pos,
"%d,", bloom_filter_ratio_))) {
SQL_PC_LOG(WARN, "failed to databuff_printf", K(ret), K(bloom_filter_ratio_));
} else {
// do nothing

2
src/sql/plan_cache/ob_plan_cache_util.h
View File

@ -1026,6 +1026,7 @@ public:
is_enable_px_fast_reclaim_(false),
enable_spf_batch_rescan_(false),
enable_var_assign_use_das_(false),
enable_das_keep_order_(false),
bloom_filter_ratio_(0),
cluster_config_version_(-1),
tenant_config_version_(-1),
@ -1070,6 +1071,7 @@ public:
bool is_enable_px_fast_reclaim_;
bool enable_spf_batch_rescan_;
bool enable_var_assign_use_das_;
bool enable_das_keep_order_;
int bloom_filter_ratio_;
private:

16
src/sql/resolver/ddl/ob_create_index_resolver.cpp
View File

@ -749,8 +749,20 @@ int ObCreateIndexResolver::set_table_option_to_stmt(bool is_partitioned)
index_arg.index_type_ = INDEX_TYPE_SPATIAL_LOCAL;
}
} else if (FTS_KEY == index_keyname_) {
// TODO hanxuan
ret = OB_NOT_SUPPORTED;
uint64_t tenant_data_version = 0;
if (OB_FAIL(GET_MIN_DATA_VERSION(session_info_->get_effective_tenant_id(), tenant_data_version))) {
LOG_WARN("get tenant data version failed", K(ret));
} else if (tenant_data_version < DATA_VERSION_4_3_2_0) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("tenant data version is less than 4.3.2, create fulltext index on existing table not supported", K(ret), K(tenant_data_version));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "tenant data version is less than 4.3.2, fulltext index");
} else if (global_) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("not support global fts index now", K(ret));
} else {
// set type to fts_index_aux first, append other fts arg later
index_arg.index_type_ = INDEX_TYPE_FTS_INDEX_LOCAL;
}
}
index_arg.data_table_id_ = data_table_id_;
index_arg.index_table_id_ = index_table_id_;

15
src/sql/resolver/ddl/ob_ddl_resolver.cpp
View File

@ -182,18 +182,19 @@ int ObDDLResolver::append_fts_args(
LOG_WARN("allocator is null", K(ret));
} else if (!fts_common_aux_table_exist) {
const int64_t num_fts_args = 4;
if (OB_FAIL(ObFtsIndexBuilderUtil::append_fts_rowkey_doc_arg(index_arg,
allocator,
index_arg_list))) {
// append fts index aux arg first, keep same logic as build fts index on existing table
if (OB_FAIL(ObFtsIndexBuilderUtil::append_fts_index_arg(index_arg,
allocator,
index_arg_list))) {
LOG_WARN("failed to append fts_index arg", K(ret));
} else if (OB_FAIL(ObFtsIndexBuilderUtil::append_fts_rowkey_doc_arg(index_arg,
allocator,
index_arg_list))) {
LOG_WARN("failed to append fts_rowkey_doc arg", K(ret));
} else if (OB_FAIL(ObFtsIndexBuilderUtil::append_fts_doc_rowkey_arg(index_arg,
allocator,
index_arg_list))) {
LOG_WARN("failed to append fts_doc_rowkey arg", K(ret));
} else if (OB_FAIL(ObFtsIndexBuilderUtil::append_fts_index_arg(index_arg,
allocator,
index_arg_list))) {
LOG_WARN("failed to append fts_index arg", K(ret));
} else if (OB_FAIL(ObFtsIndexBuilderUtil::append_fts_doc_word_arg(index_arg,
allocator,
index_arg_list))) {

5
src/sql/resolver/dml/ob_hint.cpp
View File

@ -838,6 +838,11 @@ bool ObOptParamHint::is_param_val_valid(const OptParamType param_type, const ObO
|| 0 == val.get_varchar().case_compare("false"));
break;
}
case ENABLE_DAS_KEEP_ORDER : {
is_valid = val.is_varchar() && (0 == val.get_varchar().case_compare("true")
|| 0 == val.get_varchar().case_compare("false"));
break;
}
case SPILL_COMPRESSION_CODEC: {
is_valid = val.is_varchar();
if (is_valid) {

1
src/sql/resolver/dml/ob_hint.h
View File

@ -156,6 +156,7 @@ struct ObOptParamHint
DEF(ENABLE_RICH_VECTOR_FORMAT,) \
DEF(_ENABLE_STORAGE_CARDINALITY_ESTIMATION,) \
DEF(PRESERVE_ORDER_FOR_PAGINATION,) \
DEF(ENABLE_DAS_KEEP_ORDER,) \
DEF(SPILL_COMPRESSION_CODEC,) \
DEF(INLIST_REWRITE_THRESHOLD,) \

1
src/storage/CMakeLists.txt
View File

@ -177,7 +177,6 @@ ob_set_subtarget(ob_storage fts
fts/ob_fts_stop_word.cpp
fts/ob_ngram_ft_parser.cpp
fts/ob_whitespace_ft_parser.cpp
fts/ob_text_retrieval_iterator.cpp
)
ob_set_subtarget(ob_storage high_availability

2
src/storage/access/ob_table_scan_range.cpp
View File

@ -187,7 +187,7 @@ int ObTableScanRange::init_rowkeys(const common::ObIArray<common::ObNewRange> &r
} else if (is_false) {
} else if (OB_FAIL(datum_rowkey.from_rowkey(rowkey, *allocator_))) {
STORAGE_LOG(WARN, "Failed to transfer rowkey to datum rowkey", K(ret));
} else if (FALSE_IT(datum_rowkey.set_group_idx(ranges.at(i).get_group_idx()))) {
} else if (FALSE_IT(datum_rowkey.set_group_idx(ranges.at(i).get_group_id()))) {
} else if (OB_FAIL(rowkeys_.push_back(datum_rowkey))) {
STORAGE_LOG(WARN, "Failed to push back datum rowkey", K(ret));
}

2
src/storage/blocksstable/ob_datum_range.h
View File

@ -212,7 +212,7 @@ OB_INLINE int ObDatumRange::from_range(const common::ObNewRange &range, ObIAlloc
STORAGE_LOG(WARN, "Failed to from end key", K(ret));
} else {
border_flag_ = range.border_flag_;
group_idx_ = range.get_group_idx();
group_idx_ = range.get_group_id();
}
return ret;

4
src/storage/blocksstable/ob_datum_rowkey.h
View File

@ -56,7 +56,7 @@ public:
OB_INLINE void set_max_rowkey() { *this = MAX_ROWKEY; store_rowkey_.set_max(); }
OB_INLINE void set_min_rowkey() { *this = MIN_ROWKEY; store_rowkey_.set_min(); }
OB_INLINE bool is_static_rowkey() const { return datums_ == &MIN_DATUM || datums_ == &MAX_DATUM; }
OB_INLINE void set_group_idx(const int32_t group_idx) { group_idx_ = group_idx; }
OB_INLINE void set_group_idx(const int64_t group_idx) { group_idx_ = group_idx; }
OB_INLINE int64_t get_group_idx() const { return group_idx_; }
OB_INLINE const common::ObStoreRowkey &get_store_rowkey() const { return store_rowkey_; }
//only for unittest
@ -97,7 +97,7 @@ public:
DECLARE_TO_STRING;
public:
int32_t datum_cnt_;
int32_t group_idx_;
int64_t group_idx_;
mutable uint64_t hash_;
ObStorageDatum *datums_;
common::ObStoreRowkey store_rowkey_;

177
src/storage/fts/ob_text_retrieval_iterator.h
View File

@ -1,177 +0,0 @@
/**
* Copyright (c) 2024 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OB_TEXT_RETRIEVAL_ITERATOR_H_
#define OB_TEXT_RETRIEVAL_ITERATOR_H_
#include "common/row/ob_row_iterator.h"
#include "storage/access/ob_dml_param.h"
namespace oceanbase
{
namespace sql
{
struct ObDASScanCtDef;
struct ObDASScanRtDef;
struct ObDASIRScanCtDef;
struct ObDASIRScanRtDef;
}
namespace storage
{
struct ObTokenRetrievalParam
{
public:
ObTokenRetrievalParam()
: app_avg_tablet_doc_token_cnt_(0),
ls_id_(),
inv_idx_tablet_id_(),
fwd_idx_tablet_id_(),
doc_id_idx_tablet_id_(),
ir_ctdef_(nullptr),
ir_rtdef_(nullptr)
{}
~ObTokenRetrievalParam() {}
bool need_relevance() const;
const share::ObLSID &get_ls_id() const;
const sql::ObDASIRScanCtDef *get_ir_ctdef() const;
sql::ObDASIRScanRtDef *get_ir_rtdef();
const sql::ObDASScanCtDef *get_inv_idx_scan_ctdef() const;
const sql::ObDASScanCtDef *get_inv_idx_agg_ctdef() const;
const sql::ObDASScanCtDef *get_fwd_idx_agg_ctdef() const;
const sql::ObDASScanCtDef *get_doc_id_idx_agg_ctdef() const;
const common::ObTabletID &get_inv_idx_tablet_id() const;
const common::ObTabletID &get_fwd_idx_tablet_id() const;
const common::ObTabletID &get_doc_id_idx_tablet_id() const;
inline void set_param(
const share::ObLSID &ls_id,
const ObTabletID &inv_idx_tablet_id,
const ObTabletID &fwd_idx_tablet_id,
const ObTabletID &doc_id_idx_tablet_id,
const sql::ObDASIRScanCtDef *ir_ctdef,
sql::ObDASIRScanRtDef *ir_rtdef,
const int64_t approx_avg_token_cnt = 0)
{
ls_id_ = ls_id;
inv_idx_tablet_id_ = inv_idx_tablet_id;
fwd_idx_tablet_id_ = fwd_idx_tablet_id;
doc_id_idx_tablet_id_ = doc_id_idx_tablet_id;
ir_ctdef_ = ir_ctdef;
ir_rtdef_ = ir_rtdef;
}
TO_STRING_KV(K_(app_avg_tablet_doc_token_cnt),
K_(ls_id),
K_(inv_idx_tablet_id),
K_(fwd_idx_tablet_id),
K_(doc_id_idx_tablet_id));
private:
int64_t app_avg_tablet_doc_token_cnt_; // TODO: use app avg tablet doc token cnt to calc bm25 idf
share::ObLSID ls_id_;
common::ObTabletID inv_idx_tablet_id_;
common::ObTabletID fwd_idx_tablet_id_;
common::ObTabletID doc_id_idx_tablet_id_;
const sql::ObDASIRScanCtDef *ir_ctdef_;
sql::ObDASIRScanRtDef *ir_rtdef_;
};
// Single token retrieval iter
class ObTextRetrievalIterator : public common::ObNewRowIterator
{
public:
ObTextRetrievalIterator();
virtual ~ObTextRetrievalIterator();
int init(
ObTokenRetrievalParam &retrieval_param,
const ObString &query_token,
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot);
virtual int get_next_row(ObNewRow *&row) override;
virtual int get_next_row() override;
virtual int get_next_rows(int64_t &count, int64_t capacity) override;
virtual void reset() override;
int get_curr_iter_row(const sql::ExprFixedArray *&curr_row, sql::ObEvalCtx *&curr_eval_ctx);
int get_curr_doc_id();
int forward_to_doc(const common::ObDocId &doc_id); // TODO: impl this primitive for skipping scan in conjunctive processing
TO_STRING_KV(KPC_(retrieval_param), K_(is_inited));
private:
int init_inv_idx_scan_param(const ObString &query_token);
int init_fwd_idx_scan_param();
static int init_base_idx_scan_param(
const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id,
const sql::ObDASScanCtDef *ctdef,
sql::ObDASScanRtDef *rtdef,
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot,
ObTableScanParam &scan_param);
int get_next_doc_token_cnt(const bool use_fwd_idx_agg);
int do_doc_cnt_agg();
int do_token_cnt_agg(const ObDocId &doc_id, int64_t &token_count);
int get_inv_idx_scan_doc_id(ObDocId &doc_id);
int fill_token_cnt_with_doc_len();
int fill_token_doc_cnt();
int project_relevance_expr();
int reuse_fwd_idx_iter();
int gen_inv_idx_scan_range(const ObString &query_token, ObNewRange &scan_range);
int gen_fwd_idx_scan_range(const ObDocId &doc_id, ObNewRange &scan_range);
inline bool need_calc_relevance() { return true; } // TODO: reduce tsc ops if no need to calc relevance
int init_calc_exprs();
void clear_row_wise_evaluated_flag();
// TODO: delete this after enable standard vectorized execution
inline int get_next_single_row(const bool is_vectorized, ObNewRowIterator *iter)
{
int ret = OB_SUCCESS;
if (is_vectorized) {
int64_t scan_row_cnt = 0;
ret = iter->get_next_rows(scan_row_cnt, 1);
} else {
ret = iter->get_next_row();
}
return ret;
}
private:
static const int64_t FWD_IDX_ROWKEY_COL_CNT = 2;
static const int64_t INV_IDX_ROWKEY_COL_CNT = 2;
private:
lib::MemoryContext mem_context_;
ObTokenRetrievalParam *retrieval_param_;
transaction::ObTxDesc *tx_desc_;
transaction::ObTxReadSnapshot *snapshot_;
ObTableScanParam inv_idx_scan_param_;
ObTableScanParam inv_idx_agg_param_;
ObTableScanParam fwd_idx_scan_param_;
common::ObSEArray<sql::ObExpr *, 2> calc_exprs_;
common::ObNewRowIterator *inverted_idx_iter_;
common::ObNewRowIterator *forward_idx_iter_;
ObObj *fwd_range_objs_;
sql::ObExpr *doc_token_cnt_expr_;
int64_t token_doc_cnt_;
bool need_fwd_idx_agg_;
bool need_inv_idx_agg_;
bool inv_idx_agg_evaluated_;
bool is_inited_;
};
}
}
#endif //OB_TEXT_RETRIEVAL_ITERATOR_H_

1
tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result
View File

@ -296,6 +296,7 @@ _enable_column_store
_enable_compaction_diagnose
_enable_compatible_monotonic
_enable_convert_real_to_decimal
_enable_das_keep_order
_enable_dblink_reuse_connection
_enable_dbms_job_package
_enable_dbms_lob_partial_update

2
tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_business_mysql.result
View File

@ -711,7 +711,7 @@ Outputs & filters:
-------------------------------------
0 - output([wfsydp.WSJJDM], [wfsydp.WSJWSY]), filter(nil), rowset=16
access([wfsydp.WSJJDM], [wfsydp.WSJWSY]), partitions(p0)
limit(181819), offset(0), is_index_back=false, is_global_index=true,
limit(181819), offset(0), is_index_back=false, is_global_index=true, keep_ordering=true,
range_key([wfsydp.WSJJDM], [wfsydp.WSJWSY], [wfsydp.WSJJZH], [wfsydp.WSDRDM], [wfsydp.WSJYZH]), range(666888,MIN,MIN,MIN,MIN ; 666888,MAX,MAX,MAX,
MAX),
range_cond([wfsydp.WSJJDM = '666888'])

6
tools/deploy/mysql_test/test_suite/subquery/r/mysql/subquery.result
View File

@ -760,14 +760,14 @@ Outputs & filters:
group(nil), agg_func([T_FUN_SUM(t2.c1)], [T_FUN_COUNT(t2.c1)])
4 - output([t2.c1]), filter(nil), rowset=16
access([t2.c1]), partitions(p0)
is_index_back=false, is_global_index=false,
is_index_back=false, is_global_index=false, keep_ordering=true,
range_key([t2.c1]), range(MIN ; MAX)always true,
range_cond([t2.c1 = :0])
5 - output([cast(T_FUN_COUNT(*), DECIMAL(24, 4))]), filter(nil), rowset=16
group(nil), agg_func([T_FUN_COUNT(*)])
6 - output(nil), filter(nil), rowset=16
access(nil), partitions(p0)
is_index_back=false, is_global_index=false,
is_index_back=false, is_global_index=false, keep_ordering=true,
range_key([t3.c1]), range(MIN ; MAX)always true,
range_cond([t3.c1 = :0])
explain select * from t1 where t1.c1 != (select c2 from t2 where t2.c1 = (select max(c2) from t3 where t3.c1 = t1.c1) order by t2.c2 limit 1);
@ -803,7 +803,7 @@ Outputs & filters:
group(nil), agg_func([T_FUN_MAX(t3.c2)])
6 - output([t3.c2]), filter(nil), rowset=16
access([t3.c2]), partitions(p0)
is_index_back=false, is_global_index=false,
is_index_back=false, is_global_index=false, keep_ordering=true,
range_key([t3.c1]), range(MIN ; MAX)always true,
range_cond([t3.c1 = :0])
7 - output([t2.c1], [t2.c2]), filter(nil), rowset=16

8
tools/deploy/mysql_test/test_suite/update/r/mysql/update2.result
View File

@ -338,7 +338,7 @@ Outputs & filters:
-------------------------------------
0 - output([t1.a], [t1.b]), filter(nil), rowset=16
access([t1.a], [t1.b]), partitions(p0)
is_index_back=false, is_global_index=true,
is_index_back=false, is_global_index=true, keep_ordering=true,
range_key([t1.b], [t1.a]), range(MIN,MIN ; MAX,MAX)always true
select * from t1;
+---+------+
@ -396,7 +396,7 @@ Outputs & filters:
-------------------------------------
0 - output([t1.a], [t1.b]), filter(nil), rowset=16
access([t1.a], [t1.b]), partitions(p0)
is_index_back=false, is_global_index=true,
is_index_back=false, is_global_index=true, keep_ordering=true,
range_key([t1.b], [t1.a]), range(MIN,MIN ; MAX,MAX)always true
select * from t1;
+---+------+
@ -3283,7 +3283,7 @@ Outputs & filters:
[gf_ar_mthly_bill.chked_amt], [gf_ar_mthly_bill.writingoff_amt], [gf_ar_mthly_bill.svc_bill_amt], [gf_ar_mthly_bill.svc_ccy], [gf_ar_mthly_bill.env_source],
[gf_ar_mthly_bill.metadata_source], [gf_ar_mthly_bill.setl_time_zone], [gf_ar_mthly_bill.actg_time_zone], [gf_ar_mthly_bill.inter_trade_flag], [gf_ar_mthly_bill.actg_bill_mth],
[gf_ar_mthly_bill.auto_writeoff_group_no]), partitions(p0)
is_index_back=true, is_global_index=true,
is_index_back=true, is_global_index=true, keep_ordering=true,
range_key([gf_ar_mthly_bill.tnt_inst_id], [gf_ar_mthly_bill.rcrd_id], [gf_ar_mthly_bill.shadow_pk_0], [gf_ar_mthly_bill.shadow_pk_1]), range(MIN,MIN,
MIN,MIN ; MAX,MAX,MAX,MAX)always true
select * from gf_ar_mthly_bill;
@ -3461,7 +3461,7 @@ Outputs & filters:
[gf_ar_mthly_bill.chked_amt], [gf_ar_mthly_bill.writingoff_amt], [gf_ar_mthly_bill.svc_bill_amt], [gf_ar_mthly_bill.svc_ccy], [gf_ar_mthly_bill.env_source],
[gf_ar_mthly_bill.metadata_source], [gf_ar_mthly_bill.setl_time_zone], [gf_ar_mthly_bill.actg_time_zone], [gf_ar_mthly_bill.inter_trade_flag], [gf_ar_mthly_bill.actg_bill_mth],
[gf_ar_mthly_bill.auto_writeoff_group_no]), partitions(p0)
is_index_back=true, is_global_index=true,
is_index_back=true, is_global_index=true, keep_ordering=true,
range_key([gf_ar_mthly_bill.tnt_inst_id], [gf_ar_mthly_bill.rcrd_id], [gf_ar_mthly_bill.shadow_pk_0], [gf_ar_mthly_bill.shadow_pk_1]), range(MIN,MIN,
MIN,MIN ; MAX,MAX,MAX,MAX)always true
select * from gf_ar_mthly_bill;

12
unittest/sql/rewrite/test_query_range.cpp
View File

@ -823,7 +823,7 @@ TEST_F(ObQueryRangeTest, single_filed_key_whole_range1)
OK(query_range.preliminary_extract_query_range(single_range_columns_, NULL, NULL, &exec_ctx_));
OK(query_range.final_extract_query_range(exec_ctx_, NULL));
OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params));
ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,(MIN;MAX)\"}]"));
ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)\"}]"));
}
TEST_F(ObQueryRangeTest, single_filed_key_whole_range2)
@ -840,7 +840,7 @@ TEST_F(ObQueryRangeTest, single_filed_key_whole_range2)
OK(query_range.preliminary_extract_query_range(single_range_columns_, exprs, dtc_params, &exec_ctx_));
OK(query_range.final_extract_query_range(exec_ctx_, dtc_params));
OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params));
ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,(MIN;MAX)\"}]"));
ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)\"}]"));
}
TEST_F(ObQueryRangeTest, double_filed_key_whole_range1)
@ -856,7 +856,7 @@ TEST_F(ObQueryRangeTest, double_filed_key_whole_range1)
OK(query_range.preliminary_extract_query_range(double_range_columns_, NULL, dtc_params, &exec_ctx_));
OK(query_range.final_extract_query_range(exec_ctx_, dtc_params));
OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params));
ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)\"}]"));
ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)\"}]"));
}
TEST_F(ObQueryRangeTest, double_filed_key_whole_range2)
@ -873,7 +873,7 @@ TEST_F(ObQueryRangeTest, double_filed_key_whole_range2)
OK(query_range.preliminary_extract_query_range(double_range_columns_, exprs, dtc_params, &exec_ctx_));
OK(query_range.final_extract_query_range(exec_ctx_, dtc_params));
OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params));
ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)\"}]"));
ASSERT_EQ(0, strcmp(to_cstring(ranges), "[{\"range\":\"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)\"}]"));
}
TEST_F(ObQueryRangeTest, range_column_with_like)
@ -1067,8 +1067,8 @@ TEST_F(ObQueryRangeTest, basic_test)
except_result(double_range_columns_,
params,
"(b = 6 and a < 5) or (a > 8 and b = 15)",
"[{\"range\":\"table_id:3003,group_idx:0,({\"NULL\":\"NULL\"},MAX;{\"BIGINT\":5},MIN)\"}, "
"{\"range\":\"table_id:3003,group_idx:0,({\"BIGINT\":8},MAX;MAX,{\"BIGINT\":15})\"}]",
"[{\"range\":\"table_id:3003,group_idx:0,index_ordered_idx:0,({\"NULL\":\"NULL\"},MAX;{\"BIGINT\":5},MIN)\"}, "
"{\"range\":\"table_id:3003,group_idx:0,index_ordered_idx:0,({\"BIGINT\":8},MAX;MAX,{\"BIGINT\":15})\"}]",
false);
query_range.reset();
}

414
unittest/sql/rewrite/test_query_range_collation.result
View File

File diff suppressed because it is too large Load Diff

138
unittest/sql/rewrite/test_query_range_filter.result
View File

@ -8,7 +8,7 @@ all_single_value_ranges = 1
star_border_flag[0] = 1
end_border_flag[0] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}]
**rowkey num = 2**
**filter count = 0**
@ -18,7 +18,7 @@ all_single_value_ranges = 0
star_border_flag[0] = 0
end_border_flag[0] = 0
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}]
**rowkey num = 3**
**filter count = 0**
@ -28,7 +28,7 @@ all_single_value_ranges = 0
star_border_flag[0] = 0
end_border_flag[0] = 0
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}]
[1] a in (1,2,3) or (a = 4 and b = 4)
@ -46,7 +46,7 @@ end_border_flag[2] = 1
star_border_flag[3] = 1
end_border_flag[3] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}]
**rowkey num = 2**
**filter count = 0**
@ -62,7 +62,7 @@ end_border_flag[2] = 0
star_border_flag[3] = 1
end_border_flag[3] = 1
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}]
**rowkey num = 3**
**filter count = 0**
@ -78,7 +78,7 @@ end_border_flag[2] = 0
star_border_flag[3] = 0
end_border_flag[3] = 0
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN,MIN;{"BIGINT":2},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN,MIN;{"BIGINT":3},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN,MIN;{"BIGINT":2},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN,MIN;{"BIGINT":3},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}]
[2] (a, b) in ((1,2),(2,3))
@ -92,7 +92,7 @@ end_border_flag[0] = 1
star_border_flag[1] = 1
end_border_flag[1] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}]
**rowkey num = 2**
**filter count = 1**
@ -104,7 +104,7 @@ end_border_flag[0] = 1
star_border_flag[1] = 1
end_border_flag[1] = 1
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}]
**rowkey num = 3**
**filter count = 1**
@ -116,7 +116,7 @@ end_border_flag[0] = 0
star_border_flag[1] = 0
end_border_flag[1] = 0
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},{"BIGINT":3},MIN;{"BIGINT":2},{"BIGINT":3},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},{"BIGINT":3},MIN;{"BIGINT":2},{"BIGINT":3},MAX)"}]
[3] a > 1
@ -128,7 +128,7 @@ all_single_value_ranges = 0
star_border_flag[0] = 0
end_border_flag[0] = 0
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1};MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1};MAX)"}]
**rowkey num = 2**
**filter count = 1**
@ -138,7 +138,7 @@ all_single_value_ranges = 0
star_border_flag[0] = 0
end_border_flag[0] = 0
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MAX;MAX,MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MAX;MAX,MAX)"}]
**rowkey num = 3**
**filter count = 1**
@ -148,7 +148,7 @@ all_single_value_ranges = 0
star_border_flag[0] = 0
end_border_flag[0] = 0
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MAX;MAX,MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MAX;MAX,MAX)"}]
[4] a > 1 and b = 1
@ -160,7 +160,7 @@ all_single_value_ranges = 0
star_border_flag[0] = 0
end_border_flag[0] = 0
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1};MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1};MAX)"}]
**rowkey num = 2**
**filter count = 1**
@ -170,7 +170,7 @@ all_single_value_ranges = 0
star_border_flag[0] = 0
end_border_flag[0] = 0
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MAX;MAX,{"BIGINT":1})"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MAX;MAX,{"BIGINT":1})"}]
**rowkey num = 3**
**filter count = 1**
@ -180,7 +180,7 @@ all_single_value_ranges = 0
star_border_flag[0] = 0
end_border_flag[0] = 0
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MAX,MAX;MAX,{"BIGINT":1},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MAX,MAX;MAX,{"BIGINT":1},MAX)"}]
[5] a in (1,2,3)
@ -196,7 +196,7 @@ end_border_flag[1] = 1
star_border_flag[2] = 1
end_border_flag[2] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}]
**rowkey num = 2**
**filter count = 1**
@ -210,7 +210,7 @@ end_border_flag[1] = 0
star_border_flag[2] = 0
end_border_flag[2] = 0
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}]
**rowkey num = 3**
**filter count = 1**
@ -224,7 +224,7 @@ end_border_flag[1] = 0
star_border_flag[2] = 0
end_border_flag[2] = 0
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}]
[6] a in (?{1},?{2},?{3}) or (a = ?{4})
@ -242,7 +242,7 @@ end_border_flag[2] = 1
star_border_flag[3] = 1
end_border_flag[3] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}]
**rowkey num = 2**
**filter count = 1**
@ -258,7 +258,7 @@ end_border_flag[2] = 0
star_border_flag[3] = 0
end_border_flag[3] = 0
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},MIN;{"BIGINT":4},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":4},MIN;{"BIGINT":4},MAX)"}]
**rowkey num = 3**
**filter count = 1**
@ -274,7 +274,7 @@ end_border_flag[2] = 0
star_border_flag[3] = 0
end_border_flag[3] = 0
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},MIN;{"BIGINT":4},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":4},MIN;{"BIGINT":4},MAX)"}]
[7] a in (?{1},?{2},?{3}) or (a = ?{4} and b = ?{4})
@ -292,7 +292,7 @@ end_border_flag[2] = 1
star_border_flag[3] = 1
end_border_flag[3] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}]
**rowkey num = 2**
**filter count = 0**
@ -308,7 +308,7 @@ end_border_flag[2] = 0
star_border_flag[3] = 1
end_border_flag[3] = 1
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}]
**rowkey num = 3**
**filter count = 0**
@ -324,7 +324,7 @@ end_border_flag[2] = 0
star_border_flag[3] = 0
end_border_flag[3] = 0
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN,MIN;{"BIGINT":2},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN,MIN;{"BIGINT":3},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN,MIN;{"BIGINT":2},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN,MIN;{"BIGINT":3},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}]
[8] a in (1,2,3) and b = 2
@ -340,7 +340,7 @@ end_border_flag[1] = 1
star_border_flag[2] = 1
end_border_flag[2] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}]
**rowkey num = 2**
**filter count = 2**
@ -354,7 +354,7 @@ end_border_flag[1] = 1
star_border_flag[2] = 1
end_border_flag[2] = 1
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3},{"BIGINT":2};{"BIGINT":3},{"BIGINT":2}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3},{"BIGINT":2};{"BIGINT":3},{"BIGINT":2}]"}]
**rowkey num = 3**
**filter count = 2**
@ -368,7 +368,7 @@ end_border_flag[1] = 0
star_border_flag[2] = 0
end_border_flag[2] = 0
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},{"BIGINT":2},MIN;{"BIGINT":2},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},{"BIGINT":2},MIN;{"BIGINT":3},{"BIGINT":2},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},{"BIGINT":2},MIN;{"BIGINT":2},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},{"BIGINT":2},MIN;{"BIGINT":3},{"BIGINT":2},MAX)"}]
[9] (a,b) in ((1,2),(2,3)) or (a,b) in ((3,4),(4,5))
@ -386,7 +386,7 @@ end_border_flag[2] = 1
star_border_flag[3] = 1
end_border_flag[3] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}]
**rowkey num = 2**
**filter count = 1**
@ -402,7 +402,7 @@ end_border_flag[2] = 1
star_border_flag[3] = 1
end_border_flag[3] = 1
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3},{"BIGINT":4};{"BIGINT":3},{"BIGINT":4}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":5};{"BIGINT":4},{"BIGINT":5}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3},{"BIGINT":4};{"BIGINT":3},{"BIGINT":4}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4},{"BIGINT":5};{"BIGINT":4},{"BIGINT":5}]"}]
**rowkey num = 3**
**filter count = 1**
@ -418,7 +418,7 @@ end_border_flag[2] = 0
star_border_flag[3] = 0
end_border_flag[3] = 0
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},{"BIGINT":3},MIN;{"BIGINT":2},{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},{"BIGINT":4},MIN;{"BIGINT":3},{"BIGINT":4},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},{"BIGINT":5},MIN;{"BIGINT":4},{"BIGINT":5},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},{"BIGINT":3},MIN;{"BIGINT":2},{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},{"BIGINT":4},MIN;{"BIGINT":3},{"BIGINT":4},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":4},{"BIGINT":5},MIN;{"BIGINT":4},{"BIGINT":5},MAX)"}]
[10] (a,b) in ((1,2),(2,3)) and c > 1
@ -432,7 +432,7 @@ end_border_flag[0] = 1
star_border_flag[1] = 1
end_border_flag[1] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}]
**rowkey num = 2**
**filter count = 1**
@ -444,7 +444,7 @@ end_border_flag[0] = 1
star_border_flag[1] = 1
end_border_flag[1] = 1
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}]
**rowkey num = 3**
**filter count = 0**
@ -456,7 +456,7 @@ end_border_flag[0] = 0
star_border_flag[1] = 0
end_border_flag[1] = 0
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},{"BIGINT":2},{"BIGINT":1};{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},{"BIGINT":3},{"BIGINT":1};{"BIGINT":2},{"BIGINT":3},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},{"BIGINT":2},{"BIGINT":1};{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},{"BIGINT":3},{"BIGINT":1};{"BIGINT":2},{"BIGINT":3},MAX)"}]
[11] (a,b) in ((1,2),(2,3)) and c = 1
@ -470,7 +470,7 @@ end_border_flag[0] = 1
star_border_flag[1] = 1
end_border_flag[1] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}]
**rowkey num = 2**
**filter count = 1**
@ -482,7 +482,7 @@ end_border_flag[0] = 1
star_border_flag[1] = 1
end_border_flag[1] = 1
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}]
**rowkey num = 3**
**filter count = 2**
@ -494,7 +494,7 @@ end_border_flag[0] = 1
star_border_flag[1] = 1
end_border_flag[1] = 1
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2},{"BIGINT":1};{"BIGINT":1},{"BIGINT":2},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":3},{"BIGINT":1};{"BIGINT":2},{"BIGINT":3},{"BIGINT":1}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2},{"BIGINT":1};{"BIGINT":1},{"BIGINT":2},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":3},{"BIGINT":1};{"BIGINT":2},{"BIGINT":3},{"BIGINT":1}]"}]
[12] a = 1 and b = 1
@ -506,7 +506,7 @@ all_single_value_ranges = 1
star_border_flag[0] = 1
end_border_flag[0] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}]
**rowkey num = 2**
**filter count = 2**
@ -516,7 +516,7 @@ all_single_value_ranges = 1
star_border_flag[0] = 1
end_border_flag[0] = 1
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}]
**rowkey num = 3**
**filter count = 2**
@ -526,7 +526,7 @@ all_single_value_ranges = 0
star_border_flag[0] = 0
end_border_flag[0] = 0
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},{"BIGINT":1},MIN;{"BIGINT":1},{"BIGINT":1},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},{"BIGINT":1},MIN;{"BIGINT":1},{"BIGINT":1},MAX)"}]
[13] (a, b, c) in ((?{1},?{1},?{1}),(?{2},?{2},?{2})) or (a,b) in ((?{3},?{3}),(?{4},?{4}))
@ -544,7 +544,7 @@ end_border_flag[2] = 1
star_border_flag[3] = 1
end_border_flag[3] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}]
**rowkey num = 2**
**filter count = 0**
@ -560,7 +560,7 @@ end_border_flag[2] = 1
star_border_flag[3] = 1
end_border_flag[3] = 1
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3},{"BIGINT":3};{"BIGINT":3},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3},{"BIGINT":3};{"BIGINT":3},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}]
**rowkey num = 3**
**filter count = 0**
@ -576,7 +576,7 @@ end_border_flag[2] = 0
star_border_flag[3] = 0
end_border_flag[3] = 0
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},{"BIGINT":3},MIN;{"BIGINT":3},{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},{"BIGINT":3},MIN;{"BIGINT":3},{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}]
[14] (a, b, c) in ((?{1},?{1},?{1}),(?{2},?{2},?{2})) or (a,b) in ((?{3},?{3}),(?{4},?{4})) and a = ?{1}
@ -590,7 +590,7 @@ end_border_flag[0] = 1
star_border_flag[1] = 1
end_border_flag[1] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}]
**rowkey num = 2**
**filter count = 0**
@ -602,7 +602,7 @@ end_border_flag[0] = 1
star_border_flag[1] = 1
end_border_flag[1] = 1
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}]
**rowkey num = 3**
**filter count = 0**
@ -614,7 +614,7 @@ end_border_flag[0] = 1
star_border_flag[1] = 1
end_border_flag[1] = 1
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}]
[15] (a, b, c) in ((?{1},?{1},?{1}),(?{2},?{2},?{2})) or (a,b) in ((?{3},?{3}),(?{4},?{4})) and c = ?{1}
@ -632,7 +632,7 @@ end_border_flag[2] = 1
star_border_flag[3] = 1
end_border_flag[3] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}]
**rowkey num = 2**
**filter count = 0**
@ -648,7 +648,7 @@ end_border_flag[2] = 1
star_border_flag[3] = 1
end_border_flag[3] = 1
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3},{"BIGINT":3};{"BIGINT":3},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3},{"BIGINT":3};{"BIGINT":3},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}]
**rowkey num = 3**
**filter count = 1**
@ -664,7 +664,7 @@ end_border_flag[2] = 1
star_border_flag[3] = 1
end_border_flag[3] = 1
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3},{"BIGINT":3},{"BIGINT":1};{"BIGINT":3},{"BIGINT":3},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":4},{"BIGINT":1};{"BIGINT":4},{"BIGINT":4},{"BIGINT":1}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3},{"BIGINT":3},{"BIGINT":1};{"BIGINT":3},{"BIGINT":3},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4},{"BIGINT":4},{"BIGINT":1};{"BIGINT":4},{"BIGINT":4},{"BIGINT":1}]"}]
[16] (a, b, c) in ((?{1},?{1},?{1}),(?{2},?{2},?{2})) or (a,b) in ((?{3},?{3}),(?{4},?{4})) or a = ?{1}
@ -682,7 +682,7 @@ end_border_flag[2] = 1
star_border_flag[3] = 1
end_border_flag[3] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4};{"BIGINT":4}]"}]
**rowkey num = 2**
**filter count = 0**
@ -698,7 +698,7 @@ end_border_flag[2] = 1
star_border_flag[3] = 1
end_border_flag[3] = 1
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3},{"BIGINT":3};{"BIGINT":3},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3},{"BIGINT":3};{"BIGINT":3},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}]
**rowkey num = 3**
**filter count = 0**
@ -714,7 +714,7 @@ end_border_flag[2] = 0
star_border_flag[3] = 0
end_border_flag[3] = 0
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},{"BIGINT":3},MIN;{"BIGINT":3},{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},{"BIGINT":3},MIN;{"BIGINT":3},{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}]
[17] a = 1 and c = 1 or (a = 2 and b = 2)
@ -728,7 +728,7 @@ end_border_flag[0] = 1
star_border_flag[1] = 1
end_border_flag[1] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}]
**rowkey num = 2**
**filter count = 0**
@ -740,7 +740,7 @@ end_border_flag[0] = 0
star_border_flag[1] = 1
end_border_flag[1] = 1
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}]
**rowkey num = 3**
**filter count = 0**
@ -752,7 +752,7 @@ end_border_flag[0] = 0
star_border_flag[1] = 0
end_border_flag[1] = 0
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},{"BIGINT":2},MIN;{"BIGINT":2},{"BIGINT":2},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},{"BIGINT":2},MIN;{"BIGINT":2},{"BIGINT":2},MAX)"}]
[18] a = 1 and b = 1 or (b = 2 and c = 2)
@ -764,7 +764,7 @@ all_single_value_ranges = 0
star_border_flag[0] = 0
end_border_flag[0] = 0
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,(MIN;MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN;MAX)"}]
**rowkey num = 2**
**filter count = 0**
@ -774,7 +774,7 @@ all_single_value_ranges = 0
star_border_flag[0] = 0
end_border_flag[0] = 0
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,(MIN,MIN;MAX,MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN;MAX,MAX)"}]
**rowkey num = 3**
**filter count = 0**
@ -784,7 +784,7 @@ all_single_value_ranges = 0
star_border_flag[0] = 0
end_border_flag[0] = 0
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,(MIN,MIN,MIN;MAX,MAX,MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,(MIN,MIN,MIN;MAX,MAX,MAX)"}]
[19] (a,b) in ((1,2)) or (a,c) in ((2,3),(3,4))
@ -800,7 +800,7 @@ end_border_flag[1] = 1
star_border_flag[2] = 1
end_border_flag[2] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}]
**rowkey num = 2**
**filter count = 0**
@ -814,7 +814,7 @@ end_border_flag[1] = 0
star_border_flag[2] = 0
end_border_flag[2] = 0
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN;{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}]
**rowkey num = 3**
**filter count = 0**
@ -828,7 +828,7 @@ end_border_flag[1] = 0
star_border_flag[2] = 0
end_border_flag[2] = 0
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},MIN,MIN;{"BIGINT":2},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN,MIN;{"BIGINT":3},MAX,MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":2},MIN,MIN;{"BIGINT":2},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN,MIN;{"BIGINT":3},MAX,MAX)"}]
[20] (a,b) in ((1,2)) or (a = 3 and c = 3)
@ -842,7 +842,7 @@ end_border_flag[0] = 1
star_border_flag[1] = 1
end_border_flag[1] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":3};{"BIGINT":3}]"}]
**rowkey num = 2**
**filter count = 0**
@ -854,7 +854,7 @@ end_border_flag[0] = 1
star_border_flag[1] = 0
end_border_flag[1] = 0
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN;{"BIGINT":3},MAX)"}]
**rowkey num = 3**
**filter count = 0**
@ -866,7 +866,7 @@ end_border_flag[0] = 0
star_border_flag[1] = 0
end_border_flag[1] = 0
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},MIN,MIN;{"BIGINT":3},MAX,MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":1},{"BIGINT":2},MIN;{"BIGINT":1},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":3},MIN,MIN;{"BIGINT":3},MAX,MAX)"}]
[21] (a,b,c) in ((1,2,3),(2,3,4)) or (a = 5 and b = 5) or (a = 7)
@ -884,7 +884,7 @@ end_border_flag[2] = 1
star_border_flag[3] = 1
end_border_flag[3] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":5};{"BIGINT":5}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":7};{"BIGINT":7}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1};{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2};{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":5};{"BIGINT":5}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":7};{"BIGINT":7}]"}]
**rowkey num = 2**
**filter count = 0**
@ -900,7 +900,7 @@ end_border_flag[2] = 1
star_border_flag[3] = 0
end_border_flag[3] = 0
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":5},{"BIGINT":5};{"BIGINT":5},{"BIGINT":5}]"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":7},MIN;{"BIGINT":7},MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2};{"BIGINT":1},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":3};{"BIGINT":2},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":5},{"BIGINT":5};{"BIGINT":5},{"BIGINT":5}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":7},MIN;{"BIGINT":7},MAX)"}]
**rowkey num = 3**
**filter count = 0**
@ -916,7 +916,7 @@ end_border_flag[2] = 0
star_border_flag[3] = 0
end_border_flag[3] = 0
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":2},{"BIGINT":3};{"BIGINT":1},{"BIGINT":2},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":3},{"BIGINT":4};{"BIGINT":2},{"BIGINT":3},{"BIGINT":4}]"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":5},{"BIGINT":5},MIN;{"BIGINT":5},{"BIGINT":5},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":7},MIN,MIN;{"BIGINT":7},MAX,MAX)"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":1},{"BIGINT":2},{"BIGINT":3};{"BIGINT":1},{"BIGINT":2},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":2},{"BIGINT":3},{"BIGINT":4};{"BIGINT":2},{"BIGINT":3},{"BIGINT":4}]"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":5},{"BIGINT":5},MIN;{"BIGINT":5},{"BIGINT":5},MAX)"}, {"range":"table_id:3003,group_idx:0,index_ordered_idx:0,({"BIGINT":7},MIN,MIN;{"BIGINT":7},MAX,MAX)"}]
[22] a = 8 and c = 9 and b = 1 or (a,b,c) in ((1,2,3),(2,3,4)) and (a = 10)
@ -928,7 +928,7 @@ all_single_value_ranges = 1
star_border_flag[0] = 1
end_border_flag[0] = 1
count of rang columns = 1
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":8};{"BIGINT":8}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":8};{"BIGINT":8}]"}]
**rowkey num = 2**
**filter count = 0**
@ -938,7 +938,7 @@ all_single_value_ranges = 1
star_border_flag[0] = 1
end_border_flag[0] = 1
count of rang columns = 2
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":8},{"BIGINT":1};{"BIGINT":8},{"BIGINT":1}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":8},{"BIGINT":1};{"BIGINT":8},{"BIGINT":1}]"}]
**rowkey num = 3**
**filter count = 1**
@ -948,5 +948,5 @@ all_single_value_ranges = 1
star_border_flag[0] = 1
end_border_flag[0] = 1
count of rang columns = 3
[{"range":"table_id:3003,group_idx:0,[{"BIGINT":8},{"BIGINT":1},{"BIGINT":9};{"BIGINT":8},{"BIGINT":1},{"BIGINT":9}]"}]
[{"range":"table_id:3003,group_idx:0,index_ordered_idx:0,[{"BIGINT":8},{"BIGINT":1},{"BIGINT":9};{"BIGINT":8},{"BIGINT":1},{"BIGINT":9}]"}]