From ca3cbbc9041bae4db45c00e0f7a67dfdac889228 Mon Sep 17 00:00:00 2001 From: HoniiTro19 Date: Mon, 2 Dec 2024 20:46:18 +0000 Subject: [PATCH] adjust column group array order when creating new tables and delayed column transform --- deps/oblib/src/lib/ob_define.h | 2 + src/rootserver/ob_ddl_service.cpp | 16 ++-- src/rootserver/ob_index_builder.cpp | 5 +- src/share/schema/ob_schema_utils.cpp | 5 +- src/share/schema/ob_table_param.cpp | 8 +- src/share/schema/ob_table_schema.cpp | 84 ++++++++++++++++--- src/share/schema/ob_table_schema.h | 3 + .../ddl/ob_create_table_resolver_base.cpp | 6 +- src/sql/resolver/ddl/ob_ddl_resolver.cpp | 2 +- 9 files changed, 103 insertions(+), 28 deletions(-) diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index 39f37fb79..d8758a31d 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -875,6 +875,8 @@ const int64_t MAX_COLUMN_CHAR_LENGTH = 255; const uint64_t INVALID_COLUMN_GROUP_ID = 0; const uint64_t DEFAULT_TYPE_COLUMN_GROUP_ID = 1; // reserve 2~999 const uint64_t COLUMN_GROUP_START_ID = 1000; +const uint64_t ALL_COLUMN_GROUP_ID = 1001; +const uint64_t ROWKEY_COLUMN_GROUP_ID = 1002; const uint64_t DEFAULT_CUSTOMIZED_CG_NUM = 2; const int64_t OB_CG_NAME_PREFIX_LENGTH = 5; // length of cg prefix like "__cg_" const int64_t OB_MAX_COLUMN_GROUP_NAME_LENGTH = OB_MAX_COLUMN_NAME_LENGTH * OB_MAX_CHAR_LEN + OB_CG_NAME_PREFIX_LENGTH; //(max_column_name_length(128) * ob_max_char_len(3)) + prefix diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index b13857354..1c40dbd9a 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -5129,21 +5129,20 @@ int ObDDLService::adjust_cg_for_offline(ObTableSchema &new_table_schema) } else if (col->is_virtual_generated_column()) { /* skip virtual column group*/ } else if (OB_FAIL(ObSchemaUtils::build_single_column_group(new_table_schema, col, new_table_schema.get_tenant_id(), - new_table_schema.get_max_used_column_group_id() +1, + new_table_schema.get_next_single_column_group_id(), new_single_cg))) { LOG_WARN("fail to build single column group", K(ret)); } else if (OB_FAIL(new_table_schema.add_column_group(new_single_cg))) { LOG_WARN("fail to add new column group to table schema", K(ret)); } } - /* add all column group*/ - if (OB_SUCC(ret) &&is_all_cg_exist) { + if (OB_SUCC(ret) && is_all_cg_exist) { ObColumnGroupSchema new_cg; new_cg.reset(); if (OB_FAIL(ObSchemaUtils::build_all_column_group( new_table_schema, new_table_schema.get_tenant_id(), - new_table_schema.get_max_used_column_group_id() +1, new_cg))) { + ALL_COLUMN_GROUP_ID, new_cg))) { LOG_WARN("fail to build new all column group schema", K(ret)); } else if (OB_FAIL(new_table_schema.add_column_group(new_cg))) { LOG_WARN("fail to add new column group to table schema", K(ret)); @@ -5167,6 +5166,10 @@ int ObDDLService::adjust_cg_for_offline(ObTableSchema &new_table_schema) LOG_WARN("fail to alter default column grouop schema", K(ret)); } } + // adjust column group array order + if (FAILEDx(new_table_schema.adjust_column_group_array())) { + LOG_WARN("fail to adjust column group array", K(ret), K(new_table_schema)); + } } return ret; } @@ -10735,7 +10738,7 @@ int ObDDLService::add_column_to_column_group( common::ObMySQLTransaction &trans) { int ret = OB_SUCCESS; - uint64_t cur_column_group_id = origin_table_schema.get_max_used_column_group_id(); + uint64_t cur_column_group_id = origin_table_schema.get_next_single_column_group_id(); ObArray column_ids; ObTableSchema::const_column_iterator it_begin = alter_table_schema.column_begin(); ObTableSchema::const_column_iterator it_end = alter_table_schema.column_end(); @@ -10790,7 +10793,7 @@ int ObDDLService::add_column_to_column_group( if (OB_FAIL(ObSchemaUtils::build_single_column_group(new_table_schema, new_table_schema.get_column_schema(column_ids.at(i)), new_table_schema.get_tenant_id(), - ++cur_column_group_id, + cur_column_group_id++, cg_schema))) { LOG_WARN("fail to build single column group", K(ret), K(new_table_schema), K(column_ids.at(i))); } else if (OB_FAIL(new_table_schema.add_column_group(cg_schema))) { @@ -15702,6 +15705,7 @@ int ObDDLService::do_offline_ddl_in_trans(obrpc::ObAlterTableArg &alter_table_ar } } } + // submit async build index task if (OB_FAIL(ret)) { } else if (is_double_table_long_running_ddl(ddl_type)) { diff --git a/src/rootserver/ob_index_builder.cpp b/src/rootserver/ob_index_builder.cpp index e4c8cc5f9..9e2def043 100644 --- a/src/rootserver/ob_index_builder.cpp +++ b/src/rootserver/ob_index_builder.cpp @@ -1637,7 +1637,7 @@ int ObIndexBuilder::create_index_column_group(const obrpc::ObCreateIndexArg &arg } else if (!is_all_cg_exist) { } else if (OB_FAIL(ObSchemaUtils::build_all_column_group(index_table_schema, index_table_schema.get_tenant_id(), - index_table_schema.get_max_used_column_group_id() + 1, + ALL_COLUMN_GROUP_ID, tmp_cg))) { LOG_WARN("failed to build all column group", K(ret)); } else if (OB_FAIL(index_table_schema.add_column_group(tmp_cg))) { @@ -1664,6 +1664,9 @@ int ObIndexBuilder::create_index_column_group(const obrpc::ObCreateIndexArg &arg LOG_USER_ERROR(OB_NOT_SUPPORTED, "tenant data version is less than 4.3, create index with column group"); } + if (FAILEDx(index_table_schema.adjust_column_group_array())) { + LOG_WARN("fail to adjust column group array", K(ret), K(index_table_schema)); + } return ret; } diff --git a/src/share/schema/ob_schema_utils.cpp b/src/share/schema/ob_schema_utils.cpp index 61f080607..e18ce2e80 100644 --- a/src/share/schema/ob_schema_utils.cpp +++ b/src/share/schema/ob_schema_utils.cpp @@ -803,7 +803,6 @@ int ObSchemaUtils::alter_rowkey_column_group(share::schema::ObTableSchema &table if (OB_ISNULL(rowkey_cg)) { ObColumnGroupSchema new_rowkey_cg; ObArray rowkey_ids; - uint64_t rowkey_cg_id = table_schema.get_max_used_column_group_id() + 1; ObTableSchema::const_column_iterator iter_begin = table_schema.column_begin(); ObTableSchema::const_column_iterator iter_end = table_schema.column_end(); for (; OB_SUCC(ret) && iter_begin != iter_end; ++iter_begin) { @@ -821,7 +820,7 @@ int ObSchemaUtils::alter_rowkey_column_group(share::schema::ObTableSchema &table if (OB_FAIL(ret)) { } else if (OB_FAIL(ObSchemaUtils::build_column_group( table_schema, table_schema.get_tenant_id(),ObColumnGroupType::ROWKEY_COLUMN_GROUP, - OB_ROWKEY_COLUMN_GROUP_NAME, rowkey_ids, rowkey_cg_id, new_rowkey_cg))) { + OB_ROWKEY_COLUMN_GROUP_NAME, rowkey_ids, ROWKEY_COLUMN_GROUP_ID, new_rowkey_cg))) { LOG_WARN("fail to build rowkey column group", K(ret)); } else if (OB_FAIL(table_schema.add_column_group(new_rowkey_cg))) { LOG_WARN("fail to add rowkey column group to table_schema", K(ret)); @@ -989,7 +988,7 @@ int ObSchemaUtils::build_add_each_column_group(const share::schema::ObTableSchem for (;OB_SUCC(ret) && iter_begin != iter_end; ++iter_begin) { column_group_schema.reset(); ObColumnSchemaV2 *column = (*iter_begin); - uint64_t cg_id = dst_table_schema.get_max_used_column_group_id() + 1; + uint64_t cg_id = dst_table_schema.get_next_single_column_group_id(); if (OB_ISNULL(column)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("column schema should not be null", K(ret)); diff --git a/src/share/schema/ob_table_param.cpp b/src/share/schema/ob_table_param.cpp index 9020ca455..4e71b4a8a 100644 --- a/src/share/schema/ob_table_param.cpp +++ b/src/share/schema/ob_table_param.cpp @@ -1230,12 +1230,8 @@ int ObTableParam::construct_columns_and_projector( } } - if (OB_SUCC(ret) && 0 < table_schema.get_column_group_count()) { - const ObColumnGroupType column_group_type = (*table_schema.column_group_begin())->get_column_group_type(); - if (ROWKEY_COLUMN_GROUP == column_group_type || ALL_COLUMN_GROUP == column_group_type) { - // delayed column transform already has the column group information that meets the cs replica requirements before the storage schema update - is_normal_cgs_at_the_end_ = true; - } + if (FAILEDx(table_schema.check_is_normal_cgs_at_the_end(is_normal_cgs_at_the_end_))) { + LOG_WARN("Fail to check whether normal cgs are at the end of schema array", K(ret), K(table_schema)); } return ret; } diff --git a/src/share/schema/ob_table_schema.cpp b/src/share/schema/ob_table_schema.cpp index 07b912ad6..4f7f3d6f0 100644 --- a/src/share/schema/ob_table_schema.cpp +++ b/src/share/schema/ob_table_schema.cpp @@ -9606,6 +9606,49 @@ int ObTableSchema::has_non_default_column_group(bool &has_non_default_column_gro return ret; } +int ObTableSchema::adjust_column_group_array() +{ + // after v435, all cg and rowkey cg should exist in the front of column group array + // this interface is only used in create table / create index / offline ddl with new table / delayed column transform + int ret = OB_SUCCESS; + bool is_rowkey_cg_exist = false; + bool is_all_cg_exist = false; + ObColumnGroupSchema *cg_schema = nullptr; + uint64_t compat_version = 0; + if (!is_column_store_supported()) { + // do nothing + } else if (OB_FAIL(GET_MIN_DATA_VERSION(get_tenant_id(), compat_version))) { + LOG_WARN("fail to get min data version", K(ret), K(get_tenant_id())); + } else if (compat_version >= DATA_VERSION_4_3_5_0) { + if (OB_FAIL(is_column_group_exist(OB_ROWKEY_COLUMN_GROUP_NAME, is_rowkey_cg_exist))) { + LOG_WARN("fail to check is each column group exist", K(ret)); + } else if (OB_FAIL(is_column_group_exist(OB_ALL_COLUMN_GROUP_NAME, is_all_cg_exist))) { + LOG_WARN("fail to check is all column group exist", K(ret)); + } else if (is_rowkey_cg_exist && OB_FAIL(get_column_group_by_name(OB_ROWKEY_COLUMN_GROUP_NAME, cg_schema))) { + LOG_WARN("fail to get rowkey column group", K(ret), KPC(this)); + } else if (is_all_cg_exist && OB_FAIL(get_column_group_by_name(OB_ALL_COLUMN_GROUP_NAME, cg_schema))) { + LOG_WARN("fail to get all column group", K(ret), KPC(this)); + } else if (OB_NOT_NULL(cg_schema)) { + int64_t new_loc = column_group_cnt_ - 1; + for (int64_t i = column_group_cnt_ - 1; OB_SUCC(ret) && i >= 0; --i) { + if (OB_ISNULL(column_group_arr_[i])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column_group should not be null", K(ret), K(i), K_(column_group_cnt)); + } else if (column_group_arr_[i] != cg_schema) { + column_group_arr_[new_loc] = column_group_arr_[i]; + new_loc -= 1; + } else { + // skip all cg or rowkey cg + } + } + if (OB_SUCC(ret)) { + column_group_arr_[0] = cg_schema; + } + } + } + return ret; +} + int ObTableSchema::get_column_group_by_id( const uint64_t column_group_id, ObColumnGroupSchema *&column_group) const @@ -9787,16 +9830,7 @@ int ObTableSchema::add_column_group_to_array(ObColumnGroupSchema *column_group) } if (OB_SUCC(ret)) { - if (ROWKEY_COLUMN_GROUP == column_group->get_column_group_type() - || ALL_COLUMN_GROUP == column_group->get_column_group_type()) { - for (int64_t idx = column_group_cnt_ - 1; idx >= 0; --idx) { - column_group_arr_[idx + 1] = column_group_arr_[idx]; - } - column_group_arr_[0] = column_group; - ++column_group_cnt_; - } else { - column_group_arr_[column_group_cnt_++] = column_group; - } + column_group_arr_[column_group_cnt_++] = column_group; } } @@ -10014,6 +10048,36 @@ int ObTableSchema::get_base_rowkey_column_group_index(int32_t &cg_idx) const return ret; } +int ObTableSchema::check_is_normal_cgs_at_the_end(bool &is_normal_cgs_at_the_end) const +{ + int ret = OB_SUCCESS; + is_normal_cgs_at_the_end = false; + if (0 < column_group_cnt_) { + bool found_normal_cg = false; + for (int64_t i = 0; OB_SUCC(ret) && i < column_group_cnt_; i++) { + if (OB_ISNULL(column_group_arr_[i])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column_group should not be null", K(ret), K(i), K_(column_group_cnt)); + } else if (0 == column_group_arr_[i]->get_column_id_count()) { + if (column_group_arr_[i]->get_column_group_type() != DEFAULT_COLUMN_GROUP) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected column group type", K(ret), KPC(column_group_arr_[i])); + } + // skip default column group + } else if (SINGLE_COLUMN_GROUP == column_group_arr_[i]->get_column_group_type() || + NORMAL_COLUMN_GROUP == column_group_arr_[i]->get_column_group_type()) { + found_normal_cg = true; + } else { + if (!found_normal_cg) { + is_normal_cgs_at_the_end = true; + } + break; + } + } + } + return ret; +} + int ObTableSchema::get_each_column_group(ObIArray &each_cgs) const { int ret = OB_SUCCESS; diff --git a/src/share/schema/ob_table_schema.h b/src/share/schema/ob_table_schema.h index aca6758c3..c8a545a7c 100644 --- a/src/share/schema/ob_table_schema.h +++ b/src/share/schema/ob_table_schema.h @@ -1666,6 +1666,8 @@ public: void set_column_store(const bool support_column_store) { is_column_store_supported_ = support_column_store; } int get_is_column_store(bool &is_column_store) const; uint64_t get_max_used_column_group_id() const { return max_used_column_group_id_; } + uint64_t get_next_single_column_group_id() const { return max_used_column_group_id_ > ROWKEY_COLUMN_GROUP_ID ? max_used_column_group_id_ + 1 : ROWKEY_COLUMN_GROUP_ID + 1; } + int check_is_normal_cgs_at_the_end(bool &is_normal_cgs_at_the_end) const; void set_max_used_column_group_id(const uint64_t id) { max_used_column_group_id_ = id; } int add_column_group(const ObColumnGroupSchema &other); // This function is only used when add default cg for sys_schema in 'hard-code' python script @@ -1679,6 +1681,7 @@ public: int remove_column_group(const uint64_t column_group_id); int has_all_column_group(bool &has_all_column_group) const; int has_non_default_column_group(bool &has_non_default_column_group) const; + int adjust_column_group_array(); // materialized view log related template static int build_mlog_table_name(Allocator &allocator, diff --git a/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp b/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp index 179c11752..19edb843a 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver_base.cpp @@ -381,7 +381,7 @@ int ObCreateTableResolverBase::resolve_column_group_helper(const ParseNode *cg_n } else if (!ObSchemaUtils::can_add_column_group(table_schema)) { } else if (ObTableStoreFormat::is_row_with_column_store(table_store_type)) { if (OB_FAIL(ObSchemaUtils::build_all_column_group(table_schema, table_schema.get_tenant_id(), - table_schema.get_max_used_column_group_id() + 1, all_cg))) { + ALL_COLUMN_GROUP_ID, all_cg))) { LOG_WARN("fail to add all column group", K(ret)); } else if (OB_FAIL(table_schema.add_column_group(all_cg))) { LOG_WARN("fail to build all column group", K(ret)); @@ -405,6 +405,10 @@ int ObCreateTableResolverBase::resolve_column_group_helper(const ParseNode *cg_n LOG_WARN("fail to adjust default column group", K(ret)); } } + + if (FAILEDx(table_schema.adjust_column_group_array())) { + LOG_WARN("fail to adjust column group array", K(ret), K(table_schema)); + } } return ret; } diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index 3d6017876..025984497 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -13242,7 +13242,7 @@ int ObDDLResolver::parse_column_group(const ParseNode *column_group_node, if (OB_SUCC(ret) && sql_exist_all_column_group) { column_group_schema.reset(); if (OB_FAIL(ObSchemaUtils::build_all_column_group(table_schema, session_info_->get_effective_tenant_id(), - dst_table_schema.get_max_used_column_group_id() + 1, + dst_table_schema.get_max_used_column_group_id() + 1, // add all cg with id max+1 first, adjust later in adjust_cg_for_offline column_group_schema))) { SQL_RESV_LOG(WARN, "build all column group failed", K(ret)); } else if (OB_FAIL(dst_table_schema.add_column_group(column_group_schema))) {