From fe02a282e1fccdd9686c45fad84a27be48a66c9d Mon Sep 17 00:00:00 2001 From: DengzhiLiu Date: Thu, 8 Feb 2024 11:24:50 +0000 Subject: [PATCH] Add default_table_store_format when creating table --- deps/oblib/src/common/ob_store_format.cpp | 23 ++++ deps/oblib/src/common/ob_store_format.h | 30 +++++ src/rootserver/ob_index_builder.cpp | 14 +-- src/share/config/ob_config_helper.cpp | 6 + src/share/config/ob_config_helper.h | 9 ++ src/share/parameter/ob_parameter_seed.ipp | 25 ++-- src/share/schema/ob_schema_struct.h | 8 -- src/share/schema/ob_schema_utils.cpp | 14 +-- .../resolver/ddl/ob_create_table_resolver.cpp | 117 +++++++++--------- .../resolver/ddl/ob_create_table_resolver.h | 2 +- src/sql/resolver/ddl/ob_ddl_resolver.cpp | 12 +- .../cs_encoding/ob_dict_column_decoder.cpp | 8 +- .../cs_encoding/ob_integer_column_decoder.cpp | 2 - .../cs_encoding/ob_integer_stream_encoder.h | 2 +- .../cs_encoding/ob_string_column_decoder.cpp | 2 - .../blocksstable/ob_micro_block_cache.cpp | 9 +- .../column_store/r/mysql/add_column.result | 7 +- .../mysql/basic_column_group_scan_flat.result | 1 - .../basic_column_group_scan_with_minor.result | 4 +- .../test_suite/column_store/t/add_column.test | 8 +- .../t/basic_column_group_scan.test | 1 - .../t/basic_column_group_scan_flat.test | 1 - .../t/basic_column_group_scan_with_minor.test | 7 +- .../column_store/t/tile_project.test | 1 - .../all_virtual_sys_parameter_stat.result | 6 +- 25 files changed, 159 insertions(+), 160 deletions(-) diff --git a/deps/oblib/src/common/ob_store_format.cpp b/deps/oblib/src/common/ob_store_format.cpp index f05046feab..4c4c493d28 100644 --- a/deps/oblib/src/common/ob_store_format.cpp +++ b/deps/oblib/src/common/ob_store_format.cpp @@ -118,6 +118,29 @@ int ObStoreFormat::find_store_format_type(const ObString &store_format, const bo { return is_oracle_mode ? find_store_format_type_oracle(store_format, store_format_type) : find_store_format_type_mysql(store_format, store_format_type); } + +int ObTableStoreFormat::find_table_store_type(const ObString &table_store_format, ObTableStoreType &table_store_type) +{ + int ret = OB_SUCCESS; + table_store_type = OB_TABLE_STORE_INVALID; + if (table_store_format.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Empty table store format str", K(ret), K(table_store_format)); + } else { + if (0 == table_store_format.case_compare("ROW")) { + table_store_type = OB_TABLE_STORE_ROW; + } else if (0 == table_store_format.case_compare("COLUMN")) { + table_store_type = OB_TABLE_STORE_COLUMN; + } else if (0 == table_store_format.case_compare("COMPOUND")) { + table_store_type = OB_TABLE_STORE_ROW_WITH_COLUMN; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected store format type", K(ret), K(table_store_format), K(table_store_type)); + } + } + return ret; +} + }//end namespace common }//end namespace oceanbase diff --git a/deps/oblib/src/common/ob_store_format.h b/deps/oblib/src/common/ob_store_format.h index c25aeba34b..63997afa46 100644 --- a/deps/oblib/src/common/ob_store_format.h +++ b/deps/oblib/src/common/ob_store_format.h @@ -52,6 +52,15 @@ enum ObStoreFormatType OB_STORE_FORMAT_MAX }; +enum ObTableStoreType : uint8_t +{ + OB_TABLE_STORE_INVALID = 0, + OB_TABLE_STORE_ROW = 1, + OB_TABLE_STORE_COLUMN = 2, + OB_TABLE_STORE_ROW_WITH_COLUMN= 3, + OB_TABLE_STORE_MAX +}; + struct ObStoreFormatItem { const char* format_name_; @@ -143,6 +152,27 @@ private: static const char *row_store_name[MAX_ROW_STORE]; }; +class ObTableStoreFormat { +public: + static inline bool is_row_store(const ObTableStoreType type) + { + return OB_TABLE_STORE_ROW == type; + } + static inline bool is_column_store(const ObTableStoreType type) + { + return OB_TABLE_STORE_COLUMN == type; + } + static inline bool is_row_with_column_store(const ObTableStoreType type) + { + return OB_TABLE_STORE_ROW_WITH_COLUMN == type; + } + static inline bool is_with_column (const ObTableStoreType type) + { + return type > OB_TABLE_STORE_ROW && type < OB_TABLE_STORE_MAX; + } + static int find_table_store_type(const ObString &store_format, ObTableStoreType &table_store_type); +}; + }//end namespace common }//end namespace oceanbase diff --git a/src/rootserver/ob_index_builder.cpp b/src/rootserver/ob_index_builder.cpp index 1f786a5888..7c9ab7c523 100644 --- a/src/rootserver/ob_index_builder.cpp +++ b/src/rootserver/ob_index_builder.cpp @@ -870,7 +870,6 @@ int ObIndexBuilder::create_index_column_group(const obrpc::ObCreateIndexArg &arg if (OB_FAIL(GET_MIN_DATA_VERSION(index_table_schema.get_tenant_id(), compat_version))) { LOG_WARN("fail to get min data version", K(ret)); } else if (compat_version >= DATA_VERSION_4_3_0_0) { - bool enable_table_with_cg = false; ObArray column_ids; // not include virtual column index_table_schema.set_column_store(true); if (arg.index_cgs_.count() > 0) { @@ -923,19 +922,12 @@ int ObIndexBuilder::create_index_column_group(const obrpc::ObCreateIndexArg &arg } } } - } else { - omt::ObTenantConfigGuard tenant_config(TENANT_CONF(index_table_schema.get_tenant_id())); - if (OB_SUCC(ret) && OB_LIKELY(tenant_config.is_valid())) { - if (tenant_config->enable_table_with_cg) { - enable_table_with_cg = true; // which means create each_cg and all_cg default - } - } } // add default column_group if (OB_SUCC(ret)) { ObColumnGroupSchema tmp_cg; - if (arg.index_cgs_.count() > 0 || enable_table_with_cg) { + if (arg.index_cgs_.count() > 0) { column_ids.reuse(); // if exists cg node, column_ids in default_type will be empty } else { ObTableSchema::const_column_iterator tmp_begin = index_table_schema.column_begin(); @@ -958,8 +950,8 @@ int ObIndexBuilder::create_index_column_group(const obrpc::ObCreateIndexArg &arg DEFAULT_TYPE_COLUMN_GROUP_ID, tmp_cg))) { LOG_WARN("fail to build default type column_group", KR(ret), "table_id", index_table_schema.get_table_id(), K(column_ids)); } else if (OB_FAIL(index_table_schema.add_column_group(tmp_cg))) { - LOG_WARN("fail to add default column group", KR(ret), "table_id", index_table_schema.get_table_id(), K(arg.index_cgs_.count()), - K(enable_table_with_cg), K(column_ids)); + LOG_WARN("fail to add default column group", KR(ret), "table_id", index_table_schema.get_table_id(), + K(arg.index_cgs_.count()), K(column_ids)); } } } else if (arg.index_cgs_.count() > 0) { diff --git a/src/share/config/ob_config_helper.cpp b/src/share/config/ob_config_helper.cpp index 9837b40502..25200b67e2 100644 --- a/src/share/config/ob_config_helper.cpp +++ b/src/share/config/ob_config_helper.cpp @@ -1130,5 +1130,11 @@ bool ObConfigIndexStatsModeChecker::check(const ObConfigItem &t) const { return 0 == tmp_str.case_compare("SAMPLED") || 0 == tmp_str.case_compare("ALL"); } +bool ObConfigTableStoreFormatChecker::check(const ObConfigItem &t) const { + const ObString tmp_str(t.str()); + return 0 == tmp_str.case_compare("ROW") || 0 == tmp_str.case_compare("COLUMN") || + 0 == tmp_str.case_compare("COMPOUND"); +} + } // end of namepace common } // end of namespace oceanbase diff --git a/src/share/config/ob_config_helper.h b/src/share/config/ob_config_helper.h index 859bebbd66..c92e96799e 100644 --- a/src/share/config/ob_config_helper.h +++ b/src/share/config/ob_config_helper.h @@ -721,6 +721,15 @@ private: DISALLOW_COPY_AND_ASSIGN(ObConfigIndexStatsModeChecker); }; +class ObConfigTableStoreFormatChecker: public ObConfigChecker { +public: + ObConfigTableStoreFormatChecker(){} + virtual ~ObConfigTableStoreFormatChecker(){} + bool check(const ObConfigItem &t) const; +private: + DISALLOW_COPY_AND_ASSIGN(ObConfigTableStoreFormatChecker); +}; + typedef __ObConfigContainer ObConfigContainer; } // namespace common diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index 85da7b63e3..12d853c8a9 100644 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -160,6 +160,12 @@ DEF_STR_WITH_CHECKER(default_compress, OB_CLUSTER_PARAMETER, "archive", "default compress strategy for create new table within oracle mode", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_STR_WITH_CHECKER(default_table_store_format, OB_TENANT_PARAMETER, "row", + common::ObConfigTableStoreFormatChecker, + "Specify the default storage format of creating table: row, column, compound format of row and column" + "values: row, column, compound", + ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); + DEF_TIME(weak_read_version_refresh_interval, OB_CLUSTER_PARAMETER, "100ms", "[50ms,)", "the time interval to refresh cluster weak read version " "Range: [50ms, +∞)", @@ -1716,24 +1722,9 @@ DEF_BOOL(_enable_column_store, OB_TENANT_PARAMETER, "True", DEF_BOOL(_enable_skip_index, OB_TENANT_PARAMETER, "True", "enable the skip index in storage engine", ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); -TEMP_DEF_BOOL(v4.3, enable_cs_encoding_filter, OB_CLUSTER_PARAMETER, "True", - "enables using column_store encoding filter logic. The default value is True.", - ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); -TEMP_DEF_INT(v4.3, storage_encoding_mode, OB_TENANT_PARAMETER, "0", "[0, 2]", - "0: row_store uses encoding, column_store used cs_encoding; 1: all use encoding; 2: all use cs_encoding. Range: [0, 2] in integer", - ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); -TEMP_DEF_BOOL(v4.3, enable_table_with_cg, OB_TENANT_PARAMETER, "False", - "enables creating table with all column_group and each column_group. The default value is True.", - ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); -TEMP_DEF_INT(v4.3, encoding_test_seed, OB_CLUSTER_PARAMETER, "0", "[0,)" - "The seed is used to test encoding algorithm. The default is 0, indicating that it is not test, the production environment must keep the default value", - ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); -TEMP_DEF_BOOL(v4.3, enable_table_without_all_cg, OB_TENANT_PARAMETER, "True", - "enables creating table without all column_group. The default value is False.", - ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); TEMP_DEF_BOOL(v4.3, enable_store_compression, OB_TENANT_PARAMETER, "False", - "enable compression in ObTempBlockStore", - ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); + "enable compression in ObTempBlockStore", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_BOOL(_enable_prefetch_limiting, OB_TENANT_PARAMETER, "False", "enable limiting memory in prefetch for single query", ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); diff --git a/src/share/schema/ob_schema_struct.h b/src/share/schema/ob_schema_struct.h index 17d2f82eea..4ac66dbe29 100755 --- a/src/share/schema/ob_schema_struct.h +++ b/src/share/schema/ob_schema_struct.h @@ -7590,14 +7590,6 @@ enum ObSAuditOperationType : uint64_t AUDIT_OP_MAX }; -enum ObStorageEncodingMode : uint64_t -{ - ROW_ENCODING_COL_CSENCODIGN = 0, - ALL_ENCODING, - ALL_CSENCODING, - MAX_ENCODING -}; - const char *get_audit_operation_type_str(const ObSAuditOperationType type); int get_operation_type_from_item_type(const bool is_stmt_audit, diff --git a/src/share/schema/ob_schema_utils.cpp b/src/share/schema/ob_schema_utils.cpp index 154dc2f086..377c8921c5 100644 --- a/src/share/schema/ob_schema_utils.cpp +++ b/src/share/schema/ob_schema_utils.cpp @@ -574,24 +574,16 @@ int ObSchemaUtils::build_column_group( column_group.set_block_size(table_schema.get_block_size()); column_group.set_compressor_type(table_schema.get_compressor_type()); const ObStoreFormatType store_format = table_schema.get_store_format(); - int64_t storage_encoding_mode = 0; - omt::ObTenantConfigGuard tcg(TENANT_CONF(tenant_id)); - if (OB_LIKELY(tcg.is_valid())) { - storage_encoding_mode = tcg->storage_encoding_mode; - } bool is_flat = lib::Worker::CompatMode::ORACLE == mode ? ((OB_STORE_FORMAT_NOCOMPRESS_ORACLE == store_format) || (OB_STORE_FORMAT_BASIC_ORACLE == store_format) || (OB_STORE_FORMAT_OLTP_ORACLE == store_format)) : ((OB_STORE_FORMAT_REDUNDANT_MYSQL == store_format) || (OB_STORE_FORMAT_COMPACT_MYSQL == store_format)); - if (is_flat || ( ObStorageEncodingMode::ALL_ENCODING == storage_encoding_mode)) { - // all use encoding + if (is_flat) { + // use the encoding type according to the row format column_group.set_row_store_type(table_schema.get_row_store_type()); - } else if (ObStorageEncodingMode::ALL_ENCODING == storage_encoding_mode) { - // all use cs_encoding - column_group.set_row_store_type(ObRowStoreType::CS_ENCODING_ROW_STORE); } else { - // row_store uses encoding; column_store uses cs_encoding + // row_store encoding type according to the row format; column_store uses cs_encoding if ((cg_type == ObColumnGroupType::DEFAULT_COLUMN_GROUP) || (cg_type == ObColumnGroupType::ALL_COLUMN_GROUP)) { column_group.set_row_store_type(table_schema.get_row_store_type()); } else { diff --git a/src/sql/resolver/ddl/ob_create_table_resolver.cpp b/src/sql/resolver/ddl/ob_create_table_resolver.cpp index c1d28e3ba2..cf0f373488 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver.cpp @@ -2977,7 +2977,7 @@ int ObCreateTableResolver::resolve_column_group(const ParseNode *cg_node) ObCreateTableStmt *create_table_stmt = static_cast(stmt_); ObArray column_ids; // not include virtual column uint64_t compat_version = 0; - bool enable_table_with_cg = false; + ObTableStoreType table_store_type = OB_TABLE_STORE_INVALID; if (OB_ISNULL(create_table_stmt)) { ret = OB_ERR_UNEXPECTED; @@ -2991,15 +2991,11 @@ int ObCreateTableResolver::resolve_column_group(const ParseNode *cg_node) } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { LOG_WARN("fail to get min data version", KR(ret), K(tenant_id)); } else if ((compat_version >= DATA_VERSION_4_3_0_0) - && need_column_group(table_schema)) { - int64_t num_child = 0; - bool exist_all_type = false; - bool exist_single_type = false; + && can_add_column_group(table_schema)) { table_schema.set_column_store(true); - // handle all_type column_group & single_type column_group if (OB_NOT_NULL(cg_node)) { + // handle all_type column_group & single_type column_group bool exist_all_column_group = false; - bool exist_each_column_group = false; if (OB_FAIL(parse_cg_node(*cg_node, exist_all_column_group))) { LOG_WARN("fail to parse cg node", KR(ret)); } else { @@ -3012,13 +3008,13 @@ int ObCreateTableResolver::resolve_column_group(const ParseNode *cg_node) if (column->is_virtual_generated_column()){ /*skip virtual generated column group*/ } else if (OB_FAIL(ObSchemaUtils::build_single_column_group( - table_schema, column, session_info_->get_effective_tenant_id(), - gen_column_group_id(), tmp_cg))) { + table_schema, column, session_info_->get_effective_tenant_id(), + gen_column_group_id(), tmp_cg))) { LOG_WARN("fail to build single column group"); } else if (tmp_cg.is_valid()) { if (OB_FAIL(table_schema.add_column_group(tmp_cg))) { LOG_WARN("fail to add single type column group", KR(ret), K(tmp_cg)); - } else if (column->is_rowkey_column() || exist_all_column_group) {//if not exist all cg, build rowkey cg + } else if (column->is_rowkey_column() || exist_all_column_group) { //if not exist all cg, build rowkey cg if (OB_FAIL(column_ids.push_back(column->get_column_id()))) { LOG_WARN("fail to push back", KR(ret), "column_id", column->get_column_id()); } @@ -3028,11 +3024,12 @@ int ObCreateTableResolver::resolve_column_group(const ParseNode *cg_node) if (OB_SUCC(ret)) { tmp_cg.reset(); - const ObColumnGroupType cg_type = exist_all_column_group ? ObColumnGroupType::ALL_COLUMN_GROUP - : ObColumnGroupType::ROWKEY_COLUMN_GROUP; - const ObString cg_name = exist_all_column_group ? OB_ALL_COLUMN_GROUP_NAME : OB_ROWKEY_COLUMN_GROUP_NAME; + const ObColumnGroupType cg_type = exist_all_column_group ? + ObColumnGroupType::ALL_COLUMN_GROUP : ObColumnGroupType::ROWKEY_COLUMN_GROUP; + const ObString cg_name = exist_all_column_group ? + OB_ALL_COLUMN_GROUP_NAME : OB_ROWKEY_COLUMN_GROUP_NAME; if (OB_FAIL(build_column_group(table_schema, cg_type, cg_name, column_ids, - gen_column_group_id(), tmp_cg))) { + gen_column_group_id(), tmp_cg))) { LOG_WARN("fail to build all type column_group", KR(ret), K(column_ids)); } else if (OB_FAIL(table_schema.add_column_group(tmp_cg))) { LOG_WARN("fail to add all type column group", KR(ret), K(tmp_cg)); @@ -3042,51 +3039,50 @@ int ObCreateTableResolver::resolve_column_group(const ParseNode *cg_node) } else { ObTenantConfigGuard tenant_config(TENANT_CONF(session_info_->get_effective_tenant_id())); if (OB_SUCC(ret) && OB_LIKELY(tenant_config.is_valid())) { - if (tenant_config->enable_table_with_cg) { - enable_table_with_cg = true; - exist_all_type = !tenant_config->enable_table_without_all_cg; - exist_single_type = true; - num_child = 2; + if (OB_FAIL(ObTableStoreFormat::find_table_store_type( + tenant_config->default_table_store_format.get_value_string(), + table_store_type))) { + LOG_WARN("fail to get table store format", K(ret), K(table_store_type)); } } - } - - // add all_type column_group and each_type column_group - //temp for test - if (OB_SUCC(ret)) { - if (enable_table_with_cg) { - ObColumnGroupSchema tmp_cg; - ObArray tmp_column_ids; - ObTableSchema::const_column_iterator tmp_begin = table_schema.column_begin(); - ObTableSchema::const_column_iterator tmp_end = table_schema.column_end(); - for (; OB_SUCC(ret) && (tmp_begin != tmp_end); tmp_begin++) { - ObColumnSchemaV2 *column = (*tmp_begin); - if (column->is_virtual_generated_column()){ - /*skip virtual generated column group*/ - } else if(OB_FAIL(ObSchemaUtils::build_single_column_group(table_schema, column, session_info_->get_effective_tenant_id(), - gen_column_group_id(), tmp_cg))) { - LOG_WARN("fail to build single column group", K(ret)); - } else if (tmp_cg.is_valid()) { - if (OB_FAIL(table_schema.add_column_group(tmp_cg))) { - LOG_WARN("fail to add single type column group", KR(ret), K(tmp_cg)); - } else if (column->is_rowkey_column() || exist_all_type) {//if not exist all cg, build rowkey cg - if (OB_FAIL(column_ids.push_back(column->get_column_id()))) { - LOG_WARN("fail to push back", KR(ret), "column_id", column->get_column_id()); + if (OB_SUCC(ret)) { + if (ObTableStoreFormat::is_with_column(table_store_type)) { + // add all_type column_group or each_type column_group + ObColumnGroupSchema tmp_cg; + ObArray tmp_column_ids; + ObTableSchema::const_column_iterator tmp_begin = table_schema.column_begin(); + ObTableSchema::const_column_iterator tmp_end = table_schema.column_end(); + for (; OB_SUCC(ret) && (tmp_begin != tmp_end); tmp_begin++) { + ObColumnSchemaV2 *column = (*tmp_begin); + if (column->is_virtual_generated_column()){ + /*skip virtual generated column group*/ + } else if(OB_FAIL(ObSchemaUtils::build_single_column_group( + table_schema, column, session_info_->get_effective_tenant_id(), + gen_column_group_id(), tmp_cg))) { + LOG_WARN("fail to build single column group", K(ret)); + } else if (tmp_cg.is_valid()) { + if (OB_FAIL(table_schema.add_column_group(tmp_cg))) { + LOG_WARN("fail to add single type column group", KR(ret), K(tmp_cg)); + } else if (column->is_rowkey_column() || ObTableStoreFormat::is_row_with_column_store(table_store_type)) { + if (OB_FAIL(column_ids.push_back(column->get_column_id()))) { + LOG_WARN("fail to push back", KR(ret), "column_id", column->get_column_id()); + } } } } - } - if (OB_SUCC(ret)) { - tmp_cg.reset(); - const ObColumnGroupType cg_type = exist_all_type ? ObColumnGroupType::ALL_COLUMN_GROUP - : ObColumnGroupType::ROWKEY_COLUMN_GROUP; - const ObString cg_name = exist_all_type ? OB_ALL_COLUMN_GROUP_NAME : OB_ROWKEY_COLUMN_GROUP_NAME; - if (OB_FAIL(build_column_group(table_schema, cg_type, cg_name, column_ids, - gen_column_group_id(), tmp_cg))) { - LOG_WARN("fail to build all type column_group", KR(ret), K(column_ids)); - } else if (OB_FAIL(table_schema.add_column_group(tmp_cg))) { - LOG_WARN("fail to add all type column group", KR(ret), K(tmp_cg)); + if (OB_SUCC(ret)) { + tmp_cg.reset(); + const ObColumnGroupType cg_type = ObTableStoreFormat::is_column_store(table_store_type) ? + ObColumnGroupType::ROWKEY_COLUMN_GROUP : ObColumnGroupType::ALL_COLUMN_GROUP; + const ObString cg_name = ObTableStoreFormat::is_column_store(table_store_type) ? + OB_ROWKEY_COLUMN_GROUP_NAME : OB_ALL_COLUMN_GROUP_NAME; + if (OB_FAIL(build_column_group(table_schema, cg_type, cg_name, column_ids, + gen_column_group_id(), tmp_cg))) { + LOG_WARN("fail to build all type column_group", KR(ret), K(column_ids)); + } else if (OB_FAIL(table_schema.add_column_group(tmp_cg))) { + LOG_WARN("fail to add all type column group", KR(ret), K(tmp_cg)); + } } } } @@ -3095,7 +3091,7 @@ int ObCreateTableResolver::resolve_column_group(const ParseNode *cg_node) // add default_type column_group if (OB_SUCC(ret)) { ObColumnGroupSchema tmp_cg; - if (OB_NOT_NULL(cg_node) || enable_table_with_cg) { + if (OB_NOT_NULL(cg_node) || ObTableStoreFormat::is_with_column(table_store_type)) { column_ids.reuse(); // if exists cg node, column_ids in default_type will be empty } else { ObTableSchema::const_column_iterator tmp_begin = table_schema.column_begin(); @@ -3115,8 +3111,8 @@ int ObCreateTableResolver::resolve_column_group(const ParseNode *cg_node) if (FAILEDx(build_column_group(table_schema, ObColumnGroupType::DEFAULT_COLUMN_GROUP, OB_DEFAULT_COLUMN_GROUP_NAME, column_ids, DEFAULT_TYPE_COLUMN_GROUP_ID, tmp_cg))) { - LOG_WARN("fail to build default type column_group", KR(ret), K(exist_all_type), K(exist_single_type), - "table_id", table_schema.get_table_id()); + LOG_WARN("fail to build default type column_group", KR(ret), K(table_store_type), + "table_id", table_schema.get_table_id()); } else if (OB_FAIL(table_schema.add_column_group(tmp_cg))) { LOG_WARN("fail to add default column group", KR(ret), "table_id", table_schema.get_table_id()); } @@ -3128,18 +3124,17 @@ int ObCreateTableResolver::resolve_column_group(const ParseNode *cg_node) } } } - return ret; } -bool ObCreateTableResolver::need_column_group(const ObTableSchema &table_schema) +bool ObCreateTableResolver::can_add_column_group(const ObTableSchema &table_schema) { - bool need_cg = false; + bool can_add_cg = false; if (table_schema.is_user_table() || table_schema.is_tmp_table()) { - need_cg = true; + can_add_cg = true; } - return need_cg; + return can_add_cg; } int ObCreateTableResolver::add_inner_index_for_heap_gtt() { diff --git a/src/sql/resolver/ddl/ob_create_table_resolver.h b/src/sql/resolver/ddl/ob_create_table_resolver.h index b97c4b31ce..8bc87564b3 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver.h +++ b/src/sql/resolver/ddl/ob_create_table_resolver.h @@ -134,7 +134,7 @@ private: uint64_t gen_column_group_id(); int resolve_column_group(const ParseNode *cg_node); - bool need_column_group(const ObTableSchema &table_schema); + bool can_add_column_group(const ObTableSchema &table_schema); int add_inner_index_for_heap_gtt(); private: diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index f4a89c7ca9..8bb0f2ff4f 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -11323,8 +11323,6 @@ int ObDDLResolver::parse_cg_node(const ParseNode &cg_node, bool &exist_all_colum } else { const int64_t num_child = cg_node.num_child_; bool exist_single_type = false; - bool already_exist_all_column_group = false; - bool already_exist_single_column_group = false; // handle all_type column_group & single_type column_group for (int64_t i = 0; OB_SUCC(ret) && (i < num_child); ++i) { ParseNode *node = cg_node.children_[i]; @@ -11332,25 +11330,23 @@ int ObDDLResolver::parse_cg_node(const ParseNode &cg_node, bool &exist_all_colum ret = OB_ERR_UNEXPECTED; LOG_WARN("children of column_group_list should not be null", KR(ret)); } else if (T_ALL_COLUMN_GROUP == node->type_) { - if (already_exist_all_column_group) { + if (exist_all_column_group) { ret = OB_ERR_COLUMN_GROUP_DUPLICATE; SQL_RESV_LOG(WARN, "all column group already exist in sql", - K(ret), K(node->children_[i]->type_)); + K(ret), K(node->children_[i]->type_)); const ObString error_msg = "all columns"; LOG_USER_ERROR(OB_ERR_COLUMN_GROUP_DUPLICATE, error_msg.length(), error_msg.ptr()); } else { - already_exist_all_column_group = true; exist_all_column_group = true; } } else if (T_SINGLE_COLUMN_GROUP == node->type_) { - if (already_exist_single_column_group) { + if (exist_single_type) { ret = OB_ERR_COLUMN_GROUP_DUPLICATE; SQL_RESV_LOG(WARN, "single column group already exist in sql", - K(ret), K(node->type_)); + K(ret), K(node->type_)); const ObString error_msg = "each column"; LOG_USER_ERROR(OB_ERR_COLUMN_GROUP_DUPLICATE, error_msg.length(), error_msg.ptr()); } else { - already_exist_single_column_group = true; exist_single_type = true; } } else if (T_NORMAL_COLUMN_GROUP == node->type_) { diff --git a/src/storage/blocksstable/cs_encoding/ob_dict_column_decoder.cpp b/src/storage/blocksstable/cs_encoding/ob_dict_column_decoder.cpp index cd0dc333e6..d6955a96ce 100644 --- a/src/storage/blocksstable/cs_encoding/ob_dict_column_decoder.cpp +++ b/src/storage/blocksstable/cs_encoding/ob_dict_column_decoder.cpp @@ -340,9 +340,7 @@ int ObDictColumnDecoder::pushdown_operator( int ret = OB_SUCCESS; filter_applied = false; const bool enable_rich_format = filter.get_op().enable_rich_format_; - if (!GCONF.enable_cs_encoding_filter) { - ret = OB_NOT_SUPPORTED; - } else if (OB_UNLIKELY(result_bitmap.size() != pd_filter_info.count_)) { + if (OB_UNLIKELY(result_bitmap.size() != pd_filter_info.count_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(result_bitmap.size()), K(pd_filter_info)); } else { @@ -457,9 +455,7 @@ int ObDictColumnDecoder::pushdown_operator( { int ret = OB_SUCCESS; - if (!GCONF.enable_cs_encoding_filter) { - ret = OB_NOT_SUPPORTED; - } else if (OB_UNLIKELY(result_bitmap.size() != pd_filter_info.count_)) { + if (OB_UNLIKELY(result_bitmap.size() != pd_filter_info.count_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), "result_map size", result_bitmap.size(), K(pd_filter_info)); } else { diff --git a/src/storage/blocksstable/cs_encoding/ob_integer_column_decoder.cpp b/src/storage/blocksstable/cs_encoding/ob_integer_column_decoder.cpp index 0562161156..ef2bca5550 100644 --- a/src/storage/blocksstable/cs_encoding/ob_integer_column_decoder.cpp +++ b/src/storage/blocksstable/cs_encoding/ob_integer_column_decoder.cpp @@ -115,8 +115,6 @@ int ObIntegerColumnDecoder::pushdown_operator( if (OB_UNLIKELY(row_cnt < 1 || row_cnt != result_bitmap.size())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(row_cnt), K(result_bitmap.size())); - } else if (!GCONF.enable_cs_encoding_filter) { - ret = OB_NOT_SUPPORTED; } else { if (integer_ctx.has_null_bitmap()) { for (int64_t i = 0; OB_SUCC(ret) && (i < row_cnt); ++i) { diff --git a/src/storage/blocksstable/cs_encoding/ob_integer_stream_encoder.h b/src/storage/blocksstable/cs_encoding/ob_integer_stream_encoder.h index add7c5b77d..2651889872 100644 --- a/src/storage/blocksstable/cs_encoding/ob_integer_stream_encoder.h +++ b/src/storage/blocksstable/cs_encoding/ob_integer_stream_encoder.h @@ -225,7 +225,7 @@ private: const int64_t orig_pos = buf_writer.length(); const int64_t raw_encoding_len = sizeof(T) * arr_count; int64_t curr_encoding_len = 0; - const int64_t test_seed = GCONF.encoding_test_seed; + const int64_t test_seed = 0; const int64_t remain_size = buf_writer.remain_buffer_size(); bool need_encode_with_raw = false; if (OB_UNLIKELY(remain_size < raw_encoding_len)) { diff --git a/src/storage/blocksstable/cs_encoding/ob_string_column_decoder.cpp b/src/storage/blocksstable/cs_encoding/ob_string_column_decoder.cpp index d658cb6659..a8651b20e2 100644 --- a/src/storage/blocksstable/cs_encoding/ob_string_column_decoder.cpp +++ b/src/storage/blocksstable/cs_encoding/ob_string_column_decoder.cpp @@ -402,8 +402,6 @@ int ObStringColumnDecoder::pushdown_operator( if (OB_UNLIKELY(row_cnt < 1 || row_cnt != result_bitmap.size())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(row_cnt), K(string_ctx), K(result_bitmap.size())); - } else if (!GCONF.enable_cs_encoding_filter) { - ret = OB_NOT_SUPPORTED; } else { const int64_t row_start = pd_filter_info.start_; const int64_t row_count = pd_filter_info.count_; diff --git a/src/storage/blocksstable/ob_micro_block_cache.cpp b/src/storage/blocksstable/ob_micro_block_cache.cpp index ea89f2953d..7c6db7e47c 100644 --- a/src/storage/blocksstable/ob_micro_block_cache.cpp +++ b/src/storage/blocksstable/ob_micro_block_cache.cpp @@ -424,9 +424,12 @@ int ObIMicroBlockIOCallback::read_block_and_copy( { int ret = OB_SUCCESS; block_data.type_ = cache_->get_type(); - // only full_transform for index_block with cs_encoding when don't use block cache - if (ObMicroBlockData::INDEX_BLOCK == block_data.type_ && - ObStoreFormat::is_row_store_type_with_cs_encoding(static_cast(header.row_store_type_))) { + // In normal cases, full_transform is not required if not using block cache, + // The ObMicroBlockCSDecoder can do part_transform and use the memory whose life cycle + // is consistent with decoder. However, at present, there are cases where rows obtained from + // index block or data block are continued to be used after decoder deconstruction, + // so the full_transform is also done here. + if (ObStoreFormat::is_row_store_type_with_cs_encoding(static_cast(header.row_store_type_))) { if (OB_FAIL(reader.decrypt_and_full_transform_data(header, block_des_meta_, buffer, size, block_data.get_buf(), block_data.get_buf_size(), nullptr))) { LOG_WARN("fail to decrypt_and_full_transform_data", K(ret), K(header), K_(block_des_meta)); diff --git a/tools/deploy/mysql_test/test_suite/column_store/r/mysql/add_column.result b/tools/deploy/mysql_test/test_suite/column_store/r/mysql/add_column.result index 97d6e1aedd..055bc96a4a 100644 --- a/tools/deploy/mysql_test/test_suite/column_store/r/mysql/add_column.result +++ b/tools/deploy/mysql_test/test_suite/column_store/r/mysql/add_column.result @@ -1,5 +1,3 @@ -set session _enable_rich_vector_format = true; -alter system enable_table_without_all_cg = false; create table t1(a int, b varchar(3048), primary key (a), c int) with column group (all columns, each column); create table t2(pk int, c1 int, c2 int unsigned, c3 varchar(100)) block_size=2048 with column group (all columns, each column); select count(*) from t1; @@ -329,10 +327,9 @@ select count(*),count(c1),min(c1),max(c1),sum(c1),count(c2),min(c2),max(c2) from count(*) count(c1) min(c1) max(c1) sum(c1) count(c2) min(c2) max(c2) 250 250 0 498 39900 100 300300300300300 498498498498498 drop table t1,t2; -alter system enable_table_without_all_cg = true; alter system flush plan cache; -create table t1(a int, b varchar(3048), primary key (a), c int) with column group (all columns, each column); -create table t2(pk int, c1 int, c2 int unsigned, c3 varchar(100)) block_size=2048 with column group (all columns, each column); +create table t1(a int, b varchar(3048), primary key (a), c int) with column group (each column); +create table t2(pk int, c1 int, c2 int unsigned, c3 varchar(100)) block_size=2048 with column group (each column); select count(*) from t1; count(*) 150 diff --git a/tools/deploy/mysql_test/test_suite/column_store/r/mysql/basic_column_group_scan_flat.result b/tools/deploy/mysql_test/test_suite/column_store/r/mysql/basic_column_group_scan_flat.result index ac9c2d5f11..19fe5810c2 100644 --- a/tools/deploy/mysql_test/test_suite/column_store/r/mysql/basic_column_group_scan_flat.result +++ b/tools/deploy/mysql_test/test_suite/column_store/r/mysql/basic_column_group_scan_flat.result @@ -1,6 +1,5 @@ drop table if exists ct1; set session ob_trx_timeout=100000000000; -set session _enable_rich_vector_format = true; create table ct1(c1 int primary key, c2 int, c3 number, c4 varchar(60000), c5 varchar(60000)) ROW_FORMAT = redundant COMPRESSION = 'none' with column group (all columns, each column); alter system minor freeze; alter system major freeze; diff --git a/tools/deploy/mysql_test/test_suite/column_store/r/mysql/basic_column_group_scan_with_minor.result b/tools/deploy/mysql_test/test_suite/column_store/r/mysql/basic_column_group_scan_with_minor.result index fe618514d0..03cb8d69f8 100644 --- a/tools/deploy/mysql_test/test_suite/column_store/r/mysql/basic_column_group_scan_with_minor.result +++ b/tools/deploy/mysql_test/test_suite/column_store/r/mysql/basic_column_group_scan_with_minor.result @@ -1,7 +1,6 @@ drop table if exists ct1; set session ob_trx_timeout=100000000000; -set session _enable_rich_vector_format = true; -create table ct1(c1 int primary key, c2 int, c3 number, c4 varchar(60000), c5 varchar(60000)) ROW_FORMAT = redundant COMPRESSION = 'none' with column group (all columns, each column); +create table ct1(c1 int primary key, c2 int, c3 number, c4 varchar(60000), c5 varchar(60000)) ROW_FORMAT = redundant COMPRESSION = 'none' with column group (each column); alter system minor freeze; alter system major freeze; update ct1 set c3 = 0 where c1 = 0; @@ -447,7 +446,6 @@ select count(*) from ct1; count(*) 10002 drop table ct1; -alter system enable_table_without_all_cg = false; alter system set _rowsets_enabled = true; alter system flush plan cache; create table ct1(c1 int primary key, c2 int, c3 number, c4 varchar(60000), c5 varchar(60000)) ROW_FORMAT = redundant COMPRESSION = 'none' with column group (all columns, each column); diff --git a/tools/deploy/mysql_test/test_suite/column_store/t/add_column.test b/tools/deploy/mysql_test/test_suite/column_store/t/add_column.test index 46b18e2ca6..82c15921c4 100644 --- a/tools/deploy/mysql_test/test_suite/column_store/t/add_column.test +++ b/tools/deploy/mysql_test/test_suite/column_store/t/add_column.test @@ -9,8 +9,6 @@ drop table if exists t1,t2; --enable_warnings --enable_query_log -set session _enable_rich_vector_format = true; -alter system enable_table_without_all_cg = false; create table t1(a int, b varchar(3048), primary key (a), c int) with column group (all columns, each column); create table t2(pk int, c1 int, c2 int unsigned, c3 varchar(100)) block_size=2048 with column group (all columns, each column); @@ -187,11 +185,10 @@ select count(*),count(c1),min(c1),max(c1),sum(c1),count(c2),min(c2),max(c2) from drop table t1,t2; -alter system enable_table_without_all_cg = true; alter system flush plan cache; -create table t1(a int, b varchar(3048), primary key (a), c int) with column group (all columns, each column); -create table t2(pk int, c1 int, c2 int unsigned, c3 varchar(100)) block_size=2048 with column group (all columns, each column); +create table t1(a int, b varchar(3048), primary key (a), c int) with column group (each column); +create table t2(pk int, c1 int, c2 int unsigned, c3 varchar(100)) block_size=2048 with column group (each column); sleep 10; @@ -353,7 +350,6 @@ drop table t1,t2; --disable_query_log set @@recyclebin = on; -alter system enable_table_without_all_cg = false; alter system set _pushdown_storage_level = 4; alter system set _rowsets_enabled = true; alter system flush plan cache; diff --git a/tools/deploy/mysql_test/test_suite/column_store/t/basic_column_group_scan.test b/tools/deploy/mysql_test/test_suite/column_store/t/basic_column_group_scan.test index da9947b11e..b686b66733 100644 --- a/tools/deploy/mysql_test/test_suite/column_store/t/basic_column_group_scan.test +++ b/tools/deploy/mysql_test/test_suite/column_store/t/basic_column_group_scan.test @@ -8,7 +8,6 @@ connect (syscon, $OBMYSQL_MS0,admin,$OBMYSQL_PWD,test,$OBMYSQL_PORT); connection syscon; set @@session.explicit_defaults_for_timestamp=off; set session ob_trx_timeout=100000000000; -set session _enable_rich_vector_format = true; alter system set_tp tp_no = 353, error_code = 4000, frequency = 1; sleep 2; set @@recyclebin = off; diff --git a/tools/deploy/mysql_test/test_suite/column_store/t/basic_column_group_scan_flat.test b/tools/deploy/mysql_test/test_suite/column_store/t/basic_column_group_scan_flat.test index 2eeb9e2941..e10e89afd7 100644 --- a/tools/deploy/mysql_test/test_suite/column_store/t/basic_column_group_scan_flat.test +++ b/tools/deploy/mysql_test/test_suite/column_store/t/basic_column_group_scan_flat.test @@ -15,7 +15,6 @@ set @@recyclebin = off; drop table if exists ct1; --enable_warnings set session ob_trx_timeout=100000000000; -set session _enable_rich_vector_format = true; create table ct1(c1 int primary key, c2 int, c3 number, c4 varchar(60000), c5 varchar(60000)) ROW_FORMAT = redundant COMPRESSION = 'none' with column group (all columns, each column); --disable_query_log diff --git a/tools/deploy/mysql_test/test_suite/column_store/t/basic_column_group_scan_with_minor.test b/tools/deploy/mysql_test/test_suite/column_store/t/basic_column_group_scan_with_minor.test index 73adb11baa..590428e5c7 100644 --- a/tools/deploy/mysql_test/test_suite/column_store/t/basic_column_group_scan_with_minor.test +++ b/tools/deploy/mysql_test/test_suite/column_store/t/basic_column_group_scan_with_minor.test @@ -10,7 +10,6 @@ connect (conn1,$OBMYSQL_MS0,$OBMYSQL_USR,$OBMYSQL_PWD,test,$OBMYSQL_PORT); connection conn1; --disable_query_log -alter system enable_table_without_all_cg = true; alter system set _rowsets_enabled = true; alter system flush plan cache; set @@recyclebin = off; @@ -20,9 +19,7 @@ set @@recyclebin = off; drop table if exists ct1; --enable_warnings set session ob_trx_timeout=100000000000; -set session _enable_rich_vector_format = true; -# create table without all column group -create table ct1(c1 int primary key, c2 int, c3 number, c4 varchar(60000), c5 varchar(60000)) ROW_FORMAT = redundant COMPRESSION = 'none' with column group (all columns, each column); +create table ct1(c1 int primary key, c2 int, c3 number, c4 varchar(60000), c5 varchar(60000)) ROW_FORMAT = redundant COMPRESSION = 'none' with column group (each column); --disable_query_log --let $count = 0 @@ -120,7 +117,6 @@ select c1,c2,c3 from ct1 where c2 <= 9 and ((c1 >= 0 and c1 <= 3000) or (c1 >= 4 select count(*) from ct1; drop table ct1; -alter system enable_table_without_all_cg = false; alter system set _rowsets_enabled = true; alter system flush plan cache; @@ -225,7 +221,6 @@ drop table ct1; --disable_query_log set @@recyclebin = on; -alter system enable_table_without_all_cg = false; alter system set _rowsets_enabled = true; alter system set _pushdown_storage_level = 4; alter system flush plan cache; diff --git a/tools/deploy/mysql_test/test_suite/column_store/t/tile_project.test b/tools/deploy/mysql_test/test_suite/column_store/t/tile_project.test index ea0431bf85..6170f32e30 100644 --- a/tools/deploy/mysql_test/test_suite/column_store/t/tile_project.test +++ b/tools/deploy/mysql_test/test_suite/column_store/t/tile_project.test @@ -8,7 +8,6 @@ set @@recyclebin = off; --disable_warnings set session ob_query_timeout=10000000000; -set session _enable_rich_vector_format = true; drop table if exists ct1,ct2,ct3; --enable_warnings diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result index 20e623d600..1fd9c271be 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result @@ -70,6 +70,7 @@ default_compress_func default_enable_extended_rowid default_progressive_merge_num default_row_format +default_table_store_format devname diag_syslog_per_error_limit disk_io_thread_count @@ -78,7 +79,6 @@ dump_data_dictionary_to_log_interval enable_async_syslog enable_cgroup enable_crazy_medium_compaction -enable_cs_encoding_filter enable_ddl enable_early_lock_release enable_kv_ttl @@ -100,13 +100,10 @@ enable_syslog_recycle enable_syslog_wf enable_sys_table_ddl enable_sys_unit_standalone -enable_table_without_all_cg -enable_table_with_cg enable_tcp_keepalive enable_transfer enable_upgrade_mode enable_user_defined_rewrite_rules -encoding_test_seed external_kms_info freeze_trigger_percentage fuse_row_cache_priority @@ -220,7 +217,6 @@ stack_size standby_db_fetch_log_rpc_timeout standby_db_preferred_upstream_log_region standby_fetch_log_bandwidth_limit -storage_encoding_mode storage_meta_cache_priority syslog_io_bandwidth_limit syslog_level