From 21ddf9e8b2d4201705cb3eb17aa5fcadb543868e Mon Sep 17 00:00:00 2001 From: obdev Date: Wed, 7 Feb 2024 14:44:41 +0000 Subject: [PATCH] [CP] [LOB] support inrow threshold config --- deps/oblib/src/lib/ob_define.h | 2 +- src/rootserver/ob_ddl_service.cpp | 13 +++++ src/share/schema/ob_schema_printer.cpp | 22 ++++++++ src/share/schema/ob_schema_printer.h | 6 ++ src/share/schema/ob_schema_service.cpp | 1 + src/share/schema/ob_table_schema.cpp | 4 +- src/share/schema/ob_table_sql_service.cpp | 13 +++++ .../parser/non_reserved_keywords_mysql_mode.c | 2 + src/sql/parser/sql_parser_mysql_mode.y | 18 +++++- .../resolver/ddl/ob_alter_table_resolver.cpp | 1 + .../resolver/ddl/ob_create_table_resolver.cpp | 19 ++++++- src/sql/resolver/ddl/ob_ddl_resolver.cpp | 55 ++++++++++++++++++- src/sql/resolver/ddl/ob_ddl_resolver.h | 4 ++ src/storage/ddl/ob_complement_data_task.cpp | 6 +- .../ddl/ob_direct_insert_sstable_ctx.cpp | 8 ++- .../ddl/ob_direct_insert_sstable_ctx.h | 3 +- src/storage/lob/ob_lob_manager.cpp | 4 +- src/storage/lob/ob_lob_manager.h | 2 +- src/storage/lob/ob_lob_util.cpp | 18 +++++- src/storage/lob/ob_lob_util.h | 19 ++++++- src/storage/ls/ob_ls_tablet_service.cpp | 30 +++++++++- src/storage/ls/ob_ls_tablet_service.h | 5 ++ 22 files changed, 234 insertions(+), 21 deletions(-) diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index 5f7b8e8ea..fbc5c5214 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -1757,7 +1757,7 @@ const int64_t OB_MAX_LOB_CHUNK_SIZE = 256 * 1024; // 256K const int64_t OB_DEFAULT_LOB_CHUNK_SIZE = OB_MAX_LOB_CHUNK_SIZE; const int64_t OB_MIN_LOB_INROW_THRESHOLD = 0; // 0 means disable inrow lob -const int64_t OB_MAX_LOB_INROW_THRESHOLD = OB_MAX_USER_ROW_LENGTH; // 1.5M +const int64_t OB_MAX_LOB_INROW_THRESHOLD = OB_MAX_USER_ROW_LENGTH / 2; // 1.5M/2 const int64_t OB_DEFAULT_LOB_INROW_THRESHOLD = 4096; // 4K const int64_t OB_MAX_CAST_CHAR_VARCHAR_LENGTH = 512; diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index cf304c6cc..94b9790fa 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -2791,6 +2791,19 @@ int ObDDLService::set_raw_table_options( } break; } + case ObAlterTableArg::LOB_INROW_THRESHOLD: { + uint64_t compat_version = OB_INVALID_VERSION; + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) { + LOG_WARN("get min data_version failed", K(ret), K(tenant_id)); + } else if (compat_version < DATA_VERSION_4_2_1_2) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("lob inrow threshold less than 4.2.1.2 not support", K(ret), K(compat_version)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "lob inrow threshold less than 4.2.1.2"); + } else { + new_table_schema.set_lob_inrow_threshold(alter_table_schema.get_lob_inrow_threshold()); + } + break; + } default: { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unknown option!", K(i)); diff --git a/src/share/schema/ob_schema_printer.cpp b/src/share/schema/ob_schema_printer.cpp index 88a7b927f..aec04ce99 100644 --- a/src/share/schema/ob_schema_printer.cpp +++ b/src/share/schema/ob_schema_printer.cpp @@ -1822,6 +1822,13 @@ int ObSchemaPrinter::print_table_definition_table_options(const ObTableSchema &t SHARE_SCHEMA_LOG(WARN, "fail to print kv attributes", K(ret), K(kv_attributes)); } } + + if (OB_SUCC(ret) && !strict_compat_ && !is_oracle_mode && !is_index_tbl) { + if (OB_FAIL(print_table_definition_lob_params(table_schema, buf, buf_len, pos))) { + SHARE_SCHEMA_LOG(WARN, "fail to print store format", K(ret), K(table_schema)); + } + } + if (OB_SUCC(ret) && pos > 0) { pos -= 1; buf[pos] = '\0'; // remove trailer space @@ -5472,6 +5479,21 @@ int ObSchemaPrinter::print_view_define_str(char* buf, return ret; } +int ObSchemaPrinter::print_table_definition_lob_params(const ObTableSchema &table_schema, + char* buf, + const int64_t& buf_len, + int64_t& pos) const +{ + int ret = OB_SUCCESS; + if (table_schema.get_lob_inrow_threshold() == OB_DEFAULT_LOB_INROW_THRESHOLD) { + // if is default not display + SHARE_SCHEMA_LOG(INFO, "default inrow threashold not display", K(ret), "lob inrow threshold", table_schema.get_lob_inrow_threshold()); + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "LOB_INROW_THRESHOLD=%ld ", table_schema.get_lob_inrow_threshold()))) { + SHARE_SCHEMA_LOG(WARN, "fail to print lob inrow threshold", K(ret), K(table_schema)); + } + return ret; +} + } // end namespace schema } //end of namespace share } // end namespace oceanbase diff --git a/src/share/schema/ob_schema_printer.h b/src/share/schema/ob_schema_printer.h index 9454277f3..3b01b6ef0 100644 --- a/src/share/schema/ob_schema_printer.h +++ b/src/share/schema/ob_schema_printer.h @@ -483,6 +483,12 @@ public: int64_t& pos, bool is_oracle_mode, const ObString &sql) const; + + int print_table_definition_lob_params(const ObTableSchema &table_schema, + char* buf, + const int64_t& buf_len, + int64_t& pos) const; + private: static bool is_subpartition_valid_in_mysql(const ObTableSchema &table_schema) { diff --git a/src/share/schema/ob_schema_service.cpp b/src/share/schema/ob_schema_service.cpp index 9defcb8e7..d36fe731b 100644 --- a/src/share/schema/ob_schema_service.cpp +++ b/src/share/schema/ob_schema_service.cpp @@ -402,6 +402,7 @@ int AlterTableSchema::assign(const ObTableSchema &src_schema) index_attributes_set_ = src_schema.index_attributes_set_; session_id_ = src_schema.session_id_; compressor_type_ = src_schema.compressor_type_; + lob_inrow_threshold_ = src_schema.lob_inrow_threshold_; is_column_store_supported_ = src_schema.is_column_store_supported_; max_used_column_group_id_ = src_schema.max_used_column_group_id_; if (OB_FAIL(deep_copy_str(src_schema.tablegroup_name_, tablegroup_name_))) { diff --git a/src/share/schema/ob_table_schema.cpp b/src/share/schema/ob_table_schema.cpp index eff418064..976bb13f3 100644 --- a/src/share/schema/ob_table_schema.cpp +++ b/src/share/schema/ob_table_schema.cpp @@ -1893,7 +1893,7 @@ bool ObTableSchema::is_valid() const valid_ret = false; } else if (!column->is_shadow_column()) { // TODO @hanhui need seperate inline memtable length from store length - varchar_col_total_length += min(column->get_data_length(), OB_MAX_LOB_HANDLE_LENGTH); + varchar_col_total_length += min(column->get_data_length(), get_lob_inrow_threshold()); } } } @@ -1931,7 +1931,7 @@ bool ObTableSchema::is_valid() const K(varchar_col_total_length), K(max_row_length)); const ObString &col_name = column->get_column_name_str(); LOG_USER_ERROR(OB_ERR_VARCHAR_TOO_LONG, - static_cast(varchar_col_total_length), max_rowkey_length, col_name.ptr()); + static_cast(varchar_col_total_length), max_row_length, col_name.ptr()); valid_ret = false; } else if (max_rowkey_length < rowkey_varchar_col_length) { LOG_WARN_RET(OB_INVALID_ERROR, "total length of varchar primary key columns is larger than the max allowed length", diff --git a/src/share/schema/ob_table_sql_service.cpp b/src/share/schema/ob_table_sql_service.cpp index 3fd9fa2fb..cbdca20c0 100644 --- a/src/share/schema/ob_table_sql_service.cpp +++ b/src/share/schema/ob_table_sql_service.cpp @@ -2681,6 +2681,10 @@ int ObTableSqlService::gen_table_dml( || !table.get_external_file_pattern().empty()))) { ret = OB_NOT_SUPPORTED; LOG_WARN("external table is not support before 4.2", K(ret), K(table)); + } else if (data_version < DATA_VERSION_4_2_1_2 + && OB_UNLIKELY(OB_DEFAULT_LOB_INROW_THRESHOLD != table.get_lob_inrow_threshold())) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("lob_inrow_threshold not support before 4.2.1.2", K(ret), K(table)); } else if (data_version < DATA_VERSION_4_3_0_0 && OB_UNLIKELY(ObRowStoreType::CS_ENCODING_ROW_STORE == table.get_row_store_type() || ObStoreFormatType::OB_STORE_FORMAT_ARCHIVE_HIGH_ORACLE == table.get_store_format())) { @@ -2823,10 +2827,13 @@ int ObTableSqlService::gen_table_dml( && OB_FAIL(dml.add_column("kv_attributes", ObHexEscapeSqlStr(kv_attributes)))) || (data_version >= DATA_VERSION_4_2_1_0 && OB_FAIL(dml.add_column("name_generated_type", table.get_name_generated_type()))) + || (data_version >= DATA_VERSION_4_2_1_2 + && OB_FAIL(dml.add_column("lob_inrow_threshold", table.get_lob_inrow_threshold()))) || (data_version >= DATA_VERSION_4_3_0_0 && OB_FAIL(dml.add_column("max_used_column_group_id", table.get_max_used_column_group_id()))) || (data_version >= DATA_VERSION_4_3_0_0 && OB_FAIL(dml.add_column("column_store", table.is_column_store_supported()))) + ) { LOG_WARN("add column failed", K(ret)); } @@ -2853,6 +2860,10 @@ int ObTableSqlService::gen_table_options_dml( LOG_WARN("ttl definition and kv attributes is not supported in version less than 4.2.1", "ttl_definition", table.get_ttl_definition().empty(), "kv_attributes", table.get_kv_attributes().empty()); + } else if (data_version < DATA_VERSION_4_2_1_2 + && OB_UNLIKELY(OB_DEFAULT_LOB_INROW_THRESHOLD != table.get_lob_inrow_threshold())) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("lob_inrow_threshold not support before 4.2.1.2", K(ret), K(table)); } else {} if (OB_SUCC(ret)) { const ObPartitionOption &part_option = table.get_part_option(); @@ -2952,6 +2963,8 @@ int ObTableSqlService::gen_table_options_dml( && OB_FAIL(dml.add_column("kv_attributes", ObHexEscapeSqlStr(kv_attributes)))) || ((data_version >= DATA_VERSION_4_2_1_0) && OB_FAIL(dml.add_column("name_generated_type", table.get_name_generated_type()))) + || (data_version >= DATA_VERSION_4_2_1_2 + && OB_FAIL(dml.add_column("lob_inrow_threshold", table.get_lob_inrow_threshold()))) || (data_version >= DATA_VERSION_4_3_0_0 && OB_FAIL(dml.add_column("max_used_column_group_id", table.get_max_used_column_group_id()))) || (data_version >= DATA_VERSION_4_3_0_0 diff --git a/src/sql/parser/non_reserved_keywords_mysql_mode.c b/src/sql/parser/non_reserved_keywords_mysql_mode.c index bcd7b9965..bbb709574 100644 --- a/src/sql/parser/non_reserved_keywords_mysql_mode.c +++ b/src/sql/parser/non_reserved_keywords_mysql_mode.c @@ -198,6 +198,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"decryption", DECRYPTION}, {"default", DEFAULT}, {"default_auth", DEFAULT_AUTH}, + {"default_lob_inrow_threshold", DEFAULT_LOB_INROW_THRESHOLD}, {"definer", DEFINER}, {"delay", DELAY}, {"delayed", DELAYED}, @@ -429,6 +430,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"listagg", LISTAGG}, {"load", LOAD}, {"ln", LN}, + {"lob_inrow_threshold", LOB_INROW_THRESHOLD}, {"local", LOCAL}, {"locality", LOCALITY}, {"localtime", LOCALTIME}, diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 9e0877771..105d08f92 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -269,7 +269,7 @@ END_P SET_VAR DELIMITER CTXCAT CTX_ID CUBE CURDATE CURRENT STACKED CURTIME CURSOR_NAME CUME_DIST CYCLE CALC_PARTITION_ID CONNECT DAG DATA DATAFILE DATA_TABLE_ID DATE DATE_ADD DATE_SUB DATETIME DAY DEALLOCATE DECRYPTION - DEFAULT_AUTH DEFINER DELAY DELAY_KEY_WRITE DEPTH DES_KEY_FILE DENSE_RANK DESCRIPTION DESTINATION DIAGNOSTICS + DEFAULT_AUTH DEFAULT_LOB_INROW_THRESHOLD DEFINER DELAY DELAY_KEY_WRITE DEPTH DES_KEY_FILE DENSE_RANK DESCRIPTION DESTINATION DIAGNOSTICS DIRECTORY DISABLE DISCARD DISK DISKGROUP DO DUMP DUMPFILE DUPLICATE DUPLICATE_SCOPE DYNAMIC DATABASE_ID DEFAULT_TABLEGROUP DISCONNECT @@ -295,7 +295,7 @@ END_P SET_VAR DELIMITER KEY_BLOCK_SIZE KEY_VERSION KVCACHE KV_ATTRIBUTES LAG LANGUAGE LAST LAST_VALUE LEAD LEADER LEAVES LESS LEAK LEAK_MOD LEAK_RATE LIB LINESTRING LIST_ - LISTAGG LOCAL LOCALITY LOCATION LOCKED LOCKS LOGFILE LOGONLY_REPLICA_NUM LOGS LOCK_ LOGICAL_READS + LISTAGG LOB_INROW_THRESHOLD LOCAL LOCALITY LOCATION LOCKED LOCKS LOGFILE LOGONLY_REPLICA_NUM LOGS LOCK_ LOGICAL_READS LEVEL LN LOG LS LINK LOG_RESTORE_SOURCE LINE_DELIMITER @@ -339,7 +339,7 @@ END_P SET_VAR DELIMITER SQL_CACHE SQL_NO_CACHE SQL_ID SQL_THREAD SQL_TSI_DAY SQL_TSI_HOUR SQL_TSI_MINUTE SQL_TSI_MONTH SQL_TSI_QUARTER SQL_TSI_SECOND SQL_TSI_WEEK SQL_TSI_YEAR SRID STANDBY STAT START STARTS STATS_AUTO_RECALC STATS_PERSISTENT STATS_SAMPLE_PAGES STATUS STATEMENTS STATISTICS STD STDDEV STDDEV_POP STDDEV_SAMP STRONG - SYNCHRONIZATION STOP STORAGE STORAGE_FORMAT_VERSION STORING STRING + SYNCHRONIZATION STOP STORAGE STORAGE_FORMAT_VERSION STORE STORING STRING SUBCLASS_ORIGIN SUBDATE SUBJECT SUBPARTITION SUBPARTITIONS SUBSTR SUBSTRING SUCCESSFUL SUM SUPER SUSPEND SWAPS SWITCH SWITCHES SWITCHOVER SYSTEM SYSTEM_USER SYSDATE SESSION_ALIAS SIZE SKEWONLY SEQUENCE SLOG STATEMENT_ID SKIP_HEADER SKIP_BLANK_LINES @@ -6573,6 +6573,16 @@ TABLE_MODE opt_equal_mark STRING_VALUE (void)($2); /* make bison mute*/ malloc_non_terminal_node($$, result->malloc_pool_, T_KV_ATTRIBUTES, 1, $3); } +| DEFAULT_LOB_INROW_THRESHOLD opt_equal_mark INTNUM +{ + (void)($2); /* make bison mute*/ + malloc_non_terminal_node($$, result->malloc_pool_, T_LOB_INROW_THRESHOLD, 1, $3); +} +| LOB_INROW_THRESHOLD opt_equal_mark INTNUM +{ + (void)($2); /* make bison mute*/ + malloc_non_terminal_node($$, result->malloc_pool_, T_LOB_INROW_THRESHOLD, 1, $3); +} ; parallel_option: @@ -18630,6 +18640,7 @@ ACCOUNT | DUPLICATE_SCOPE | DYNAMIC | DEFAULT_TABLEGROUP +| DEFAULT_LOB_INROW_THRESHOLD | EFFECTIVE | EMPTY | EMPTY_FIELD_AS_NULL @@ -18762,6 +18773,7 @@ ACCOUNT | LIST_ | LISTAGG | LN +| LOB_INROW_THRESHOLD | LOCAL | LOCALITY | LOCKED diff --git a/src/sql/resolver/ddl/ob_alter_table_resolver.cpp b/src/sql/resolver/ddl/ob_alter_table_resolver.cpp index 940b36c68..4ea37b2f7 100644 --- a/src/sql/resolver/ddl/ob_alter_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_alter_table_resolver.cpp @@ -391,6 +391,7 @@ int ObAlterTableResolver::set_table_options() alter_table_schema.set_table_mode_struct(table_mode_); alter_table_schema.set_tablespace_id(tablespace_id_); alter_table_schema.set_dop(table_dop_); + alter_table_schema.set_lob_inrow_threshold(lob_inrow_threshold_); //deep copy if (OB_FAIL(ret)) { //do nothing diff --git a/src/sql/resolver/ddl/ob_create_table_resolver.cpp b/src/sql/resolver/ddl/ob_create_table_resolver.cpp index 91e3f1b4b..fd0e4c40a 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver.cpp @@ -1340,7 +1340,7 @@ int ObCreateTableResolver::resolve_table_elements(const ParseNode *node, LOG_USER_ERROR(OB_ERR_TOO_LONG_COLUMN_LENGTH, column.get_column_name(), ObAccuracy::MAX_ACCURACY2[is_oracle_mode][column.get_data_type()].get_length()); } else { - length = min(length, OB_MAX_LOB_HANDLE_LENGTH); + length = min(length, table_schema.get_lob_inrow_threshold()); } } if (OB_SUCC(ret) && (row_data_length += length) > OB_MAX_USER_ROW_LENGTH) { @@ -2425,6 +2425,23 @@ int ObCreateTableResolver::set_table_option_to_schema(ObTableSchema &table_schem } } + if (OB_SUCC(ret)) { + // if lob_inrow_threshold not set, used config default_lob_inrow_threshold + if (is_set_lob_inrow_threshold_) { + table_schema.set_lob_inrow_threshold(lob_inrow_threshold_); + } else if (OB_ISNULL(session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session if NULL", K(ret)); + } else if (OB_FALSE_IT((lob_inrow_threshold_ = session_info_->get_default_lob_inrow_threshold()))) { + } else if (lob_inrow_threshold_ < OB_MIN_LOB_INROW_THRESHOLD || lob_inrow_threshold_ > OB_MAX_LOB_INROW_THRESHOLD) { + ret = OB_INVALID_ARGUMENT; + SQL_RESV_LOG(ERROR, "invalid inrow threshold", K(ret), K(lob_inrow_threshold_)); + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "invalid inrow threshold"); + } else { + table_schema.set_lob_inrow_threshold(lob_inrow_threshold_); + } + } + if (OB_SUCC(ret) && table_schema.is_external_table()) { if (table_schema.get_external_file_format().empty() || table_schema.get_external_file_location().empty()) { diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index b11aa4c5d..4fda6343f 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -113,7 +113,9 @@ ObDDLResolver::ObDDLResolver(ObResolverParams ¶ms) is_external_table_(false), ttl_definition_(), kv_attributes_(), - name_generated_type_(GENERATED_TYPE_UNKNOWN) + name_generated_type_(GENERATED_TYPE_UNKNOWN), + is_set_lob_inrow_threshold_(false), + lob_inrow_threshold_(OB_DEFAULT_LOB_INROW_THRESHOLD) { table_mode_.reset(); } @@ -2236,6 +2238,10 @@ int ObDDLResolver::resolve_table_option(const ParseNode *option_node, const bool } break; } + case T_LOB_INROW_THRESHOLD: { + ret = resolve_lob_inrow_threshold(option_node, is_index_option); + break; + } default: { /* won't be here */ ret = OB_ERR_UNEXPECTED; @@ -3820,6 +3826,52 @@ int ObDDLResolver::resolve_srid_node(share::schema::ObColumnSchemaV2 &column, return ret; } +int ObDDLResolver::resolve_lob_inrow_threshold(const ParseNode *option_node, const bool is_index_option) +{ + int ret = OB_SUCCESS; + uint64_t tenant_id = 0; + uint64_t tenant_data_version = 0;; + if (OB_ISNULL(session_info_)) { + ret = OB_ERR_UNEXPECTED; + SQL_RESV_LOG(WARN, "session_info_ is null", K(ret)); + } else if (OB_FALSE_IT(tenant_id = session_info_->get_effective_tenant_id())) { + } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) { + LOG_WARN("get tenant data version failed", K(ret)); + } else if (tenant_data_version < DATA_VERSION_4_2_1_2) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("lob inrow threshold is not supported in data version less than 4.2.1.2", K(ret), K(tenant_data_version)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "lob inrow threshold is not supported in data version less than 4.2.1.2"); + } else if (is_index_option) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("index option should not specify lob inrow threshold", K(ret)); + } else if (OB_ISNULL(option_node)) { + ret = OB_ERR_UNEXPECTED; + SQL_RESV_LOG(WARN, "option_node is null", K(ret)); + } else if (OB_ISNULL(option_node->children_)) { + ret = OB_ERR_UNEXPECTED; + SQL_RESV_LOG(WARN, "the children of option_node is null", K(option_node->children_), K(ret)); + } else if (OB_ISNULL(option_node->children_[0])) { + ret = OB_ERR_UNEXPECTED; + SQL_RESV_LOG(ERROR,"children can't be null", K(ret)); + } else if (OB_ISNULL(stmt_)) { + ret = OB_ERR_UNEXPECTED; + SQL_RESV_LOG(ERROR,"stmt_ is null", K(ret)); + } else { + lob_inrow_threshold_ = option_node->children_[0]->value_; + is_set_lob_inrow_threshold_ = true; + if (lob_inrow_threshold_ < OB_MIN_LOB_INROW_THRESHOLD || lob_inrow_threshold_ > OB_MAX_LOB_INROW_THRESHOLD) { + ret = OB_INVALID_ARGUMENT; + SQL_RESV_LOG(ERROR, "invalid inrow threshold", K(ret), K(lob_inrow_threshold_)); + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "lob inrow threshold, should be [0, 786432]"); + } else if (stmt::T_ALTER_TABLE == stmt_->get_stmt_type()) { + if (OB_FAIL(alter_table_bitset_.add_member(ObAlterTableArg::LOB_INROW_THRESHOLD))) { + SQL_RESV_LOG(WARN, "failed to add member to bitset!", K(ret)); + } + } + } + return ret; +} + /* int ObDDLResolver::resolve_generated_column_definition(ObColumnSchemaV2 &column, ParseNode *node, ObColumnResolveStat &resolve_stat) @@ -4380,6 +4432,7 @@ void ObDDLResolver::reset() { hash_subpart_num_ = -1; ttl_definition_.reset(); kv_attributes_.reset(); + lob_inrow_threshold_ = OB_DEFAULT_LOB_INROW_THRESHOLD; } bool ObDDLResolver::is_valid_prefix_key_type(const ObObjTypeClass column_type_class) diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.h b/src/sql/resolver/ddl/ob_ddl_resolver.h index 3f2082fa2..0675c3e2e 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.h +++ b/src/sql/resolver/ddl/ob_ddl_resolver.h @@ -536,6 +536,8 @@ protected: const ParseNode &skip_index_node, share::schema::ObColumnSchemaV2 &column_schema); int check_skip_index(share::schema::ObTableSchema &table_schema); + int resolve_lob_inrow_threshold(const ParseNode *option_node, const bool is_index_option); + /* int resolve_generated_column_definition( share::schema::ObColumnSchemaV2 &column, @@ -937,6 +939,8 @@ protected: common::ObString ttl_definition_; common::ObString kv_attributes_; ObNameGeneratedType name_generated_type_; + bool is_set_lob_inrow_threshold_; + int64_t lob_inrow_threshold_; private: template DISALLOW_COPY_AND_ASSIGN(ObDDLResolver); diff --git a/src/storage/ddl/ob_complement_data_task.cpp b/src/storage/ddl/ob_complement_data_task.cpp index cee45b0c4..2ba8a989f 100644 --- a/src/storage/ddl/ob_complement_data_task.cpp +++ b/src/storage/ddl/ob_complement_data_task.cpp @@ -1163,6 +1163,7 @@ int ObComplementWriteTask::append_row(ObScan *scan) const int64_t extra_rowkey_cnt = storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); bool ddl_committed = false; blocksstable::ObNewRowBuilder new_row_builder; + int64_t lob_inrow_threshold = OB_DEFAULT_LOB_INROW_THRESHOLD; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObComplementWriteTask is not inited", K(ret)); @@ -1208,6 +1209,7 @@ int ObComplementWriteTask::append_row(ObScan *scan) LOG_WARN("fail to open macro block writer", K(ret), K(data_desc)); } else { rowkey_column_cnt = hidden_table_schema->get_rowkey_column_num(); + lob_inrow_threshold = hidden_table_schema->get_lob_inrow_threshold(); } ObTableSchemaParam schema_param(allocator); // Hack to prevent row reshaping from converting empty string to null. @@ -1261,9 +1263,11 @@ int ObComplementWriteTask::append_row(ObScan *scan) } else if (org_col_ids_.at(i).col_type_.is_lob_storage()) { lob_cnt++; const int64_t timeout_ts = ObTimeUtility::current_time() + 60000000; // 60s + ObLobStorageParam lob_storage_param; + lob_storage_param.inrow_threshold_ = lob_inrow_threshold; if (OB_FAIL(ObInsertLobColumnHelper::insert_lob_column( lob_allocator, param_->dest_ls_id_, param_->dest_tablet_id_, - org_col_ids_.at(i), datum, timeout_ts, true, param_->orig_tenant_id_))) { + org_col_ids_.at(i), lob_storage_param, datum, timeout_ts, true, param_->orig_tenant_id_))) { LOG_WARN("fail to insert_lob_col", K(ret), K(datum)); } } diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx.cpp b/src/storage/ddl/ob_direct_insert_sstable_ctx.cpp index 51f273f20..d1115d65e 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx.cpp +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx.cpp @@ -224,7 +224,8 @@ ObSSTableInsertSliceWriter::ObSSTableInsertSliceWriter() sql_mode_for_ddl_reshape_(0), reshape_ptr_(nullptr), is_inited_(false), - new_row_builder_() + new_row_builder_(), + lob_inrow_threshold_(OB_DEFAULT_LOB_INROW_THRESHOLD) { } @@ -264,6 +265,7 @@ int ObSSTableInsertSliceWriter::init(const ObSSTableInsertSliceParam &slice_para 0/*cluster_version*/))) { LOG_WARN("fail to init data desc", KR(ret), K_(data_desc)); } else { + lob_inrow_threshold_ = table_schema->get_lob_inrow_threshold(); data_desc_.get_desc().sstable_index_builder_ = slice_param.sstable_index_builder_; if (OB_FAIL(macro_block_writer_.open(data_desc_.get_desc(), slice_param.start_seq_, &redo_log_writer_callback_))) { @@ -356,8 +358,10 @@ int ObSSTableInsertSliceWriter::append_row(const ObNewRow &row_val) const int64_t timeout_ts = ObTimeUtility::current_time() + ObInsertLobColumnHelper::LOB_ACCESS_TX_TIMEOUT; bool has_lob_header = store_row_.row_val_.cells_[i].has_lob_header(); + ObLobStorageParam lob_storage_param; + lob_storage_param.inrow_threshold_ = lob_inrow_threshold_; if (OB_FAIL(ObInsertLobColumnHelper::insert_lob_column( - lob_allocator_, ls_id_, tablet_id_, col_descs_->at(i), datum, timeout_ts, has_lob_header, + lob_allocator_, ls_id_, tablet_id_, col_descs_->at(i), lob_storage_param, datum, timeout_ts, has_lob_header, MTL_ID()))) { LOG_WARN("fail to insert_lob_col", KR(ret), K(datum)); } diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx.h b/src/storage/ddl/ob_direct_insert_sstable_ctx.h index 57fd00e16..d0c4cb630 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx.h +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx.h @@ -144,7 +144,7 @@ public: OB_INLINE int64_t get_snapshot_version() const { return snapshot_version_; } TO_STRING_KV(K_(tablet_id), K_(ls_id), K_(rowkey_column_num), K_(is_index_table), KP_(col_descs), K_(snapshot_version), K_(data_desc), K_(lob_cnt), K_(sql_mode_for_ddl_reshape), - KP_(reshape_ptr)); + KP_(reshape_ptr), K_(lob_inrow_threshold)); private: int prepare_reshape( const common::ObTabletID &tablet_id, @@ -177,6 +177,7 @@ private: blocksstable::ObDatumRow datum_row_; bool is_inited_; blocksstable::ObNewRowBuilder new_row_builder_; + int64_t lob_inrow_threshold_; }; class ObSSTableInsertTabletContext final diff --git a/src/storage/lob/ob_lob_manager.cpp b/src/storage/lob/ob_lob_manager.cpp index 36a9acf0a..83aecad4e 100644 --- a/src/storage/lob/ob_lob_manager.cpp +++ b/src/storage/lob/ob_lob_manager.cpp @@ -1056,7 +1056,7 @@ int ObLobManager::check_need_out_row( bool &need_out_row) { int ret = OB_SUCCESS; - need_out_row = (param.byte_size_ + add_len) > LOB_IN_ROW_MAX_LENGTH; + need_out_row = (param.byte_size_ + add_len) > param.get_inrow_threshold(); if (param.lob_locator_ != nullptr) { // TODO @lhd remove after tmp lob support outrow if (!param.lob_locator_->is_persist_lob()) { @@ -1758,7 +1758,7 @@ int ObLobManager::prepare_for_write( modified_end *= max_bytes_in_char; } uint64_t total_size = param.byte_size_ > modified_end ? param.byte_size_ : modified_end; - need_out_row = (total_size > LOB_IN_ROW_MAX_LENGTH); + need_out_row = (total_size > param.get_inrow_threshold()); if (param.lob_common_->in_row_) { old_data.assign_ptr(param.lob_common_->get_inrow_data_ptr(), param.byte_size_); } diff --git a/src/storage/lob/ob_lob_manager.h b/src/storage/lob/ob_lob_manager.h index 748020739..1c95cd41d 100644 --- a/src/storage/lob/ob_lob_manager.h +++ b/src/storage/lob/ob_lob_manager.h @@ -241,7 +241,7 @@ public: uint64_t len, int64_t timeout, ObLobLocatorV2 &lob); - inline bool can_write_inrow(uint64_t len) { return len <= LOB_IN_ROW_MAX_LENGTH; } + inline bool can_write_inrow(uint64_t len, int64_t inrow_threshold) { return len <= inrow_threshold; } private: // private function int write_inrow_inner(ObLobAccessParam& param, ObString& data, ObString& old_data); diff --git a/src/storage/lob/ob_lob_util.cpp b/src/storage/lob/ob_lob_util.cpp index ef097acb6..85b0a6668 100644 --- a/src/storage/lob/ob_lob_util.cpp +++ b/src/storage/lob/ob_lob_util.cpp @@ -49,6 +49,16 @@ int ObLobAccessParam::set_lob_locator(common::ObLobLocatorV2 *lob_locator) return ret; } +int64_t ObLobAccessParam::get_inrow_threshold() +{ + int64_t res = inrow_threshold_; + if (res < OB_MIN_LOB_INROW_THRESHOLD || res > OB_MAX_LOB_INROW_THRESHOLD) { + LIB_LOG_RET(WARN, OB_ERR_UNEXPECTED, "invalid inrow threshold, use default inrow threshold", K(res)); + res = OB_DEFAULT_LOB_INROW_THRESHOLD; + } + return res; +} + int ObInsertLobColumnHelper::start_trans(const share::ObLSID &ls_id, const bool is_for_read, const int64_t timeout_ts, @@ -107,6 +117,7 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, const share::ObLSID ls_id, const common::ObTabletID tablet_id, const ObColDesc &column, + const ObLobStorageParam &lob_storage_param, blocksstable::ObStorageDatum &datum, const int64_t timeout_ts, const bool has_lob_header, @@ -130,7 +141,7 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, // datum with null ptr and zero len should treat as no lob header bool set_has_lob_header = has_lob_header && data.length() > 0; ObLobLocatorV2 src(data, set_has_lob_header); - if (src.has_inrow_data() && lob_mngr->can_write_inrow(data.length())) { + if (src.has_inrow_data() && lob_mngr->can_write_inrow(data.length(), lob_storage_param.inrow_threshold_)) { // fast path for inrow data if (OB_FAIL(src.get_inrow_data(data))) { LOG_WARN("fail to get inrow data", K(ret), K(src)); @@ -165,6 +176,8 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, lob_param.timeout_ = timeout_ts; lob_param.scan_backward_ = false; lob_param.offset_ = 0; + lob_param.inrow_threshold_ = lob_storage_param.inrow_threshold_; + LOG_DEBUG("lob storage param", K(lob_storage_param), K(column)); if (!src.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid src lob locator.", K(ret)); @@ -187,13 +200,14 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, const share::ObLSID ls_id, const common::ObTabletID tablet_id, const ObColDesc &column, + const ObLobStorageParam &lob_storage_param, ObObj &obj, const int64_t timeout_ts) { int ret = OB_SUCCESS; ObStorageDatum datum; datum.from_obj(obj); - if (OB_SUCC(insert_lob_column(allocator, ls_id, tablet_id, column, datum, timeout_ts, obj.has_lob_header(), MTL_ID()))) { + if (OB_SUCC(insert_lob_column(allocator, ls_id, tablet_id, column, lob_storage_param, datum, timeout_ts, obj.has_lob_header(), MTL_ID()))) { obj.set_lob_value(obj.get_type(), datum.get_string().ptr(), datum.get_string().length()); } return ret; diff --git a/src/storage/lob/ob_lob_util.h b/src/storage/lob/ob_lob_util.h index 6c0483158..c893dc3eb 100644 --- a/src/storage/lob/ob_lob_util.h +++ b/src/storage/lob/ob_lob_util.h @@ -30,6 +30,17 @@ namespace oceanbase namespace storage { +struct ObLobStorageParam +{ + ObLobStorageParam(): + inrow_threshold_(OB_DEFAULT_LOB_INROW_THRESHOLD) + {} + + TO_STRING_KV(K_(inrow_threshold)); + + int64_t inrow_threshold_; +}; + struct ObLobAccessParam { ObLobAccessParam() : tx_desc_(nullptr), snapshot_(), tx_id_(), sql_mode_(SMO_DEFAULT), allocator_(nullptr), @@ -43,7 +54,7 @@ struct ObLobAccessParam { scan_backward_(false), asscess_ptable_(false), offset_(0), len_(0), parent_seq_no_(), seq_no_st_(), used_seq_cnt_(0), total_seq_cnt_(0), checksum_(0), update_len_(0), op_type_(ObLobDataOutRowCtx::OpType::SQL), is_fill_zero_(false), from_rpc_(false), - inrow_read_nocopy_(false) + inrow_read_nocopy_(false), inrow_threshold_(OB_DEFAULT_LOB_INROW_THRESHOLD) {} ~ObLobAccessParam() { if (OB_NOT_NULL(dml_base_param_)) { @@ -52,10 +63,11 @@ struct ObLobAccessParam { } public: int set_lob_locator(common::ObLobLocatorV2 *lob_locator); + int64_t get_inrow_threshold(); TO_STRING_KV(K_(tenant_id), K_(src_tenant_id), K_(ls_id), K_(tablet_id), KPC_(lob_locator), KPC_(lob_common), KPC_(lob_data), K_(byte_size), K_(handle_size), K_(coll_type), K_(scan_backward), K_(offset), K_(len), K_(parent_seq_no), K_(seq_no_st), K_(used_seq_cnt), K_(total_seq_cnt), K_(checksum), K_(update_len), K_(op_type), - K_(is_fill_zero), K_(from_rpc), K_(snapshot), K_(tx_id), K_(inrow_read_nocopy)); + K_(is_fill_zero), K_(from_rpc), K_(snapshot), K_(tx_id), K_(inrow_read_nocopy), K_(inrow_threshold)); public: transaction::ObTxDesc *tx_desc_; // for write/update/delete transaction::ObTxReadSnapshot snapshot_; // for read @@ -100,6 +112,7 @@ public: bool is_fill_zero_; // fill zero when erase bool from_rpc_; bool inrow_read_nocopy_; + int64_t inrow_threshold_; }; struct ObLobMetaInfo { @@ -198,6 +211,7 @@ public: const share::ObLSID ls_id, const common::ObTabletID tablet_id, const share::schema::ObColDesc &column, + const ObLobStorageParam &lob_storage_param, blocksstable::ObStorageDatum &datum, const int64_t timeout_ts, const bool has_lob_header, @@ -206,6 +220,7 @@ public: const share::ObLSID ls_id, const common::ObTabletID tablet_id, const share::schema::ObColDesc &column, + const ObLobStorageParam &lob_storage_param, ObObj &obj, const int64_t timeout_ts); }; diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index dd15d2cde..c0144b525 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -4012,7 +4012,9 @@ int ObLSTabletService::insert_lob_col( // for not strict sql mode, will insert empty string without lob header bool has_lob_header = obj.has_lob_header() && raw_data.length() > 0; ObLobLocatorV2 loc(raw_data, has_lob_header); - if (OB_FAIL(lob_mngr->append(lob_param, loc))) { + if (OB_FAIL(set_lob_storage_params(run_ctx, column, lob_param))) { + LOG_WARN("set_lob_storage_params fail", K(ret), K(column)); + } else if (OB_FAIL(lob_mngr->append(lob_param, loc))) { LOG_WARN("[STORAGE_LOB]lob append failed.", K(ret)); } else { ObLobCommon *res_lob_common = lob_param.lob_common_; @@ -4335,7 +4337,9 @@ int ObLSTabletService::process_delta_lob( // should use old obj lob ObLobLocatorV2 old_lob; ObString old_disk_lob; - if (OB_FAIL(old_obj.get_lob_locatorv2(old_lob))) { + if (OB_FAIL(set_lob_storage_params(run_ctx, column, lob_param))) { + LOG_WARN("set_lob_storage_params fail", K(ret), K(column)); + } else if (OB_FAIL(old_obj.get_lob_locatorv2(old_lob))) { LOG_WARN("get old lob locator failed.", K(ret), K(old_obj)); } else if (!old_lob.is_valid()) { ret = OB_ERR_UNEXPECTED; @@ -4361,6 +4365,26 @@ int ObLSTabletService::process_delta_lob( return ret; } +int ObLSTabletService::set_lob_storage_params( + ObDMLRunningCtx &run_ctx, + const ObColDesc &column, + ObLobAccessParam &lob_param) +{ + int ret = OB_SUCCESS; + const ObTableDMLParam *table_param = run_ctx.dml_param_.table_param_; + const ObColumnParam *column_param = nullptr; + if (OB_ISNULL(table_param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table_param is null", K(ret)); + } else if (OB_ISNULL(column_param = table_param->get_data_table().get_column(column.col_id_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column_param is null", K(ret), K(table_param)); + } else { + lob_param.inrow_threshold_ = table_param->get_data_table().get_lob_inrow_threshold(); + } + return ret; +} + int ObLSTabletService::process_lob_row( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, @@ -5346,6 +5370,8 @@ int ObLSTabletService::delete_lob_col( if (data.length() < sizeof(ObLobCommon)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("[STORAGE_LOB]Invalid Lob data.", K(ret), K(obj), K(data)); + } else if (OB_FAIL(set_lob_storage_params(run_ctx, column, lob_param))) { + LOG_WARN("set_lob_storage_params fail", K(ret), K(column)); } else { void *buf = run_ctx.lob_allocator_.alloc(data.length()); if (OB_ISNULL(buf)) { diff --git a/src/storage/ls/ob_ls_tablet_service.h b/src/storage/ls/ob_ls_tablet_service.h index e8ef4db9c..48e2c0547 100644 --- a/src/storage/ls/ob_ls_tablet_service.h +++ b/src/storage/ls/ob_ls_tablet_service.h @@ -652,6 +652,11 @@ private: ObObj &old_obj, ObLobLocatorV2 &delta_lob, ObObj &obj); + static int set_lob_storage_params( + ObDMLRunningCtx &run_ctx, + const ObColDesc &column, + ObLobAccessParam &lob_param); + static int process_lob_row( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx,