hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[CP] [LOB] support inrow threshold config

Browse Source
This commit is contained in:
obdev 2024-02-07 14:44:41 +00:00 committed by ob-robot
parent faf440e6ea
commit 21ddf9e8b2
22 changed files with 234 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deps/oblib/src/lib/ob_define.h vendored
View File

@ -1757,7 +1757,7 @@ const int64_t OB_MAX_LOB_CHUNK_SIZE = 256 * 1024; // 256K
const int64_t OB_DEFAULT_LOB_CHUNK_SIZE = OB_MAX_LOB_CHUNK_SIZE;
const int64_t OB_MIN_LOB_INROW_THRESHOLD = 0; // 0 means disable inrow lob
const int64_t OB_MAX_LOB_INROW_THRESHOLD = OB_MAX_USER_ROW_LENGTH; // 1.5M
const int64_t OB_MAX_LOB_INROW_THRESHOLD = OB_MAX_USER_ROW_LENGTH / 2; // 1.5M/2
const int64_t OB_DEFAULT_LOB_INROW_THRESHOLD = 4096; // 4K
const int64_t OB_MAX_CAST_CHAR_VARCHAR_LENGTH = 512;

13
src/rootserver/ob_ddl_service.cpp
View File

@ -2791,6 +2791,19 @@ int ObDDLService::set_raw_table_options(
}
break;
}
case ObAlterTableArg::LOB_INROW_THRESHOLD: {
uint64_t compat_version = OB_INVALID_VERSION;
if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) {
LOG_WARN("get min data_version failed", K(ret), K(tenant_id));
} else if (compat_version < DATA_VERSION_4_2_1_2) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("lob inrow threshold less than 4.2.1.2 not support", K(ret), K(compat_version));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "lob inrow threshold less than 4.2.1.2");
} else {
new_table_schema.set_lob_inrow_threshold(alter_table_schema.get_lob_inrow_threshold());
}
break;
}
default: {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unknown option!", K(i));

22
src/share/schema/ob_schema_printer.cpp
View File

@ -1822,6 +1822,13 @@ int ObSchemaPrinter::print_table_definition_table_options(const ObTableSchema &t
SHARE_SCHEMA_LOG(WARN, "fail to print kv attributes", K(ret), K(kv_attributes));
}
}
if (OB_SUCC(ret) && !strict_compat_ && !is_oracle_mode && !is_index_tbl) {
if (OB_FAIL(print_table_definition_lob_params(table_schema, buf, buf_len, pos))) {
SHARE_SCHEMA_LOG(WARN, "fail to print store format", K(ret), K(table_schema));
}
}
if (OB_SUCC(ret) && pos > 0) {
pos -= 1;
buf[pos] = '\0'; // remove trailer space
@ -5472,6 +5479,21 @@ int ObSchemaPrinter::print_view_define_str(char* buf,
return ret;
}
int ObSchemaPrinter::print_table_definition_lob_params(const ObTableSchema &table_schema,
char* buf,
const int64_t& buf_len,
int64_t& pos) const
{
int ret = OB_SUCCESS;
if (table_schema.get_lob_inrow_threshold() == OB_DEFAULT_LOB_INROW_THRESHOLD) {
// if is default not display
SHARE_SCHEMA_LOG(INFO, "default inrow threashold not display", K(ret), "lob inrow threshold", table_schema.get_lob_inrow_threshold());
} else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "LOB_INROW_THRESHOLD=%ld ", table_schema.get_lob_inrow_threshold()))) {
SHARE_SCHEMA_LOG(WARN, "fail to print lob inrow threshold", K(ret), K(table_schema));
}
return ret;
}
} // end namespace schema
} //end of namespace share
} // end namespace oceanbase

6
src/share/schema/ob_schema_printer.h
View File

@ -483,6 +483,12 @@ public:
int64_t& pos,
bool is_oracle_mode,
const ObString &sql) const;
int print_table_definition_lob_params(const ObTableSchema &table_schema,
char* buf,
const int64_t& buf_len,
int64_t& pos) const;
private:
static bool is_subpartition_valid_in_mysql(const ObTableSchema &table_schema)
{

1
src/share/schema/ob_schema_service.cpp
View File

@ -402,6 +402,7 @@ int AlterTableSchema::assign(const ObTableSchema &src_schema)
index_attributes_set_ = src_schema.index_attributes_set_;
session_id_ = src_schema.session_id_;
compressor_type_ = src_schema.compressor_type_;
lob_inrow_threshold_ = src_schema.lob_inrow_threshold_;
is_column_store_supported_ = src_schema.is_column_store_supported_;
max_used_column_group_id_ = src_schema.max_used_column_group_id_;
if (OB_FAIL(deep_copy_str(src_schema.tablegroup_name_, tablegroup_name_))) {

4
src/share/schema/ob_table_schema.cpp
View File

@ -1893,7 +1893,7 @@ bool ObTableSchema::is_valid() const
valid_ret = false;
} else if (!column->is_shadow_column()) {
// TODO @hanhui need seperate inline memtable length from store length
varchar_col_total_length += min(column->get_data_length(), OB_MAX_LOB_HANDLE_LENGTH);
varchar_col_total_length += min(column->get_data_length(), get_lob_inrow_threshold());
}
}
}
@ -1931,7 +1931,7 @@ bool ObTableSchema::is_valid() const
K(varchar_col_total_length), K(max_row_length));
const ObString &col_name = column->get_column_name_str();
LOG_USER_ERROR(OB_ERR_VARCHAR_TOO_LONG,
static_cast<int>(varchar_col_total_length), max_rowkey_length, col_name.ptr());
static_cast<int>(varchar_col_total_length), max_row_length, col_name.ptr());
valid_ret = false;
} else if (max_rowkey_length < rowkey_varchar_col_length) {
LOG_WARN_RET(OB_INVALID_ERROR, "total length of varchar primary key columns is larger than the max allowed length",

13
src/share/schema/ob_table_sql_service.cpp
View File

@ -2681,6 +2681,10 @@ int ObTableSqlService::gen_table_dml(
|| !table.get_external_file_pattern().empty()))) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("external table is not support before 4.2", K(ret), K(table));
} else if (data_version < DATA_VERSION_4_2_1_2
&& OB_UNLIKELY(OB_DEFAULT_LOB_INROW_THRESHOLD != table.get_lob_inrow_threshold())) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("lob_inrow_threshold not support before 4.2.1.2", K(ret), K(table));
} else if (data_version < DATA_VERSION_4_3_0_0
&& OB_UNLIKELY(ObRowStoreType::CS_ENCODING_ROW_STORE == table.get_row_store_type() ||
ObStoreFormatType::OB_STORE_FORMAT_ARCHIVE_HIGH_ORACLE == table.get_store_format())) {
@ -2823,10 +2827,13 @@ int ObTableSqlService::gen_table_dml(
&& OB_FAIL(dml.add_column("kv_attributes", ObHexEscapeSqlStr(kv_attributes))))
|| (data_version >= DATA_VERSION_4_2_1_0
&& OB_FAIL(dml.add_column("name_generated_type", table.get_name_generated_type())))
|| (data_version >= DATA_VERSION_4_2_1_2
&& OB_FAIL(dml.add_column("lob_inrow_threshold", table.get_lob_inrow_threshold())))
|| (data_version >= DATA_VERSION_4_3_0_0
&& OB_FAIL(dml.add_column("max_used_column_group_id", table.get_max_used_column_group_id())))
|| (data_version >= DATA_VERSION_4_3_0_0
&& OB_FAIL(dml.add_column("column_store", table.is_column_store_supported())))
) {
LOG_WARN("add column failed", K(ret));
}
@ -2853,6 +2860,10 @@ int ObTableSqlService::gen_table_options_dml(
LOG_WARN("ttl definition and kv attributes is not supported in version less than 4.2.1",
"ttl_definition", table.get_ttl_definition().empty(),
"kv_attributes", table.get_kv_attributes().empty());
} else if (data_version < DATA_VERSION_4_2_1_2
&& OB_UNLIKELY(OB_DEFAULT_LOB_INROW_THRESHOLD != table.get_lob_inrow_threshold())) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("lob_inrow_threshold not support before 4.2.1.2", K(ret), K(table));
} else {}
if (OB_SUCC(ret)) {
const ObPartitionOption &part_option = table.get_part_option();
@ -2952,6 +2963,8 @@ int ObTableSqlService::gen_table_options_dml(
&& OB_FAIL(dml.add_column("kv_attributes", ObHexEscapeSqlStr(kv_attributes))))
|| ((data_version >= DATA_VERSION_4_2_1_0)
&& OB_FAIL(dml.add_column("name_generated_type", table.get_name_generated_type())))
|| (data_version >= DATA_VERSION_4_2_1_2
&& OB_FAIL(dml.add_column("lob_inrow_threshold", table.get_lob_inrow_threshold())))
|| (data_version >= DATA_VERSION_4_3_0_0
&& OB_FAIL(dml.add_column("max_used_column_group_id", table.get_max_used_column_group_id())))
|| (data_version >= DATA_VERSION_4_3_0_0

2
src/sql/parser/non_reserved_keywords_mysql_mode.c
View File

@ -198,6 +198,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] =
{"decryption", DECRYPTION},
{"default", DEFAULT},
{"default_auth", DEFAULT_AUTH},
{"default_lob_inrow_threshold", DEFAULT_LOB_INROW_THRESHOLD},
{"definer", DEFINER},
{"delay", DELAY},
{"delayed", DELAYED},
@ -429,6 +430,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] =
{"listagg", LISTAGG},
{"load", LOAD},
{"ln", LN},
{"lob_inrow_threshold", LOB_INROW_THRESHOLD},
{"local", LOCAL},
{"locality", LOCALITY},
{"localtime", LOCALTIME},

18
src/sql/parser/sql_parser_mysql_mode.y
View File

@ -269,7 +269,7 @@ END_P SET_VAR DELIMITER
CTXCAT CTX_ID CUBE CURDATE CURRENT STACKED CURTIME CURSOR_NAME CUME_DIST CYCLE CALC_PARTITION_ID CONNECT
DAG DATA DATAFILE DATA_TABLE_ID DATE DATE_ADD DATE_SUB DATETIME DAY DEALLOCATE DECRYPTION
DEFAULT_AUTH DEFINER DELAY DELAY_KEY_WRITE DEPTH DES_KEY_FILE DENSE_RANK DESCRIPTION DESTINATION DIAGNOSTICS
DEFAULT_AUTH DEFAULT_LOB_INROW_THRESHOLD DEFINER DELAY DELAY_KEY_WRITE DEPTH DES_KEY_FILE DENSE_RANK DESCRIPTION DESTINATION DIAGNOSTICS
DIRECTORY DISABLE DISCARD DISK DISKGROUP DO DUMP DUMPFILE DUPLICATE DUPLICATE_SCOPE DYNAMIC
DATABASE_ID DEFAULT_TABLEGROUP DISCONNECT
@ -295,7 +295,7 @@ END_P SET_VAR DELIMITER
KEY_BLOCK_SIZE KEY_VERSION KVCACHE KV_ATTRIBUTES
LAG LANGUAGE LAST LAST_VALUE LEAD LEADER LEAVES LESS LEAK LEAK_MOD LEAK_RATE LIB LINESTRING LIST_
LISTAGG LOCAL LOCALITY LOCATION LOCKED LOCKS LOGFILE LOGONLY_REPLICA_NUM LOGS LOCK_ LOGICAL_READS
LISTAGG LOB_INROW_THRESHOLD LOCAL LOCALITY LOCATION LOCKED LOCKS LOGFILE LOGONLY_REPLICA_NUM LOGS LOCK_ LOGICAL_READS
LEVEL LN LOG LS LINK LOG_RESTORE_SOURCE LINE_DELIMITER
@ -339,7 +339,7 @@ END_P SET_VAR DELIMITER
SQL_CACHE SQL_NO_CACHE SQL_ID SQL_THREAD SQL_TSI_DAY SQL_TSI_HOUR SQL_TSI_MINUTE SQL_TSI_MONTH
SQL_TSI_QUARTER SQL_TSI_SECOND SQL_TSI_WEEK SQL_TSI_YEAR SRID STANDBY STAT START STARTS STATS_AUTO_RECALC
STATS_PERSISTENT STATS_SAMPLE_PAGES STATUS STATEMENTS STATISTICS STD STDDEV STDDEV_POP STDDEV_SAMP STRONG
SYNCHRONIZATION STOP STORAGE STORAGE_FORMAT_VERSION STORING STRING
SYNCHRONIZATION STOP STORAGE STORAGE_FORMAT_VERSION STORE STORING STRING
SUBCLASS_ORIGIN SUBDATE SUBJECT SUBPARTITION SUBPARTITIONS SUBSTR SUBSTRING SUCCESSFUL SUM
SUPER SUSPEND SWAPS SWITCH SWITCHES SWITCHOVER SYSTEM SYSTEM_USER SYSDATE SESSION_ALIAS
SIZE SKEWONLY SEQUENCE SLOG STATEMENT_ID SKIP_HEADER SKIP_BLANK_LINES
@ -6573,6 +6573,16 @@ TABLE_MODE opt_equal_mark STRING_VALUE
(void)($2); /* make bison mute*/
malloc_non_terminal_node($$, result->malloc_pool_, T_KV_ATTRIBUTES, 1, $3);
}
| DEFAULT_LOB_INROW_THRESHOLD opt_equal_mark INTNUM
{
(void)($2); /* make bison mute*/
malloc_non_terminal_node($$, result->malloc_pool_, T_LOB_INROW_THRESHOLD, 1, $3);
}
| LOB_INROW_THRESHOLD opt_equal_mark INTNUM
{
(void)($2); /* make bison mute*/
malloc_non_terminal_node($$, result->malloc_pool_, T_LOB_INROW_THRESHOLD, 1, $3);
}
;
parallel_option:
@ -18630,6 +18640,7 @@ ACCOUNT
| DUPLICATE_SCOPE
| DYNAMIC
| DEFAULT_TABLEGROUP
| DEFAULT_LOB_INROW_THRESHOLD
| EFFECTIVE
| EMPTY
| EMPTY_FIELD_AS_NULL
@ -18762,6 +18773,7 @@ ACCOUNT
| LIST_
| LISTAGG
| LN
| LOB_INROW_THRESHOLD
| LOCAL
| LOCALITY
| LOCKED

1
src/sql/resolver/ddl/ob_alter_table_resolver.cpp
View File

@ -391,6 +391,7 @@ int ObAlterTableResolver::set_table_options()
alter_table_schema.set_table_mode_struct(table_mode_);
alter_table_schema.set_tablespace_id(tablespace_id_);
alter_table_schema.set_dop(table_dop_);
alter_table_schema.set_lob_inrow_threshold(lob_inrow_threshold_);
//deep copy
if (OB_FAIL(ret)) {
//do nothing

19
src/sql/resolver/ddl/ob_create_table_resolver.cpp
View File

@ -1340,7 +1340,7 @@ int ObCreateTableResolver::resolve_table_elements(const ParseNode *node,
LOG_USER_ERROR(OB_ERR_TOO_LONG_COLUMN_LENGTH, column.get_column_name(),
ObAccuracy::MAX_ACCURACY2[is_oracle_mode][column.get_data_type()].get_length());
} else {
length = min(length, OB_MAX_LOB_HANDLE_LENGTH);
length = min(length, table_schema.get_lob_inrow_threshold());
}
}
if (OB_SUCC(ret) && (row_data_length += length) > OB_MAX_USER_ROW_LENGTH) {
@ -2425,6 +2425,23 @@ int ObCreateTableResolver::set_table_option_to_schema(ObTableSchema &table_schem
}
}
if (OB_SUCC(ret)) {
// if lob_inrow_threshold not set, used config default_lob_inrow_threshold
if (is_set_lob_inrow_threshold_) {
table_schema.set_lob_inrow_threshold(lob_inrow_threshold_);
} else if (OB_ISNULL(session_info_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("session if NULL", K(ret));
} else if (OB_FALSE_IT((lob_inrow_threshold_ = session_info_->get_default_lob_inrow_threshold()))) {
} else if (lob_inrow_threshold_ < OB_MIN_LOB_INROW_THRESHOLD || lob_inrow_threshold_ > OB_MAX_LOB_INROW_THRESHOLD) {
ret = OB_INVALID_ARGUMENT;
SQL_RESV_LOG(ERROR, "invalid inrow threshold", K(ret), K(lob_inrow_threshold_));
LOG_USER_ERROR(OB_INVALID_ARGUMENT, "invalid inrow threshold");
} else {
table_schema.set_lob_inrow_threshold(lob_inrow_threshold_);
}
}
if (OB_SUCC(ret) && table_schema.is_external_table()) {
if (table_schema.get_external_file_format().empty()
|| table_schema.get_external_file_location().empty()) {

55
src/sql/resolver/ddl/ob_ddl_resolver.cpp
View File

@ -113,7 +113,9 @@ ObDDLResolver::ObDDLResolver(ObResolverParams &params)
is_external_table_(false),
ttl_definition_(),
kv_attributes_(),
name_generated_type_(GENERATED_TYPE_UNKNOWN)
name_generated_type_(GENERATED_TYPE_UNKNOWN),
is_set_lob_inrow_threshold_(false),
lob_inrow_threshold_(OB_DEFAULT_LOB_INROW_THRESHOLD)
{
table_mode_.reset();
}
@ -2236,6 +2238,10 @@ int ObDDLResolver::resolve_table_option(const ParseNode *option_node, const bool
}
break;
}
case T_LOB_INROW_THRESHOLD: {
ret = resolve_lob_inrow_threshold(option_node, is_index_option);
break;
}
default: {
/* won't be here */
ret = OB_ERR_UNEXPECTED;
@ -3820,6 +3826,52 @@ int ObDDLResolver::resolve_srid_node(share::schema::ObColumnSchemaV2 &column,
return ret;
}
int ObDDLResolver::resolve_lob_inrow_threshold(const ParseNode *option_node, const bool is_index_option)
{
int ret = OB_SUCCESS;
uint64_t tenant_id = 0;
uint64_t tenant_data_version = 0;;
if (OB_ISNULL(session_info_)) {
ret = OB_ERR_UNEXPECTED;
SQL_RESV_LOG(WARN, "session_info_ is null", K(ret));
} else if (OB_FALSE_IT(tenant_id = session_info_->get_effective_tenant_id())) {
} else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) {
LOG_WARN("get tenant data version failed", K(ret));
} else if (tenant_data_version < DATA_VERSION_4_2_1_2) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("lob inrow threshold is not supported in data version less than 4.2.1.2", K(ret), K(tenant_data_version));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "lob inrow threshold is not supported in data version less than 4.2.1.2");
} else if (is_index_option) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("index option should not specify lob inrow threshold", K(ret));
} else if (OB_ISNULL(option_node)) {
ret = OB_ERR_UNEXPECTED;
SQL_RESV_LOG(WARN, "option_node is null", K(ret));
} else if (OB_ISNULL(option_node->children_)) {
ret = OB_ERR_UNEXPECTED;
SQL_RESV_LOG(WARN, "the children of option_node is null", K(option_node->children_), K(ret));
} else if (OB_ISNULL(option_node->children_[0])) {
ret = OB_ERR_UNEXPECTED;
SQL_RESV_LOG(ERROR,"children can't be null", K(ret));
} else if (OB_ISNULL(stmt_)) {
ret = OB_ERR_UNEXPECTED;
SQL_RESV_LOG(ERROR,"stmt_ is null", K(ret));
} else {
lob_inrow_threshold_ = option_node->children_[0]->value_;
is_set_lob_inrow_threshold_ = true;
if (lob_inrow_threshold_ < OB_MIN_LOB_INROW_THRESHOLD || lob_inrow_threshold_ > OB_MAX_LOB_INROW_THRESHOLD) {
ret = OB_INVALID_ARGUMENT;
SQL_RESV_LOG(ERROR, "invalid inrow threshold", K(ret), K(lob_inrow_threshold_));
LOG_USER_ERROR(OB_INVALID_ARGUMENT, "lob inrow threshold, should be [0, 786432]");
} else if (stmt::T_ALTER_TABLE == stmt_->get_stmt_type()) {
if (OB_FAIL(alter_table_bitset_.add_member(ObAlterTableArg::LOB_INROW_THRESHOLD))) {
SQL_RESV_LOG(WARN, "failed to add member to bitset!", K(ret));
}
}
}
return ret;
}
/*
int ObDDLResolver::resolve_generated_column_definition(ObColumnSchemaV2 &column,
ParseNode *node, ObColumnResolveStat &resolve_stat)
@ -4380,6 +4432,7 @@ void ObDDLResolver::reset() {
hash_subpart_num_ = -1;
ttl_definition_.reset();
kv_attributes_.reset();
lob_inrow_threshold_ = OB_DEFAULT_LOB_INROW_THRESHOLD;
}
bool ObDDLResolver::is_valid_prefix_key_type(const ObObjTypeClass column_type_class)

4
src/sql/resolver/ddl/ob_ddl_resolver.h
View File

@ -536,6 +536,8 @@ protected:
const ParseNode &skip_index_node,
share::schema::ObColumnSchemaV2 &column_schema);
int check_skip_index(share::schema::ObTableSchema &table_schema);
int resolve_lob_inrow_threshold(const ParseNode *option_node, const bool is_index_option);
/*
int resolve_generated_column_definition(
share::schema::ObColumnSchemaV2 &column,
@ -937,6 +939,8 @@ protected:
common::ObString ttl_definition_;
common::ObString kv_attributes_;
ObNameGeneratedType name_generated_type_;
bool is_set_lob_inrow_threshold_;
int64_t lob_inrow_threshold_;
private:
template <typename STMT>
DISALLOW_COPY_AND_ASSIGN(ObDDLResolver);

6
src/storage/ddl/ob_complement_data_task.cpp
View File

@ -1163,6 +1163,7 @@ int ObComplementWriteTask::append_row(ObScan *scan)
const int64_t extra_rowkey_cnt = storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt();
bool ddl_committed = false;
blocksstable::ObNewRowBuilder new_row_builder;
int64_t lob_inrow_threshold = OB_DEFAULT_LOB_INROW_THRESHOLD;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObComplementWriteTask is not inited", K(ret));
@ -1208,6 +1209,7 @@ int ObComplementWriteTask::append_row(ObScan *scan)
LOG_WARN("fail to open macro block writer", K(ret), K(data_desc));
} else {
rowkey_column_cnt = hidden_table_schema->get_rowkey_column_num();
lob_inrow_threshold = hidden_table_schema->get_lob_inrow_threshold();
}
ObTableSchemaParam schema_param(allocator);
// Hack to prevent row reshaping from converting empty string to null.
@ -1261,9 +1263,11 @@ int ObComplementWriteTask::append_row(ObScan *scan)
} else if (org_col_ids_.at(i).col_type_.is_lob_storage()) {
lob_cnt++;
const int64_t timeout_ts = ObTimeUtility::current_time() + 60000000; // 60s
ObLobStorageParam lob_storage_param;
lob_storage_param.inrow_threshold_ = lob_inrow_threshold;
if (OB_FAIL(ObInsertLobColumnHelper::insert_lob_column(
lob_allocator, param_->dest_ls_id_, param_->dest_tablet_id_,
org_col_ids_.at(i), datum, timeout_ts, true, param_->orig_tenant_id_))) {
org_col_ids_.at(i), lob_storage_param, datum, timeout_ts, true, param_->orig_tenant_id_))) {
LOG_WARN("fail to insert_lob_col", K(ret), K(datum));
}
}

8
src/storage/ddl/ob_direct_insert_sstable_ctx.cpp
View File

@ -224,7 +224,8 @@ ObSSTableInsertSliceWriter::ObSSTableInsertSliceWriter()
sql_mode_for_ddl_reshape_(0),
reshape_ptr_(nullptr),
is_inited_(false),
new_row_builder_()
new_row_builder_(),
lob_inrow_threshold_(OB_DEFAULT_LOB_INROW_THRESHOLD)
{
}
@ -264,6 +265,7 @@ int ObSSTableInsertSliceWriter::init(const ObSSTableInsertSliceParam &slice_para
0/*cluster_version*/))) {
LOG_WARN("fail to init data desc", KR(ret), K_(data_desc));
} else {
lob_inrow_threshold_ = table_schema->get_lob_inrow_threshold();
data_desc_.get_desc().sstable_index_builder_ = slice_param.sstable_index_builder_;
if (OB_FAIL(macro_block_writer_.open(data_desc_.get_desc(), slice_param.start_seq_,
&redo_log_writer_callback_))) {
@ -356,8 +358,10 @@ int ObSSTableInsertSliceWriter::append_row(const ObNewRow &row_val)
const int64_t timeout_ts =
ObTimeUtility::current_time() + ObInsertLobColumnHelper::LOB_ACCESS_TX_TIMEOUT;
bool has_lob_header = store_row_.row_val_.cells_[i].has_lob_header();
ObLobStorageParam lob_storage_param;
lob_storage_param.inrow_threshold_ = lob_inrow_threshold_;
if (OB_FAIL(ObInsertLobColumnHelper::insert_lob_column(
lob_allocator_, ls_id_, tablet_id_, col_descs_->at(i), datum, timeout_ts, has_lob_header,
lob_allocator_, ls_id_, tablet_id_, col_descs_->at(i), lob_storage_param, datum, timeout_ts, has_lob_header,
MTL_ID()))) {
LOG_WARN("fail to insert_lob_col", KR(ret), K(datum));
}

3
src/storage/ddl/ob_direct_insert_sstable_ctx.h
View File

@ -144,7 +144,7 @@ public:
OB_INLINE int64_t get_snapshot_version() const { return snapshot_version_; }
TO_STRING_KV(K_(tablet_id), K_(ls_id), K_(rowkey_column_num), K_(is_index_table), KP_(col_descs),
K_(snapshot_version), K_(data_desc), K_(lob_cnt), K_(sql_mode_for_ddl_reshape),
KP_(reshape_ptr));
KP_(reshape_ptr), K_(lob_inrow_threshold));
private:
int prepare_reshape(
const common::ObTabletID &tablet_id,
@ -177,6 +177,7 @@ private:
blocksstable::ObDatumRow datum_row_;
bool is_inited_;
blocksstable::ObNewRowBuilder new_row_builder_;
int64_t lob_inrow_threshold_;
};
class ObSSTableInsertTabletContext final

4
src/storage/lob/ob_lob_manager.cpp
View File

@ -1056,7 +1056,7 @@ int ObLobManager::check_need_out_row(
bool &need_out_row)
{
int ret = OB_SUCCESS;
need_out_row = (param.byte_size_ + add_len) > LOB_IN_ROW_MAX_LENGTH;
need_out_row = (param.byte_size_ + add_len) > param.get_inrow_threshold();
if (param.lob_locator_ != nullptr) {
// TODO @lhd remove after tmp lob support outrow
if (!param.lob_locator_->is_persist_lob()) {
@ -1758,7 +1758,7 @@ int ObLobManager::prepare_for_write(
modified_end *= max_bytes_in_char;
}
uint64_t total_size = param.byte_size_ > modified_end ? param.byte_size_ : modified_end;
need_out_row = (total_size > LOB_IN_ROW_MAX_LENGTH);
need_out_row = (total_size > param.get_inrow_threshold());
if (param.lob_common_->in_row_) {
old_data.assign_ptr(param.lob_common_->get_inrow_data_ptr(), param.byte_size_);
}

2
src/storage/lob/ob_lob_manager.h
View File

@ -241,7 +241,7 @@ public:
uint64_t len,
int64_t timeout,
ObLobLocatorV2 &lob);
inline bool can_write_inrow(uint64_t len) { return len <= LOB_IN_ROW_MAX_LENGTH; }
inline bool can_write_inrow(uint64_t len, int64_t inrow_threshold) { return len <= inrow_threshold; }
private:
// private function
int write_inrow_inner(ObLobAccessParam& param, ObString& data, ObString& old_data);

18
src/storage/lob/ob_lob_util.cpp
View File

@ -49,6 +49,16 @@ int ObLobAccessParam::set_lob_locator(common::ObLobLocatorV2 *lob_locator)
return ret;
}
int64_t ObLobAccessParam::get_inrow_threshold()
{
int64_t res = inrow_threshold_;
if (res < OB_MIN_LOB_INROW_THRESHOLD || res > OB_MAX_LOB_INROW_THRESHOLD) {
LIB_LOG_RET(WARN, OB_ERR_UNEXPECTED, "invalid inrow threshold, use default inrow threshold", K(res));
res = OB_DEFAULT_LOB_INROW_THRESHOLD;
}
return res;
}
int ObInsertLobColumnHelper::start_trans(const share::ObLSID &ls_id,
const bool is_for_read,
const int64_t timeout_ts,
@ -107,6 +117,7 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator,
const share::ObLSID ls_id,
const common::ObTabletID tablet_id,
const ObColDesc &column,
const ObLobStorageParam &lob_storage_param,
blocksstable::ObStorageDatum &datum,
const int64_t timeout_ts,
const bool has_lob_header,
@ -130,7 +141,7 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator,
// datum with null ptr and zero len should treat as no lob header
bool set_has_lob_header = has_lob_header && data.length() > 0;
ObLobLocatorV2 src(data, set_has_lob_header);
if (src.has_inrow_data() && lob_mngr->can_write_inrow(data.length())) {
if (src.has_inrow_data() && lob_mngr->can_write_inrow(data.length(), lob_storage_param.inrow_threshold_)) {
// fast path for inrow data
if (OB_FAIL(src.get_inrow_data(data))) {
LOG_WARN("fail to get inrow data", K(ret), K(src));
@ -165,6 +176,8 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator,
lob_param.timeout_ = timeout_ts;
lob_param.scan_backward_ = false;
lob_param.offset_ = 0;
lob_param.inrow_threshold_ = lob_storage_param.inrow_threshold_;
LOG_DEBUG("lob storage param", K(lob_storage_param), K(column));
if (!src.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid src lob locator.", K(ret));
@ -187,13 +200,14 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator,
const share::ObLSID ls_id,
const common::ObTabletID tablet_id,
const ObColDesc &column,
const ObLobStorageParam &lob_storage_param,
ObObj &obj,
const int64_t timeout_ts)
{
int ret = OB_SUCCESS;
ObStorageDatum datum;
datum.from_obj(obj);
if (OB_SUCC(insert_lob_column(allocator, ls_id, tablet_id, column, datum, timeout_ts, obj.has_lob_header(), MTL_ID()))) {
if (OB_SUCC(insert_lob_column(allocator, ls_id, tablet_id, column, lob_storage_param, datum, timeout_ts, obj.has_lob_header(), MTL_ID()))) {
obj.set_lob_value(obj.get_type(), datum.get_string().ptr(), datum.get_string().length());
}
return ret;

19
src/storage/lob/ob_lob_util.h
View File

@ -30,6 +30,17 @@ namespace oceanbase
namespace storage
{
struct ObLobStorageParam
{
ObLobStorageParam():
inrow_threshold_(OB_DEFAULT_LOB_INROW_THRESHOLD)
{}
TO_STRING_KV(K_(inrow_threshold));
int64_t inrow_threshold_;
};
struct ObLobAccessParam {
ObLobAccessParam()
: tx_desc_(nullptr), snapshot_(), tx_id_(), sql_mode_(SMO_DEFAULT), allocator_(nullptr),
@ -43,7 +54,7 @@ struct ObLobAccessParam {
scan_backward_(false), asscess_ptable_(false), offset_(0), len_(0),
parent_seq_no_(), seq_no_st_(), used_seq_cnt_(0), total_seq_cnt_(0), checksum_(0), update_len_(0),
op_type_(ObLobDataOutRowCtx::OpType::SQL), is_fill_zero_(false), from_rpc_(false),
inrow_read_nocopy_(false)
inrow_read_nocopy_(false), inrow_threshold_(OB_DEFAULT_LOB_INROW_THRESHOLD)
{}
~ObLobAccessParam() {
if (OB_NOT_NULL(dml_base_param_)) {
@ -52,10 +63,11 @@ struct ObLobAccessParam {
}
public:
int set_lob_locator(common::ObLobLocatorV2 *lob_locator);
int64_t get_inrow_threshold();
TO_STRING_KV(K_(tenant_id), K_(src_tenant_id), K_(ls_id), K_(tablet_id), KPC_(lob_locator), KPC_(lob_common),
KPC_(lob_data), K_(byte_size), K_(handle_size), K_(coll_type), K_(scan_backward), K_(offset), K_(len),
K_(parent_seq_no), K_(seq_no_st), K_(used_seq_cnt), K_(total_seq_cnt), K_(checksum), K_(update_len), K_(op_type),
K_(is_fill_zero), K_(from_rpc), K_(snapshot), K_(tx_id), K_(inrow_read_nocopy));
K_(is_fill_zero), K_(from_rpc), K_(snapshot), K_(tx_id), K_(inrow_read_nocopy), K_(inrow_threshold));
public:
transaction::ObTxDesc *tx_desc_; // for write/update/delete
transaction::ObTxReadSnapshot snapshot_; // for read
@ -100,6 +112,7 @@ public:
bool is_fill_zero_; // fill zero when erase
bool from_rpc_;
bool inrow_read_nocopy_;
int64_t inrow_threshold_;
};
struct ObLobMetaInfo {
@ -198,6 +211,7 @@ public:
const share::ObLSID ls_id,
const common::ObTabletID tablet_id,
const share::schema::ObColDesc &column,
const ObLobStorageParam &lob_storage_param,
blocksstable::ObStorageDatum &datum,
const int64_t timeout_ts,
const bool has_lob_header,
@ -206,6 +220,7 @@ public:
const share::ObLSID ls_id,
const common::ObTabletID tablet_id,
const share::schema::ObColDesc &column,
const ObLobStorageParam &lob_storage_param,
ObObj &obj,
const int64_t timeout_ts);
};

30
src/storage/ls/ob_ls_tablet_service.cpp
View File

@ -4012,7 +4012,9 @@ int ObLSTabletService::insert_lob_col(
// for not strict sql mode, will insert empty string without lob header
bool has_lob_header = obj.has_lob_header() && raw_data.length() > 0;
ObLobLocatorV2 loc(raw_data, has_lob_header);
if (OB_FAIL(lob_mngr->append(lob_param, loc))) {
if (OB_FAIL(set_lob_storage_params(run_ctx, column, lob_param))) {
LOG_WARN("set_lob_storage_params fail", K(ret), K(column));
} else if (OB_FAIL(lob_mngr->append(lob_param, loc))) {
LOG_WARN("[STORAGE_LOB]lob append failed.", K(ret));
} else {
ObLobCommon *res_lob_common = lob_param.lob_common_;
@ -4335,7 +4337,9 @@ int ObLSTabletService::process_delta_lob(
// should use old obj lob
ObLobLocatorV2 old_lob;
ObString old_disk_lob;
if (OB_FAIL(old_obj.get_lob_locatorv2(old_lob))) {
if (OB_FAIL(set_lob_storage_params(run_ctx, column, lob_param))) {
LOG_WARN("set_lob_storage_params fail", K(ret), K(column));
} else if (OB_FAIL(old_obj.get_lob_locatorv2(old_lob))) {
LOG_WARN("get old lob locator failed.", K(ret), K(old_obj));
} else if (!old_lob.is_valid()) {
ret = OB_ERR_UNEXPECTED;
@ -4361,6 +4365,26 @@ int ObLSTabletService::process_delta_lob(
return ret;
}
int ObLSTabletService::set_lob_storage_params(
ObDMLRunningCtx &run_ctx,
const ObColDesc &column,
ObLobAccessParam &lob_param)
{
int ret = OB_SUCCESS;
const ObTableDMLParam *table_param = run_ctx.dml_param_.table_param_;
const ObColumnParam *column_param = nullptr;
if (OB_ISNULL(table_param)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table_param is null", K(ret));
} else if (OB_ISNULL(column_param = table_param->get_data_table().get_column(column.col_id_))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("column_param is null", K(ret), K(table_param));
} else {
lob_param.inrow_threshold_ = table_param->get_data_table().get_lob_inrow_threshold();
}
return ret;
}
int ObLSTabletService::process_lob_row(
ObTabletHandle &tablet_handle,
ObDMLRunningCtx &run_ctx,
@ -5346,6 +5370,8 @@ int ObLSTabletService::delete_lob_col(
if (data.length() < sizeof(ObLobCommon)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("[STORAGE_LOB]Invalid Lob data.", K(ret), K(obj), K(data));
} else if (OB_FAIL(set_lob_storage_params(run_ctx, column, lob_param))) {
LOG_WARN("set_lob_storage_params fail", K(ret), K(column));
} else {
void *buf = run_ctx.lob_allocator_.alloc(data.length());
if (OB_ISNULL(buf)) {

5
src/storage/ls/ob_ls_tablet_service.h
View File

@ -652,6 +652,11 @@ private:
ObObj &old_obj,
ObLobLocatorV2 &delta_lob,
ObObj &obj);
static int set_lob_storage_params(
ObDMLRunningCtx &run_ctx,
const ObColDesc &column,
ObLobAccessParam &lob_param);
static int process_lob_row(
ObTabletHandle &tablet_handle,
ObDMLRunningCtx &run_ctx,