From 4900683cff7f3242e4faf7eb6ebc61575e6773c5 Mon Sep 17 00:00:00 2001 From: Hooper9973 Date: Wed, 15 Nov 2023 11:40:31 +0000 Subject: [PATCH] [SKIP INDEX] Add Type validation and fallback path. --- src/share/schema/ob_table_schema.cpp | 3 +- src/sql/resolver/ddl/ob_ddl_resolver.cpp | 2 +- .../index_block/ob_index_block_aggregator.cpp | 54 ++++++++++++------- .../index_block/ob_index_block_aggregator.h | 11 ++-- .../index_block/ob_index_block_util.h | 12 +++++ 5 files changed, 56 insertions(+), 26 deletions(-) diff --git a/src/share/schema/ob_table_schema.cpp b/src/share/schema/ob_table_schema.cpp index df94ecc6a7..280f11444c 100644 --- a/src/share/schema/ob_table_schema.cpp +++ b/src/share/schema/ob_table_schema.cpp @@ -7190,8 +7190,7 @@ int ObTableSchema::check_skip_index_valid() const ret = OB_ERR_UNEXPECTED; LOG_USER_ERROR(OB_ERR_UNEXPECTED, "skip index on virtual generated column"); LOG_WARN("unexpected skip index on virtual generated column", K(ret), KPC(column_schema)); - } else if (OB_UNLIKELY(is_lob_storage(column_schema->get_meta_type().get_type()) && - !ob_is_large_text(column_schema->get_meta_type().get_type()))) { + } else if (OB_UNLIKELY(is_skip_index_black_list_type(column_schema->get_meta_type().get_type()))) { ret = OB_NOT_SUPPORTED; LOG_USER_ERROR(OB_NOT_SUPPORTED, "skip index on column with invalid column type"); LOG_USER_ERROR(OB_NOT_SUPPORTED, ob_obj_type_str(column_schema->get_meta_type().get_type())); diff --git a/src/sql/resolver/ddl/ob_ddl_resolver.cpp b/src/sql/resolver/ddl/ob_ddl_resolver.cpp index a9a167d611..c3bcf0e199 100644 --- a/src/sql/resolver/ddl/ob_ddl_resolver.cpp +++ b/src/sql/resolver/ddl/ob_ddl_resolver.cpp @@ -11273,7 +11273,7 @@ int ObDDLResolver::resolve_column_skip_index( ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected invalid type list node", K(ret), K(type_list_node->num_child_), K(type_list_node->type_)); - } else if (is_lob_storage(column_schema.get_data_type()) && !ob_is_large_text(column_schema.get_data_type())) { + } else if (is_skip_index_black_list_type(column_schema.get_data_type())) { ret = OB_NOT_SUPPORTED; LOG_WARN("invalid column type", K(ret), K(column_schema)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "build skip index on invalid type"); diff --git a/src/storage/blocksstable/index_block/ob_index_block_aggregator.cpp b/src/storage/blocksstable/index_block/ob_index_block_aggregator.cpp index 72971c27af..101af65528 100644 --- a/src/storage/blocksstable/index_block/ob_index_block_aggregator.cpp +++ b/src/storage/blocksstable/index_block/ob_index_block_aggregator.cpp @@ -46,6 +46,9 @@ int ObColNullCountAggregator::init(const ObColDesc &col_desc, ObStorageDatum &re } else { null_count_ = 0; result_ = &result; + if (is_skip_index_black_list_type(col_desc.col_type_.get_type())) { + set_not_aggregate(); + } } return ret; } @@ -53,7 +56,11 @@ int ObColNullCountAggregator::init(const ObColDesc &col_desc, ObStorageDatum &re void ObColNullCountAggregator::reuse() { null_count_ = 0; - can_aggregate_ = true; + if (is_skip_index_black_list_type(col_desc_.col_type_.get_type())) { + set_not_aggregate(); + } else { + can_aggregate_ = true; + } } int ObColNullCountAggregator::eval(const ObStorageDatum &datum, const bool is_data) @@ -101,11 +108,11 @@ int ObColMaxAggregator::init(const ObColDesc &col_desc, ObStorageDatum &result) cmp_func_ = basic_funcs->null_first_cmp_; result_ = &result; result_->set_null(); - if (col_desc.col_type_.is_lob_storage() && !ob_is_large_text(col_desc.col_type_.get_type())) { + if (is_skip_index_black_list_type(col_desc.col_type_.get_type())) { set_not_aggregate(); } - obj_type_ = col_desc.col_type_.get_type(); - LOG_DEBUG("[SKIP INDEX] init max aggregator", K(obj_type_), K(can_aggregate_)); + col_desc_ = col_desc; + LOG_DEBUG("[SKIP INDEX] init max aggregator", K(col_desc_), K(can_aggregate_)); } return ret; } @@ -115,7 +122,7 @@ void ObColMaxAggregator::reuse() if (nullptr != result_) { result_->set_null(); } - if (is_lob_storage(obj_type_) && !ob_is_large_text(obj_type_)) { + if (is_skip_index_black_list_type(col_desc_.col_type_.get_type())) { set_not_aggregate(); } else { can_aggregate_ = true; @@ -130,15 +137,15 @@ int ObColMaxAggregator::eval(const ObStorageDatum &datum, const bool is_data) LOG_WARN("Not init", K(ret)); } else if (!can_aggregate_ || datum.is_nop()) { // Skip - } else if (need_set_not_aggregate(obj_type_, datum)){ + } else if (need_set_not_aggregate(col_desc_.col_type_.get_type(), datum)){ set_not_aggregate(); } else { int cmp_res = 0; if (OB_FAIL(cmp_func_(datum, *result_, cmp_res))){ - LOG_WARN("Failed to compare datum", K(ret), K(datum), K(*result_), K(obj_type_)); + LOG_WARN("Failed to compare datum", K(ret), K(datum), K(*result_), K(col_desc_)); } else if (cmp_res > 0) { if (OB_FAIL(copy_agg_datum(datum, *result_))) { - LOG_WARN("Fail to copy aggregated datum", K(ret), K(datum), KPC(result_), K(obj_type_)); + LOG_WARN("Fail to copy aggregated datum", K(ret), K(datum), KPC(result_), K(col_desc_)); } } } @@ -172,11 +179,11 @@ int ObColMinAggregator::init(const ObColDesc &col_desc, ObStorageDatum &result) cmp_func_ = basic_funcs->null_last_cmp_; result_ = &result; result_->set_null(); - if (col_desc.col_type_.is_lob_storage() && !ob_is_large_text(col_desc.col_type_.get_type())) { + if (is_skip_index_black_list_type(col_desc.col_type_.get_type())) { set_not_aggregate(); } - obj_type_ = col_desc.col_type_.get_type(); - LOG_DEBUG("[SKIP INDEX] init min aggregator", K(obj_type_), K(can_aggregate_)); + col_desc_ = col_desc; + LOG_DEBUG("[SKIP INDEX] init min aggregator", K(col_desc_), K(can_aggregate_)); } return ret; } @@ -186,7 +193,7 @@ void ObColMinAggregator::reuse() if (nullptr != result_) { result_->set_null(); } - if (is_lob_storage(obj_type_) && !ob_is_large_text(obj_type_)) { + if (is_skip_index_black_list_type(col_desc_.col_type_.get_type())) { set_not_aggregate(); } else { can_aggregate_ = true; @@ -201,15 +208,15 @@ int ObColMinAggregator::eval(const ObStorageDatum &datum, const bool is_data) LOG_WARN("Not init", K(ret)); } else if (!can_aggregate_ || datum.is_nop()) { // Skip - } else if (need_set_not_aggregate(obj_type_, datum)){ + } else if (need_set_not_aggregate(col_desc_.col_type_.get_type(), datum)){ set_not_aggregate(); } else { int cmp_res = 0; if (OB_FAIL(cmp_func_(datum, *result_, cmp_res))){ - LOG_WARN("Failed to compare datum", K(ret), K(datum), K(*result_), K(obj_type_)); + LOG_WARN("Failed to compare datum", K(ret), K(datum), K(*result_), K(col_desc_)); } else if (cmp_res < 0) { if (OB_FAIL(copy_agg_datum(datum, *result_))) { - LOG_WARN("Fail to copy aggregated datum", K(ret), K(datum), KPC(result_), K(obj_type_)); + LOG_WARN("Fail to copy aggregated datum", K(ret), K(datum), KPC(result_), K(col_desc_)); } } } @@ -331,7 +338,10 @@ int ObSkipIndexAggregator::eval(const ObDatumRow &datum_row) LOG_WARN("Unexcepted extended datum" , K(ret), K(datum), K(datum_row), K(idx_col_meta)); } } else if (OB_FAIL(col_aggs_.at(i)->eval(datum, is_data_))) { - LOG_WARN("Fail to eval aggregate column", K(ret), K(datum), K_(is_data), K(idx_col_meta), K(i)); + col_aggs_.at(i)->set_not_aggregate(); + LOG_ERROR("Fail to eval aggregate column", K(ret), K(datum), K_(is_data), + K(idx_col_meta), K(i), K(col_aggs_.at(i)->get_col_decs())); + ret = OB_SUCCESS; } } } @@ -361,7 +371,10 @@ int ObSkipIndexAggregator::eval(const char *buf, const int64_t buf_size) ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected ext agg datum", K(ret), K(tmp_datum)); } else if (OB_FAIL(col_aggs_.at(i)->eval(tmp_datum, false))) { - LOG_WARN("Fail to eval aggregate column", K(ret), K(tmp_datum), K(idx_col_meta), K(i)); + col_aggs_.at(i)->set_not_aggregate(); + LOG_ERROR("Fail to eval aggregate column", K(ret), K(tmp_datum), K_(is_data), + K(idx_col_meta), K(i), K(col_aggs_.at(i)->get_col_decs())); + ret = OB_SUCCESS; } } } @@ -533,7 +546,12 @@ int ObSkipIndexAggregator::init_col_aggregator( int ret = OB_SUCCESS; ObIColAggregator *col_aggregator = nullptr; void *buf = nullptr; - if (OB_ISNULL(buf = allocator.alloc(sizeof(T)))) { + const ObObjType col_type = col_desc.col_type_.get_type(); + if (!is_skip_index_while_list_type(col_type) && + !is_skip_index_black_list_type(col_type)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not supported skip index on column with unknown column type", K(col_desc), K(col_type)); + } else if (OB_ISNULL(buf = allocator.alloc(sizeof(T)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("Fail to alloc memory for column aggregator", K(ret)); } else if (FALSE_IT(col_aggregator = new (buf) T())) { diff --git a/src/storage/blocksstable/index_block/ob_index_block_aggregator.h b/src/storage/blocksstable/index_block/ob_index_block_aggregator.h index bcc8763e81..3746d99b41 100644 --- a/src/storage/blocksstable/index_block/ob_index_block_aggregator.h +++ b/src/storage/blocksstable/index_block/ob_index_block_aggregator.h @@ -24,7 +24,7 @@ namespace blocksstable class ObIColAggregator { public: - ObIColAggregator() : can_aggregate_(true) {} + ObIColAggregator() : col_desc_(), can_aggregate_(true) {} virtual ~ObIColAggregator() {} virtual int init(const ObColDesc &col_desc, ObStorageDatum &result) = 0; @@ -35,7 +35,9 @@ public: VIRTUAL_TO_STRING_KV(K_(can_aggregate)); void set_not_aggregate() { can_aggregate_ = false; } + inline ObColDesc get_col_decs() const { return col_desc_; } protected: + int inner_init(const ObColDesc &col_desc, ObStorageDatum &result); static int copy_agg_datum(const ObDatum &src, ObDatum &dst); static bool need_set_not_aggregate(const ObObjType type, const ObDatum &datum) { @@ -44,6 +46,7 @@ protected: (is_lob_storage(type) && !datum.is_null() && !datum.get_lob_data().in_row_); } protected: + ObColDesc col_desc_; bool can_aggregate_; }; @@ -67,7 +70,7 @@ private: class ObColMaxAggregator : public ObIColAggregator { public: - ObColMaxAggregator() : cmp_func_(nullptr), result_(nullptr), obj_type_(ObObjType::ObMaxType) {} + ObColMaxAggregator() : cmp_func_(nullptr), result_(nullptr) {} virtual ~ObColMaxAggregator() {} int init(const ObColDesc &col_desc, ObStorageDatum &result) override; @@ -78,14 +81,13 @@ public: private: common::ObDatumCmpFuncType cmp_func_; ObStorageDatum *result_; - ObObjType obj_type_; DISALLOW_COPY_AND_ASSIGN(ObColMaxAggregator); }; class ObColMinAggregator : public ObIColAggregator { public: - ObColMinAggregator() : cmp_func_(nullptr), result_(nullptr), obj_type_(ObObjType::ObMaxType) {} + ObColMinAggregator() : cmp_func_(nullptr), result_(nullptr) {} virtual ~ObColMinAggregator() {} int init(const ObColDesc &col_desc, ObStorageDatum &result) override; @@ -96,7 +98,6 @@ public: private: common::ObDatumCmpFuncType cmp_func_; ObStorageDatum *result_; - ObObjType obj_type_; DISALLOW_COPY_AND_ASSIGN(ObColMinAggregator); }; diff --git a/src/storage/blocksstable/index_block/ob_index_block_util.h b/src/storage/blocksstable/index_block/ob_index_block_util.h index b73a313ed3..5f49e52a41 100644 --- a/src/storage/blocksstable/index_block/ob_index_block_util.h +++ b/src/storage/blocksstable/index_block/ob_index_block_util.h @@ -154,6 +154,18 @@ OB_INLINE static int get_skip_index_store_upper_size( return ret; } +OB_INLINE static bool is_skip_index_black_list_type(const ObObjType &obj_type) +{ + return ob_is_json_tc(obj_type) || ob_is_geometry_tc(obj_type) + || ob_is_user_defined_sql_type(obj_type); +} + +OB_INLINE static bool is_skip_index_while_list_type(const ObObjType &obj_type) +{ + const ObObjTypeClass tc = ob_obj_type_class(obj_type); + return (ObNullTC <= tc && tc <= ObLobTC) || ObDecimalIntTC == tc; +} + } // blocksstable } // oceanbase