diff --git a/src/pl/sys_package/ob_dbms_stats.cpp b/src/pl/sys_package/ob_dbms_stats.cpp index 8159ad4fb3..3fb5651bd9 100644 --- a/src/pl/sys_package/ob_dbms_stats.cpp +++ b/src/pl/sys_package/ob_dbms_stats.cpp @@ -3263,11 +3263,6 @@ int ObDbmsStats::init_column_stat_params(ObIAllocator &allocator, if (ObColumnStatParam::is_valid_avglen_type(col->get_meta_type().get_type())) { col_param.set_need_avg_len(); } - //check need truncate str - if (ob_is_string_type(col->get_meta_type().get_type()) && - col->get_data_length() > OPT_STATS_MAX_VALUE_CHAR_LEN) { - col_param.set_need_truncate_str(); - } } if (col->is_rowkey_column() && !table_schema.is_heap_table()) { col_param.set_is_index_column(); diff --git a/src/share/stat/ob_dbms_stats_utils.cpp b/src/share/stat/ob_dbms_stats_utils.cpp index d02eddfbf0..7f27dc77c9 100644 --- a/src/share/stat/ob_dbms_stats_utils.cpp +++ b/src/share/stat/ob_dbms_stats_utils.cpp @@ -776,5 +776,80 @@ int ObDbmsStatsUtils::get_subpart_infos(const ObTableSchema &table_schema, return ret; } +int ObDbmsStatsUtils::truncate_string_for_opt_stats(const ObObj *old_obj, + ObIAllocator &alloc, + ObObj *&new_obj) +{ + int ret = OB_SUCCESS; + bool is_truncated = false; + if (OB_ISNULL(old_obj)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null"); + } else if (ObColumnStatParam::is_valid_opt_col_type(old_obj->get_type()) && old_obj->is_string_type()) { + ObString str; + if (OB_FAIL(old_obj->get_string(str))) { + LOG_WARN("failed to get string", K(ret), K(str)); + } else { + ObObj *tmp_obj = NULL; + int64_t truncated_str_len = get_truncated_str_len(str, old_obj->get_collation_type()); + if (truncated_str_len == str.length()) { + //do nothing + } else if (OB_UNLIKELY(truncated_str_len < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(old_obj), K(str), K(truncated_str_len)); + } else if (OB_ISNULL(tmp_obj = static_cast(alloc.alloc(sizeof(ObObj))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc buf", K(ret)); + } else { + tmp_obj->set_varchar(str.ptr(), static_cast(truncated_str_len)); + tmp_obj->set_meta_type(old_obj->get_meta()); + new_obj = tmp_obj; + is_truncated = true; + } + } + } + if (OB_SUCC(ret) && !is_truncated) { + new_obj = const_cast(old_obj); + } + LOG_TRACE("Succeed to truncate string obj for opt stats", KPC(old_obj), KPC(new_obj), K(is_truncated)); + return ret; +} + + +int ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(ObObj &obj) +{ + int ret = OB_SUCCESS; + if (ObColumnStatParam::is_valid_opt_col_type(obj.get_type()) && obj.is_string_type()) { + ObString str; + if (OB_FAIL(obj.get_string(str))) { + LOG_WARN("failed to get string", K(ret), K(str), K(obj)); + } else { + int64_t truncated_str_len = get_truncated_str_len(str, obj.get_collation_type()); + if (OB_UNLIKELY(truncated_str_len < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(obj), K(str), K(truncated_str_len)); + } else { + str.assign_ptr(str.ptr(), static_cast(truncated_str_len)); + obj.set_common_value(str); + LOG_TRACE("Succeed to shadow truncate string obj for opt stats", K(obj)); + } + } + } + return ret; +} + +int64_t ObDbmsStatsUtils::get_truncated_str_len(const ObString &str, const ObCollationType cs_type) +{ + int64_t truncated_str_len = 0; + int64_t mb_len = ObCharset::strlen_char(cs_type, str.ptr(), str.length()); + if (mb_len <= OPT_STATS_MAX_VALUE_CHAR_LEN) {//keep origin str + truncated_str_len = str.length(); + } else {//get max truncate str len + truncated_str_len = ObCharset::charpos(cs_type, str.ptr(), str.length(), OPT_STATS_MAX_VALUE_CHAR_LEN); + } + LOG_TRACE("Succeed to get truncated str len", K(str), K(cs_type), K(truncated_str_len)); + return truncated_str_len; +} + } } diff --git a/src/share/stat/ob_dbms_stats_utils.h b/src/share/stat/ob_dbms_stats_utils.h index 824be55f77..c60d6b389e 100644 --- a/src/share/stat/ob_dbms_stats_utils.h +++ b/src/share/stat/ob_dbms_stats_utils.h @@ -124,6 +124,14 @@ public: ObIArray &subpart_ids, OSGPartMap *part_map = NULL); + static int truncate_string_for_opt_stats(const ObObj *old_obj, + ObIAllocator &alloc, + ObObj *&new_obj); + + static int shadow_truncate_string_for_opt_stats(ObObj &obj); + + static int64_t get_truncated_str_len(const ObString &str, const ObCollationType cs_type); + private: static int batch_write(share::schema::ObSchemaGetterGuard *schema_guard, const uint64_t tenant_id, diff --git a/src/share/stat/ob_incremental_stat_estimator.cpp b/src/share/stat/ob_incremental_stat_estimator.cpp index a183836dae..c198460dd3 100644 --- a/src/share/stat/ob_incremental_stat_estimator.cpp +++ b/src/share/stat/ob_incremental_stat_estimator.cpp @@ -763,6 +763,7 @@ int ObIncrementalStatEstimator::derive_global_histogram(ObIArray &a } } if (OB_SUCC(ret)) { + ObHistogram tmp_histogram; if (OB_FAIL(top_k_fre_hist->create_topk_fre_items())) { LOG_WARN("failed to adjust frequency sort", K(ret)); } else if (top_k_fre_hist->get_buckets().count() == 0) { @@ -774,12 +775,18 @@ int ObIncrementalStatEstimator::derive_global_histogram(ObIArray &a total_row_count, not_null_count, num_distinct, - histogram))) { + tmp_histogram))) { LOG_WARN("failed to try build topk histogram", K(ret)); + } else if (OB_FAIL(histogram.deep_copy(allocator, tmp_histogram))) { + LOG_WARN("failed to deep copy", K(ret)); } else { need_gather_hybrid_hist |= histogram.is_hybrid(); } } + if (top_k_fre_hist != NULL) { + top_k_fre_hist->~ObTopKFrequencyHistograms(); + top_k_fre_hist = NULL; + } } return ret; } diff --git a/src/share/stat/ob_opt_osg_column_stat.cpp b/src/share/stat/ob_opt_osg_column_stat.cpp index d98533ee7f..c1f6b124e1 100644 --- a/src/share/stat/ob_opt_osg_column_stat.cpp +++ b/src/share/stat/ob_opt_osg_column_stat.cpp @@ -17,7 +17,7 @@ #include "share/stat/ob_stat_define.h" #include "share/stat/ob_stat_item.h" #include "sql/engine/aggregate/ob_aggregate_processor.h" -#include "sql/optimizer/ob_optimizer_util.h" +#include "share/stat/ob_dbms_stats_utils.h" #include "sql/engine/expr/ob_expr_sys_op_opnsize.h" namespace oceanbase { namespace common { @@ -117,14 +117,14 @@ int ObOptOSGColumnStat::set_min_max_datum_to_obj() LOG_WARN("failed to get min obj"); } else if (OB_FAIL(max_val_.get_obj(*max_obj))) { LOG_WARN("failed to get max obj"); - } else if (OB_FAIL(ObOptimizerUtil::truncate_string_for_opt_stats(min_obj, allocator_, min_obj))) { + } else if (OB_FAIL(ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(*min_obj))) { LOG_WARN("fail to truncate string", K(ret)); - } else if (OB_FAIL(ObOptimizerUtil::truncate_string_for_opt_stats(max_obj, allocator_, max_obj))) { + } else if (OB_FAIL(ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(*max_obj))) { LOG_WARN("fail to truncate string", K(ret)); } else { const ObObj &min_val = col_stat_->get_min_value(); const ObObj &max_val = col_stat_->get_max_value(); - LOG_TRACE("set min/max val", K(min_obj), K(max_obj), K(min_val), K(max_val)); + LOG_TRACE("set min/max val", KPC(min_obj), KPC(max_obj), K(min_val), K(max_val)); if (min_val.is_null() || (!min_obj->is_null() && *min_obj < min_val)) { col_stat_->set_min_value(*min_obj); } diff --git a/src/share/stat/ob_opt_stat_monitor_manager.cpp b/src/share/stat/ob_opt_stat_monitor_manager.cpp index bfa7abe542..bbb3f323ed 100644 --- a/src/share/stat/ob_opt_stat_monitor_manager.cpp +++ b/src/share/stat/ob_opt_stat_monitor_manager.cpp @@ -961,27 +961,39 @@ int ObOptStatMonitorManager::generate_opt_stat_monitoring_info_rows(observer::Ob int ObOptStatMonitorManager::clean_useless_dml_stat_info(uint64_t tenant_id) { int ret = OB_SUCCESS; - ObSqlString delete_sql; - int64_t affected_rows = 0; + ObSqlString delete_table_sql; + ObSqlString delete_part_sql; + int64_t affected_rows1 = 0; + int64_t affected_rows2 = 0; const char* all_table_name = NULL; if (OB_FAIL(ObSchemaUtils::get_all_table_name(tenant_id, all_table_name))) { LOG_WARN("failed to get all table name", K(ret)); - } else if (OB_FAIL(delete_sql.append_fmt("DELETE FROM %s m WHERE (NOT EXISTS (SELECT 1 " \ + } else if (OB_FAIL(delete_table_sql.append_fmt("DELETE FROM %s m WHERE (NOT EXISTS (SELECT 1 " \ "FROM %s t, %s db WHERE t.tenant_id = db.tenant_id AND t.database_id = db.database_id "\ - "AND t.table_id = m.table_id AND t.tenant_id = m.tenant_id AND db.database_name != '__recyclebin') "\ - "OR (tenant_id, table_id, tablet_id) IN (SELECT m.tenant_id, m.table_id, m.tablet_id FROM "\ - "%s m, %s t WHERE t.table_id = m.table_id AND t.tenant_id = m.tenant_id AND t.part_level > 0 "\ - "AND NOT EXISTS (SELECT 1 FROM %s p WHERE p.table_id = m.table_id AND p.tenant_id = m.tenant_id AND p.tablet_id = m.tablet_id) "\ - "AND NOT EXISTS (SELECT 1 FROM %s sp WHERE sp.table_id = m.table_id AND sp.tenant_id = m.tenant_id AND sp.tablet_id = m.tablet_id))) "\ + "AND t.table_id = m.table_id AND t.tenant_id = m.tenant_id AND db.database_name != '__recyclebin')) "\ "AND table_id > %ld;", share::OB_ALL_MONITOR_MODIFIED_TNAME, all_table_name, share::OB_ALL_DATABASE_TNAME, - share::OB_ALL_MONITOR_MODIFIED_TNAME, all_table_name, share::OB_ALL_PART_TNAME, + OB_MAX_INNER_TABLE_ID))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(delete_part_sql.append_fmt("DELETE FROM %s m WHERE (tenant_id, table_id, tablet_id) IN ( "\ + "SELECT m.tenant_id, m.table_id, m.tablet_id FROM "\ + "%s m, %s t, %s db WHERE t.table_id = m.table_id AND t.tenant_id = m.tenant_id AND t.part_level > 0 "\ + "AND t.tenant_id = db.tenant_id AND t.database_id = db.database_id AND db.database_name != '__recyclebin' "\ + "AND NOT EXISTS (SELECT 1 FROM %s p WHERE p.table_id = m.table_id AND p.tenant_id = m.tenant_id AND p.tablet_id = m.tablet_id) "\ + "AND NOT EXISTS (SELECT 1 FROM %s sp WHERE sp.table_id = m.table_id AND sp.tenant_id = m.tenant_id AND sp.tablet_id = m.tablet_id)) "\ + "AND table_id > %ld;", + share::OB_ALL_MONITOR_MODIFIED_TNAME, share::OB_ALL_MONITOR_MODIFIED_TNAME, + all_table_name, share::OB_ALL_DATABASE_TNAME, share::OB_ALL_PART_TNAME, share::OB_ALL_SUB_PART_TNAME, OB_MAX_INNER_TABLE_ID))) { LOG_WARN("failed to append fmt", K(ret)); - } else if (OB_FAIL(mysql_proxy_->write(tenant_id, delete_sql.ptr(), affected_rows))) { - LOG_WARN("failed to execute sql", K(ret), K(delete_sql)); + } else if (OB_FAIL(mysql_proxy_->write(tenant_id, delete_table_sql.ptr(), affected_rows1))) { + LOG_WARN("failed to execute sql", K(ret), K(delete_table_sql)); + } else if (OB_FAIL(mysql_proxy_->write(tenant_id, delete_part_sql.ptr(), affected_rows2))) { + LOG_WARN("failed to execute sql", K(ret), K(delete_part_sql)); } else { - LOG_TRACE("succeed to clean useless monitor modified_data", K(tenant_id), K(delete_sql), K(affected_rows)); + LOG_TRACE("succeed to clean useless monitor modified_data", K(tenant_id), K(delete_table_sql), + K(affected_rows1), K(delete_part_sql), + K(affected_rows2)); } return ret; } diff --git a/src/share/stat/ob_stat_define.h b/src/share/stat/ob_stat_define.h index ccdfe77cf4..e70e4819bb 100644 --- a/src/share/stat/ob_stat_define.h +++ b/src/share/stat/ob_stat_define.h @@ -113,8 +113,7 @@ enum ColumnGatherFlag NO_NEED_STAT = 0, VALID_OPT_COL = 1, NEED_BASIC_STAT = 1 << 1, - NEED_AVG_LEN = 1 << 2, - NEED_TRUNCATE_STR = 1 << 3 + NEED_AVG_LEN = 1 << 2 }; enum ObGranularityType @@ -378,11 +377,9 @@ struct ObColumnStatParam { inline void set_valid_opt_col() { gather_flag_ |= ColumnGatherFlag::VALID_OPT_COL; } inline void set_need_basic_stat() { gather_flag_ |= ColumnGatherFlag::NEED_BASIC_STAT; } inline void set_need_avg_len() { gather_flag_ |= ColumnGatherFlag::NEED_AVG_LEN; } - inline void set_need_truncate_str() { gather_flag_ |= ColumnGatherFlag::NEED_TRUNCATE_STR; } inline bool is_valid_opt_col() const { return gather_flag_ & ColumnGatherFlag::VALID_OPT_COL; } inline bool need_basic_stat() const { return gather_flag_ & ColumnGatherFlag::NEED_BASIC_STAT; } inline bool need_avg_len() const { return gather_flag_ & ColumnGatherFlag::NEED_AVG_LEN; } - inline bool need_truncate_str() const { return gather_flag_ & ColumnGatherFlag::NEED_TRUNCATE_STR; } ObString column_name_; uint64_t column_id_; diff --git a/src/share/stat/ob_stat_item.cpp b/src/share/stat/ob_stat_item.cpp index ed0519f339..2a39ceb770 100644 --- a/src/share/stat/ob_stat_item.cpp +++ b/src/share/stat/ob_stat_item.cpp @@ -100,20 +100,11 @@ int ObStatMaxValue::gen_expr(char *buf, const int64_t buf_len, int64_t &pos) if (OB_ISNULL(col_param_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("column param is null", K(ret)); - } else if (!col_param_->need_truncate_str() && - OB_FAIL(databuff_printf(buf, buf_len, pos, + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, lib::is_oracle_mode() ? " MAX(\"%.*s\")" : " MAX(`%.*s`)", col_param_->column_name_.length(), col_param_->column_name_.ptr()))) { LOG_WARN("failed to print max(col) expr", K(ret)); - } else if (col_param_->need_truncate_str() && - OB_FAIL(databuff_printf(buf, buf_len, pos, - lib::is_oracle_mode() ? " MAX(SUBSTR(\"%.*s\", 1, %ld))" : - " MAX(SUBSTR(`%.*s`, 1, %ld))", - col_param_->column_name_.length(), - col_param_->column_name_.ptr(), - OPT_STATS_MAX_VALUE_CHAR_LEN))) { - LOG_WARN("failed to print max(col) expr", K(ret)); } return ret; } @@ -125,6 +116,8 @@ int ObStatMaxValue::decode(ObObj &obj) if (OB_ISNULL(col_stat_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("col stat is not given", K(ret), K(col_stat_)); + } else if (OB_FAIL(ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(obj))) { + LOG_WARN("fail to truncate string", K(ret)); } else { col_stat_->set_max_value(obj); } @@ -137,20 +130,11 @@ int ObStatMinValue::gen_expr(char *buf, const int64_t buf_len, int64_t &pos) if (OB_ISNULL(col_param_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("column param is null", K(ret)); - } else if (!col_param_->need_truncate_str() && - OB_FAIL(databuff_printf(buf, buf_len, pos, + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, lib::is_oracle_mode() ? " MIN(\"%.*s\")" : " MIN(`%.*s`)", col_param_->column_name_.length(), col_param_->column_name_.ptr()))) { LOG_WARN("failed to print max(col) expr", K(ret)); - } else if (col_param_->need_truncate_str() && - OB_FAIL(databuff_printf(buf, buf_len, pos, - lib::is_oracle_mode() ? " MIN(SUBSTR(\"%.*s\", 1, %ld))" : - " MIN(SUBSTR(`%.*s`, 1, %ld))", - col_param_->column_name_.length(), - col_param_->column_name_.ptr(), - OPT_STATS_MAX_VALUE_CHAR_LEN))) { - LOG_WARN("failed to print max(col) expr", K(ret)); } return ret; } @@ -162,6 +146,8 @@ int ObStatMinValue::decode(ObObj &obj) if (OB_ISNULL(col_stat_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("col stat is not given", K(ret), K(col_stat_)); + } else if (OB_FAIL(ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(obj))) { + LOG_WARN("fail to truncate string", K(ret)); } else { col_stat_->set_min_value(obj); } @@ -261,8 +247,7 @@ int ObStatTopKHist::gen_expr(char *buf, const int64_t buf_len, int64_t &pos) } double err_rate = 1.0 / (1000 * (bkt_num / MIN_BUCKET_SIZE)); if (OB_SUCC(ret)) { - if (!col_param_->need_truncate_str() && - OB_FAIL(databuff_printf(buf, buf_len, pos, + if (OB_FAIL(databuff_printf(buf, buf_len, pos, lib::is_oracle_mode() ? " TOP_K_FRE_HIST(%lf, \"%.*s\", %ld)" : " TOP_K_FRE_HIST(%lf, `%.*s`, %ld)", err_rate, @@ -270,16 +255,6 @@ int ObStatTopKHist::gen_expr(char *buf, const int64_t buf_len, int64_t &pos) col_param_->column_name_.ptr(), bkt_num))) { LOG_WARN("failed to print buf topk hist expr", K(ret)); - } else if (col_param_->need_truncate_str() && - OB_FAIL(databuff_printf(buf, buf_len, pos, - lib::is_oracle_mode() ? " TOP_K_FRE_HIST(%lf, SUBSTR(\"%.*s\", 1, %ld), %ld)" : - " TOP_K_FRE_HIST(%lf, SUBSTR(`%.*s`, 1, %ld), %ld)", - err_rate, - col_param_->column_name_.length(), - col_param_->column_name_.ptr(), - OPT_STATS_MAX_VALUE_CHAR_LEN, - bkt_num))) { - LOG_WARN("failed to print buf topk hist expr", K(ret)); } } } @@ -682,24 +657,14 @@ int ObStatHybridHist::gen_expr(char *buf, const int64_t buf_len, int64_t &pos) if (OB_FAIL(databuff_printf(buf, buf_len, pos, " NULL"))) { LOG_WARN("failed to print buf", K(ret)); } else {/*do nothing*/} - } else if (!col_param_->need_truncate_str() && - OB_FAIL(databuff_printf(buf, buf_len, pos, + } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, lib::is_oracle_mode() ? " HYBRID_HIST(\"%.*s\", %ld)" : " HYBRID_HIST(`%.*s`, %ld)", col_param_->column_name_.length(), col_param_->column_name_.ptr(), col_param_->bucket_num_))) { LOG_WARN("failed to print buf", K(ret)); - } else if (col_param_->need_truncate_str() && - OB_FAIL(databuff_printf(buf, buf_len, pos, - lib::is_oracle_mode() ? " HYBRID_HIST(SUBSTR(\"%.*s\", 1, %ld), %ld)" : - " HYBRID_HIST(SUBSTR(`%.*s`, 1, %ld), %ld)", - col_param_->column_name_.length(), - col_param_->column_name_.ptr(), - OPT_STATS_MAX_VALUE_CHAR_LEN, - col_param_->bucket_num_))) { - LOG_WARN("failed to print buf", K(ret)); - } else {/*do nothing*/} + } return ret; } diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.cpp b/src/sql/engine/aggregate/ob_aggregate_processor.cpp index 35e16e4a05..7e51900d6b 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.cpp +++ b/src/sql/engine/aggregate/ob_aggregate_processor.cpp @@ -29,6 +29,7 @@ #include "sql/engine/expr/ob_expr_lob_utils.h" #include "sql/engine/basic/ob_material_op_impl.h" #include "share/stat/ob_hybrid_hist_estimator.h" +#include "share/stat/ob_dbms_stats_utils.h" namespace oceanbase { @@ -2538,6 +2539,8 @@ int ObAggregateProcessor::prepare_aggr_result(const ObChunkDatumStore::StoredRow ObObj obj; if (OB_FAIL(stored_row.cells()[0].to_obj(obj, aggr_info.param_exprs_.at(0)->obj_meta_))) { LOG_WARN("failed to obj", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(obj))) { + LOG_WARN("fail to truncate string", K(ret)); } else if (OB_FAIL(extra->topk_fre_hist_.add_top_k_frequency_item(obj))) { LOG_WARN("failed to process row", K(ret)); } else {/*do nothing*/} @@ -2984,6 +2987,8 @@ int ObAggregateProcessor::process_aggr_result(const ObChunkDatumStore::StoredRow ObObj obj; if (OB_FAIL(stored_row.cells()[0].to_obj(obj, aggr_info.param_exprs_.at(0)->obj_meta_))) { LOG_WARN("failed to obj", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(obj))) { + LOG_WARN("fail to truncate string", K(ret)); } else if (OB_FAIL(extra->topk_fre_hist_.add_top_k_frequency_item(obj))) { LOG_WARN("failed to process row", K(ret)); } else {/*do nothing*/} @@ -4854,6 +4859,8 @@ int ObAggregateProcessor::top_fre_hist_calc_batch( ObObj obj; if (OB_FAIL(datum->to_obj(obj, obj_meta))) { LOG_WARN("failed to obj", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(obj))) { + LOG_WARN("fail to truncate string", K(ret)); } else if (OB_FAIL(extra_info->topk_fre_hist_.add_top_k_frequency_item(obj))) { LOG_WARN("failed to process row", K(ret)); } else {/*do nothing*/} @@ -6048,6 +6055,9 @@ int ObAggregateProcessor::compute_hybrid_hist_result(const ObAggrInfo &aggr_info LOG_WARN("get unexpected null", K(ret), K(stored_row)); } else if (stored_row->cells()[0].is_null()) { ++ null_count; + } else if (OB_FAIL(shadow_truncate_string_for_hybird_hist(aggr_info.param_exprs_.at(0)->obj_meta_, + const_cast(stored_row->cells()[0])))) { + LOG_WARN("failed to shadow truncate string for hybird hist", K(ret)); } else if (OB_FAIL(prev_row.save_store_row(*stored_row))) { LOG_WARN("failed to deep copy limit last rows", K(ret)); } else { @@ -6064,6 +6074,9 @@ int ObAggregateProcessor::compute_hybrid_hist_result(const ObAggrInfo &aggr_info if (OB_ISNULL(stored_row) || OB_UNLIKELY(stored_row->cnt_ != 1)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(stored_row)); + } else if (OB_FAIL(shadow_truncate_string_for_hybird_hist(aggr_info.param_exprs_.at(0)->obj_meta_, + const_cast(stored_row->cells()[0])))) { + LOG_WARN("failed to shadow truncate string for hybrid hist", K(ret)); } else if (OB_FAIL(check_rows_equal(prev_row, *stored_row, aggr_info, is_equal))) { LOG_WARN("failed to is order by item equal with prev row", K(ret)); } else if (is_equal) { @@ -6133,6 +6146,24 @@ int ObAggregateProcessor::compute_hybrid_hist_result(const ObAggrInfo &aggr_info return ret; } +int ObAggregateProcessor::shadow_truncate_string_for_hybird_hist(const ObObjMeta obj_meta, + ObDatum &datum) +{ + int ret = OB_SUCCESS; + if (ObColumnStatParam::is_valid_opt_col_type(obj_meta.get_type()) && obj_meta.is_string_type() && !datum.is_null()) { + const ObString &str = datum.get_string(); + int64_t truncated_str_len = ObDbmsStatsUtils::get_truncated_str_len(str, obj_meta.get_collation_type()); + if (OB_UNLIKELY(truncated_str_len < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(obj_meta), K(datum), K(truncated_str_len)); + } else { + datum.set_string(str.ptr(), static_cast(truncated_str_len)); + LOG_TRACE("Succeed to shadow truncate string for hybird hist", K(datum)); + } + } + return ret; +} + int ObAggregateProcessor::get_hybrid_hist_result(ObHybridHistograms *hybrid_hist, bool has_lob_header, ObDatum &result_datum) diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.h b/src/sql/engine/aggregate/ob_aggregate_processor.h index fda5a452b9..2987526dd1 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.h +++ b/src/sql/engine/aggregate/ob_aggregate_processor.h @@ -961,6 +961,8 @@ private: int check_key_valid(common::hash::ObHashSet &view_key_names, const ObString& key); + int shadow_truncate_string_for_hybird_hist(const ObObjMeta obj_meta, ObDatum &datum); + OB_INLINE void clear_op_evaluated_flag() { if (OB_NOT_NULL(op_eval_infos_)) { diff --git a/src/sql/engine/cmd/ob_analyze_executor.cpp b/src/sql/engine/cmd/ob_analyze_executor.cpp index faac7be6d6..793368d273 100644 --- a/src/sql/engine/cmd/ob_analyze_executor.cpp +++ b/src/sql/engine/cmd/ob_analyze_executor.cpp @@ -71,7 +71,7 @@ int ObAnalyzeExecutor::execute(ObExecContext &ctx, ObAnalyzeStmt &stmt) } else { ObOptStatGatherStat gather_stat(task_info); ObOptStatGatherStatList::instance().push(gather_stat); - ObOptStatRunningMonitor running_monitor(ctx.get_allocator(), start_time, param.allocator_->total(), gather_stat); + ObOptStatRunningMonitor running_monitor(ctx.get_allocator(), start_time, param.allocator_->used(), gather_stat); if (OB_FAIL(running_monitor.add_table_info(param))) { LOG_WARN("failed to add table info", K(ret)); } else if (OB_FAIL(ObDbmsStatsLockUnlock::check_stat_locked(ctx, param))) { diff --git a/src/sql/optimizer/ob_access_path_estimation.cpp b/src/sql/optimizer/ob_access_path_estimation.cpp index 6c6cedeae6..e92b4a97e9 100644 --- a/src/sql/optimizer/ob_access_path_estimation.cpp +++ b/src/sql/optimizer/ob_access_path_estimation.cpp @@ -341,9 +341,13 @@ int ObAccessPathEstimation::process_storage_estimation(ObOptimizerContext &ctx, ret = OB_ERR_UNEXPECTED; LOG_WARN("task is null", K(ret)); } else if (OB_FAIL(do_storage_estimation(ctx, *tasks.at(i)))) { - LOG_WARN("failed to process storage estimation", K(ret)); - need_fallback = true; - ret = OB_SUCCESS; + if (is_retry_ret(ret)) { + //retry code throw error, and retry + } else { + LOG_WARN("failed to process storage estimation", K(ret)); + need_fallback = true; + ret = OB_SUCCESS; + } break; } else if (!tasks.at(i)->check_result_reliable()) { need_fallback = true; @@ -1114,8 +1118,12 @@ int ObAccessPathEstimation::storage_estimate_full_table_rowcount(ObOptimizerCont } else if (OB_FAIL(arg.index_params_.push_back(path_arg))) { LOG_WARN("failed to add primary key estimation arg", K(ret)); } else if (OB_FAIL(do_storage_estimation(ctx, task))) { - LOG_WARN("failed to do storage estimation", K(ret)); - ret = OB_SUCCESS; + if (is_retry_ret(ret)) { + //retry code throw error, and retry + } else { + LOG_WARN("failed to do storage estimation", K(ret)); + ret = OB_SUCCESS; + } } else if (OB_UNLIKELY(res.index_param_res_.count() != 1)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("storage estimation result size is unexpected", K(ret)); @@ -1814,5 +1822,15 @@ int ObAccessPathEstimation::update_column_metas_by_ds_col_stat(const int64_t row return ret; } +bool ObAccessPathEstimation::is_retry_ret(int ret) +{ + return ret == OB_NOT_MASTER || + ret == OB_RS_NOT_MASTER || + ret == OB_TENANT_NOT_IN_SERVER || + ret == OB_NO_READABLE_REPLICA || + ret == OB_LS_NOT_EXIST || + ret == OB_TABLET_NOT_EXIST; +} + } // end of sql } // end of oceanbase diff --git a/src/sql/optimizer/ob_access_path_estimation.h b/src/sql/optimizer/ob_access_path_estimation.h index 01e254c03b..2d8e2982e2 100644 --- a/src/sql/optimizer/ob_access_path_estimation.h +++ b/src/sql/optimizer/ob_access_path_estimation.h @@ -207,6 +207,8 @@ private: common::ObIArray &ds_paths, common::ObIArray &no_ds_paths, bool &all_path_is_get); + + static bool is_retry_ret(int ret); }; } diff --git a/src/sql/optimizer/ob_opt_selectivity.cpp b/src/sql/optimizer/ob_opt_selectivity.cpp index 072a0e6b71..a56eaa957a 100644 --- a/src/sql/optimizer/ob_opt_selectivity.cpp +++ b/src/sql/optimizer/ob_opt_selectivity.cpp @@ -28,6 +28,7 @@ #include "sql/optimizer/ob_logical_operator.h" #include "sql/optimizer/ob_join_order.h" #include "common/ob_smart_call.h" +#include "share/stat/ob_dbms_stats_utils.h" using namespace oceanbase::common; using namespace oceanbase::share::schema; @@ -2357,8 +2358,8 @@ int ObOptSelectivity::calc_column_range_selectivity(const OptTableMetas &table_m ObObj *new_start_obj = NULL; ObObj *new_end_obj = NULL; ObArenaAllocator tmp_alloc("ObOptSel"); - if (OB_FAIL(ObOptimizerUtil::truncate_string_for_opt_stats(&start_obj, tmp_alloc, new_start_obj)) || - OB_FAIL(ObOptimizerUtil::truncate_string_for_opt_stats(&end_obj, tmp_alloc, new_end_obj))) { + if (OB_FAIL(ObDbmsStatsUtils::truncate_string_for_opt_stats(&start_obj, tmp_alloc, new_start_obj)) || + OB_FAIL(ObDbmsStatsUtils::truncate_string_for_opt_stats(&end_obj, tmp_alloc, new_end_obj))) { LOG_WARN("failed to convert valid obj for opt stats", K(ret), K(start_obj), K(end_obj), KPC(new_start_obj), KPC(new_end_obj)); } else if (OB_ISNULL(new_start_obj) || OB_ISNULL(new_end_obj)) { @@ -3605,7 +3606,7 @@ int ObOptSelectivity::get_equal_pred_sel(const ObHistogram &histogram, bool is_equal = false; ObObj *new_value = NULL; ObArenaAllocator tmp_alloc("ObOptSel"); - if (OB_FAIL(ObOptimizerUtil::truncate_string_for_opt_stats(&value, tmp_alloc, new_value))) { + if (OB_FAIL(ObDbmsStatsUtils::truncate_string_for_opt_stats(&value, tmp_alloc, new_value))) { LOG_WARN("failed to convert valid obj for opt stats", K(ret), K(value), KPC(new_value)); } else if (OB_ISNULL(new_value)) { ret = OB_ERR_UNEXPECTED; @@ -3652,8 +3653,8 @@ int ObOptSelectivity::get_range_sel_by_histogram(const ObHistogram &histogram, ObObj *new_startobj = NULL; ObObj *new_endobj = NULL; ObArenaAllocator tmp_alloc("ObOptSel"); - if (OB_FAIL(ObOptimizerUtil::truncate_string_for_opt_stats(startobj, tmp_alloc, new_startobj)) || - OB_FAIL(ObOptimizerUtil::truncate_string_for_opt_stats(endobj, tmp_alloc, new_endobj))) { + if (OB_FAIL(ObDbmsStatsUtils::truncate_string_for_opt_stats(startobj, tmp_alloc, new_startobj)) || + OB_FAIL(ObDbmsStatsUtils::truncate_string_for_opt_stats(endobj, tmp_alloc, new_endobj))) { LOG_WARN("failed to convert valid obj for opt stats", K(ret), KPC(startobj), KPC(endobj), KPC(new_startobj), KPC(new_endobj)); } else if (OB_ISNULL(new_startobj) || OB_ISNULL(new_endobj)) { diff --git a/src/sql/optimizer/ob_optimizer_util.cpp b/src/sql/optimizer/ob_optimizer_util.cpp index b5608ee547..b584916861 100644 --- a/src/sql/optimizer/ob_optimizer_util.cpp +++ b/src/sql/optimizer/ob_optimizer_util.cpp @@ -8601,51 +8601,6 @@ int ObOptimizerUtil::replace_gen_column(ObLogPlan *log_plan, ObRawExpr *part_exp return ret; } -int ObOptimizerUtil::truncate_string_for_opt_stats(const ObObj *old_obj, - ObIAllocator &alloc, - ObObj *&new_obj) -{ - int ret = OB_SUCCESS; - bool is_truncated = false; - if (OB_ISNULL(old_obj)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null"); - } else if (old_obj->is_string_type()) { - ObString str; - if (OB_FAIL(old_obj->get_string(str))) { - LOG_WARN("failed to get string", K(ret), K(str)); - } else { - int64_t mb_len = ObCharset::strlen_char(old_obj->get_collation_type(), str.ptr(), str.length()); - if (mb_len <= OPT_STATS_MAX_VALUE_CHAR_LEN) { - } else { - //need truncate string, because the opt stats is gathered after truncate string, such as: max_value、min_value、histogram. - ObObj *tmp_obj = NULL; - if (OB_ISNULL(tmp_obj = static_cast(alloc.alloc(sizeof(ObObj))))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc buf", K(ret)); - } else { - int64_t valid_len = ObCharset::charpos(old_obj->get_collation_type(), str.ptr(), - str.length(), OPT_STATS_MAX_VALUE_CHAR_LEN); - if (OB_UNLIKELY(valid_len <= 0)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(valid_len), K(str), K(OPT_STATS_MAX_VALUE_CHAR_LEN)); - } else { - tmp_obj->set_varchar(str.ptr(), valid_len); - tmp_obj->set_meta_type(old_obj->get_meta()); - new_obj = tmp_obj; - is_truncated = true; - } - } - } - } - } - if (OB_SUCC(ret) && !is_truncated) { - new_obj = const_cast(old_obj); - } - LOG_TRACE("Succeed to truncate string obj for opt stats", KPC(old_obj), KPC(new_obj), K(is_truncated)); - return ret; -} - int ObOptimizerUtil::generate_pseudo_trans_info_expr(ObOptimizerContext &opt_ctx, const common::ObString &table_name, ObOpPseudoColumnRawExpr *&expr) diff --git a/src/sql/optimizer/ob_optimizer_util.h b/src/sql/optimizer/ob_optimizer_util.h index 9ae057884f..ffc47e36dc 100644 --- a/src/sql/optimizer/ob_optimizer_util.h +++ b/src/sql/optimizer/ob_optimizer_util.h @@ -1437,10 +1437,6 @@ public: static int check_contain_my_exec_param(ObRawExpr* expr, const common::ObIArray & my_exec_params, bool &contain); - static int truncate_string_for_opt_stats(const ObObj *old_obj, - ObIAllocator &alloc, - ObObj *&new_obj); - static int generate_pseudo_trans_info_expr(ObOptimizerContext &opt_ctx, const common::ObString &table_name, ObOpPseudoColumnRawExpr *&expr);