diff --git a/src/share/stat/ob_incremental_stat_estimator.cpp b/src/share/stat/ob_incremental_stat_estimator.cpp index cb9d81f4aa..4d40a96689 100644 --- a/src/share/stat/ob_incremental_stat_estimator.cpp +++ b/src/share/stat/ob_incremental_stat_estimator.cpp @@ -679,6 +679,7 @@ int ObIncrementalStatEstimator::derive_global_histogram(ObIArray &a ObTopKFrequencyHistograms *top_k_fre_hist = new (ptr) ObTopKFrequencyHistograms(); top_k_fre_hist->set_window_size(1000); top_k_fre_hist->set_item_size(256); + top_k_fre_hist->set_is_topk_hist_need_des_row(true); for (int64_t i = 0; OB_SUCC(ret) && i < all_part_histograms.count(); ++i) { if (all_part_histograms.at(i).is_valid()) { if (all_part_histograms.at(i).get_type() == ObHistType::FREQUENCY || diff --git a/src/share/stat/ob_stat_define.h b/src/share/stat/ob_stat_define.h index e70e4819bb..099a4b3b15 100644 --- a/src/share/stat/ob_stat_define.h +++ b/src/share/stat/ob_stat_define.h @@ -577,13 +577,15 @@ struct ObTableStatParam { struct ObOptStat { + ObOptStat() : table_stat_(NULL), column_stats_() { + column_stats_.set_attr(lib::ObMemAttr(MTL_ID(), "ObOptStat")); + } + virtual ~ObOptStat(); ObOptTableStat *table_stat_; // turn the column stat into pointer ObArray column_stats_; - - virtual ~ObOptStat(); - - TO_STRING_KV("",""); + TO_STRING_KV(K(table_stat_), + K(column_stats_.count())); }; struct ObHistogramParam diff --git a/src/share/stat/ob_stat_item.cpp b/src/share/stat/ob_stat_item.cpp index 2a39ceb770..ef33217752 100644 --- a/src/share/stat/ob_stat_item.cpp +++ b/src/share/stat/ob_stat_item.cpp @@ -253,7 +253,7 @@ int ObStatTopKHist::gen_expr(char *buf, const int64_t buf_len, int64_t &pos) err_rate, col_param_->column_name_.length(), col_param_->column_name_.ptr(), - bkt_num))) { + col_param_->bucket_num_))) { LOG_WARN("failed to print buf topk hist expr", K(ret)); } } diff --git a/src/share/stat/ob_topk_hist_estimator.cpp b/src/share/stat/ob_topk_hist_estimator.cpp index ea9fe543a1..154a909c98 100644 --- a/src/share/stat/ob_topk_hist_estimator.cpp +++ b/src/share/stat/ob_topk_hist_estimator.cpp @@ -12,6 +12,7 @@ #define USING_LOG_PREFIX SQL_ENG #include "ob_topk_hist_estimator.h" +#include namespace oceanbase { @@ -54,19 +55,35 @@ struct CopyableComparer }; //sort and reserve topk sort -int ObTopKFrequencyHistograms::create_topk_fre_items() +int ObTopKFrequencyHistograms::create_topk_fre_items(const ObObjMeta *obj_meta/*default null*/) { int ret = OB_SUCCESS; topk_fre_items_.set_allocator(&topk_buf_); - if (OB_FAIL(topk_fre_items_.init(used_list_.count()))) { + if (OB_FAIL(topk_fre_items_.prepare_allocate(used_list_.count()))) { LOG_WARN("failed to init fixed array", K(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < used_list_.count(); ++i) { - if (OB_ISNULL(used_list_.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("used list is null", K(ret)); - } else if (OB_FAIL(topk_fre_items_.push_back(*used_list_.at(i)))) { - LOG_WARN("failed to push back topk item", K(ret)); + if (is_need_merge_topk_hist()) { + ObTopkItem *topk_item = NULL; + if (OB_ISNULL(topk_item = static_cast(used_list_.at(i)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), KPC(topk_item)); + } else { + topk_fre_items_.at(i) = *topk_item; + } + } else { + ObTopkDatumItem *datum_item = NULL; + if (OB_ISNULL(datum_item = static_cast(used_list_.at(i))) || + OB_ISNULL(datum_item->datum_) || + OB_ISNULL(obj_meta)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(datum_item), K(obj_meta)); + } else if (OB_FAIL(datum_item->datum_->to_obj(topk_fre_items_.at(i).col_obj_, *obj_meta))) { + LOG_WARN("failed to obj", K(ret)); + } else { + topk_fre_items_.at(i).fre_times_ = datum_item->fre_times_; + topk_fre_items_.at(i).delta_ = datum_item->delta_; + } } } if (OB_SUCC(ret) && topk_fre_items_.count() > 0) { @@ -76,10 +93,10 @@ int ObTopKFrequencyHistograms::create_topk_fre_items() while (topk_fre_items_.count() > item_size_) { topk_fre_items_.pop_back(); } - LOG_TRACE("Succeed to adjust topk fre items", K(topk_fre_items_)); + LOG_TRACE("Succeed to adjust topk fre items", K(topk_fre_items_), K(N_), K(by_pass_), K(disuse_cnt_)); } } else { - LOG_TRACE("Succeed to adjust topk fre items", K(topk_fre_items_)); + LOG_TRACE("Succeed to adjust topk fre items", K(topk_fre_items_), K(N_), K(by_pass_), K(disuse_cnt_)); } return ret; } @@ -98,27 +115,62 @@ int ObTopKFrequencyHistograms::sort_topk_fre_items(ObIArray &items) return ret; } +void ObTopKFrequencyHistograms::reset_memory_usage() +{ + used_list_.reset(); + free_list_.reset(); + topk_map_.destroy(); + topk_buf_.reset(); + obj_buf1_.reset(); + obj_buf2_.reset(); +} + int ObTopKFrequencyHistograms::merge_distribute_top_k_fre_items(const ObObj &obj) { int ret = OB_SUCCESS; ObTopKFrequencyHistograms topk_hist; - //1.calc frequency times - if (OB_UNLIKELY(!obj.is_lob())) { + if (obj.is_null() || by_pass_) { + //do nothing + } else if (OB_UNLIKELY(!obj.is_lob())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(obj), K(ret)); - } else if (OB_UNLIKELY(window_size_ <= 0 || item_size_ <= 0)) { - /*do nothing*/ + } else if (OB_UNLIKELY(window_size_ <= 0 || + item_size_ <= 0 || + get_max_reserved_item_size() >= window_size_ || + !is_need_merge_topk_hist())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid argument", K(window_size_), K(item_size_), K(!is_need_merge_topk_hist()), K(ret)); } else if (OB_FAIL(topk_hist.read_result(obj))) { LOG_WARN("failed to read result from lob data", K(ret)); + } else if (topk_hist.is_by_pass()) { + N_ += topk_hist.get_total_fre_item(); + disuse_cnt_ += topk_hist.get_total_disuse_fre_item(); + if (is_satisfied_by_pass()) { + by_pass_ = true; + reset_memory_usage(); + } } else { + if (OB_UNLIKELY(!topk_map_.created())) { + int64_t hash_map_size = window_size_ + get_max_reserved_item_size(); + if (OB_FAIL(topk_map_.create(hash_map_size, "TopkMap", "TopkNode", MTL_ID()))) { + LOG_WARN("failed to create hash map", K(ret)); + } else { + set_the_obj_memory_use_limit(); + } + } const ObIArray &topk_fre_items = topk_hist.get_buckets(); for (int64_t i = 0; OB_SUCC(ret) && i < topk_fre_items.count(); ++i) { const ObTopkItem &cur_item = topk_fre_items.at(i); - ObTopkItem *item = NULL; - if (NULL != (item = get_entry(topk_fre_items.at(i).col_obj_))) { + ObTopkBaseItem *item = NULL; + uint64_t obj_hash = 0; + uint64_t seed = 0; + if (OB_FAIL(topk_fre_items.at(i).col_obj_.hash_murmur(obj_hash, seed))) { + LOG_WARN("fail to do hash", K(ret)); + } else if (NULL != (item = get_entry(obj_hash))) { item->fre_times_ += cur_item.fre_times_; item->delta_ += cur_item.delta_; - } else if (OB_FAIL(add_entry(cur_item.col_obj_, + } else if (OB_FAIL(add_entry(obj_hash, + cur_item.col_obj_, cur_item.fre_times_, cur_item.delta_))) { LOG_WARN("failed to add entry", K(ret)); @@ -126,7 +178,8 @@ int ObTopKFrequencyHistograms::merge_distribute_top_k_fre_items(const ObObj &obj } if (OB_SUCC(ret)) { int64_t bucket_num_old = bucket_num_; - N_ = N_ + topk_hist.get_total_fre_item(); + N_ += topk_hist.get_total_fre_item(); + disuse_cnt_ += topk_hist.get_total_disuse_fre_item(); bucket_num_ = N_ / window_size_; if (bucket_num_ > bucket_num_old) { if (OB_FAIL(shrink_topk_items())) { @@ -139,38 +192,81 @@ int ObTopKFrequencyHistograms::merge_distribute_top_k_fre_items(const ObObj &obj return ret; } +int ObTopKFrequencyHistograms::add_top_k_frequency_item(uint64_t datum_hash, const ObDatum &datum) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(window_size_ <= 0 || + item_size_ <= 0 || + get_max_reserved_item_size() >= window_size_ || + is_need_merge_topk_hist())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid argument", K(window_size_), K(item_size_), K(is_need_merge_topk_hist()), K(ret)); + } else if (!datum.is_null()) { + if (OB_UNLIKELY(!topk_map_.created())) { + int64_t hash_map_size = window_size_ + get_max_reserved_item_size(); + if (OB_FAIL(topk_map_.create(hash_map_size, "TopkMap", "TopkNode", MTL_ID()))) { + LOG_WARN("failed to create hash map", K(ret)); + } else { + set_the_obj_memory_use_limit(); + } + } + if (OB_SUCC(ret)) { + ++ N_; + if (!by_pass_) { + ObTopkBaseItem *item = NULL; + if (NULL != (item = get_entry(datum_hash))) { + item->fre_times_ ++; + } else if (OB_FAIL(add_entry(datum_hash, datum, 1L, bucket_num_ - 1))) { + LOG_WARN("failed to add new entry", K(ret)); + } + if (OB_SUCC(ret) && N_ % window_size_ == 0) { + if (OB_FAIL(shrink_topk_items())) { + LOG_WARN("failed to try shrinking topk fre item", K(ret)); + } + } + } else { + ++ disuse_cnt_; + } + } + } + return ret; +} + int ObTopKFrequencyHistograms::add_top_k_frequency_item(const ObObj &obj) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(window_size_ <= 0 || item_size_ <= 0)) { - // TODO jiangxiu.wt, better to directly return error - // do nothing + if (OB_UNLIKELY(window_size_ <= 0 || + item_size_ <= 0 || + get_max_reserved_item_size() >= window_size_ || + !is_need_merge_topk_hist())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid argument", K(window_size_), K(item_size_), K(!is_need_merge_topk_hist()), K(ret)); } else if (!obj.is_null()) { if (OB_UNLIKELY(!topk_map_.created())) { - const int64_t hashmap_size = 2 * (window_size_ + item_size_); - if (OB_FAIL(topk_map_.create(hashmap_size, "TopkMap"))) { + int64_t hash_map_size = window_size_ + get_max_reserved_item_size(); + if (OB_FAIL(topk_map_.create(hash_map_size, "TopkMap", "TopkNode", MTL_ID()))) { LOG_WARN("failed to create hash map", K(ret)); } else { - need_deep_copy_ = obj.need_deep_copy(); + set_the_obj_memory_use_limit(); } } - if (OB_FAIL(ret)) { - // do nothing - } else if (OB_UNLIKELY(is_topk_hist_need_des_row_)) { - if (OB_FAIL(merge_distribute_top_k_fre_items(obj))) { - LOG_WARN("failed to process distribute row", K(ret)); - } - } else { + if (OB_SUCC(ret)) { ++ N_; - ObTopkItem *item = NULL; - if (NULL != (item = get_entry(obj))) { - item->fre_times_ ++; - } else if (OB_FAIL(add_entry(obj, 1L, bucket_num_ - 1))) { - LOG_WARN("failed to add new entry", K(ret)); - } - if (OB_SUCC(ret) && N_ % window_size_ == 0) { - if (OB_FAIL(shrink_topk_items())) { - LOG_WARN("failed to try shrinking topk fre item"); + if (!by_pass_) { + ObTopkBaseItem *item = NULL; + uint64_t obj_hash = 0; + uint64_t seed = 0; + if (OB_FAIL(obj.hash_murmur(obj_hash, seed))) { + LOG_WARN("fail to do hash", K(ret)); + } else if (NULL != (item = get_entry(obj_hash))) { + item->fre_times_ ++; + } else if (OB_FAIL(add_entry(obj_hash, obj, 1L, bucket_num_ - 1))) { + LOG_WARN("failed to add new entry", K(ret)); + } + if (OB_SUCC(ret) && N_ % window_size_ == 0) { + if (OB_FAIL(shrink_topk_items())) { + LOG_WARN("failed to try shrinking topk fre item"); + } } } } @@ -178,17 +274,44 @@ int ObTopKFrequencyHistograms::add_top_k_frequency_item(const ObObj &obj) return ret; } -ObTopkItem* ObTopKFrequencyHistograms::get_entry(const ObObj &obj) +ObTopkBaseItem* ObTopKFrequencyHistograms::get_entry(const uint64_t hash_val) { - ObTopkItem *ret = NULL; + ObTopkBaseItem *ret = NULL; const uint64_t* ptr = NULL; - if (NULL != (ptr = topk_map_.get(obj))) { - ret = reinterpret_cast(*ptr); + if (NULL != (ptr = topk_map_.get(hash_val))) { + ret = reinterpret_cast(*ptr); } return ret; } -int ObTopKFrequencyHistograms::add_entry(const ObObj &obj, +int ObTopKFrequencyHistograms::add_entry(const uint64_t &datum_hash, + const ObDatum &datum, + const int64_t fre_time, + const int64_t delta) +{ + int ret = OB_SUCCESS; + ObTopkDatumItem *item = NULL; + if (OB_ISNULL(item = alloc_topk_datum_item()) || + OB_ISNULL(item->datum_ = OB_NEWx(ObDatum, (&get_allocator())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory", K(ret)); + } else { + item->fre_times_ = fre_time; + item->delta_ = delta; + item->datum_hash_ = datum_hash; + if (OB_FAIL(item->datum_->deep_copy(datum, get_allocator()))) { + LOG_WARN("failed to deep copy datum"); + } else if (OB_FAIL(topk_map_.set_refactored(datum_hash, reinterpret_cast(item)))) { + LOG_WARN("failed to add item", K(ret)); + } else if (OB_FAIL(used_list_.push_back(item))) { + LOG_WARN("failed to push back item", K(ret)); + } + } + return ret; +} + +int ObTopKFrequencyHistograms::add_entry(const uint64_t &obj_hash, + const ObObj &obj, const int64_t fre_time, const int64_t delta) { @@ -200,20 +323,33 @@ int ObTopKFrequencyHistograms::add_entry(const ObObj &obj, } else { item->fre_times_ = fre_time; item->delta_ = delta; + item->obj_hash_ = obj_hash; if (OB_FAIL(ob_write_obj(get_allocator(), obj, item->col_obj_))) { LOG_WARN("failed to write obj", K(ret)); - } else if (OB_FAIL(topk_map_.set_refactored(item->col_obj_, - reinterpret_cast(item)))) { + } else if (OB_FAIL(topk_map_.set_refactored(obj_hash, reinterpret_cast(item)))) { LOG_WARN("failed to add item", K(ret)); } else if (OB_FAIL(used_list_.push_back(item))) { LOG_WARN("failed to push back item", K(ret)); - } else if (need_deep_copy_) { - ++ copied_count_; } } return ret; } +ObTopkDatumItem* ObTopKFrequencyHistograms::alloc_topk_datum_item() +{ + ObTopkDatumItem *item = NULL; + if (free_list_.empty()) { + void *ptr = topk_buf_.alloc(sizeof(ObTopkDatumItem)); + if (NULL != ptr) { + item = new (ptr) ObTopkDatumItem(); + } + } else { + item = static_cast(free_list_.at(free_list_.count() - 1)); + free_list_.pop_back(); + } + return item; +} + ObTopkItem* ObTopKFrequencyHistograms::alloc_topk_item() { ObTopkItem *item = NULL; @@ -223,20 +359,24 @@ ObTopkItem* ObTopKFrequencyHistograms::alloc_topk_item() item = new (ptr) ObTopkItem(); } } else { - item = free_list_.at(free_list_.count() - 1); + item = static_cast(free_list_.at(free_list_.count() - 1)); free_list_.pop_back(); } return item; } -int ObTopKFrequencyHistograms::remove_entry(ObTopkItem *item) +int ObTopKFrequencyHistograms::remove_entry(ObTopkBaseItem *item) { int ret = OB_SUCCESS; if (OB_ISNULL(item)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("item is null", K(ret)); - } else if (OB_FAIL(topk_map_.erase_refactored(item->col_obj_))) { - LOG_WARN("failed to erase item", K(ret), K(item->col_obj_)); + } else if (is_need_merge_topk_hist() && + OB_FAIL(topk_map_.erase_refactored(static_cast(item)->obj_hash_))) { + LOG_WARN("failed to erase item", K(ret)); + } else if (!is_need_merge_topk_hist() && + OB_FAIL(topk_map_.erase_refactored(static_cast(item)->datum_hash_))) { + LOG_WARN("failed to erase item", K(ret)); } else if (OB_FAIL(free_list_.push_back(item))) { LOG_WARN("faild to push back item", K(ret)); } @@ -253,9 +393,9 @@ int ObTopKFrequencyHistograms::shrink_topk_items() int ret = OB_SUCCESS; int64_t pos = 0; int64_t rm_cnt = 0; - int64_t max_rm_cnt = used_list_.count() - item_size_; + int64_t max_rm_cnt = used_list_.count() - get_max_reserved_item_size(); for (int64_t i = 0; OB_SUCC(ret) && i < used_list_.count(); ++i) { - ObTopkItem *item = NULL; + ObTopkBaseItem *item = NULL; if (OB_ISNULL(item = used_list_.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(i)); @@ -266,20 +406,91 @@ int ObTopKFrequencyHistograms::shrink_topk_items() LOG_WARN("failed to remove item", K(ret)); } else { ++ rm_cnt; + disuse_cnt_ += item->fre_times_; } } - while (used_list_.count() > pos) { - used_list_.pop_back(); - } - if (OB_SUCC(ret) && OB_UNLIKELY(copied_count_ >= 100 * window_size_)) { - if (OB_FAIL(shrink_memory_usage())) { - LOG_WARN("failed to shrink memory usage", K(ret)); + if (OB_SUCC(ret)) { + if (is_satisfied_by_pass()) { + by_pass_ = true; + reset_memory_usage(); + } else { + while (used_list_.count() > pos) { + used_list_.pop_back(); + } + if (OB_UNLIKELY(obj_memory_limit_ < MIN_OBJ_MEMORY_LIMIT || + obj_memory_limit_ > MAX_OBJ_MEMORY_LIMIT)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(obj_memory_limit_)); + } else if (OB_UNLIKELY(get_allocator().used() > obj_memory_limit_)) { + if (OB_FAIL(shrink_memory_usage())) { + LOG_WARN("failed to shrink memory usage", K(ret)); + } + } + bucket_num_ = N_ / window_size_ + 1;//next window size should add 1. } } - bucket_num_ = N_ / window_size_ + 1;//next window size should add 1. return ret; } +bool ObTopKFrequencyHistograms::is_satisfied_by_pass() const +{ + bool is_satisfied = false; + if (N_ % (window_size_ * 100) == 0) { + if (1.0 * disuse_cnt_ / N_ > get_current_min_topk_ratio()) { + is_satisfied = true; + LOG_INFO("topk hist is statisfied by pass", K(disuse_cnt_), K(N_), K(get_current_min_topk_ratio())); + } + } + return is_satisfied; +} + +/* as following rule to set min topk ratio, Assume the total min topk ratio is: r + * if N_ < window_size_ * 1000: + * min_ratio at least r/2; + * if N_ < window_size_ * 10000: + * min_ratio at least r/2 + r/4 + * if N_ between window_size_ * 10000 and window_size_ * 100000: + * increase per window_size_ * 10000, the min_ratio increase: r/2 + r/4+...r/(2^x) + * if N_ more than window_size_ * 100000, at least 100000000 rows: + * min_ratio set the r. + * +*/ +double ObTopKFrequencyHistograms::get_current_min_topk_ratio() const +{ + double min_topk_ratio = 1.0 - 1.0 / item_size_; + if (N_ < window_size_ * 1000) { + min_topk_ratio = min_topk_ratio / 2; + } else if (N_ < window_size_ * 10000) { + min_topk_ratio = min_topk_ratio / 2 + min_topk_ratio / 4; + } else if (N_ >= window_size_ * 10000 && N_ < window_size_ * 100000) { + int64_t n = N_ / (window_size_ * 10000) + 3; + int64_t power = 2; + double new_min_topk_ratio = 0.0; + for (int i = 1; i < n; i++) { + power *= i; + new_min_topk_ratio += min_topk_ratio / power; + } + min_topk_ratio = new_min_topk_ratio; + } + LOG_TRACE("get_current_min_topk_ratio", K(min_topk_ratio), K(item_size_), K(N_)); + return min_topk_ratio; +} + +//allocte object total memory used restict[1MB ~ 10MB] +void ObTopKFrequencyHistograms::set_the_obj_memory_use_limit() +{ + //default is current tenant limit's 1/1000 + obj_memory_limit_ = lib::get_tenant_memory_limit(MTL_ID()) / 1000; + //then see the sql work arena limit, set the limit's 1/100 + if (lib::ObMallocAllocator::get_instance() != NULL) { + auto ta = lib::ObMallocAllocator::get_instance()->get_tenant_ctx_allocator(MTL_ID(), common::ObCtxIds::WORK_AREA); + obj_memory_limit_ = std::min(obj_memory_limit_, ta->get_limit() / 100); + } + obj_memory_limit_ = std::max(obj_memory_limit_, MIN_OBJ_MEMORY_LIMIT); + obj_memory_limit_ = std::min(obj_memory_limit_, MAX_OBJ_MEMORY_LIMIT); + LOG_TRACE("set the memory use limit", K(obj_memory_limit_)); +} + /** * @brief ObTopKFrequencyHistograms::shrink_memory_usage * we use two buffer in turn. @@ -291,22 +502,41 @@ int ObTopKFrequencyHistograms::shrink_memory_usage() int ret = OB_SUCCESS; ObIAllocator &next_allocator = get_next_allocator(); ObIAllocator &curr_allocator = get_allocator(); - copied_count_ = 0; topk_map_.reuse(); + LOG_TRACE("begine to shrink memory usage", K(obj_memory_limit_), K(curr_allocator.used())); for (int64_t i = 0; OB_SUCC(ret) && i < used_list_.count(); ++i) { - ObTopkItem *item = NULL; - if (OB_ISNULL(item = used_list_.at(i))) { + if (OB_ISNULL(used_list_.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("topk item is null", K(ret)); - } else if (OB_FAIL(ob_write_obj(next_allocator, - item->col_obj_, - item->col_obj_))) { - LOG_WARN("failed to write obj", K(ret)); - } else if (OB_FAIL(topk_map_.set_refactored(item->col_obj_, - reinterpret_cast(item)))) { - LOG_WARN("failed to add item", K(ret)); - } else if (need_deep_copy_) { - ++copied_count_; + } else if (is_need_merge_topk_hist()) { + ObTopkItem *topk_item = NULL; + if (OB_ISNULL(topk_item = static_cast(used_list_.at(i)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), KPC(topk_item)); + } else if (OB_FAIL(ob_write_obj(next_allocator, + topk_item->col_obj_, + topk_item->col_obj_))) { + LOG_WARN("failed to write obj", K(ret)); + } else if (OB_FAIL(topk_map_.set_refactored(topk_item->obj_hash_, + reinterpret_cast(topk_item)))) { + LOG_WARN("failed to add item", K(ret)); + } + } else { + ObTopkDatumItem *datum_item = NULL; + ObDatum *tmp_datum = NULL; + if (OB_ISNULL(datum_item = static_cast(used_list_.at(i))) || + OB_ISNULL(tmp_datum = datum_item->datum_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), KPC(datum_item)); + } else if (OB_ISNULL(datum_item->datum_ = OB_NEWx(ObDatum, (&next_allocator)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory", K(ret)); + } else if (OB_FAIL(datum_item->datum_->deep_copy(*tmp_datum, next_allocator))) { + LOG_WARN("failed to write obj", K(ret)); + } else if (OB_FAIL(topk_map_.set_refactored(datum_item->datum_hash_, + reinterpret_cast(datum_item)))) { + LOG_WARN("failed to add item", K(ret)); + } } } if (OB_SUCC(ret)) { @@ -346,6 +576,8 @@ OB_DEF_SERIALIZE(ObTopKFrequencyHistograms) OB_UNIS_ENCODE(top_k_fre_item.fre_times_); OB_UNIS_ENCODE(top_k_fre_item.delta_); } + OB_UNIS_ENCODE(by_pass_); + OB_UNIS_ENCODE(disuse_cnt_); return ret; } @@ -361,6 +593,8 @@ OB_DEF_SERIALIZE_SIZE(ObTopKFrequencyHistograms) OB_UNIS_ADD_LEN(top_k_fre_item.fre_times_); OB_UNIS_ADD_LEN(top_k_fre_item.delta_); } + OB_UNIS_ADD_LEN(by_pass_); + OB_UNIS_ADD_LEN(disuse_cnt_); return len; } @@ -383,6 +617,8 @@ OB_DEF_DESERIALIZE(ObTopKFrequencyHistograms) LOG_WARN("failed to push back", K(ret)); } } + OB_UNIS_DECODE(by_pass_); + OB_UNIS_DECODE(disuse_cnt_); return ret; } } diff --git a/src/share/stat/ob_topk_hist_estimator.h b/src/share/stat/ob_topk_hist_estimator.h index c2e60f7d9d..9cdcb19976 100644 --- a/src/share/stat/ob_topk_hist_estimator.h +++ b/src/share/stat/ob_topk_hist_estimator.h @@ -21,24 +21,55 @@ namespace oceanbase namespace common { -struct ObTopkItem +struct ObTopkBaseItem +{ + ObTopkBaseItem() + : fre_times_(0), + delta_(0) + { + } + + TO_STRING_KV(K_(fre_times), + K_(delta)); + int64_t fre_times_; + int64_t delta_; +}; + +struct ObTopkItem : public ObTopkBaseItem { ObTopkItem() - : fre_times_(0), - delta_(0), + : ObTopkBaseItem(), + obj_hash_(0), col_obj_() { } TO_STRING_KV(K_(fre_times), K_(delta), + K_(obj_hash), K_(col_obj)); - int64_t fre_times_; - int64_t delta_; + uint64_t obj_hash_; common::ObObj col_obj_; }; -typedef common::hash::ObHashMap ObTopkMap; +struct ObTopkDatumItem : public ObTopkBaseItem +{ + ObTopkDatumItem() + : ObTopkBaseItem(), + datum_hash_(0), + datum_(NULL) + { + } + + TO_STRING_KV(K_(fre_times), + K_(delta), + K_(datum_hash), + KPC_(datum)); + uint64_t datum_hash_; + ObDatum *datum_; +}; + +typedef common::hash::ObHashMap ObTopkMap; class ObTopKFrequencyHistograms { @@ -54,40 +85,49 @@ class ObTopKFrequencyHistograms used_list_(), free_list_(), topk_map_(), - topk_buf_("OptTopkHist"), + topk_buf_("OptTopkHistbuf"), use_first_alloc_(true), - obj_buf1_("OptTopkHist"), - obj_buf2_("OptTopkHist"), - copied_count_(), - need_deep_copy_(false) + obj_buf1_("OptTopObjbuf1"), + obj_buf2_("OptTopObjbuf2"), + by_pass_(false), + disuse_cnt_(0), + obj_memory_limit_(0) { topk_buf_.set_tenant_id(MTL_ID()); obj_buf1_.set_tenant_id(MTL_ID()); obj_buf2_.set_tenant_id(MTL_ID()); + used_list_.set_attr(lib::ObMemAttr(MTL_ID(), "ObTopkUsedlist")); + free_list_.set_attr(lib::ObMemAttr(MTL_ID(), "ObTopkfreelist")); } int read_result(const ObObj &result_obj); - - int create_topk_fre_items(); + int create_topk_fre_items(const ObObjMeta *obj_meta = NULL); int sort_topk_fre_items(ObIArray &items); int shrink_topk_items(); int add_top_k_frequency_item(const ObObj &obj); + int add_top_k_frequency_item(uint64_t datum_hash, const ObDatum &datum); int merge_distribute_top_k_fre_items(const ObObj &obj); void set_window_size(int64_t window_size) { window_size_ = window_size; } void set_item_size(int64_t item_size) { item_size_ = item_size; } void set_is_topk_hist_need_des_row(bool is_true) { is_topk_hist_need_des_row_ = is_true; } + bool is_need_merge_topk_hist() const { return is_topk_hist_need_des_row_; } const ObIArray &get_buckets() const { return topk_fre_items_; } bool has_bucket() const { return !used_list_.empty(); } int64_t get_total_fre_item() const { return N_; } - - ObTopkItem *get_entry(const ObObj &obj); - int add_entry(const ObObj &obj, const int64_t fre, const int64_t delta); - int remove_entry(ObTopkItem *item); - + ObTopkBaseItem *get_entry(const uint64_t hash_val); + int add_entry(const uint64_t &datum_hash, const ObDatum &datum, const int64_t fre_time, const int64_t delta); + int add_entry(const uint64_t &obj_hash, const ObObj &obj, const int64_t fre_time, const int64_t delta); + int remove_entry(ObTopkBaseItem *item); int shrink_memory_usage(); - + void reset_memory_usage(); + bool is_by_pass() const { return by_pass_; } + bool is_satisfied_by_pass() const; + int64_t get_total_disuse_fre_item() const { return disuse_cnt_; } + double get_current_min_topk_ratio() const; + void set_the_obj_memory_use_limit(); + int64_t get_max_reserved_item_size() const { return item_size_ * 1.5; } ObTopkItem *alloc_topk_item(); - + ObTopkDatumItem *alloc_topk_datum_item(); ObIAllocator &get_allocator() { if (use_first_alloc_) { @@ -96,7 +136,6 @@ class ObTopKFrequencyHistograms return obj_buf2_; } } - ObIAllocator &get_next_allocator() { if (use_first_alloc_) { @@ -105,40 +144,41 @@ class ObTopKFrequencyHistograms return obj_buf1_; } } - void swap_allocator() { use_first_alloc_ = !use_first_alloc_; } - TO_STRING_KV(K_(N), K_(bucket_num), K_(window_size), K_(item_size), K_(is_topk_hist_need_des_row), - K_(topk_fre_items)); + K_(topk_fre_items), + K_(by_pass), + K_(disuse_cnt), + K_(obj_memory_limit)); private: + const int64_t MIN_OBJ_MEMORY_LIMIT = 1LL * 1024LL * 1024LL; // 1MB + const int64_t MAX_OBJ_MEMORY_LIMIT = 10LL * 1024LL * 1024LL; // 10MB int64_t N_; int64_t bucket_num_; int64_t window_size_; // in default, the window size is 1000 int64_t item_size_; bool is_topk_hist_need_des_row_; // for parallel topk histogram compuation common::ObFixedArray topk_fre_items_; - // the following are used to computation - common::ObSEArray used_list_; - common::ObSEArray free_list_; + common::ObSEArray used_list_; + common::ObSEArray free_list_; ObTopkMap topk_map_; ObArenaAllocator topk_buf_; - // obj_buf1_ and obj_buf2_ are used in turn // true for obj_buf1_, false for obj_buf2_ bool use_first_alloc_; ObArenaAllocator obj_buf1_; ObArenaAllocator obj_buf2_; - - int64_t copied_count_; - bool need_deep_copy_; + bool by_pass_; + int64_t disuse_cnt_; + int64_t obj_memory_limit_; }; } // end of namespace common diff --git a/src/sql/code_generator/ob_static_engine_expr_cg.cpp b/src/sql/code_generator/ob_static_engine_expr_cg.cpp index 1cb8194b93..a7a9216135 100644 --- a/src/sql/code_generator/ob_static_engine_expr_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_expr_cg.cpp @@ -1446,7 +1446,9 @@ ObStaticEngineExprCG::ObExprBatchSize ObStaticEngineExprCG::get_expr_execute_siz size = ObExprBatchSize::one; break; } - if (is_large_data(raw_exprs.at(i)->get_data_type())) { + if (is_large_data(raw_exprs.at(i)->get_data_type()) && + raw_exprs.at(i)->get_expr_type() != T_FUN_TOP_FRE_HIST && + raw_exprs.at(i)->get_expr_type() != T_FUN_HYBRID_HIST) { // 1. batchsize should be scale down when longtext/mediumtext/lob shows up // 2. keep searching size = ObExprBatchSize::small; diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.cpp b/src/sql/engine/aggregate/ob_aggregate_processor.cpp index cc2611a06a..997393e09c 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.cpp +++ b/src/sql/engine/aggregate/ob_aggregate_processor.cpp @@ -2530,21 +2530,31 @@ int ObAggregateProcessor::prepare_aggr_result(const ObChunkDatumStore::StoredRow ret = OB_ERR_UNEXPECTED; LOG_WARN("extra_ is NULL", K(ret), K(aggr_cell)); } else if (OB_UNLIKELY(stored_row.cnt_ != 1 || - aggr_info.param_exprs_.count() != 1 || - OB_ISNULL(aggr_info.param_exprs_.at(0)))) { + aggr_info.param_exprs_.count() != 1) || + OB_ISNULL(aggr_info.param_exprs_.at(0)) || + OB_ISNULL(aggr_info.param_exprs_.at(0)->basic_funcs_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(stored_row.cnt_), K(aggr_info)); } else if (OB_FAIL(init_topk_fre_histogram_item(aggr_info, &(extra->topk_fre_hist_)))) { LOG_WARN("failed to init topk fre histogram", K(ret)); - } else { + } else if (extra->topk_fre_hist_.is_need_merge_topk_hist()) { ObObj obj; if (OB_FAIL(stored_row.cells()[0].to_obj(obj, aggr_info.param_exprs_.at(0)->obj_meta_))) { LOG_WARN("failed to obj", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(obj))) { - LOG_WARN("fail to truncate string", K(ret)); - } else if (OB_FAIL(extra->topk_fre_hist_.add_top_k_frequency_item(obj))) { + } else if (OB_FAIL(extra->topk_fre_hist_.merge_distribute_top_k_fre_items(obj))) { LOG_WARN("failed to process row", K(ret)); - } else {/*do nothing*/} + } + } else if (OB_FAIL(shadow_truncate_string_for_hist(aggr_info.param_exprs_.at(0)->obj_meta_, + const_cast(stored_row.cells()[0])))) { + LOG_WARN("failed to shadow truncate string for hist", K(ret)); + } else { + ObExprHashFuncType hash_func = aggr_info.param_exprs_.at(0)->basic_funcs_->murmur_hash_; + uint64_t datum_value = 0; + if (OB_FAIL(hash_func(stored_row.cells()[0], datum_value, datum_value))) { + LOG_WARN("fail to do hash", K(ret)); + } else if (OB_FAIL(extra->topk_fre_hist_.add_top_k_frequency_item(datum_value, stored_row.cells()[0]))) { + LOG_WARN("failed to process row", K(ret)); + } } break; } @@ -2980,19 +2990,29 @@ int ObAggregateProcessor::process_aggr_result(const ObChunkDatumStore::StoredRow ret = OB_ERR_UNEXPECTED; LOG_WARN("extra_ is NULL", K(ret), K(aggr_cell)); } else if (OB_UNLIKELY(stored_row.cnt_ != 1 || - aggr_info.param_exprs_.count() != 1 || - OB_ISNULL(aggr_info.param_exprs_.at(0)))) { + aggr_info.param_exprs_.count() != 1 || + OB_ISNULL(aggr_info.param_exprs_.at(0))) || + OB_ISNULL(aggr_info.param_exprs_.at(0)->basic_funcs_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(stored_row.cnt_), K(aggr_info)); - } else { + } else if (extra->topk_fre_hist_.is_need_merge_topk_hist()) { ObObj obj; if (OB_FAIL(stored_row.cells()[0].to_obj(obj, aggr_info.param_exprs_.at(0)->obj_meta_))) { LOG_WARN("failed to obj", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(obj))) { - LOG_WARN("fail to truncate string", K(ret)); - } else if (OB_FAIL(extra->topk_fre_hist_.add_top_k_frequency_item(obj))) { + } else if (OB_FAIL(extra->topk_fre_hist_.merge_distribute_top_k_fre_items(obj))) { LOG_WARN("failed to process row", K(ret)); - } else {/*do nothing*/} + } + } else if (OB_FAIL(shadow_truncate_string_for_hist(aggr_info.param_exprs_.at(0)->obj_meta_, + const_cast(stored_row.cells()[0])))) { + LOG_WARN("failed to shadow truncate string for hist", K(ret)); + } else { + ObExprHashFuncType hash_func = aggr_info.param_exprs_.at(0)->basic_funcs_->murmur_hash_; + uint64_t datum_value = 0; + if (OB_FAIL(hash_func(stored_row.cells()[0], datum_value, datum_value))) { + LOG_WARN("fail to do hash", K(ret)); + } else if (OB_FAIL(extra->topk_fre_hist_.add_top_k_frequency_item(datum_value, stored_row.cells()[0]))) { + LOG_WARN("failed to process row", K(ret)); + } } break; } @@ -3811,7 +3831,10 @@ int ObAggregateProcessor::collect_aggr_result( if (OB_ISNULL(extra)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("extra info is null", K(ret)); - } else if (OB_FAIL(get_top_k_fre_hist_result(extra->topk_fre_hist_, has_lob_header, result))) { + } else if (OB_FAIL(get_top_k_fre_hist_result(extra->topk_fre_hist_, + aggr_info.param_exprs_.at(0)->obj_meta_, + has_lob_header, + result))) { LOG_WARN("failed to get topk fre hist result", K(ret)); } else { } @@ -4857,14 +4880,24 @@ int ObAggregateProcessor::top_fre_hist_calc_batch( if (datum->is_null()) { continue; } - ObObj obj; - if (OB_FAIL(datum->to_obj(obj, obj_meta))) { - LOG_WARN("failed to obj", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(obj))) { - LOG_WARN("fail to truncate string", K(ret)); - } else if (OB_FAIL(extra_info->topk_fre_hist_.add_top_k_frequency_item(obj))) { - LOG_WARN("failed to process row", K(ret)); - } else {/*do nothing*/} + if (extra_info->topk_fre_hist_.is_need_merge_topk_hist()) {//merge + ObObj obj; + if (OB_FAIL(datum->to_obj(obj, obj_meta))) { + LOG_WARN("failed to obj", K(ret)); + } else if (OB_FAIL(extra_info->topk_fre_hist_.merge_distribute_top_k_fre_items(obj))) { + LOG_WARN("failed to process row", K(ret)); + } + } else if (OB_FAIL(shadow_truncate_string_for_hist(obj_meta, *datum))) { + LOG_WARN("failed to shadow truncate string for hist", K(ret)); + } else { + ObExprHashFuncType hash_func = aggr_info.param_exprs_.at(0)->basic_funcs_->murmur_hash_; + uint64_t datum_value = 0; + if (OB_FAIL(hash_func(*datum, datum_value, datum_value))) { + LOG_WARN("fail to do hash", K(ret)); + } else if (OB_FAIL(extra_info->topk_fre_hist_.add_top_k_frequency_item(datum_value, *datum))) { + LOG_WARN("failed to process row", K(ret)); + } + } } return ret; } @@ -5958,13 +5991,15 @@ int ObAggregateProcessor::get_pl_agg_udf_result(const ObAggrInfo &aggr_info, } int ObAggregateProcessor::get_top_k_fre_hist_result(ObTopKFrequencyHistograms &top_k_fre_hist, + const ObObjMeta &obj_meta, bool has_lob_header, ObDatum &result_datum) { int ret = OB_SUCCESS; - if (top_k_fre_hist.has_bucket()) { + if (top_k_fre_hist.has_bucket() || + (top_k_fre_hist.is_by_pass() && !top_k_fre_hist.is_need_merge_topk_hist())) { //sort and reserve topk item - if (OB_FAIL(top_k_fre_hist.create_topk_fre_items())) { + if (OB_FAIL(top_k_fre_hist.create_topk_fre_items(&obj_meta))) { LOG_WARN("failed to adjust frequency sort", K(ret)); } else { char *buf = NULL; @@ -6061,9 +6096,9 @@ int ObAggregateProcessor::compute_hybrid_hist_result(const ObAggrInfo &aggr_info LOG_WARN("get unexpected null", K(ret), K(stored_row)); } else if (stored_row->cells()[0].is_null()) { ++ null_count; - } else if (OB_FAIL(shadow_truncate_string_for_hybird_hist(aggr_info.param_exprs_.at(0)->obj_meta_, - const_cast(stored_row->cells()[0])))) { - LOG_WARN("failed to shadow truncate string for hybird hist", K(ret)); + } else if (OB_FAIL(shadow_truncate_string_for_hist(aggr_info.param_exprs_.at(0)->obj_meta_, + const_cast(stored_row->cells()[0])))) { + LOG_WARN("failed to shadow truncate string for hist", K(ret)); } else if (OB_FAIL(prev_row.save_store_row(*stored_row))) { LOG_WARN("failed to deep copy limit last rows", K(ret)); } else { @@ -6080,9 +6115,9 @@ int ObAggregateProcessor::compute_hybrid_hist_result(const ObAggrInfo &aggr_info if (OB_ISNULL(stored_row) || OB_UNLIKELY(stored_row->cnt_ != 1)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(stored_row)); - } else if (OB_FAIL(shadow_truncate_string_for_hybird_hist(aggr_info.param_exprs_.at(0)->obj_meta_, - const_cast(stored_row->cells()[0])))) { - LOG_WARN("failed to shadow truncate string for hybrid hist", K(ret)); + } else if (OB_FAIL(shadow_truncate_string_for_hist(aggr_info.param_exprs_.at(0)->obj_meta_, + const_cast(stored_row->cells()[0])))) { + LOG_WARN("failed to shadow truncate string for hist", K(ret)); } else if (OB_FAIL(check_rows_equal(prev_row, *stored_row, aggr_info, is_equal))) { LOG_WARN("failed to is order by item equal with prev row", K(ret)); } else if (is_equal) { @@ -6152,8 +6187,8 @@ int ObAggregateProcessor::compute_hybrid_hist_result(const ObAggrInfo &aggr_info return ret; } -int ObAggregateProcessor::shadow_truncate_string_for_hybird_hist(const ObObjMeta obj_meta, - ObDatum &datum) +int ObAggregateProcessor::shadow_truncate_string_for_hist(const ObObjMeta obj_meta, + ObDatum &datum) { int ret = OB_SUCCESS; if (ObColumnStatParam::is_valid_opt_col_type(obj_meta.get_type()) && obj_meta.is_string_type() && !datum.is_null()) { @@ -6164,7 +6199,7 @@ int ObAggregateProcessor::shadow_truncate_string_for_hybird_hist(const ObObjMeta LOG_WARN("get unexpected error", K(ret), K(obj_meta), K(datum), K(truncated_str_len)); } else { datum.set_string(str.ptr(), static_cast(truncated_str_len)); - LOG_TRACE("Succeed to shadow truncate string for hybird hist", K(datum)); + LOG_TRACE("Succeed to shadow truncate string for hist", K(datum)); } } return ret; diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.h b/src/sql/engine/aggregate/ob_aggregate_processor.h index c1f52befc1..92073f8c09 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.h +++ b/src/sql/engine/aggregate/ob_aggregate_processor.h @@ -931,6 +931,7 @@ private: int init_topk_fre_histogram_item(const ObAggrInfo &aggr_info, ObTopKFrequencyHistograms *topk_fre_hist); int get_top_k_fre_hist_result(ObTopKFrequencyHistograms &top_k_fre_hist, + const ObObjMeta &obj_meta, bool has_lob_header, ObDatum &result_datum); @@ -961,7 +962,7 @@ private: int check_key_valid(common::hash::ObHashSet &view_key_names, const ObString& key); - int shadow_truncate_string_for_hybird_hist(const ObObjMeta obj_meta, ObDatum &datum); + int shadow_truncate_string_for_hist(const ObObjMeta obj_meta, ObDatum &datum); OB_INLINE void clear_op_evaluated_flag() { diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 3382632033..cdfd9de8a6 100755 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -1956,12 +1956,12 @@ COUNT '(' opt_all '*' ')' OVER new_generalized_window_clause malloc_non_terminal_node($$, result->malloc_pool_, T_WIN_FUN_NTH_VALUE, 4, $3, $5, $7, $8); malloc_non_terminal_node($$, result->malloc_pool_, T_WINDOW_FUNCTION, 2, $$, $10); } -| TOP_K_FRE_HIST '(' bit_expr ',' bit_expr ',' bit_expr ')' OVER new_generalized_window_clause +| TOP_K_FRE_HIST '(' DECIMAL_VAL ',' bit_expr ',' INTNUM ')' OVER new_generalized_window_clause { malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_TOP_FRE_HIST, 3, $3, $5, $7); malloc_non_terminal_node($$, result->malloc_pool_, T_WINDOW_FUNCTION, 2, $$, $10); } -| HYBRID_HIST '(' bit_expr ',' bit_expr ')' OVER new_generalized_window_clause +| HYBRID_HIST '(' bit_expr ',' INTNUM ')' OVER new_generalized_window_clause { malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_HYBRID_HIST, 2, $3, $5); malloc_non_terminal_node($$, result->malloc_pool_, T_WINDOW_FUNCTION, 2, $$, $8); @@ -2358,11 +2358,11 @@ MOD '(' expr ',' expr ')' merge_nodes(group_concat_exprs, result, T_EXPR_LIST, $4); malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_GROUP_CONCAT, 4, $3, group_concat_exprs, $5, $6); } -| TOP_K_FRE_HIST '(' bit_expr ',' bit_expr ',' bit_expr ')' +| TOP_K_FRE_HIST '(' DECIMAL_VAL ',' bit_expr ',' INTNUM ')' { malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_TOP_FRE_HIST, 3, $3, $5, $7); } -| HYBRID_HIST '(' bit_expr ',' bit_expr ')' +| HYBRID_HIST '(' bit_expr ',' INTNUM ')' { malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_HYBRID_HIST, 2, $3, $5); } diff --git a/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp b/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp index 8a8531f3b3..f247618d1b 100644 --- a/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp @@ -3252,7 +3252,11 @@ int ObRawExprDeduceType::add_implicit_cast(ObAggFunRawExpr &parent, (parent.get_expr_type() == T_FUN_REGR_SXY && i == 1) || (parent.get_expr_type() == T_FUN_JSON_OBJECTAGG && i == 1) || (parent.get_expr_type() == T_FUN_ORA_JSON_OBJECTAGG && i > 0) || - (parent.get_expr_type() == T_FUN_ORA_XMLAGG && i > 0)) { + (parent.get_expr_type() == T_FUN_ORA_XMLAGG && i > 0) || + ((parent.get_expr_type() == T_FUN_SUM || + parent.get_expr_type() == T_FUN_AVG || + parent.get_expr_type() == T_FUN_COUNT) && + child_ptr->get_expr_type() == T_FUN_SYS_OP_OPNSIZE)) { //do nothing } else if (parent.get_expr_type() == T_FUN_WM_CONCAT || parent.get_expr_type() == T_FUN_KEEP_WM_CONCAT ||