hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

support json type

Browse Source
This commit is contained in:
xj0
2022-02-08 14:58:13 +08:00
committed by LINxiansheng
parent 4b25bac8d0
commit e5f59ea074
241 changed files with 46116 additions and 749 deletions
Download Patch File Download Diff File Expand all files Collapse all files

320
src/sql/engine/aggregate/ob_aggregate_function.cpp
View File

@ -25,6 +25,8 @@
#include "sql/engine/expr/ob_expr_estimate_ndv.h"
#include "sql/engine/user_defined_function/ob_udf_util.h"
#include "sql/engine/expr/ob_expr_operator.h"
#include "sql/engine/expr/ob_expr_json_func_helper.h"
namespace oceanbase {
using namespace common;
namespace sql {
@ -142,7 +144,7 @@ ObGroupConcatRowStore::~ObGroupConcatRowStore()
{}
int ObGroupConcatRowStore::init(const uint64_t tenant_id, const ObIArray<ObSortColumn>& sort_columns,
const ObSortImpl::SortExtraInfos* extra_infos, const bool rewind)
const ObSortImpl::SortExtraInfos* extra_infos, const bool rewind, int64_t dir_id)
{
int ret = OB_SUCCESS;
rows_ = 0;
@ -158,6 +160,8 @@ int ObGroupConcatRowStore::init(const uint64_t tenant_id, const ObIArray<ObSortC
true /* enable dump */,
ObChunkRowStore::FULL))) {
LOG_WARN("row store failed", K(ret));
} else {
rs_.set_dir_id(dir_id);
}
need_sort_ = false;
} else {
@ -254,7 +258,8 @@ ObAggregateFunction::ObAggregateFunction()
first_rollup_cols_(),
agg_udf_buf_(ObModIds::OB_SQL_UDF),
agg_udf_metas_(),
agg_udf_()
agg_udf_(),
dir_id_(-1)
{
if (share::is_oracle_mode()) {
group_concat_max_len_ = OB_DEFAULT_GROUP_CONCAT_MAX_LEN_FOR_ORACLE;
@ -373,7 +378,9 @@ int ObAggregateFunction::init(const int64_t input_column_count, const ObAggrExpr
case T_FUN_KEEP_SUM:
case T_FUN_KEEP_COUNT:
case T_FUN_KEEP_WM_CONCAT:
case T_FUN_WM_CONCAT: {
case T_FUN_WM_CONCAT:
case T_FUN_JSON_ARRAYAGG:
case T_FUN_JSON_OBJECTAGG: {
aggr_fun_need_cell_ctx_ = true;
break;
}
@ -1016,7 +1023,9 @@ int ObAggregateFunction::init_one_group(const int64_t group_id)
case T_FUN_KEEP_SUM:
case T_FUN_KEEP_COUNT:
case T_FUN_KEEP_WM_CONCAT:
case T_FUN_WM_CONCAT: {
case T_FUN_WM_CONCAT:
case T_FUN_JSON_ARRAYAGG:
case T_FUN_JSON_OBJECTAGG: {
void* mem1 = NULL;
void* mem2 = NULL;
ObGroupConcatCtx* gc_ctx = NULL;
@ -1036,7 +1045,7 @@ int ObAggregateFunction::init_one_group(const int64_t group_id)
if (OB_FAIL(gc_ctx->gc_rs_->init(expr_ctx_.my_session_->get_effective_tenant_id(),
cexpr->get_sort_columns(),
&cexpr->get_sort_extra_infos(),
need_rewind))) {
need_rewind, dir_id_))) {
LOG_WARN("init group concat store failed", K(ret));
} else {
gr.ctx_[node->ctx_idx_] = gc_ctx;
@ -1180,7 +1189,9 @@ int ObAggregateFunction::rollup_process(const ObTimeZoneInfo* tz_info, const int
case T_FUN_GROUP_PERCENTILE_DISC:
case T_FUN_MEDIAN:
case T_FUN_KEEP_WM_CONCAT:
case T_FUN_WM_CONCAT: {
case T_FUN_WM_CONCAT:
case T_FUN_JSON_ARRAYAGG:
case T_FUN_JSON_OBJECTAGG: {
if (NULL == cell_ctx1 || NULL == cell_ctx2 || NULL == cell_ctx1->gc_rs_ || NULL == cell_ctx2->gc_rs_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("NULL group concat row store", K(ret));
@ -1338,7 +1349,9 @@ int ObAggregateFunction::rollup_aggregation(const ObItemType aggr_fun, const boo
case T_FUN_KEEP_COUNT:
case T_FUN_KEEP_SUM:
case T_FUN_KEEP_WM_CONCAT:
case T_FUN_WM_CONCAT: {
case T_FUN_WM_CONCAT:
case T_FUN_JSON_ARRAYAGG:
case T_FUN_JSON_OBJECTAGG: {
if (OB_ISNULL(group_concat_row_store1) || OB_ISNULL(group_concat_row_store2)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("group concat row store is null", K(ret));
@ -1498,7 +1511,9 @@ int ObAggregateFunction::init_aggr_cell(const ObItemType aggr_fun, const ObNewRo
case T_FUN_KEEP_SUM:
case T_FUN_KEEP_COUNT:
case T_FUN_KEEP_WM_CONCAT:
case T_FUN_WM_CONCAT: {
case T_FUN_WM_CONCAT:
case T_FUN_JSON_ARRAYAGG:
case T_FUN_JSON_OBJECTAGG: {
if (OB_ISNULL(cell_ctx)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("group concat row store is null", K(ret));
@ -1586,7 +1601,9 @@ int ObAggregateFunction::calc_aggr_cell(const ObItemType aggr_fun, const ObNewRo
case T_FUN_KEEP_SUM:
case T_FUN_KEEP_COUNT:
case T_FUN_KEEP_WM_CONCAT:
case T_FUN_WM_CONCAT: {
case T_FUN_WM_CONCAT:
case T_FUN_JSON_ARRAYAGG:
case T_FUN_JSON_OBJECTAGG: {
if (OB_UNLIKELY(oprands.is_invalid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("oprands is invalid", K(oprands));
@ -1668,7 +1685,9 @@ int ObAggregateFunction::calc_aggr_cell(const ObItemType aggr_fun, const ObNewRo
case T_FUN_KEEP_SUM:
case T_FUN_KEEP_COUNT:
case T_FUN_KEEP_WM_CONCAT:
case T_FUN_WM_CONCAT: {
case T_FUN_WM_CONCAT:
case T_FUN_JSON_ARRAYAGG:
case T_FUN_JSON_OBJECTAGG: {
if (OB_ISNULL(cell_ctx)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("group concat row store is null", K(ret));
@ -2256,6 +2275,36 @@ int ObAggregateFunction::get_result(ObNewRow& row, const common::ObTimeZoneInfo*
LOG_DEBUG("get udf result", K(agg_result), K(agg_udf_id));
break;
}
case T_FUN_JSON_ARRAYAGG: {
ObGroupConcatCtx *cell_ctx = static_cast<ObGroupConcatCtx *>(get_agg_cell_ctx(group_id, node->ctx_idx_));
ObObj concat_obj;
if (OB_FAIL(get_json_arrayagg_result(cexpr,
cell_ctx,
concat_obj))) {
LOG_WARN("failed to get json_arrayagg result", K(ret));
} else if (OB_FAIL(clone_cell(concat_obj, stored_row->reserved_cells_[aggr_idx]))) {
LOG_WARN("fail to clone cell", K(ret), K(concat_obj));
} else {
LOG_TRACE("concat_obj", K(stored_row->reserved_cells_[aggr_idx]));
*aggr_cell = stored_row->reserved_cells_[aggr_idx];
}
break;
}
case T_FUN_JSON_OBJECTAGG: {
ObGroupConcatCtx *cell_ctx = static_cast<ObGroupConcatCtx *>(get_agg_cell_ctx(group_id, node->ctx_idx_));
ObObj concat_obj;
if (OB_FAIL(get_json_objectagg_result(cexpr,
cell_ctx,
concat_obj))) {
LOG_WARN("failed to get json_arrayagg result", K(ret));
} else if (OB_FAIL(clone_cell(concat_obj, stored_row->reserved_cells_[aggr_idx]))) {
LOG_WARN("fail to clone cell", K(ret), K(concat_obj));
} else {
LOG_TRACE("concat_obj", K(stored_row->reserved_cells_[aggr_idx]));
*aggr_cell = stored_row->reserved_cells_[aggr_idx];
}
break;
}
case T_FUN_GROUP_RANK:
case T_FUN_GROUP_DENSE_RANK:
case T_FUN_GROUP_PERCENT_RANK:
@ -3184,5 +3233,256 @@ int ObAggregateFunction::get_wm_concat_result(
return ret;
}
int ObAggregateFunction::get_json_arrayagg_result(const ObAggregateExpression *&cexpr,
ObGroupConcatCtx *&cell_ctx,
ObObj &concat_obj)
{
UNUSED(cexpr);
int ret = OB_SUCCESS;
ObSEArray<ObSortColumn, 4> sort_columns;
common::ObArenaAllocator tmp_alloc;
ObGroupConcatRowStore *json_row_store = NULL;
if (OB_ISNULL(cell_ctx) || OB_ISNULL(expr_ctx_.my_session_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret));
} else if (OB_ISNULL(json_row_store = cell_ctx->gc_rs_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("group concat row store is NULL", K(ret));
} else if (json_row_store->is_allocate_sort() &&
OB_FAIL(json_row_store->get_sort_columns(sort_columns))) {
LOG_WARN("failed to get sort columns", K(ret));
} else {
//concat row may be iterated in rollup_process(), rewind here.
if (json_row_store->is_iterated()) {
if (OB_FAIL(json_row_store->rewind())) {
LOG_WARN("group concat row store rewind failed", K(ret));
}
} else {
if (OB_FAIL(json_row_store->finish_add_row())) {
LOG_WARN("finish add row to group concat row store failed", K(ret));
}
}
const ObNewRow *sort_row = NULL;
ObJsonArray json_array(&tmp_alloc);
while (OB_SUCC(ret) && OB_SUCC(json_row_store->get_next_row(sort_row))) {
if (OB_ISNULL(sort_row)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret), K(sort_row));
} else if (sort_row->get_count() != 1) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected column count", K(ret), K(sort_row));
} else {
ObObjType val_type = sort_row->cells_[0].get_type();
ObCollationType cs_type = sort_row->cells_[0].get_collation_type();
ObScale scale = sort_row->cells_[0].get_scale();
ObObj converted_obj(sort_row->cells_[0]);
ObString converted_str;
// get value and add to json_array
ObIJsonBase *json_val = NULL;
if (ob_is_string_type(val_type)
&& (ObCharset::charset_type_by_coll(cs_type) != CHARSET_UTF8MB4)) {
ObString origin_str = converted_obj.get_string();
if (OB_FAIL(ObExprUtil::convert_string_collation(origin_str, cs_type, converted_str,
CS_TYPE_UTF8MB4_BIN, tmp_alloc))) {
LOG_WARN("convert string collation failed", K(ret), K(cs_type), K(origin_str.length()));
} else {
converted_obj.set_string(val_type, converted_str);
converted_obj.set_collation_type(CS_TYPE_UTF8MB4_BIN);
cs_type = CS_TYPE_UTF8MB4_BIN;
}
}
if (OB_FAIL(ret)) {
} else if (ObJsonExprHelper::is_convertible_to_json(val_type)) {
if (OB_FAIL(ObJsonExprHelper::transform_convertible_2jsonBase(converted_obj, val_type,
&tmp_alloc, cs_type,
json_val, false, true))) {
LOG_WARN("failed: parse value to jsonBase", K(ret), K(val_type));
}
} else {
if (OB_FAIL(ObJsonExprHelper::transform_scalar_2jsonBase(converted_obj, val_type,
&tmp_alloc, scale,
expr_ctx_.my_session_->get_timezone_info(),
json_val, false))) {
LOG_WARN("failed: parse value to jsonBase", K(ret), K(val_type));
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(json_array.array_append(json_val))) {
LOG_WARN("failed: json array append json value", K(ret));
} else if(json_array.get_serialize_size() > OB_MAX_PACKET_LENGTH) {
ret = OB_ERR_TOO_LONG_STRING_IN_CONCAT;
LOG_WARN("result of json_arrayagg is too long", K(ret), K(json_array.get_serialize_size()),
K(OB_MAX_PACKET_LENGTH));
}
}
}
if (ret != OB_ITER_END && ret != OB_SUCCESS) {
LOG_WARN("fail to get next row", K(ret));
} else {
ret = OB_SUCCESS;
ObString str;
// output res
if (OB_FAIL(json_array.get_raw_binary(str, &stored_row_buf_))) {
LOG_WARN("get result binary failed", K(ret));
} else {
concat_obj.set_string(ObJsonType, str);
}
}
}
return ret;
}
int ObAggregateFunction::get_json_objectagg_result(const ObAggregateExpression *&cexpr,
ObGroupConcatCtx *&cell_ctx,
ObObj &concat_obj)
{
UNUSED(cexpr);
int ret = OB_SUCCESS;
ObSEArray<ObSortColumn, 4> sort_columns;
common::ObArenaAllocator tmp_alloc;
ObGroupConcatRowStore *json_row_store = NULL;
if (OB_ISNULL(cell_ctx) || OB_ISNULL(expr_ctx_.my_session_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret));
} else if (OB_ISNULL(json_row_store = cell_ctx->gc_rs_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("group concat row store is NULL", K(ret));
} else if (json_row_store->is_allocate_sort() &&
OB_FAIL(json_row_store->get_sort_columns(sort_columns))) {
LOG_WARN("failed to get sort columns", K(ret));
} else {
//concat row may be iterated in rollup_process(), rewind here.
if (json_row_store->is_iterated()) {
if (OB_FAIL(json_row_store->rewind())) {
LOG_WARN("group concat row store rewind failed", K(ret));
}
} else {
if (OB_FAIL(json_row_store->finish_add_row())) {
LOG_WARN("finish add row to group concat row store failed", K(ret));
}
}
const ObNewRow *sort_row = NULL;
ObJsonObject json_object(&tmp_alloc);
while (OB_SUCC(ret) && OB_SUCC(json_row_store->get_next_row(sort_row))) {
if (OB_ISNULL(sort_row)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret), K(sort_row));
} else {
const ObObj &param_obj = sort_row->get_cell(0);
const ObObj *cast_para_obj = &param_obj;
ObObj tmp_obj;
ObObjType val_type0 = sort_row->get_cell(0).get_type();
ObCollationType cs_type0 = param_obj.get_collation_type();
if (val_type0 == ObNullType) {
ret = OB_ERR_JSON_DOCUMENT_NULL_KEY;
LOG_WARN("null type for json_objectagg key");
} else if (!param_obj.is_string_type()) {
// if not string convert to string
EXPR_DEFINE_CAST_CTX(expr_ctx_, CM_WARN_ON_FAIL);
cast_ctx.dest_collation_ = CS_TYPE_UTF8MB4_BIN;
if (OB_FAIL(ObObjCaster::to_type(ObLongTextType,
cast_ctx,
param_obj,
tmp_obj,
cast_para_obj))) {
LOG_WARN("cast failed.", K(ret), K(param_obj), K(ObLongTextType));
}
} else if (ob_is_string_type(param_obj.get_type()) && cs_type0 == CS_TYPE_BINARY) {
// not support binary charset as mysql
LOG_WARN("unsuport json string type with binary charset", K(param_obj.get_type()), K(cs_type0));
ret = OB_ERR_INVALID_JSON_CHARSET;
LOG_USER_ERROR(OB_ERR_INVALID_JSON_CHARSET);
}
ObString key_str = cast_para_obj->get_string();
if (OB_SUCC(ret) && ObCharset::charset_type_by_coll(cs_type0) != CHARSET_UTF8MB4) {
// string, other non-utf8 charsets
ObString converted_key_str;
if (OB_FAIL(ObExprUtil::convert_string_collation(key_str, cs_type0, converted_key_str,
CS_TYPE_UTF8MB4_BIN, tmp_alloc))) {
LOG_WARN("convert key string collation failed", K(ret), K(cs_type0), K(key_str.length()));
} else {
key_str = converted_key_str;
}
}
if (OB_FAIL(ret)) {
} else if (sort_row->get_count() < 2) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected column count", K(ret), K(sort_row->get_count()));
} else if (OB_SUCC(ret)) {
ObObjType val_type1 = sort_row->get_cell(1).get_type();
ObScale scale1 = sort_row->get_cell(1).get_scale();
ObCollationType cs_type1 = sort_row->get_cell(1).get_collation_type();
// get key and value, and append to wrapper
ObString key_data;
ObIJsonBase *json_val = NULL;
if (OB_FAIL(deep_copy_ob_string(tmp_alloc, key_str, key_data))) {
LOG_WARN("fail copy string", K(ret), K(key_str.length()));
} else {
ObObj converted_obj(sort_row->get_cell(1));
ObString converted_str;
if (ob_is_string_type(val_type1)
&& (ObCharset::charset_type_by_coll(cs_type1) != CHARSET_UTF8MB4)) {
ObString origin_str = converted_obj.get_string();
if (OB_FAIL(ObExprUtil::convert_string_collation(origin_str, cs_type1, converted_str,
CS_TYPE_UTF8MB4_BIN, tmp_alloc))) {
LOG_WARN("convert string collation failed", K(ret), K(cs_type1), K(origin_str.length()));
} else {
converted_obj.set_string(val_type1, converted_str);
converted_obj.set_collation_type(CS_TYPE_UTF8MB4_BIN);
cs_type1 = CS_TYPE_UTF8MB4_BIN;
}
}
if (OB_FAIL(ret)) {
} else if (ObJsonExprHelper::is_convertible_to_json(val_type1)) {
if (OB_FAIL(ObJsonExprHelper::transform_convertible_2jsonBase(converted_obj, val_type1,
&tmp_alloc, cs_type1,
json_val, false, true))) {
LOG_WARN("failed: parse value to jsonBase", K(ret), K(val_type1));
}
} else {
if (OB_FAIL(ObJsonExprHelper::transform_scalar_2jsonBase(converted_obj, val_type1,
&tmp_alloc, val_type1,
expr_ctx_.my_session_->get_timezone_info(),
json_val, false))) {
LOG_WARN("failed: parse value to jsonBase", K(ret), K(val_type1));
}
}
if (OB_FAIL(ret)) {
}else if (OB_FAIL(json_object.object_add(key_data, json_val))) {
LOG_WARN("failed: json object add json value", K(ret));
} else if(json_object.get_serialize_size() > OB_MAX_PACKET_LENGTH) {
ret = OB_ERR_TOO_LONG_STRING_IN_CONCAT;
LOG_WARN("result of json_arrayagg is too long", K(ret), K(json_object.get_serialize_size()),
K(OB_MAX_PACKET_LENGTH));
}
}
}
}
}
if (ret == OB_ERR_JSON_DOCUMENT_NULL_KEY) {
LOG_USER_ERROR(OB_ERR_JSON_DOCUMENT_NULL_KEY);
}
if (ret != OB_ITER_END && ret != OB_SUCCESS) {
LOG_WARN("fail to get next row", K(ret));
} else {
ret = OB_SUCCESS;
ObString str;
// output res
if (OB_FAIL(json_object.get_raw_binary(str, &stored_row_buf_))) {
LOG_WARN("get result binary failed", K(ret));
} else {
concat_obj.set_string(ObJsonType, str);
}
}
}
return ret;
}
} // namespace sql
} // namespace oceanbase

8
src/sql/engine/aggregate/ob_aggregate_function.h
View File

@ -89,7 +89,7 @@ public:
~ObGroupConcatRowStore();
int init(const uint64_t tenant_id, const common::ObIArray<ObSortColumn>& sort_columns,
const ObSortImpl::SortExtraInfos* extra_infos, const bool rewind);
const ObSortImpl::SortExtraInfos* extra_infos, const bool rewind, int64_t dir_id);
void reuse();
int add_row(const common::ObNewRow& row)
@ -190,6 +190,7 @@ public:
~ObAggregateFunction();
void set_int_div_as_double(bool did);
bool get_int_div_as_double() const;
void set_dir_id(int64_t dir_id) { dir_id_ = dir_id; }
void set_sort_based_gby()
{
is_sort_based_gby_ = true;
@ -299,6 +300,10 @@ private:
int get_wm_concat_result(
const ObAggregateExpression*& cexpr, ObGroupConcatCtx*& cell_ctx, bool is_keep_group_concat, ObObj& concat_obj);
int get_json_arrayagg_result(const ObAggregateExpression *&cexpr, ObGroupConcatCtx *&cell_ctx,
ObObj &concat_obj);
int get_json_objectagg_result(const ObAggregateExpression *&cexpr, ObGroupConcatCtx *&cell_ctx,
ObObj &concat_obj);
// disallow copy
DISALLOW_COPY_AND_ASSIGN(ObAggregateFunction);
typedef common::hash::ObHashSet<ObAggregateDistinctItem, common::hash::NoPthreadDefendMode> AggrDistinctBucket;
@ -341,6 +346,7 @@ private:
common::ObArenaAllocator agg_udf_buf_;
common::ObSEArray<ObAggUdfMeta, 16> agg_udf_metas_;
common::hash::ObHashMap<int64_t, ObAggUdfExeUnit, common::hash::NoPthreadDefendMode> agg_udf_;
int64_t dir_id_;
};
inline void ObAggregateFunction::set_int_div_as_double(bool did)

383
src/sql/engine/aggregate/ob_aggregate_processor.cpp
View File

@ -24,6 +24,7 @@
#include "sql/parser/ob_item_type_str.h"
#include "sql/engine/expr/ob_expr_util.h"
#include "sql/engine/sort/ob_sort_op_impl.h"
#include "sql/engine/expr/ob_expr_json_func_helper.h"
namespace oceanbase {
using namespace common;
@ -240,8 +241,8 @@ int ObAggregateProcessor::ExtraResult::init_distinct_set(
return ret;
}
int ObAggregateProcessor::GroupConcatExtraResult::init(
const uint64_t tenant_id, const ObAggrInfo& aggr_info, ObEvalCtx& eval_ctx, const bool need_rewind)
int ObAggregateProcessor::GroupConcatExtraResult::init(const uint64_t tenant_id,
const ObAggrInfo& aggr_info, ObEvalCtx& eval_ctx, const bool need_rewind, int64_t dir_id)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(OB_INVALID_ID == tenant_id)) {
@ -272,6 +273,8 @@ int ObAggregateProcessor::GroupConcatExtraResult::init(
ObModIds::OB_SQL_AGGR_FUN_GROUP_CONCAT,
true /* enable dump */))) {
LOG_WARN("row store failed", K(ret));
} else {
row_store_.set_dir_id(dir_id);
}
}
}
@ -305,6 +308,7 @@ void ObAggregateProcessor::GroupConcatExtraResult::reuse_self()
row_store_iter_.reset();
row_store_.reset();
}
bool_mark_.reset();
row_count_ = 0;
iter_idx_ = 0;
};
@ -354,6 +358,33 @@ int64_t ObAggregateProcessor::GroupConcatExtraResult::to_string(char* buf, const
return pos;
}
int ObAggregateProcessor::GroupConcatExtraResult::get_bool_mark(int64_t col_index, bool &is_bool)
{
INIT_SUCC(ret);
if (col_index + 1 > bool_mark_.count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("index is overflow", K(ret), K(col_index), K(bool_mark_.count()));
} else {
is_bool = bool_mark_[col_index];
}
return ret;
}
int ObAggregateProcessor::GroupConcatExtraResult::set_bool_mark(int64_t col_index, bool is_bool)
{
INIT_SUCC(ret);
if (col_index > bool_mark_.count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("index is overflow", K(ret), K(col_index), K(bool_mark_.count()));
} else if (col_index == bool_mark_.count()) {
bool_mark_.push_back(is_bool);
} else {
bool_mark_[col_index] = is_bool;
}
return ret;
}
int64_t ObAggregateProcessor::ExtraResult::to_string(char* buf, const int64_t buf_len) const
{
int64_t pos = 0;
@ -389,7 +420,8 @@ ObAggregateProcessor::ObAggregateProcessor(ObEvalCtx& eval_ctx, ObIArray<ObAggrI
group_rows_(),
concat_str_max_len_(OB_DEFAULT_GROUP_CONCAT_MAX_LEN_FOR_ORACLE),
cur_concat_buf_len_(0),
concat_str_buf_(NULL)
concat_str_buf_(NULL),
dir_id_(-1)
{}
int ObAggregateProcessor::init()
@ -416,8 +448,11 @@ int ObAggregateProcessor::init()
} else {
has_distinct_ |= aggr_info.has_distinct_;
has_group_concat_ |=
(T_FUN_GROUP_CONCAT == aggr_info.get_expr_type() || T_FUN_KEEP_WM_CONCAT == aggr_info.get_expr_type() ||
T_FUN_WM_CONCAT == aggr_info.get_expr_type());
(T_FUN_GROUP_CONCAT == aggr_info.get_expr_type() ||
T_FUN_KEEP_WM_CONCAT == aggr_info.get_expr_type() ||
T_FUN_WM_CONCAT == aggr_info.get_expr_type() ||
T_FUN_JSON_ARRAYAGG == aggr_info.get_expr_type() ||
T_FUN_JSON_OBJECTAGG == aggr_info.get_expr_type());
has_order_by_ |= aggr_info.has_order_by_;
if (!has_extra_) {
has_extra_ |= aggr_info.has_distinct_;
@ -935,7 +970,9 @@ int ObAggregateProcessor::init_one_group(const int64_t group_id)
case T_FUN_KEEP_SUM:
case T_FUN_KEEP_COUNT:
case T_FUN_KEEP_WM_CONCAT:
case T_FUN_WM_CONCAT: {
case T_FUN_WM_CONCAT:
case T_FUN_JSON_ARRAYAGG:
case T_FUN_JSON_OBJECTAGG: {
void* tmp_buf = NULL;
if (OB_ISNULL(tmp_buf = aggr_alloc_.alloc(sizeof(GroupConcatExtraResult)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
@ -948,7 +985,7 @@ int ObAggregateProcessor::init_one_group(const int64_t group_id)
if (OB_FAIL(result->init(eval_ctx_.exec_ctx_.get_my_session()->get_effective_tenant_id(),
aggr_info,
eval_ctx_,
need_rewind))) {
need_rewind, dir_id_))) {
LOG_WARN("init GroupConcatExtraResult failed", K(ret));
} else if (aggr_info.separator_expr_ != NULL) {
ObDatum* separator_result = NULL;
@ -1142,7 +1179,9 @@ int ObAggregateProcessor::rollup_aggregation(
case T_FUN_KEEP_MIN:
case T_FUN_KEEP_SUM:
case T_FUN_KEEP_WM_CONCAT:
case T_FUN_WM_CONCAT: {
case T_FUN_WM_CONCAT:
case T_FUN_JSON_ARRAYAGG:
case T_FUN_JSON_OBJECTAGG: {
GroupConcatExtraResult* aggr_extra = NULL;
GroupConcatExtraResult* rollup_extra = NULL;
if (OB_ISNULL(aggr_extra = static_cast<GroupConcatExtraResult*>(aggr_cell.get_extra())) ||
@ -1153,6 +1192,20 @@ int ObAggregateProcessor::rollup_aggregation(
LOG_WARN("finish add row failed", K(ret));
} else {
const ObChunkDatumStore::StoredRow* stored_row = NULL;
if (aggr_fun == T_FUN_JSON_ARRAYAGG || aggr_fun == T_FUN_JSON_OBJECTAGG) {
int64_t len = aggr_extra->get_bool_mark_size();
if (OB_FAIL(rollup_extra->reserve_bool_mark_count(len))) {
LOG_WARN("reserve_bool_mark_count failed", K(ret), K(len));
}
for (int64_t i = 0; OB_SUCC(ret) && i < len; i++) {
bool is_bool = false;
if (OB_FAIL(aggr_extra->get_bool_mark(i, is_bool))) {
LOG_WARN("get_bool_mark failed", K(ret));
} else if (OB_FAIL(rollup_extra->set_bool_mark(i, is_bool))) {
LOG_WARN("set_bool_mark failed", K(ret));
}
}
}
while (OB_SUCC(ret) && OB_SUCC(aggr_extra->get_next_row(stored_row))) {
if (OB_ISNULL(stored_row)) {
ret = OB_ERR_UNEXPECTED;
@ -1316,7 +1369,9 @@ int ObAggregateProcessor::prepare_aggr_result(const ObChunkDatumStore::StoredRow
case T_FUN_KEEP_COUNT:
case T_FUN_KEEP_SUM:
case T_FUN_KEEP_WM_CONCAT:
case T_FUN_WM_CONCAT: {
case T_FUN_WM_CONCAT:
case T_FUN_JSON_ARRAYAGG:
case T_FUN_JSON_OBJECTAGG: {
GroupConcatExtraResult* extra = NULL;
if (OB_ISNULL(extra = static_cast<GroupConcatExtraResult*>(aggr_cell.get_extra()))) {
ret = OB_ERR_UNEXPECTED;
@ -1328,6 +1383,23 @@ int ObAggregateProcessor::prepare_aggr_result(const ObChunkDatumStore::StoredRow
} else if (param_exprs != NULL && OB_FAIL(extra->add_row(*param_exprs, eval_ctx_))) {
LOG_WARN("fail to add row", K(ret));
} else {
if (aggr_fun == T_FUN_JSON_ARRAYAGG || aggr_fun == T_FUN_JSON_OBJECTAGG) {
int64_t len = param_exprs->count();
if (OB_FAIL(extra->reserve_bool_mark_count(len))) {
LOG_WARN("reserve_bool_mark_count failed", K(ret), K(len));
}
for (int64_t i = 0; OB_SUCC(ret) && i < len; i++) {
ObExpr *tmp = NULL;
if (OB_FAIL(param_exprs->at(i, tmp))){
LOG_WARN("fail to get param_exprs[i]", K(ret));
} else {
bool is_bool = (tmp->is_boolean_ == 1);
if (OB_FAIL(extra->set_bool_mark(i, is_bool))){
LOG_WARN("fail to set_bool_mark", K(ret));
}
}
}
}
LOG_DEBUG("succ to add row", K(stored_row), KPC(extra));
}
}
@ -1482,7 +1554,9 @@ int ObAggregateProcessor::process_aggr_result(const ObChunkDatumStore::StoredRow
case T_FUN_KEEP_COUNT:
case T_FUN_KEEP_SUM:
case T_FUN_KEEP_WM_CONCAT:
case T_FUN_WM_CONCAT: {
case T_FUN_WM_CONCAT:
case T_FUN_JSON_ARRAYAGG:
case T_FUN_JSON_OBJECTAGG: {
GroupConcatExtraResult* extra = NULL;
if (OB_ISNULL(extra = static_cast<GroupConcatExtraResult*>(aggr_cell.get_extra()))) {
ret = OB_ERR_UNEXPECTED;
@ -1653,7 +1727,24 @@ int ObAggregateProcessor::collect_aggr_result(AggrCell& aggr_cell, const ObExpr*
}
break;
}
case T_FUN_JSON_ARRAYAGG: {
GroupConcatExtraResult *extra = static_cast<GroupConcatExtraResult *>(aggr_cell.get_extra());
if (OB_FAIL(get_json_arrayagg_result(aggr_info, extra, result))) {
LOG_WARN("failed to get json_arrayagg result", K(ret));
} else {
eval_info.evaluated_ = true;
}
break;
}
case T_FUN_JSON_OBJECTAGG: {
GroupConcatExtraResult *extra = static_cast<GroupConcatExtraResult *>(aggr_cell.get_extra());
if (OB_FAIL(get_json_objectagg_result(aggr_info, extra, result))) {
LOG_WARN("failed to get json_objectagg result", K(ret));
} else {
eval_info.evaluated_ = true;
}
break;
}
case T_FUN_WM_CONCAT:
case T_FUN_KEEP_WM_CONCAT: {
GroupConcatExtraResult* extra = static_cast<GroupConcatExtraResult*>(aggr_cell.get_extra());
@ -3173,5 +3264,275 @@ int ObAggregateProcessor::get_wm_concat_result(
return ret;
}
int ObAggregateProcessor::convert_datum_to_obj(const ObAggrInfo &aggr_info,
const ObChunkDatumStore::StoredRow &stored_row,
ObObj *tmp_obj,
int64_t obj_cnt)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(obj_cnt != stored_row.cnt_ || obj_cnt != aggr_info.param_exprs_.count())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected error", K(ret), K(obj_cnt), K(stored_row.cnt_),
K(aggr_info.param_exprs_.count()));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < stored_row.cnt_; ++i) {
if (OB_FAIL(stored_row.cells()[i].to_obj(tmp_obj[i],
aggr_info.param_exprs_.at(i)->obj_meta_))) {
LOG_WARN("failed to obj", K(ret));
} else {/*do nothing*/}
}
}
return ret;
}
int ObAggregateProcessor::get_json_arrayagg_result(const ObAggrInfo &aggr_info,
GroupConcatExtraResult *&extra,
ObDatum &concat_result)
{
int ret = OB_SUCCESS;
common::ObArenaAllocator tmp_alloc;
if (OB_ISNULL(extra) || OB_UNLIKELY(extra->empty())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unpexcted null", K(ret), K(extra));
} else if (extra->is_iterated() && OB_FAIL(extra->rewind())) {
// Group concat row may be iterated in rollup_process(), rewind here.
LOG_WARN("rewind failed", KPC(extra), K(ret));
} else if (!extra->is_iterated() && OB_FAIL(extra->finish_add_row())) {
LOG_WARN("finish_add_row failed", KPC(extra), K(ret));
} else {
const ObChunkDatumStore::StoredRow *storted_row = NULL;
ObJsonArray json_array(&tmp_alloc);
bool is_bool = false;
if (OB_FAIL(extra->get_bool_mark(0, is_bool))) {
LOG_WARN("get_bool info failed, may not distinguish between bool and int", K(ret));
}
while (OB_SUCC(ret) && OB_SUCC(extra->get_next_row(storted_row))) {
if (OB_ISNULL(storted_row)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret), K(storted_row));
} else {
// get type
ObObj *tmp_obj = NULL;
if (OB_ISNULL(tmp_obj = static_cast<ObObj*>(tmp_alloc.alloc(
sizeof(ObObj) * (storted_row->cnt_))))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate memory", K(ret), K(tmp_obj));
} else if (OB_FAIL(convert_datum_to_obj(aggr_info, *storted_row, tmp_obj, storted_row->cnt_))) {
LOG_WARN("failed to convert datum to obj", K(ret));
} else if (storted_row->cnt_ < 1) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected column count", K(ret), K(storted_row));
} else {
ObObjType val_type = tmp_obj->get_type();
ObCollationType cs_type = tmp_obj->get_collation_type();
ObScale scale = tmp_obj->get_scale();
ObIJsonBase *json_val = NULL;
ObDatum converted_datum;
converted_datum.set_datum(storted_row->cells()[0]);
// convert string charset if needed
if (ob_is_string_type(val_type)
&& (ObCharset::charset_type_by_coll(cs_type) != CHARSET_UTF8MB4)) {
ObString origin_str = converted_datum.get_string();
ObString converted_str;
if (OB_FAIL(ObExprUtil::convert_string_collation(origin_str, cs_type, converted_str,
CS_TYPE_UTF8MB4_BIN, tmp_alloc))) {
LOG_WARN("convert string collation failed", K(ret), K(cs_type), K(origin_str.length()));
} else {
converted_datum.set_string(converted_str);
cs_type = CS_TYPE_UTF8MB4_BIN;
}
}
// get json value
if (OB_FAIL(ret)) {
} else if (is_bool) {
void *json_node_buf = tmp_alloc.alloc(sizeof(ObJsonBoolean));
if (OB_ISNULL(json_node_buf)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed: allocate jsonboolean", K(ret));
} else {
ObJsonBoolean *bool_node = (ObJsonBoolean*)new(json_node_buf)ObJsonBoolean(converted_datum.get_bool());
json_val = bool_node;
}
} else if (ObJsonExprHelper::is_convertible_to_json(val_type)) {
if (OB_FAIL(ObJsonExprHelper::transform_convertible_2jsonBase(converted_datum, val_type,
&tmp_alloc, cs_type,
json_val, false, true))) {
LOG_WARN("failed: parse value to jsonBase", K(ret), K(val_type));
}
} else {
if (OB_FAIL(ObJsonExprHelper::transform_scalar_2jsonBase(converted_datum, val_type,
&tmp_alloc, scale,
eval_ctx_.exec_ctx_.get_my_session()->get_timezone_info(),
json_val, false))) {
LOG_WARN("failed: parse value to jsonBase", K(ret), K(val_type));
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(json_array.array_append(json_val))) {
LOG_WARN("failed: json array append json value", K(ret));
} else if(json_array.get_serialize_size() > OB_MAX_PACKET_LENGTH) {
ret = OB_ERR_TOO_LONG_STRING_IN_CONCAT;
LOG_WARN("result of json_arrayagg is too long", K(ret), K(json_array.get_serialize_size()),
K(OB_MAX_PACKET_LENGTH));
}
}
}
}//end of while
if (ret != OB_ITER_END && ret != OB_SUCCESS) {
LOG_WARN("fail to get next row", K(ret));
} else {
ret = OB_SUCCESS;
ObString str;
// output res
if (OB_FAIL(json_array.get_raw_binary(str, &aggr_alloc_))) {
LOG_WARN("get result binary failed", K(ret));
} else {
concat_result.set_string(str);
}
}
}
return ret;
}
int ObAggregateProcessor::get_json_objectagg_result(const ObAggrInfo &aggr_info,
GroupConcatExtraResult *&extra,
ObDatum &concat_result)
{
int ret = OB_SUCCESS;
const int col_num = 2;
common::ObArenaAllocator tmp_alloc;
if (OB_ISNULL(extra) || OB_UNLIKELY(extra->empty())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unpexcted null", K(ret), K(extra));
} else if (extra->is_iterated() && OB_FAIL(extra->rewind())) {
// Group concat row may be iterated in rollup_process(), rewind here.
LOG_WARN("rewind failed", KPC(extra), K(ret));
} else if (!extra->is_iterated() && OB_FAIL(extra->finish_add_row())) {
LOG_WARN("finish_add_row failed", KPC(extra), K(ret));
} else {
const ObChunkDatumStore::StoredRow *storted_row = NULL;
ObJsonObject json_object(&tmp_alloc);
ObObj tmp_obj[col_num];
bool is_bool = false;
if (OB_FAIL(extra->get_bool_mark(1, is_bool))) {
LOG_WARN("get_bool info failed, may not distinguish between bool and int", K(ret));
}
while (OB_SUCC(ret) && OB_SUCC(extra->get_next_row(storted_row))) {
if (OB_ISNULL(storted_row)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret), K(storted_row));
} else if(storted_row->cnt_ != col_num) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected column count", K(ret), K(storted_row->cnt_));
} else {
// get obj
if (OB_FAIL(convert_datum_to_obj(aggr_info, *storted_row, tmp_obj, storted_row->cnt_))) {
LOG_WARN("failed to convert datum to obj", K(ret));
} else if (tmp_obj[0].get_type() == ObNullType) {
ret = OB_ERR_JSON_DOCUMENT_NULL_KEY;
LOG_WARN("null type for json_objectagg key");
} else if (ob_is_string_type(tmp_obj[0].get_type())
&& tmp_obj[0].get_collation_type() == CS_TYPE_BINARY) {
// not support binary charset as mysql
LOG_WARN("unsuport json string type with binary charset",
K(tmp_obj[0].get_type()), K(tmp_obj[0].get_collation_type()));
ret = OB_ERR_INVALID_JSON_CHARSET;
LOG_USER_ERROR(OB_ERR_INVALID_JSON_CHARSET);
} else {
ObObjType val_type0 = tmp_obj[0].get_type();
ObCollationType cs_type0 = tmp_obj[0].get_collation_type();
ObObjType val_type1 = tmp_obj[1].get_type();
ObScale scale1 = tmp_obj[1].get_scale();
ObCollationType cs_type1 = tmp_obj[1].get_collation_type();
ObString key_string = tmp_obj[0].get_string();
if (OB_SUCC(ret) && ObCharset::charset_type_by_coll(cs_type0) != CHARSET_UTF8MB4) {
ObString converted_key_str;
if (OB_FAIL(ObExprUtil::convert_string_collation(key_string, cs_type0, converted_key_str,
CS_TYPE_UTF8MB4_BIN, tmp_alloc))) {
LOG_WARN("convert key string collation failed", K(ret), K(cs_type0), K(key_string.length()));
} else {
key_string = converted_key_str;
}
}
// get key and value, and append to json_object
ObString key_data;
ObIJsonBase *json_val = NULL;
if (OB_SUCC(ret) && OB_FAIL(deep_copy_ob_string(tmp_alloc, key_string, key_data))) {
LOG_WARN("fail copy string", K(ret), K(key_string.length()));
} else {
ObDatum converted_datum;
converted_datum.set_datum(storted_row->cells()[1]);
if (ob_is_string_type(val_type1)
&& (ObCharset::charset_type_by_coll(cs_type1) != CHARSET_UTF8MB4)) {
ObString origin_str = converted_datum.get_string();
ObString converted_str;
if (OB_FAIL(ObExprUtil::convert_string_collation(origin_str, cs_type1, converted_str,
CS_TYPE_UTF8MB4_BIN, tmp_alloc))) {
LOG_WARN("convert string collation failed", K(ret), K(cs_type1), K(origin_str.length()));
} else {
converted_datum.set_string(converted_str);
cs_type1 = CS_TYPE_UTF8MB4_BIN;
}
}
if (OB_FAIL(ret)) {
} else if (is_bool) {
void *json_node_buf = tmp_alloc.alloc(sizeof(ObJsonBoolean));
if (OB_ISNULL(json_node_buf)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed: allocate jsonboolean", K(ret));
} else {
ObJsonBoolean *bool_node = (ObJsonBoolean*)new(json_node_buf)ObJsonBoolean(storted_row->cells()[1].get_bool());
json_val = bool_node;
}
} else if (ObJsonExprHelper::is_convertible_to_json(val_type1)) {
if (OB_FAIL(ObJsonExprHelper::transform_convertible_2jsonBase(converted_datum, val_type1,
&tmp_alloc, cs_type1,
json_val, false, true))) {
LOG_WARN("failed: parse value to jsonBase", K(ret), K(val_type1));
}
} else {
if (OB_FAIL(ObJsonExprHelper::transform_scalar_2jsonBase(converted_datum, val_type1,
&tmp_alloc, scale1,
eval_ctx_.exec_ctx_.get_my_session()->get_timezone_info(),
json_val, false))) {
LOG_WARN("failed: parse value to jsonBase", K(ret), K(val_type1));
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(json_object.object_add(key_data, json_val))) {
LOG_WARN("failed: json object add json value", K(ret));
} else if(json_object.get_serialize_size() > OB_MAX_PACKET_LENGTH) {
ret = OB_ERR_TOO_LONG_STRING_IN_CONCAT;
LOG_WARN("result of json_objectagg is too long", K(ret), K(json_object.get_serialize_size()),
K(OB_MAX_PACKET_LENGTH));
}
}
}
}
}//end of while
if (ret == OB_ERR_JSON_DOCUMENT_NULL_KEY) {
LOG_USER_ERROR(OB_ERR_JSON_DOCUMENT_NULL_KEY);
}
if (ret != OB_ITER_END && ret != OB_SUCCESS) {
LOG_WARN("fail to get next row", K(ret));
} else {
ret = OB_SUCCESS;
ObString str;
// output res
if (OB_FAIL(json_object.get_raw_binary(str, &aggr_alloc_))) {
LOG_WARN("get result binary failed", K(ret));
} else {
concat_result.set_string(str);
}
}
}
return ret;
}
} // namespace sql
} // namespace oceanbase

25
src/sql/engine/aggregate/ob_aggregate_processor.h
View File

@ -158,13 +158,14 @@ public:
class GroupConcatExtraResult : public ExtraResult {
public:
explicit GroupConcatExtraResult(common::ObIAllocator& alloc)
: ExtraResult(alloc), row_count_(0), iter_idx_(0), sort_op_(NULL), separator_datum_(NULL)
: ExtraResult(alloc), row_count_(0), iter_idx_(0), sort_op_(NULL), separator_datum_(NULL), bool_mark_(alloc)
{}
virtual ~GroupConcatExtraResult();
void reuse_self();
virtual void reuse() override;
int init(const uint64_t tenant_id, const ObAggrInfo& aggr_info, ObEvalCtx& eval_ctx, const bool need_rewind);
int init(const uint64_t tenant_id, const ObAggrInfo& aggr_info,
ObEvalCtx& eval_ctx, const bool need_rewind, int64_t dir_id_);
int add_row(const ObIArray<ObExpr*>& expr, ObEvalCtx& eval_ctx)
{
@ -207,6 +208,10 @@ public:
return row_count_;
}
ObDatum *&get_separator_datum() { return separator_datum_; }
int reserve_bool_mark_count(int64_t count) { return bool_mark_.reserve(count); }
int get_bool_mark_size() { return bool_mark_.count(); }
int get_bool_mark(int64_t col_index, bool &is_bool);
int set_bool_mark(int64_t col_index, bool is_bool);
DECLARE_VIRTUAL_TO_STRING;
private:
@ -218,6 +223,7 @@ public:
ObSortOpImpl* sort_op_;
ObDatum *separator_datum_;
common::ObFixedArray<bool, common::ObIAllocator> bool_mark_;
};
class AggrCell {
@ -332,7 +338,6 @@ public:
uint64_t tiny_num_uint_;
};
bool is_tiny_num_used_;
// for T_FUN_APPROX_COUNT_DISTINCT
ObDatum llc_bitmap_;
@ -381,6 +386,7 @@ public:
void destroy();
void reuse();
void set_dir_id(int64_t dir_id) { dir_id_ = dir_id; }
int prepare(GroupRow& group_row);
int process(GroupRow& group_row);
int collect(const int64_t group_id = 0, const ObExpr* diff_expr = NULL);
@ -464,6 +470,14 @@ private:
const ObAggrInfo& aggr_info, bool& is_equal);
int get_wm_concat_result(
const ObAggrInfo& aggr_info, GroupConcatExtraResult*& extra, bool is_keep_group_concat, ObDatum& concat_result);
int convert_datum_to_obj(const ObAggrInfo &aggr_info,
const ObChunkDatumStore::StoredRow &stored_row,
ObObj *tmp_obj,
int64_t obj_cnt);
int get_json_arrayagg_result(const ObAggrInfo &aggr_info, GroupConcatExtraResult *&extra,
ObDatum &concat_result);
int get_json_objectagg_result(const ObAggrInfo &aggr_info, GroupConcatExtraResult *&extra,
ObDatum &concat_result);
// HyperLogLogCount-related functions
int llc_init(ObDatum& datum);
@ -501,6 +515,7 @@ private:
uint64_t concat_str_max_len_;
uint64_t cur_concat_buf_len_;
char* concat_str_buf_;
int64_t dir_id_;
// common::ObArenaAllocator aggr_udf_buf_;
// common::ObSEArray<ObAggUdfMeta, 16> aggr_udf_metas_;
// common::hash::ObHashMap<int64_t, ObAggUdfExeUnit, common::hash::NoPthreadDefendMode> aggr_udf_;
@ -578,7 +593,9 @@ OB_INLINE bool ObAggregateProcessor::need_extra_info(const ObExprOperatorType ex
case T_FUN_KEEP_SUM:
case T_FUN_KEEP_COUNT:
case T_FUN_KEEP_WM_CONCAT:
case T_FUN_WM_CONCAT: {
case T_FUN_WM_CONCAT:
case T_FUN_JSON_ARRAYAGG:
case T_FUN_JSON_OBJECTAGG: {
need_extra = true;
break;
}

6
src/sql/engine/aggregate/ob_groupby.cpp
View File

@ -87,7 +87,8 @@ int ObGroupBy::init_group_by(ObExecContext& ctx) const
int64_t optimizer_est_input_rows = 0;
ObExprCtx expr_ctx;
ObGroupByCtx* groupby_ctx = NULL;
int64_t dir_id = -1;
if (OB_FAIL(init_op_ctx(ctx))) {
LOG_WARN("init operator context failed", K(ret));
} else if (OB_FAIL(wrap_expr_ctx(ctx, expr_ctx))) {
@ -98,6 +99,9 @@ int ObGroupBy::init_group_by(ObExecContext& ctx) const
} else if (OB_ISNULL(my_phy_plan_) || OB_ISNULL(child_op_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K_(my_phy_plan), K_(child_op));
} else if (ObChunkStoreUtil::alloc_dir_id(dir_id)) {
LOG_WARN("failed to alloc dir id", K(ret));
} else if (FALSE_IT(groupby_ctx->get_aggr_func().set_dir_id(dir_id))) {
} else if (OB_FAIL(ObSQLUtils::get_default_cast_mode(my_phy_plan_, expr_ctx.my_session_, expr_ctx.cast_mode_))) {
LOG_WARN("set cast mode failed", K(ret));
} else if (OB_ISNULL(get_child(0))) {

1
src/sql/engine/aggregate/ob_hash_groupby_op.cpp
View File

@ -87,6 +87,7 @@ int ObHashGroupByOp::inner_open()
MY_SPEC.id_,
&ctx_))) {
LOG_WARN("failed to init sql mem processor", K(ret));
} else if (FALSE_IT(aggr_processor_.set_dir_id(sql_mem_processor_.get_dir_id()))) {
} else if (FALSE_IT(init_size = estimate_hash_bucket_cnt_by_mem_size(est_group_cnt,
sql_mem_processor_.get_mem_bound(),
est_hash_mem_size * 1. / estimate_mem_size))) {

5
src/sql/engine/aggregate/ob_merge_groupby_op.cpp
View File

@ -72,7 +72,10 @@ int ObMergeGroupByOp::init()
int ret = OB_SUCCESS;
const int64_t col_count =
(MY_SPEC.has_rollup_ ? (MY_SPEC.group_exprs_.count() + MY_SPEC.rollup_exprs_.count() + 1) : 0);
if (OB_FAIL(aggr_processor_.init_group_rows(col_count))) {
if (OB_FAIL(ObChunkStoreUtil::alloc_dir_id(dir_id_))) {
LOG_WARN("failed to alloc dir id", K(ret));
} else if (FALSE_IT(aggr_processor_.set_dir_id(dir_id_))) {
} else if (OB_FAIL(aggr_processor_.init_group_rows(col_count))) {
LOG_WARN("failed to initialize init_group_rows", K(ret));
} else if (OB_FAIL(
curr_groupby_datums_.prepare_allocate(MY_SPEC.group_exprs_.count() + MY_SPEC.rollup_exprs_.count()))) {

4
src/sql/engine/aggregate/ob_merge_groupby_op.h
View File

@ -71,7 +71,8 @@ public:
cur_output_group_id_(common::OB_INVALID_INDEX),
first_output_group_id_(0),
last_child_output_(aggr_processor_.get_aggr_alloc()),
curr_groupby_datums_(aggr_processor_.get_aggr_alloc())
curr_groupby_datums_(aggr_processor_.get_aggr_alloc()),
dir_id_(-1)
{}
void reset();
virtual int inner_open() override;
@ -98,6 +99,7 @@ private:
int64_t first_output_group_id_;
ObChunkDatumStore::LastStoredRow<> last_child_output_;
DatumFixedArray curr_groupby_datums_;
int64_t dir_id_;
};
} // end namespace sql

3
src/sql/engine/aggregate/ob_scalar_aggregate_op.cpp
View File

@ -25,6 +25,9 @@ int ObScalarAggregateOp::inner_open()
int ret = OB_SUCCESS;
if (OB_FAIL(ObGroupByOp::inner_open())) {
LOG_WARN("failed to inner_open", K(ret));
} else if (OB_FAIL(ObChunkStoreUtil::alloc_dir_id(dir_id_))) {
LOG_WARN("failed to alloc dir id", K(ret));
} else if (FALSE_IT(aggr_processor_.set_dir_id(dir_id_))) {
} else if (OB_FAIL(aggr_processor_.init_one_group())) {
LOG_WARN("failed to init one group", K(ret));
} else {

1
src/sql/engine/aggregate/ob_scalar_aggregate_op.h
View File

@ -50,6 +50,7 @@ private:
private:
bool started_;
int64_t dir_id_;
};
} // end namespace sql