hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[FEAT MERGE] Roaringbitmap type phase-2, performance optimization

Browse Source
This commit is contained in:
qijiax
2024-08-28 03:45:02 +00:00
committed by ob-robot
parent e7cd603e19
commit c76fda0bef
25 changed files with 1958 additions and 770 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/sql/engine/expr/ob_expr_rb_build_empty.cpp
View File

@ -52,18 +52,12 @@ int ObExprRbBuildEmpty::eval_rb_build_empty(const ObExpr &expr,
ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx);
common::ObArenaAllocator &tmp_allocator = tmp_alloc_g.get_allocator();
lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(ObRbExprHelper::get_tenant_id(ctx.exec_ctx_.get_my_session()), "ROARINGBITMAP"));
ObString rb_bin;
ObRoaringBitmap *rb_empty;
if (OB_ISNULL(rb_empty = OB_NEWx(ObRoaringBitmap, &tmp_allocator, (&tmp_allocator)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to create alloc memory to roaringbitmap", K(ret));
} else if (OB_FAIL(ObRbUtils::rb_serialize(tmp_allocator, rb_bin, rb_empty))) {
LOG_WARN("failed to serialize empty roaringbitmap", K(ret));
if (OB_FAIL(ObRbUtils::build_empty_binary(tmp_allocator, rb_bin))) {
LOG_WARN("fail to build empty rb binary", K(ret));
} else if (OB_FAIL(ObRbExprHelper::pack_rb_res(expr, ctx, res, rb_bin))) {
LOG_WARN("fail to pack roaringbitmap res", K(ret));
}
ObRbUtils::rb_destroy(rb_empty);
return ret;
}

13
src/sql/engine/expr/ob_expr_rb_build_varbinary.cpp
View File

@ -69,21 +69,14 @@ int ObExprRbBuildVarbinary::eval_rb_build_varbinary(const ObExpr &expr,
ObExpr *rb_arg = expr.args_[0];
bool is_null_result = false;
bool is_rb_null = false;
ObString rb_bin = nullptr;
ObString res_rb_bin = nullptr;
ObString rb_bin;
ObString res_rb_bin;
if (OB_FAIL(ObRbExprHelper::get_input_roaringbitmap_bin(ctx, tmp_allocator, rb_arg, rb_bin, is_rb_null))) {
LOG_WARN("fail to get input roaringbitmap", K(ret));
} else if (is_rb_null || rb_bin == nullptr) {
is_null_result = true;
} else if (OB_FAIL(ObRbUtils::build_binary(tmp_allocator, rb_bin, res_rb_bin))) {
LOG_WARN("failed to build rb binary", K(ret));
}
if (OB_FAIL(ret)) {
} else if (is_null_result) {
res.set_null();
} else if (OB_FAIL(ObRbExprHelper::pack_rb_res(expr, ctx, res, res_rb_bin))) {
} else if (OB_FAIL(ObRbExprHelper::pack_rb_res(expr, ctx, res, rb_bin))) {
LOG_WARN("fail to pack roaringbitmap res", K(ret));
}

75
src/sql/engine/expr/ob_expr_rb_calc.cpp
View File

@ -69,39 +69,62 @@ int ObExprRbCalc::eval_rb_calc(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res,
bool is_rb1_null = false;
bool is_rb2_null = false;
bool is_res_null = false;
ObRoaringBitmap *rb1 = nullptr;
ObRoaringBitmap *rb2 = nullptr;
ObString rb_res;
if (OB_FAIL(ObRbExprHelper::get_input_roaringbitmap(ctx, tmp_allocator, rb1_arg, rb1, is_rb1_null))) {
LOG_WARN("failed to get left input roaringbitmap", K(ret));
} else if (is_rb1_null && !is_null2empty) {
is_res_null = true;
} else if (is_rb1_null && is_null2empty
&& OB_ISNULL(rb1 = OB_NEWx(ObRoaringBitmap, &tmp_allocator, (&tmp_allocator)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to create alloc memory to roaringbitmap", K(ret));
} else if (OB_FAIL(ObRbExprHelper::get_input_roaringbitmap(ctx, tmp_allocator, rb2_arg, rb2, is_rb2_null))) {
LOG_WARN("failed to get right input roaringbitmap", K(ret));
} else if (is_rb2_null && !is_null2empty) {
is_res_null = true;
} else if (is_rb2_null && is_null2empty
&& OB_ISNULL(rb2 = OB_NEWx(ObRoaringBitmap, &tmp_allocator, (&tmp_allocator)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to create alloc memory to roaringbitmap", K(ret));
} else if (OB_FAIL(rb1->value_calc(rb2, op))) {
LOG_WARN("failed to calcutlate roaringbitmap value_and", K(ret));
ObString res_rb_bin;
if (op == ObRbOperation::AND || op == ObRbOperation::ANDNOT) {
ObString rb1_bin;
ObString rb2_bin;
if (OB_FAIL(ObRbExprHelper::get_input_roaringbitmap_bin(ctx, tmp_allocator, rb1_arg, rb1_bin, is_rb1_null))) {
LOG_WARN("fail to get left input roaringbitmap", K(ret));
} else if (is_rb1_null && !is_null2empty) {
is_res_null = true;
} else if (is_rb1_null && is_null2empty && OB_FAIL(ObRbUtils::build_empty_binary(tmp_allocator, rb1_bin))) {
LOG_WARN("failed to build empty roaringbitmap binary", K(ret));
} else if (OB_FAIL(ObRbExprHelper::get_input_roaringbitmap_bin(ctx, tmp_allocator, rb2_arg, rb2_bin, is_rb2_null))) {
LOG_WARN("fail to get right input roaringbitmap", K(ret));
} else if (is_rb2_null && !is_null2empty) {
is_res_null = true;
} else if (is_rb2_null && is_null2empty && OB_FAIL(ObRbUtils::build_empty_binary(tmp_allocator, rb2_bin))) {
LOG_WARN("failed to build empty roaringbitmap binary", K(ret));
} else if (OB_FAIL(ObRbUtils::binary_calc(tmp_allocator, rb1_bin, rb2_bin, res_rb_bin, op))) {
LOG_WARN("failed to calculate roaringbitmap", K(ret), K(rb1_bin), K(rb2_bin), K(op));
}
} else if (op == ObRbOperation::OR || op == ObRbOperation::XOR) {
ObRoaringBitmap *rb1 = nullptr;
ObRoaringBitmap *rb2 = nullptr;
if (OB_FAIL(ObRbExprHelper::get_input_roaringbitmap(ctx, tmp_allocator, rb1_arg, rb1, is_rb1_null))) {
LOG_WARN("failed to get left input roaringbitmap", K(ret));
} else if (is_rb1_null && !is_null2empty) {
is_res_null = true;
} else if (is_rb1_null && is_null2empty
&& OB_ISNULL(rb1 = OB_NEWx(ObRoaringBitmap, &tmp_allocator, (&tmp_allocator)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to create alloc memory to roaringbitmap", K(ret));
} else if (OB_FAIL(ObRbExprHelper::get_input_roaringbitmap(ctx, tmp_allocator, rb2_arg, rb2, is_rb2_null))) {
LOG_WARN("failed to get right input roaringbitmap", K(ret));
} else if (is_rb2_null && !is_null2empty) {
is_res_null = true;
} else if (is_rb2_null && is_null2empty
&& OB_ISNULL(rb2 = OB_NEWx(ObRoaringBitmap, &tmp_allocator, (&tmp_allocator)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to create alloc memory to roaringbitmap", K(ret));
} else if (OB_FAIL(ObRbUtils::calc_inplace(rb1, rb2, op))) {
LOG_WARN("failed to calcutlate roaringbitmap inplace", K(ret));
} else if (OB_FAIL(ObRbUtils::rb_serialize(tmp_allocator, res_rb_bin, rb1))) {
LOG_WARN("failed to serialize roaringbitmap", K(ret));
}
ObRbUtils::rb_destroy(rb1);
ObRbUtils::rb_destroy(rb2);
} else {
ret = OB_NOT_SUPPORTED;
LOG_WARN("operation is not supported", K(ret), K(op));
}
if (OB_FAIL(ret)) {
} else if (is_res_null) {
res.set_null();
} else if (OB_FAIL(ObRbUtils::rb_serialize(tmp_allocator, rb_res, rb1))) {
LOG_WARN("failed to serialize roaringbitmap", K(ret));
} else if (OB_FAIL(ObRbExprHelper::pack_rb_res(expr, ctx, res, rb_res))) {
} else if (OB_FAIL(ObRbExprHelper::pack_rb_res(expr, ctx, res, res_rb_bin))) {
LOG_WARN("fail to pack roaringbitmap res", K(ret));
}
ObRbUtils::rb_destroy(rb1);
ObRbUtils::rb_destroy(rb2);
return ret;
}

30
src/sql/engine/expr/ob_expr_rb_calc_cardinality.cpp
View File

@ -70,36 +70,30 @@ int ObExprRbCalcCardinality::eval_rb_calc_cardinality(const ObExpr &expr, ObEval
bool is_rb1_null = false;
bool is_rb2_null = false;
bool is_res_null = false;
ObRoaringBitmap *rb1 = nullptr;
ObRoaringBitmap *rb2 = nullptr;
ObString rb1_bin;
ObString rb2_bin;
uint64_t cardinality = 0;
if (OB_FAIL(ObRbExprHelper::get_input_roaringbitmap(ctx, tmp_allocator, rb1_arg, rb1, is_rb1_null))) {
LOG_WARN("failed to get left input roaringbitmap", K(ret));
if (OB_FAIL(ObRbExprHelper::get_input_roaringbitmap_bin(ctx, tmp_allocator, rb1_arg, rb1_bin, is_rb1_null))) {
LOG_WARN("fail to get left input roaringbitmap", K(ret));
} else if (is_rb1_null && !is_null2empty) {
is_res_null = true;
} else if (is_rb1_null && is_null2empty
&& OB_ISNULL(rb1 = OB_NEWx(ObRoaringBitmap, &tmp_allocator, (&tmp_allocator)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to create alloc memory to roaringbitmap", K(ret));
} else if (OB_FAIL(ObRbExprHelper::get_input_roaringbitmap(ctx, tmp_allocator, rb2_arg, rb2, is_rb2_null))) {
LOG_WARN("failed to get right input roaringbitmap", K(ret));
} else if (is_rb1_null && is_null2empty && OB_FAIL(ObRbUtils::build_empty_binary(tmp_allocator, rb1_bin))) {
LOG_WARN("failed to build empty roaringbitmap binary", K(ret));
} else if (OB_FAIL(ObRbExprHelper::get_input_roaringbitmap_bin(ctx, tmp_allocator, rb2_arg, rb2_bin, is_rb2_null))) {
LOG_WARN("fail to get right input roaringbitmap", K(ret));
} else if (is_rb2_null && !is_null2empty) {
is_res_null = true;
} else if (is_rb2_null && is_null2empty
&& OB_ISNULL(rb2 = OB_NEWx(ObRoaringBitmap, &tmp_allocator, (&tmp_allocator)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to create alloc memory to roaringbitmap", K(ret));
} else if (is_rb2_null && is_null2empty && OB_FAIL(ObRbUtils::build_empty_binary(tmp_allocator, rb2_bin))) {
LOG_WARN("failed to build empty roaringbitmap binary", K(ret));
} else if (OB_FAIL(ObRbUtils::get_calc_cardinality(tmp_allocator, rb1_bin, rb2_bin, cardinality, op))) {
LOG_WARN("failed to get roaringbitmap claculate cardinality", K(ret), K(op));
}
if (OB_FAIL(ret)) {
} else if (is_res_null) {
res.set_null();
} else {
ObRbUtils::calc_cardinality(rb1, rb2, cardinality, op);
res.set_uint(cardinality);
}
ObRbUtils::rb_destroy(rb1);
ObRbUtils::rb_destroy(rb2);
return ret;
}

5
src/sql/engine/expr/ob_expr_rb_cardinality.cpp
View File

@ -63,15 +63,12 @@ int ObExprRbCardinality::eval_rb_cardinality(const ObExpr &expr, ObEvalCtx &ctx,
ObExpr *rb_arg = expr.args_[0];
bool is_rb_null = false;
ObString rb_bin;
ObRbBinType bin_type;
uint64_t cardinality = 0;
if (OB_FAIL(ObRbExprHelper::get_input_roaringbitmap_bin(ctx, tmp_allocator, rb_arg, rb_bin, is_rb_null))) {
LOG_WARN("fail to get input roaringbitmap", K(ret));
} else if (is_rb_null || rb_bin == nullptr) {
res.set_null();
} else if (OB_FAIL(ObRbUtils::check_get_bin_type(rb_bin, bin_type))) {
LOG_WARN("invalid roaringbitmap binary string", K(ret));
} else if (OB_FAIL(ObRbUtils::get_cardinality(tmp_allocator, rb_bin, bin_type, cardinality))){
} else if (OB_FAIL(ObRbUtils::get_cardinality(tmp_allocator, rb_bin, cardinality))){
LOG_WARN("failed to get cardinality from roaringbitmap binary", K(ret));
} else {
res.set_uint(cardinality);

46
src/sql/engine/expr/ob_expr_rb_func_helper.cpp
View File

@ -33,22 +33,25 @@ namespace sql
int ObRbExprHelper::get_input_roaringbitmap_bin(ObEvalCtx &ctx, ObIAllocator &allocator, ObExpr *rb_arg, ObString &rb_bin, bool &is_rb_null)
{
INIT_SUCC(ret);
ObDatum *rb_datum;
ObDatum *rb_datum = nullptr;
ObString get_str;
if (OB_FAIL(rb_arg->eval(ctx, rb_datum))) {
LOG_WARN("eval roaringbitmap args failed", K(ret));
} else if (rb_datum->is_null()) {
is_rb_null = true;
} else if (OB_FALSE_IT(rb_bin = rb_datum->get_string())) {
} else if (OB_FAIL(ObTextStringHelper::read_real_string_data(
allocator,
*rb_datum,
rb_arg->datum_meta_,
rb_arg->obj_meta_.has_lob_header(),
rb_bin))) {
LOG_WARN("fail to get real string data", K(ret), K(rb_bin));
} else if (rb_bin.empty()) {
ret = OB_INVALID_DATA;
LOG_WARN("roaringbitmap binary is empty", K(ret), K(rb_bin));
get_str))) {
LOG_WARN("fail to get real string data", K(ret), K(get_str));
} else if (rb_arg->datum_meta_.type_ != ObRoaringBitmapType) {
if (OB_FAIL(ObRbUtils::build_binary(allocator, get_str, rb_bin))) {
LOG_WARN("failed to build roaringbitmap from binary", K(ret), K(get_str));
}
} else {
rb_bin.assign_ptr(get_str.ptr(), get_str.length());
}
return ret;
}
@ -56,11 +59,30 @@ int ObRbExprHelper::get_input_roaringbitmap_bin(ObEvalCtx &ctx, ObIAllocator &al
int ObRbExprHelper::get_input_roaringbitmap(ObEvalCtx &ctx, ObIAllocator &allocator, ObExpr *rb_arg, ObRoaringBitmap *&rb, bool &is_rb_null)
{
INIT_SUCC(ret);
ObString rb_bin = nullptr;
if (OB_FAIL(get_input_roaringbitmap_bin(ctx, allocator, rb_arg, rb_bin, is_rb_null))) {
LOG_WARN("failed to get input roaringbitmap binary", K(ret));
} else if (!is_rb_null && OB_FAIL(ObRbUtils::rb_deserialize(allocator, rb_bin, rb))) {
LOG_WARN("failed to deserialize roaringbitmap", K(ret));
ObDatum *rb_datum = nullptr;
ObString get_str;
if (OB_FAIL(rb_arg->eval(ctx, rb_datum))) {
LOG_WARN("eval roaringbitmap args failed", K(ret));
} else if (rb_datum->is_null()) {
is_rb_null = true;
} else if (OB_FAIL(ObTextStringHelper::read_real_string_data(
allocator,
*rb_datum,
rb_arg->datum_meta_,
rb_arg->obj_meta_.has_lob_header(),
get_str))) {
LOG_WARN("fail to get real string data", K(ret), K(get_str));
} else if (rb_arg->datum_meta_.type_ != ObRoaringBitmapType) {
bool need_validate = true;
if (OB_FAIL(ObRbUtils::check_binary(get_str))) {
LOG_WARN("invalid roaringbitmap binary string", K(ret));
} else if (OB_FAIL(ObRbUtils::rb_deserialize(allocator, get_str, rb, need_validate))) {
LOG_WARN("failed to deserialize roaringbitmap", K(ret));
}
} else {
if (OB_FAIL(ObRbUtils::rb_deserialize(allocator, get_str, rb))) {
LOG_WARN("failed to deserialize roaringbitmap", K(ret));
}
}
return ret;
}

20
src/sql/engine/expr/ob_expr_rb_is_empty.cpp
View File

@ -61,21 +61,19 @@ int ObExprRbIsEmpty::eval_rb_is_empty(const ObExpr &expr, ObEvalCtx &ctx, ObDatu
lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(ObRbExprHelper::get_tenant_id(ctx.exec_ctx_.get_my_session()), "ROARINGBITMAP"));
ObExpr *rb_arg = expr.args_[0];
bool is_rb_null = false;
ObRoaringBitmap *rb = nullptr;
ObDatum *rb_datum = nullptr;
if (OB_FAIL(ObRbExprHelper::get_input_roaringbitmap(ctx, tmp_allocator, rb_arg, rb, is_rb_null))) {
ObString rb_bin;
uint64_t cardinality = 0;
if (OB_FAIL(ObRbExprHelper::get_input_roaringbitmap_bin(ctx, tmp_allocator, rb_arg, rb_bin, is_rb_null))) {
LOG_WARN("fail to get input roaringbitmap", K(ret));
} else if (is_rb_null) {
} else if (is_rb_null || rb_bin == nullptr) {
res.set_null();
} else if (OB_FAIL(ObRbUtils::get_cardinality(tmp_allocator, rb_bin, cardinality))){
LOG_WARN("failed to get cardinality from roaringbitmap binary", K(ret));
} else if (cardinality == 0) {
res.set_bool(true);
} else {
uint64_t cardinality = rb->get_cardinality();
if (cardinality == 0) {
res.set_bool(true);
} else {
res.set_bool(false);
}
res.set_bool(false);
}
ObRbUtils::rb_destroy(rb);
return ret;
}

21
src/sql/engine/expr/ob_expr_rb_to_string.cpp
View File

@ -64,27 +64,14 @@ int ObExprRbToString::eval_rb_to_string(const ObExpr &expr,
ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx);
common::ObArenaAllocator &tmp_allocator = tmp_alloc_g.get_allocator();
lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(ObRbExprHelper::get_tenant_id(ctx.exec_ctx_.get_my_session()), "ROARINGBITMAP"));
ObExpr *arg = expr.args_[0];
ObExpr *rb_arg = expr.args_[0];
bool is_rb_null = false;
ObDatum *datum = nullptr;
ObString rb_bin;
ObString rb_str;
if (OB_FAIL(arg->eval(ctx, datum))) {
LOG_WARN("eval roaringbitmap args failed", K(ret));
} else if (datum->is_null()) {
is_rb_null = true;
} else if (OB_FALSE_IT(rb_bin = datum->get_string())) {
} else if (OB_FAIL(ObTextStringHelper::read_real_string_data(tmp_allocator,
*datum,
arg->datum_meta_,
arg->obj_meta_.has_lob_header(),
rb_bin))) {
LOG_WARN("failed to get real string data", K(ret), K(rb_bin));
}
if (OB_FAIL(ret)) {
} else if (is_rb_null) {
if (OB_FAIL(ObRbExprHelper::get_input_roaringbitmap_bin(ctx, tmp_allocator, rb_arg, rb_bin, is_rb_null))) {
LOG_WARN("fail to get input roaringbitmap", K(ret));
} else if (is_rb_null || rb_bin == nullptr) {
res.set_null();
} else if (OB_FAIL(ObRbUtils::rb_to_string(tmp_allocator, rb_bin, rb_str))) {
LOG_WARN("failed to print roaringbitmap to string", K(ret));

20
src/sql/engine/expr/ob_expr_rb_to_varbinary.cpp
View File

@ -72,27 +72,17 @@ int ObExprRbToVarbinary::eval_rb_to_varbinary(const ObExpr &expr,
ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx);
common::ObArenaAllocator &tmp_allocator = tmp_alloc_g.get_allocator();
lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(ObRbExprHelper::get_tenant_id(ctx.exec_ctx_.get_my_session()), "ROARINGBITMAP"));
ObExpr *arg = expr.args_[0];
ObExpr *rb_arg = expr.args_[0];
bool is_rb_null = false;
ObDatum *datum = nullptr;
ObString rb_bin;
ObString expected_format;
ObString res_bin;
if (OB_FAIL(arg->eval(ctx, datum))) {
LOG_WARN("eval roaringbitmap args failed", K(ret));
} else if (datum->is_null()) {
is_rb_null = true;
} else if (OB_FALSE_IT(rb_bin = datum->get_string())) {
} else if (OB_FAIL(ObTextStringHelper::read_real_string_data(
tmp_allocator,
*datum,
arg->datum_meta_,
arg->obj_meta_.has_lob_header(),
rb_bin))) {
LOG_WARN("fail to get real string data", K(ret), K(rb_bin));
if (OB_FAIL(ObRbExprHelper::get_input_roaringbitmap_bin(ctx, tmp_allocator, rb_arg, rb_bin, is_rb_null))) {
LOG_WARN("fail to get input roaringbitmap", K(ret));
} else if (expr.arg_cnt_ == 1) {
res_bin = rb_bin;
res_bin.assign(rb_bin.ptr(), rb_bin.length());
} else {
ObExpr *format_arg = expr.args_[1];
ObDatum *format_datum = nullptr;
@ -112,7 +102,7 @@ int ObExprRbToVarbinary::eval_rb_to_varbinary(const ObExpr &expr,
} else if (expected_format.case_compare("roaring") != 0) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("not supported expected format", K(ret), K(expected_format));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "expected format expect 'roaring' is");
LOG_USER_ERROR(OB_NOT_SUPPORTED, "expected format except 'roaring' is");
} else if (OB_FAIL(ObRbUtils::binary_format_convert(tmp_allocator, rb_bin, res_bin))) {
LOG_WARN("failed to convert binary to roaring format", K(ret), K(rb_bin));
}