hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE]4_1_sql_feature

Browse Source 
Co-authored-by: leslieyuchen <leslieyuchen@gmail.com>
Co-authored-by: Charles0429 <xiezhenjiang@gmail.com>
Co-authored-by: raywill <hustos@gmail.com>
This commit is contained in:
obdev
2023-01-28 16:01:26 +08:00
committed by ob-robot
parent 3080f2b66f
commit 2d19a9d8f5
846 changed files with 161957 additions and 116661 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/sql/executor/ob_remote_executor_processor.cpp
View File

@ -424,6 +424,7 @@ int ObRemoteBaseExecuteP<T>::execute_remote_plan(ObExecContext &exec_ctx,
ObPhysicalPlanCtx *plan_ctx = exec_ctx.get_physical_plan_ctx();
ObOperator *se_op = nullptr; // static engine operator
exec_ctx.set_use_temp_expr_ctx_cache(true);
FLTSpanGuard(remote_execute);
if (OB_ISNULL(plan_ctx) || OB_ISNULL(session)) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("op is NULL", K(ret), K(plan_ctx), K(session));

104
src/sql/executor/ob_slice_calc.cpp
View File

@ -1276,6 +1276,42 @@ bool ObRangeSliceIdCalc::Compare::operator()(
return less;
}
int ObWfHybridDistSliceIdCalc::get_slice_idx_vec(
const ObIArray<ObExpr*> &exprs, ObEvalCtx &eval_ctx,
ObBitVector &skip, const int64_t batch_size,
int64_t *&indexes)
{
UNUSEDx(exprs, eval_ctx, skip, batch_size, indexes);
return common::OB_NOT_SUPPORTED;
}
int ObWfHybridDistSliceIdCalc::get_slice_indexes(const ObIArray<ObExpr*> &exprs,
ObEvalCtx &eval_ctx,
SliceIdxArray &slice_idx_array)
{
int ret = OB_SUCCESS;
if (slice_id_calc_type_ <= SliceIdCalcType::INVALID
|| slice_id_calc_type_ >= SliceIdCalcType::MAX) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("slice_id_calc_type_ is invalid", K(ret), K(slice_id_calc_type_));
} else if (SliceIdCalcType::BROADCAST == slice_id_calc_type_
&& OB_FAIL(broadcast_slice_id_calc_.get_slice_indexes(
exprs, eval_ctx, slice_idx_array))) {
LOG_WARN("get_slice_indexes failed", K(ret), K(slice_id_calc_type_));
} else if (SliceIdCalcType::RANDOM == slice_id_calc_type_
&& OB_FAIL(random_slice_id_calc_.get_slice_indexes(
exprs, eval_ctx, slice_idx_array))) {
LOG_WARN("get_slice_indexes failed", K(ret), K(slice_id_calc_type_));
} else if (SliceIdCalcType::HASH == slice_id_calc_type_
&& OB_FAIL(hash_slice_id_calc_.get_slice_indexes(exprs, eval_ctx, slice_idx_array))) {
LOG_WARN("get_slice_indexes failed", K(ret), K(slice_id_calc_type_));
}
return ret;
}
int ObNullAwareHashSliceIdCalc::get_slice_indexes(const ObIArray<ObExpr*> &exprs,
ObEvalCtx &eval_ctx,
SliceIdxArray &slice_idx_array)
@ -1340,3 +1376,71 @@ int ObNullAwareAffinitizedRepartSliceIdxCalc::get_slice_indexes(
return ret;
}
int ObHybridHashSliceIdCalcBase::check_if_popular_value(ObEvalCtx &eval_ctx, bool &is_popular)
{
int ret = OB_SUCCESS;
uint64_t hash_val = 0;
is_popular = false;
if (OB_ISNULL(popular_values_hash_) || popular_values_hash_->count() <= 0) {
// assume not popular, do nothing
} else if (OB_UNLIKELY(hash_calc_.hash_funcs_->count() != 1)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("only support 1 condition for hybrid hash for now. this may change later",
K(ret), K(hash_calc_.hash_funcs_->count()));
} else if (OB_FAIL(hash_calc_.calc_hash_value(eval_ctx, hash_val))) {
LOG_WARN("fail get hash value", K(ret));
} else {
// build a small hash table to accelerate the lookup.
// if popular_values_hash_->count() <= 3, we use array lookup instead
if (use_hash_lookup_) {
if (OB_HASH_EXIST == (ret = popular_values_map_.exist_refactored(hash_val))) {
is_popular = true;
ret = OB_SUCCESS; // popular value
} else if (OB_HASH_NOT_EXIST == ret) {
ret = OB_SUCCESS; // not popular value
} else {
LOG_WARN("fail lookup hash map", K(ret));
}
} else {
for (int64_t i = 0; i < popular_values_hash_->count(); ++i) {
if (hash_val == popular_values_hash_->at(i)) {
is_popular = true;
break;
}
}
}
}
return ret;
}
int ObHybridHashRandomSliceIdCalc::get_slice_idx(
const ObIArray<ObExpr*> &exprs, ObEvalCtx &eval_ctx, int64_t &slice_idx)
{
int ret = OB_SUCCESS;
bool is_popular = false;
if (OB_FAIL(check_if_popular_value(eval_ctx, is_popular))) {
LOG_WARN("fail check if value popular", K(ret));
} else if (is_popular) {
ret = random_calc_.get_slice_idx(exprs, eval_ctx, slice_idx);
} else {
ret = hash_calc_.get_slice_idx(exprs, eval_ctx, slice_idx);
}
return ret;
}
int ObHybridHashBroadcastSliceIdCalc::get_slice_indexes(
const ObIArray<ObExpr*> &exprs, ObEvalCtx &eval_ctx, SliceIdxArray &slice_idx_array)
{
int ret = OB_SUCCESS;
bool is_popular = false;
if (OB_FAIL(check_if_popular_value(eval_ctx, is_popular))) {
LOG_WARN("fail check if value popular", K(ret));
} else if (is_popular) {
ret = broadcast_calc_.get_slice_indexes(exprs, eval_ctx, slice_idx_array);
} else {
ret = hash_calc_.get_slice_indexes(exprs, eval_ctx, slice_idx_array);
}
return ret;
}

139
src/sql/executor/ob_slice_calc.h
View File

@ -82,7 +82,6 @@ public:
// support vectorized slice indexes calculation.
bool support_vectorized_calc() const { return support_vectorized_calc_; }
virtual void set_calc_hash_keys(int64_t n_keys) { UNUSED(n_keys); }
// Calculate slice index vector for batch rows.
// The function is called only support_vectorized_calc() is true.
virtual int get_slice_idx_vec(const ObIArray<ObExpr*> &exprs, ObEvalCtx &eval_ctx,
@ -695,6 +694,95 @@ public:
int64_t n_keys_;
};
class ObHybridHashSliceIdCalcBase
{
public:
ObHybridHashSliceIdCalcBase(common::ObIAllocator &alloc,
const int64_t slice_cnt,
ObNullDistributeMethod::Type null_row_dist_method,
const ObIArray<ObExpr*> *dist_exprs,
const ObIArray<ObHashFunc> *hash_funcs,
const ObIArray<uint64_t> *popular_values_hash)
: hash_calc_(alloc, slice_cnt, null_row_dist_method, dist_exprs, hash_funcs),
popular_values_hash_(popular_values_hash),
use_hash_lookup_(false)
{
int ret = OB_SUCCESS;
if (popular_values_hash && popular_values_hash->count() > 3) {
// popular value is not ususally not large. 2x the hash bucket size for better performance
if (OB_FAIL(popular_values_map_.create(popular_values_hash->count() * 2, "PopValBkt", "PopValNode"))) {
SQL_LOG(WARN, "fail create popular values map", K(ret), K(popular_values_hash->count()));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < popular_values_hash_->count(); ++i) {
if (OB_FAIL(popular_values_map_.set_refactored(popular_values_hash_->at(i), 0))) {
SQL_LOG(WARN, "fail init popular values map",
K(ret), K(i), K(popular_values_hash->count()));
}
}
}
if (OB_SUCC(ret)) {
use_hash_lookup_ = true;
}
}
}
~ObHybridHashSliceIdCalcBase()
{
if (popular_values_map_.created()) {
(void) popular_values_map_.destroy();
}
}
protected:
int check_if_popular_value(ObEvalCtx &eval_ctx, bool &is_popular);
ObHashSliceIdCalc hash_calc_;
const common::ObIArray<uint64_t> *popular_values_hash_;
common::hash::ObHashSet<uint64_t, common::hash::NoPthreadDefendMode> popular_values_map_;
bool use_hash_lookup_;
};
// broadcast side of px hybrid hash send
class ObHybridHashBroadcastSliceIdCalc : public ObHybridHashSliceIdCalcBase,
public ObMultiSliceIdxCalc
{
public:
ObHybridHashBroadcastSliceIdCalc(common::ObIAllocator &alloc,
const int64_t slice_cnt,
ObNullDistributeMethod::Type null_row_dist_method,
const ObIArray<ObExpr*> *dist_exprs,
const ObIArray<ObHashFunc> *hash_funcs,
const ObIArray<uint64_t> *popular_values_hash)
: ObHybridHashSliceIdCalcBase(alloc, slice_cnt, null_row_dist_method, dist_exprs, hash_funcs, popular_values_hash),
ObMultiSliceIdxCalc(alloc, null_row_dist_method),
broadcast_calc_(alloc, slice_cnt, null_row_dist_method)
{}
virtual int get_slice_indexes(
const ObIArray<ObExpr*> &exprs, ObEvalCtx &eval_ctx, SliceIdxArray &slice_idx_array);
private:
ObBroadcastSliceIdCalc broadcast_calc_;
};
// random side of px hybrid hash send
class ObHybridHashRandomSliceIdCalc : public ObHybridHashSliceIdCalcBase,
public ObSliceIdxCalc
{
public:
ObHybridHashRandomSliceIdCalc(common::ObIAllocator &alloc,
const int64_t slice_cnt,
ObNullDistributeMethod::Type null_row_dist_method,
const ObIArray<ObExpr*> *dist_exprs,
const ObIArray<ObHashFunc> *hash_funcs,
const ObIArray<uint64_t> *popular_values_hash)
: ObHybridHashSliceIdCalcBase(alloc, slice_cnt, null_row_dist_method, dist_exprs, hash_funcs, popular_values_hash),
ObSliceIdxCalc(alloc, null_row_dist_method),
random_calc_(alloc, slice_cnt)
{}
virtual int get_slice_idx(
const ObIArray<ObExpr*> &exprs, ObEvalCtx &eval_ctx, int64_t &slice_idx) override;
private:
ObRandomSliceIdCalc random_calc_;
};
class ObSlaveMapPkeyRangeIdxCalc : public ObSlaveMapRepartIdxCalcBase
{
public:
@ -867,6 +955,55 @@ private:
hash::ObHashMap<int64_t, ObPxPartChMapItem> affi_hash_map_;
};
class ObWfHybridDistSliceIdCalc : public ObSliceIdxCalc
{
public:
enum SliceIdCalcType {
INVALID = 0,
BROADCAST = 1,
RANDOM = 2,
HASH = 3,
MAX = 4,
};
ObWfHybridDistSliceIdCalc(
ObIAllocator &alloc, const int64_t task_cnt, ObNullDistributeMethod::Type null_row_dist_method,
const ObIArray<ObExpr*> *dist_exprs, const ObIArray<ObHashFunc> *hash_funcs)
: ObSliceIdxCalc(alloc, null_row_dist_method),
slice_id_calc_type_(SliceIdCalcType::INVALID),
broadcast_slice_id_calc_(alloc, task_cnt, null_row_dist_method),
random_slice_id_calc_(alloc, task_cnt),
hash_slice_id_calc_(alloc, task_cnt, null_row_dist_method, dist_exprs, hash_funcs)
{
support_vectorized_calc_ = false;
}
virtual int get_slice_idx(const ObIArray<ObExpr*> &exprs, ObEvalCtx &eval_ctx, int64_t &slice_idx)
{
UNUSED(exprs);
UNUSED(eval_ctx);
UNUSED(slice_idx);
return common::OB_NOT_IMPLEMENT;
}
virtual int get_slice_idx_vec(const ObIArray<ObExpr*> &exprs, ObEvalCtx &eval_ctx,
ObBitVector &skip, const int64_t batch_size,
int64_t *&indexes) override;
virtual int get_slice_indexes(const ObIArray<ObExpr*> &exprs,
ObEvalCtx &eval_ctx,
SliceIdxArray &slice_idx_array);
virtual void set_calc_hash_keys(int64_t n_keys)
{
hash_slice_id_calc_.set_calc_hash_keys(n_keys);
}
void set_slice_id_calc_type(SliceIdCalcType slice_id_calc_type)
{
slice_id_calc_type_ = slice_id_calc_type;
}
private:
SliceIdCalcType slice_id_calc_type_;
ObBroadcastSliceIdCalc broadcast_slice_id_calc_;
ObRandomSliceIdCalc random_slice_id_calc_;
ObHashSliceIdCalc hash_slice_id_calc_;
};
class ObNullAwareHashSliceIdCalc : public ObHashSliceIdCalc
{
public:

4
src/sql/executor/ob_task.cpp
View File

@ -83,7 +83,7 @@ OB_DEF_SERIALIZE(ObTask)
buf, buf_len, pos, *exec_ctx_, *const_cast<ObExprFrameInfo *>(frame_info)))) {
LOG_WARN("failed to serialize rt expr", K(ret));
} else if (OB_FAIL(ObPxTreeSerializer::serialize_tree(
buf, buf_len, pos, *root_spec_, false /**is full tree*/))) {
buf, buf_len, pos, *root_spec_, false /**is full tree*/, runner_svr_))) {
LOG_WARN("fail serialize root_op", K(ret), K(buf_len), K(pos));
} else if (OB_FAIL(ObPxTreeSerializer::serialize_op_input(
buf, buf_len, pos, *root_spec_, exec_ctx_->get_kit_store(), false/*is full tree*/))) {
@ -214,7 +214,7 @@ OB_DEF_SERIALIZE(ObMiniTask)
OB_UNIS_ENCODE(has_extend_root_spec);
if (OB_SUCC(ret) && has_extend_root_spec) {
if (OB_FAIL(ObPxTreeSerializer::serialize_tree(
buf, buf_len, pos, *extend_root_spec_ , false /**is full tree*/))) {
buf, buf_len, pos, *extend_root_spec_ , false /**is full tree*/, runner_svr_))) {
LOG_WARN("fail serialize root_op", K(ret), K(buf_len), K(pos));
} else if (OB_FAIL(ObPxTreeSerializer::serialize_op_input(
buf, buf_len, pos, *extend_root_spec_, exec_ctx_->get_kit_store(), false/*is full tree*/))) {