hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

patch 4.0

Browse Source
This commit is contained in:
wangzelin.wzl
2022-10-24 10:34:53 +08:00
parent 4ad6e00ec3
commit 93a1074b0c
10533 changed files with 2588271 additions and 2299373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

267
src/sql/executor/ob_shuffle_service.cpp
View File

@ -21,23 +21,50 @@
using namespace oceanbase::common;
using namespace oceanbase::share::schema;
using namespace oceanbase::observer;
namespace oceanbase {
namespace sql {
namespace oceanbase
{
namespace sql
{
int ObShuffleService::get_partition_ids(ObExecContext& exec_ctx, const share::schema::ObTableSchema& table_schema,
const common::ObNewRow& row, const ObSqlExpression& part_func, const ObSqlExpression& subpart_func,
const ObIArray<ObTransmitRepartColumn>& repart_columns, const ObIArray<ObTransmitRepartColumn>& repart_sub_columns,
int64_t& part_id, int64_t& subpart_id, bool& no_match_partiton)
// 问：为什么 ObShuffleService 里，key 分区的处理总是走一个单独路径？
//
// 答：实现原因。理论上二者可以统一。
// 2017年本逻辑的作者没有努力把二者融合起来，做了两段分支逻辑，导致了现在的局面。
//
// key 分区依赖于 key 表达式对多列做计算，然后对计算结果取模，
// hash 分区对一个 hash function 做计算，然后对计算结果取模。
//
// part_func, subpart_func 这两个参数是专门给 key 分区使用的，用他们计算出
// 一个值，然后再调用 hash 函数计算出最终的 part id。
//
// 对于 hash 分区，只需要传入 repart_columns, repart_sub_columns 即可，
// 其计算 hash 值的表达式是固定的函数，函数的参数就是 repart_columns 指定
//
//
int ObShuffleService::get_partition_ids(ObExecContext &exec_ctx,
const share::schema::ObTableSchema &table_schema,
const common::ObNewRow &row,
const ObSqlExpression &part_func,
const ObSqlExpression &subpart_func,
const ObIArray<ObTransmitRepartColumn> &repart_columns,
const ObIArray<ObTransmitRepartColumn> &repart_sub_columns,
int64_t &part_id,
int64_t &subpart_id,
bool &no_match_partiton)
{
int ret = OB_SUCCESS;
if (OB_FAIL(init_expr_ctx(exec_ctx))) {
LOG_WARN("Failed to init expr calculation context", K(ret));
} else if (OB_FAIL(get_part_id(exec_ctx, table_schema, row, part_func, repart_columns, part_id))) {
} else if (OB_FAIL(get_part_id(exec_ctx, table_schema, row,
part_func, repart_columns, part_id))) {
LOG_WARN("failed to get part id", K(ret));
} else if (PARTITION_LEVEL_TWO != table_schema.get_part_level() || NO_MATCH_PARTITION == part_id) {
} else if (PARTITION_LEVEL_TWO != table_schema.get_part_level() ||
NO_MATCH_PARTITION == part_id) {
// do nothing
} else if (OB_FAIL(
get_subpart_id(exec_ctx, table_schema, row, part_id, subpart_func, repart_sub_columns, subpart_id))) {
} else if (OB_FAIL(get_subpart_id(exec_ctx, table_schema, row, part_id, subpart_func,
repart_sub_columns, subpart_id))) {
LOG_WARN("failed to get subpart id", K(ret));
}
if (OB_SUCC(ret)) {
@ -49,51 +76,24 @@ int ObShuffleService::get_partition_ids(ObExecContext& exec_ctx, const share::sc
return ret;
}
int ObShuffleService::get_partition_ids(ObExecContext& exec_ctx, const share::schema::ObTableSchema& table_schema,
const common::ObNewRow& row, const ObSqlExpression& part_func, const ObSqlExpression& subpart_func,
const ObIArray<ObTransmitRepartColumn>& repart_columns, const ObIArray<ObTransmitRepartColumn>& repart_sub_columns,
const ObPxPartChMap& ch_map, int64_t& part_id, int64_t& subpart_id, bool& no_match_partiton,
ObRepartitionType part_type)
{
int ret = OB_SUCCESS;
if (OB_FAIL(init_expr_ctx(exec_ctx))) {
LOG_WARN("Failed to init expr calculation context", K(ret));
} else if (OB_REPARTITION_ONE_SIDE_ONE_LEVEL_SUB == part_type &&
OB_FAIL(ObShuffleService::get_part_id_by_ch_map(ch_map, part_id))) {
LOG_WARN("fail to get part id", K(ret));
} else if (OB_REPARTITION_ONE_SIDE_ONE_LEVEL_SUB != part_type &&
OB_FAIL(get_part_id(exec_ctx, table_schema, row, part_func, repart_columns, part_id))) {
LOG_WARN("failed to get part id", K(ret));
} else if (PARTITION_LEVEL_TWO != table_schema.get_part_level() || NO_MATCH_PARTITION == part_id) {
// do nothing
} else if (OB_REPARTITION_ONE_SIDE_ONE_LEVEL_FIRST == part_type &&
OB_FAIL(ObShuffleService::get_sub_part_id_by_ch_map(ch_map, part_id, subpart_id))) {
LOG_WARN("fail to get sub part id", K(ret));
} else if (OB_REPARTITION_ONE_SIDE_ONE_LEVEL_FIRST != part_type &&
OB_FAIL(
get_subpart_id(exec_ctx, table_schema, row, part_id, subpart_func, repart_sub_columns, subpart_id))) {
LOG_WARN("failed to get subpart id", K(ret));
}
if (OB_SUCC(ret)) {
if (part_id == NO_MATCH_PARTITION || subpart_id == NO_MATCH_PARTITION) {
no_match_partiton = true;
}
}
LOG_DEBUG("get part id and subpart id", K(part_id), K(subpart_id), K(no_match_partiton));
return ret;
}
int ObShuffleService::get_part_id(ObExecContext& exec_ctx, const share::schema::ObTableSchema& table_schema,
const common::ObNewRow& row, const ObSqlExpression& part_func,
const ObIArray<ObTransmitRepartColumn>& repart_columns, int64_t& part_id)
int ObShuffleService::get_part_id(ObExecContext &exec_ctx,
const share::schema::ObTableSchema &table_schema,
const common::ObNewRow &row,
const ObSqlExpression &part_func,
const ObIArray<ObTransmitRepartColumn> &repart_columns,
int64_t &part_id)
{
int ret = OB_SUCCESS;
if (table_schema.is_key_part()) {
if (OB_FAIL(get_key_part_id(exec_ctx, table_schema, row, part_func, part_id))) {
LOG_WARN("get key part id failed");
}
} else if (table_schema.is_list_part() || table_schema.is_range_part() || table_schema.is_hash_part()) {
if (OB_FAIL(get_non_key_partition_part_id(exec_ctx, table_schema, row, repart_columns, part_id))) {
} else if (table_schema.is_list_part() ||
table_schema.is_range_part() ||
table_schema.is_hash_part()) {
if (OB_FAIL(get_non_key_partition_part_id(exec_ctx, table_schema, row,
repart_columns, part_id))) {
LOG_WARN("failed to get non key partition part id", K(ret));
}
} else {
@ -103,14 +103,18 @@ int ObShuffleService::get_part_id(ObExecContext& exec_ctx, const share::schema::
return ret;
}
int ObShuffleService::get_key_part_id(ObExecContext& exec_ctx, const share::schema::ObTableSchema& table_schema,
const common::ObNewRow& row, const ObSqlExpression& part_func, int64_t& part_id)
int ObShuffleService::get_key_part_id(ObExecContext &exec_ctx,
const share::schema::ObTableSchema &table_schema,
const common::ObNewRow &row,
const ObSqlExpression &part_func,
int64_t &part_id)
{
int ret = OB_SUCCESS;
UNUSED(exec_ctx);
int64_t calc_result = 0;
ObObj func_result;
int64_t part_count = table_schema.get_part_option().get_part_num();
//一级分区
if (part_count <= 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the part num can not be null", K(part_count), K(part_func), K(ret));
@ -119,7 +123,7 @@ int ObShuffleService::get_key_part_id(ObExecContext& exec_ctx, const share::sche
} else if (OB_FAIL(func_result.get_int(calc_result))) {
LOG_WARN("Fail to get int64 from result", K(func_result), K(ret));
} else if (calc_result < 0) {
ret = OB_INVALID_ARGUMENT;
ret = OB_INVALID_ARGUMENT;
LOG_WARN("Input arguments is invalid", K(calc_result), K(part_count), K(ret));
} else if (OB_FAIL(ObPartitionUtils::calc_hash_part_idx(calc_result, part_count, part_id))) {
LOG_WARN("calc_hash_part_idx failed", K(ret));
@ -127,45 +131,34 @@ int ObShuffleService::get_key_part_id(ObExecContext& exec_ctx, const share::sche
return ret;
}
int ObShuffleService::get_non_key_partition_part_id(ObExecContext& exec_ctx, const ObTableSchema& table_schema,
const ObNewRow& row, const ObIArray<ObTransmitRepartColumn>& repart_columns, int64_t& part_id)
int ObShuffleService::get_non_key_partition_part_id(ObExecContext &exec_ctx,
const ObTableSchema &table_schema,
const ObNewRow &row,
const ObIArray<ObTransmitRepartColumn> &repart_columns,
int64_t &part_id)
{
int ret = OB_SUCCESS;
ObNewRow part_row;
ObSEArray<int64_t, 1> part_ids;
bool is_hash_v2 = (PARTITION_FUNC_TYPE_HASH_V2 == table_schema.get_part_option().get_part_func_type());
if (OB_FAIL(get_repart_row(exec_ctx, row, repart_columns, part_row, table_schema.is_hash_part(), is_hash_v2))) {
LOG_WARN("fail to get part and subpart obj idxs", K(ret));
} else {
if (OB_FAIL(table_schema.get_part(part_row, part_ids))) {
LOG_WARN("get part idxs from table schema error", K(ret));
} else if (0 == part_ids.count()) {
part_id = NO_MATCH_PARTITION;
} else if (1 != part_ids.count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the part idxs is invalid", K(ret), K(part_ids.count()));
} else {
part_id = part_ids.at(0);
}
if (OB_NOT_NULL(part_row.cells_)) {
exec_ctx.get_allocator().free(part_row.cells_);
part_row.cells_ = NULL;
}
}
return ret;
return OB_NOT_SUPPORTED;
}
int ObShuffleService::get_subpart_id(ObExecContext& exec_ctx, const share::schema::ObTableSchema& table_schema,
const common::ObNewRow& row, int64_t part_id, const ObSqlExpression& subpart_func,
const ObIArray<ObTransmitRepartColumn>& repart_sub_columns, int64_t& subpart_id)
int ObShuffleService::get_subpart_id(ObExecContext &exec_ctx,
const share::schema::ObTableSchema &table_schema,
const common::ObNewRow &row,
int64_t part_id,
const ObSqlExpression &subpart_func,
const ObIArray<ObTransmitRepartColumn> &repart_sub_columns,
int64_t &subpart_id)
{
int ret = OB_SUCCESS;
if (table_schema.is_key_subpart()) {
if (OB_FAIL(get_key_subpart_id(exec_ctx, table_schema, row, part_id, subpart_func, subpart_id))) {
if (OB_FAIL(get_key_subpart_id(exec_ctx, table_schema, row,
part_id, subpart_func, subpart_id))) {
LOG_WARN("get key subpart id failed");
}
} else if (table_schema.is_list_subpart() || table_schema.is_range_subpart() || table_schema.is_hash_subpart()) {
if (OB_FAIL(get_non_key_subpart_id(exec_ctx, table_schema, row, part_id, repart_sub_columns, subpart_id))) {
} else if (table_schema.is_list_subpart()
|| table_schema.is_range_subpart()
|| table_schema.is_hash_subpart()) {
if (OB_FAIL(get_non_key_subpart_id(exec_ctx, table_schema, row, part_id,
repart_sub_columns, subpart_id))) {
LOG_WARN("get range or hash subpart id failed");
}
} else {
@ -175,78 +168,39 @@ int ObShuffleService::get_subpart_id(ObExecContext& exec_ctx, const share::schem
return ret;
}
int ObShuffleService::get_key_subpart_id(ObExecContext& exec_ctx, const ObTableSchema& table_schema,
const ObNewRow& row, int64_t part_id, const ObSqlExpression& subpart_func, int64_t& subpart_id)
// FIXME: 支持非模版化二级分区
int ObShuffleService::get_key_subpart_id(ObExecContext &exec_ctx,
const ObTableSchema &table_schema,
const ObNewRow &row,
int64_t part_id,
const ObSqlExpression &subpart_func,
int64_t &subpart_id)
{
int ret = OB_SUCCESS;
UNUSED(exec_ctx);
int64_t sub_calc_result = 0;
ObObj sub_func_result;
int64_t sub_part_num = 0;
const ObPartition* partition = nullptr;
if (PARTITION_LEVEL_TWO != table_schema.get_part_level()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("should not call this func without sub partition", K(ret), "level", table_schema.get_part_level());
// do nothing
} else if (OB_FAIL(table_schema.get_partition_by_part_id(part_id, false, partition))) {
LOG_WARN("fail get partition", K(ret), K(part_id));
} else if (OB_ISNULL(partition)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("partition should exist", K(sub_part_num), K(part_id), K(ret));
} else if ((sub_part_num = partition->get_sub_part_num()) <= 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the part num can not be null", K(sub_part_num), K(subpart_func));
} else if (OB_FAIL(subpart_func.calc(expr_ctx_, row, sub_func_result))) {
LOG_WARN("Failed to calc hash expr", K(ret), K(row));
} else if (OB_FAIL(sub_func_result.get_int(sub_calc_result))) {
LOG_WARN("Fail to get int64 from result", K(sub_func_result), K(ret));
} else if (sub_calc_result < 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("Input arguments is invalid", K(sub_calc_result), K(ret));
} else {
subpart_id = sub_calc_result % sub_part_num;
}
return ret;
return OB_NOT_SUPPORTED;
}
int ObShuffleService::get_non_key_subpart_id(ObExecContext& exec_ctx, const ObTableSchema& table_schema,
const ObNewRow& row, int64_t part_id, const ObIArray<ObTransmitRepartColumn>& repart_sub_columns,
int64_t& subpart_id)
//FIXME:此处和22x实现不一致，需要check一下
int ObShuffleService::get_non_key_subpart_id(ObExecContext &exec_ctx,
const ObTableSchema &table_schema,
const ObNewRow &row,
int64_t part_id,
const ObIArray<ObTransmitRepartColumn> &repart_sub_columns,
int64_t &subpart_id)
{
int ret = OB_SUCCESS;
ObNewRow subpart_row;
ObSEArray<int64_t, 1> subpart_ids;
bool is_hash_v2 = (PARTITION_FUNC_TYPE_HASH_V2 == table_schema.get_sub_part_option().get_sub_part_func_type());
if (OB_FAIL(
get_repart_row(exec_ctx, row, repart_sub_columns, subpart_row, table_schema.is_hash_subpart(), is_hash_v2))) {
LOG_WARN("fail to get part and subpart obj idxs", K(ret));
} else {
if (OB_FAIL(table_schema.get_subpart(part_id, subpart_row, subpart_ids))) {
LOG_WARN("get part idxs from table schema error", K(part_id), K(ret));
} else if (0 == subpart_ids.count()) {
subpart_id = NO_MATCH_PARTITION;
} else if (1 != subpart_ids.count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the part idxs is invalid", K(ret), K(subpart_ids.count()));
} else {
subpart_id = subpart_ids.at(0);
}
if (OB_NOT_NULL(subpart_row.cells_)) {
exec_ctx.get_allocator().free(subpart_row.cells_);
subpart_row.cells_ = NULL;
}
}
return ret;
return OB_NOT_SUPPORTED;
}
int ObShuffleService::get_repart_row(ObExecContext& exec_ctx, const common::ObNewRow& in_row,
const ObIArray<ObTransmitRepartColumn>& repart_columns, common::ObNewRow& out_row, bool hash_part, bool is_hash_v2)
int ObShuffleService::get_repart_row(ObExecContext &exec_ctx,
const common::ObNewRow &in_row,
const ObIArray<ObTransmitRepartColumn> &repart_columns,
common::ObNewRow &out_row,
bool hash_part)
{
int ret = OB_SUCCESS;
UNUSED(exec_ctx);
if (current_cell_count_ < repart_columns.count()) {
current_cell_count_ = repart_columns.count();
row_cache_.cells_ = static_cast<ObObj*>(allocator_.alloc(sizeof(ObObj) * current_cell_count_));
row_cache_.cells_ = static_cast<ObObj*>(allocator_.alloc(sizeof(ObObj)*current_cell_count_));
}
row_cache_.count_ = repart_columns.count();
if (OB_ISNULL(row_cache_.cells_)) {
@ -254,6 +208,7 @@ int ObShuffleService::get_repart_row(ObExecContext& exec_ctx, const common::ObNe
LOG_WARN("alloc mem failed", K(ret));
} else {
for (int64_t i = 0; i < repart_columns.count() && OB_SUCC(ret); ++i) {
//为什么远端传过来的是int64?
int64_t idx = (repart_columns.at(i).index_);
row_cache_.cells_[i] = in_row.get_cell(idx);
}
@ -262,11 +217,16 @@ int ObShuffleService::get_repart_row(ObExecContext& exec_ctx, const common::ObNe
}
}
if (OB_SUCC(ret) && hash_part) {
/*
* create table t1 (c1 int, c2 int);
* insert into t1 values(1,1);
* 这里的1，1在进入之后会被先解析称为int64_t，也就是int type。
* 在真正插入存储之前会cast为真实类型也就是int32 type。
* 重分区时候会使用到这个接口，所以这里的类型也要打开int32 type类型。
*
* */
ObObj result;
if (is_hash_v2 && OB_FAIL(ObExprFuncPartHash::calc_value(expr_ctx_, out_row.cells_, out_row.count_, result))) {
LOG_WARN("Failed to calc hash value", K(ret));
} else if (!is_hash_v2 &&
OB_FAIL(ObExprFuncPartOldHash::calc_value(expr_ctx_, out_row.cells_, out_row.count_, result))) {
if (OB_FAIL(ObExprFuncPartHash::calc_value(expr_ctx_, out_row.cells_, out_row.count_, result))) {
LOG_WARN("Failed to calc hash value", K(ret));
} else {
out_row.cells_[0] = result;
@ -276,14 +236,14 @@ int ObShuffleService::get_repart_row(ObExecContext& exec_ctx, const common::ObNe
return ret;
}
int ObShuffleService::init_expr_ctx(ObExecContext& exec_ctx)
int ObShuffleService::init_expr_ctx(ObExecContext &exec_ctx)
{
int ret = OB_SUCCESS;
ObSQLSessionInfo* my_session = NULL;
const ObTimeZoneInfo* tz_info = NULL;
ObSQLSessionInfo *my_session = NULL;
const ObTimeZoneInfo *tz_info = NULL;
int64_t tz_offset = 0;
if (nullptr != expr_ctx_.exec_ctx_) {
// Has been inited, do nothing.
//Has been inited, do nothing.
} else if (OB_ISNULL(my_session = exec_ctx.get_my_session())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("session is NULL", K(ret));
@ -304,5 +264,6 @@ int ObShuffleService::init_expr_ctx(ObExecContext& exec_ctx)
return ret;
}
} // namespace sql
} // namespace oceanbase
}
}