hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Do not report ERROR when sample rate equals 100

Browse Source
This commit is contained in:
ZenoWang
2024-02-06 14:49:31 +00:00
committed by ob-robot
parent 1f1d5c08ae
commit c8ef409bf3
3710 changed files with 486984 additions and 3083329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

581
unittest/storage/blocksstable/encoding/test_column_decoder.h
View File

@ -39,32 +39,6 @@ using namespace common;
using namespace storage;
using namespace share::schema;
class VectorDecodeTestUtil
{
public:
static int generate_column_output_expr(
const int64_t batch_cnt,
const ObObjMeta &obj_meta,
const VectorFormat &format,
sql::ObEvalCtx &eval_ctx,
sql::ObExpr &col_expr,
ObIAllocator &frame_allocator);
static bool verify_vector_and_datum_match(
const ObIVector &vector,
const int64_t vec_idx,
const ObDatum &datum);
static bool need_test_vec_with_type(
const VectorFormat &format,
const VecValueTypeClass &vec_tc);
static int test_batch_decode_perf(
ObMicroBlockDecoder &decoder,
const int64_t col_idx,
const ObObjMeta obj_meta,
const int64_t decode_cnt,
const VectorFormat vector_format);
};
class TestColumnDecoder : public ::testing::Test
{
public:
@ -139,13 +113,7 @@ public:
void filter_pushdown_comaprison_neg_test();
void batch_decode_to_datum_test(bool is_condensed = false);
void batch_decode_to_vector_test(
const bool is_condensed,
const bool has_null,
const bool align_row_id,
const VectorFormat vector_format);
void col_equal_batch_decode_to_vector_test(const VectorFormat vector_format);
void col_substr_batch_decode_to_vector_test(const VectorFormat vector_format);
void cell_decode_to_datum_test();
void cell_decode_to_datum_test_without_hex();
@ -1723,553 +1691,6 @@ void TestColumnDecoder::cell_inter_column_substring_to_datum_test()
// std::cout << "Batch decode by column cost time: " << batch_end_time - batch_start_time << std::endl;
// }
void TestColumnDecoder::batch_decode_to_vector_test(
const bool is_condensed,
const bool has_null,
const bool align_row_id,
const VectorFormat vector_format)
{
FLOG_INFO("start one batch decode to vector test", K(is_condensed), K(has_null), K(vector_format));
ObArenaAllocator test_allocator;
encoder_.reuse();
void *row_buf = allocator_.alloc(sizeof(ObDatumRow) * ROW_CNT);
ObDatumRow *rows = new (row_buf) ObDatumRow[ROW_CNT];
for (int64_t i = 0; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, rows[i].init(test_allocator, full_column_cnt_));
}
ObDatumRow row;
ASSERT_EQ(OB_SUCCESS, row.init(test_allocator, full_column_cnt_));
int64_t seed0 = 10000;
int64_t seed1 = 10001;
for (int64_t i = 0; i < ROW_CNT - 35; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
for (int64_t i = ROW_CNT - 35; i < ROW_CNT - 32; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed1, row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
if (has_null) {
for (int64_t j = 0; j < full_column_cnt_; ++j) {
row.storage_datums_[j].set_null();
}
for (int64_t i = ROW_CNT - 32; i < ROW_CNT - 30; ++i) {
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
} else {
for (int64_t i = ROW_CNT - 32; i < ROW_CNT - 30; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed1, row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
}
for (int64_t i = ROW_CNT - 30; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
if (is_condensed) {
const_cast<bool &>(encoder_.ctx_.encoder_opt_.enable_bit_packing_) = false;
} else {
const_cast<bool &>(encoder_.ctx_.encoder_opt_.enable_bit_packing_) = true;
}
char *buf = NULL;
int64_t size = 0;
ASSERT_EQ(OB_SUCCESS, encoder_.build_block(buf, size));
ObMicroBlockDecoder decoder;
ObMicroBlockData data(encoder_.data_buffer_.data(), encoder_.data_buffer_.length());
ASSERT_EQ(OB_SUCCESS, decoder.init(data, read_info_));
ObArenaAllocator frame_allocator;
sql::ObExecContext exec_context(test_allocator);
sql::ObEvalCtx eval_ctx(exec_context);
const char *ptr_arr[ROW_CNT];
uint32_t len_arr[ROW_CNT];
for (int64_t i = 0; i < full_column_cnt_; ++i) {
bool need_test_column = true;
ObObjMeta col_meta = col_descs_.at(i).col_type_;
const int16_t precision = col_meta.is_decimal_int() ? col_meta.get_stored_precision() : PRECISION_UNKNOWN_YET;
VecValueTypeClass vec_tc = common::get_vec_value_tc(
col_meta.get_type(),
col_meta.get_scale(),
precision);
if (i >= ROWKEY_CNT && i < read_info_.get_rowkey_count()) {
need_test_column = false;
} else {
need_test_column = VectorDecodeTestUtil::need_test_vec_with_type(vector_format, vec_tc);
}
if (!need_test_column) {
continue;
}
sql::ObExpr col_expr;
int64_t test_row_cnt = align_row_id ? ROW_CNT : ROW_CNT / 2;
ASSERT_EQ(OB_SUCCESS, VectorDecodeTestUtil::generate_column_output_expr(
ROW_CNT, col_meta, vector_format, eval_ctx, col_expr, frame_allocator));
int32_t col_offset = i;
LOG_INFO("Current col: ", K(i), K(col_meta), K(*decoder.decoders_[col_offset].ctx_), K(precision), K(vec_tc));
int64_t row_ids[test_row_cnt];
int64_t row_id_idx = 0;
for (int64_t datum_idx = 0; datum_idx < ROW_CNT; ++datum_idx) {
if (!align_row_id && 0 == datum_idx % 2) {
// skip
} else if (row_id_idx == test_row_cnt) {
// skip
} else {
row_ids[row_id_idx] = datum_idx;
++row_id_idx;
}
}
ObVectorDecodeCtx vector_ctx(ptr_arr, len_arr, row_ids, test_row_cnt, 0, col_expr.get_vector_header(eval_ctx));
ASSERT_EQ(OB_SUCCESS, decoder.decoders_[col_offset].decode_vector(decoder.row_index_, vector_ctx));
for (int64_t vec_idx = 0; vec_idx < test_row_cnt; ++vec_idx) {
ASSERT_TRUE(VectorDecodeTestUtil::verify_vector_and_datum_match(*(col_expr.get_vector_header(eval_ctx).get_vector()),
vec_idx, rows[row_ids[vec_idx]].storage_datums_[col_offset]));
}
// ASSERT_EQ(OB_SUCCESS, VectorDecodeTestUtil::test_batch_decode_perf(decoder, col_offset, col_meta, 100000, vector_format));
decoder.decoder_allocator_.reuse();
}
}
void TestColumnDecoder::col_equal_batch_decode_to_vector_test(const VectorFormat vector_format)
{
FLOG_INFO("start one column equal batch decode to vector test", K(vector_format));
ObArenaAllocator test_allocator;
encoder_.reuse();
void *row_buf = allocator_.alloc(sizeof(ObDatumRow) * ROW_CNT);
ObDatumRow *rows = new (row_buf) ObDatumRow[ROW_CNT];
for (int64_t i = 0; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, rows[i].init(allocator_, full_column_cnt_));
}
ObDatumRow row;
ASSERT_EQ(OB_SUCCESS, row.init(allocator_, full_column_cnt_));
int64_t seed0 = 10000;
int64_t seed1 = 10001;
for (int64_t i = 0; i < ROW_CNT - 35; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
for (int64_t j = read_info_.get_rowkey_count() + 1; j < full_column_cnt_; j += 2) {
row.storage_datums_[j] = row.storage_datums_[j - 1];
}
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
for (int64_t i = ROW_CNT - 35; i < ROW_CNT - 32; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed1, row));
for (int64_t j = read_info_.get_rowkey_count() + 1; j < full_column_cnt_; j += 2) {
row.storage_datums_[j] = row.storage_datums_[j - 1];
}
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
for (int64_t j = 0; j < full_column_cnt_; ++j) {
row.storage_datums_[j].set_null();
}
for (int64_t i = ROW_CNT - 32; i < ROW_CNT - 30; ++i) {
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
for (int64_t i = ROW_CNT - 30; i < ROW_CNT - 2; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
for (int64_t j = read_info_.get_rowkey_count() + 1; j < full_column_cnt_; j += 2) {
row.storage_datums_[j] = row.storage_datums_[j - 1];
}
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
// generate exception data
for (int64_t i = ROW_CNT - 2; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
char *buf = NULL;
int64_t size = 0;
ASSERT_EQ(OB_SUCCESS, encoder_.build_block(buf, size));
ObMicroBlockDecoder decoder;
ObMicroBlockData data(encoder_.data_buffer_.data(), encoder_.data_buffer_.length());
ASSERT_EQ(OB_SUCCESS, decoder.init(data, read_info_));
ObArenaAllocator frame_allocator;
sql::ObExecContext exec_context(test_allocator);
sql::ObEvalCtx eval_ctx(exec_context);
const char *ptr_arr[ROW_CNT];
uint32_t len_arr[ROW_CNT];
for (int64_t i = 0; i < full_column_cnt_; ++i) {
bool need_test_column = true;
ObObjMeta col_meta = col_descs_.at(i).col_type_;
const int16_t precision = col_meta.is_decimal_int() ? col_meta.get_stored_precision() : PRECISION_UNKNOWN_YET;
VecValueTypeClass vec_tc = common::get_vec_value_tc(
col_meta.get_type(),
col_meta.get_scale(),
precision);
if (i >= ROWKEY_CNT && i < read_info_.get_rowkey_count()) {
need_test_column = false;
} else {
need_test_column = VectorDecodeTestUtil::need_test_vec_with_type(vector_format, vec_tc);
}
if (!need_test_column) {
continue;
}
sql::ObExpr col_expr;
ASSERT_EQ(OB_SUCCESS, VectorDecodeTestUtil::generate_column_output_expr(
ROW_CNT, col_meta, vector_format, eval_ctx, col_expr, frame_allocator));
int32_t col_offset = i;
LOG_INFO("Current col: ", K(i), K(col_meta), K(*decoder.decoders_[col_offset].ctx_), K(precision), K(vec_tc));
int64_t row_ids[ROW_CNT];
for (int64_t datum_idx = 0; datum_idx < ROW_CNT; ++datum_idx) {
row_ids[datum_idx] = datum_idx;
}
ObVectorDecodeCtx vector_ctx(ptr_arr, len_arr, row_ids, ROW_CNT, 0, col_expr.get_vector_header(eval_ctx));
ASSERT_EQ(OB_SUCCESS, decoder.decoders_[col_offset].decode_vector(decoder.row_index_, vector_ctx));
for (int64_t vec_idx = 0; vec_idx < ROW_CNT; ++vec_idx) {
ASSERT_TRUE(VectorDecodeTestUtil::verify_vector_and_datum_match(*(col_expr.get_vector_header(eval_ctx).get_vector()),
vec_idx, rows[vec_idx].storage_datums_[col_offset]));
}
}
}
void TestColumnDecoder::col_substr_batch_decode_to_vector_test(const VectorFormat vector_format)
{
FLOG_INFO("start one column substring batch decode to vector test", K(vector_format));
ObArenaAllocator test_allocator;
encoder_.reuse();
void *row_buf = allocator_.alloc(sizeof(ObDatumRow) * ROW_CNT);
ObDatumRow *rows = new (row_buf) ObDatumRow[ROW_CNT];
for (int64_t i = 0; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, rows[i].init(allocator_, full_column_cnt_));
}
ObDatumRow row;
ASSERT_EQ(OB_SUCCESS, row.init(allocator_, full_column_cnt_));
int64_t seed0 = 10000;
int64_t seed1 = 10001;
for (int64_t i = 0; i < ROW_CNT - 35; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
// generate different start point data
for (int64_t j = read_info_.get_rowkey_count() + 1; j < full_column_cnt_ - 2; j += 2) {
ObString str = row.storage_datums_[j].get_string();
ObString sub_str;
ob_sub_str(allocator_, str, i / 5, sub_str);
row.storage_datums_[j - 1].set_string(sub_str);
}
// generate same start point data
ObString str = row.storage_datums_[full_column_cnt_ - 1].get_string();
ObString sub_str;
ob_sub_str(allocator_,str,1,sub_str);
row.storage_datums_[full_column_cnt_ - 2].set_string(sub_str);
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
for (int64_t i = ROW_CNT - 35; i < ROW_CNT - 32; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed1, row));
for (int64_t j = read_info_.get_rowkey_count() + 1; j < full_column_cnt_ - 2; j += 2) {
ObString str = row.storage_datums_[j].get_string();
ObString sub_str;
ob_sub_str(allocator_, str, i / 5, sub_str);
row.storage_datums_[j - 1].set_string(sub_str);
}
ObString str = row.storage_datums_[full_column_cnt_ - 1].get_string();
ObString sub_str;
ob_sub_str(allocator_,str,1,sub_str);
row.storage_datums_[full_column_cnt_ - 2].set_string(sub_str);
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
for (int64_t j = 0; j < full_column_cnt_; ++j) {
row.storage_datums_[j].set_null();
}
for (int64_t i = ROW_CNT - 32; i < ROW_CNT - 30; ++i) {
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
for (int64_t i = ROW_CNT - 30; i < ROW_CNT - 2; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
for (int64_t j = read_info_.get_rowkey_count() + 1; j < full_column_cnt_ - 2; j += 2) {
ObString str = row.storage_datums_[j].get_string();
ObString sub_str;
ob_sub_str(allocator_, str, i / 5, sub_str);
row.storage_datums_[j - 1].set_string(sub_str);
}
ObString str = row.storage_datums_[full_column_cnt_ - 1].get_string();
ObString sub_str;
ob_sub_str(allocator_,str,1,sub_str);
row.storage_datums_[full_column_cnt_ - 2].set_string(sub_str);
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
// generate exception data
for (int64_t i = ROW_CNT - 2; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
char *buf = NULL;
int64_t size = 0;
ASSERT_EQ(OB_SUCCESS, encoder_.build_block(buf, size));
ObMicroBlockDecoder decoder;
ObMicroBlockData data(encoder_.data_buffer_.data(), encoder_.data_buffer_.length());
ASSERT_EQ(OB_SUCCESS, decoder.init(data, read_info_));
ObArenaAllocator frame_allocator;
sql::ObExecContext exec_context(test_allocator);
sql::ObEvalCtx eval_ctx(exec_context);
const char *ptr_arr[ROW_CNT];
uint32_t len_arr[ROW_CNT];
for (int64_t i = 0; i < full_column_cnt_; ++i) {
bool need_test_column = true;
ObObjMeta col_meta = col_descs_.at(i).col_type_;
const int16_t precision = col_meta.is_decimal_int() ? col_meta.get_stored_precision() : PRECISION_UNKNOWN_YET;
VecValueTypeClass vec_tc = common::get_vec_value_tc(
col_meta.get_type(),
col_meta.get_scale(),
precision);
if (i >= ROWKEY_CNT && i < read_info_.get_rowkey_count()) {
need_test_column = false;
} else {
need_test_column = VectorDecodeTestUtil::need_test_vec_with_type(vector_format, vec_tc);
}
if (!need_test_column) {
continue;
}
sql::ObExpr col_expr;
ASSERT_EQ(OB_SUCCESS, VectorDecodeTestUtil::generate_column_output_expr(
ROW_CNT, col_meta, vector_format, eval_ctx, col_expr, frame_allocator));
int32_t col_offset = i;
LOG_INFO("Current col: ", K(i), K(col_meta), K(*decoder.decoders_[col_offset].ctx_), K(precision), K(vec_tc));
int64_t row_ids[ROW_CNT];
for (int64_t datum_idx = 0; datum_idx < ROW_CNT; ++datum_idx) {
row_ids[datum_idx] = datum_idx;
}
ObVectorDecodeCtx vector_ctx(ptr_arr, len_arr, row_ids, ROW_CNT, 0, col_expr.get_vector_header(eval_ctx));
ASSERT_EQ(OB_SUCCESS, decoder.decoders_[col_offset].decode_vector(decoder.row_index_, vector_ctx));
for (int64_t vec_idx = 0; vec_idx < ROW_CNT; ++vec_idx) {
ASSERT_TRUE(VectorDecodeTestUtil::verify_vector_and_datum_match(*(col_expr.get_vector_header(eval_ctx).get_vector()),
vec_idx, rows[vec_idx].storage_datums_[col_offset]));
}
}
}
int VectorDecodeTestUtil::generate_column_output_expr(
const int64_t batch_cnt,
const ObObjMeta &obj_meta,
const VectorFormat &format,
sql::ObEvalCtx &eval_ctx,
sql::ObExpr &col_expr,
ObIAllocator &frame_allocator)
{
// int ObStaticEngineExprCG::arrange_datums_data
int ret = OB_SUCCESS;
int64_t cur_total_size = 0;
col_expr.reset();
col_expr.batch_result_ = 1;
// hard coded
col_expr.frame_idx_ = 0;
col_expr.res_buf_len_ = 128;
const int16_t precision = obj_meta.is_decimal_int() ? obj_meta.get_stored_precision() : PRECISION_UNKNOWN_YET;
col_expr.vec_value_tc_ = get_vec_value_tc(obj_meta.get_type(), obj_meta.get_scale(), precision);
col_expr.is_fixed_length_data_ = common::is_fixed_length_vec(col_expr.vec_value_tc_);
col_expr.datum_off_ = cur_total_size;
cur_total_size += sizeof(ObDatum) * batch_cnt;
col_expr.pvt_skip_off_ = cur_total_size;
cur_total_size += sql::ObBitVector::memory_size(batch_cnt);
if (!col_expr.is_fixed_length_data_) {
col_expr.len_arr_off_ = cur_total_size;
cur_total_size += sizeof(uint32_t) * (batch_cnt + 1); // offsets size
col_expr.offset_off_ = cur_total_size;
cur_total_size += sizeof(char *) * (batch_cnt); // ptrs size
}
col_expr.vector_header_off_ = cur_total_size;
cur_total_size += sizeof(sql::VectorHeader);
col_expr.null_bitmap_off_ = cur_total_size;
cur_total_size += sql::ObBitVector::memory_size(batch_cnt);
if (!col_expr.is_fixed_length_data_) {
col_expr.cont_buf_off_ = cur_total_size;
cur_total_size += sizeof(sql::ObDynReserveBuf);
}
col_expr.eval_info_off_ = cur_total_size;
cur_total_size += sizeof(sql::ObEvalInfo);
col_expr.eval_flags_off_ = cur_total_size;
cur_total_size += sql::ObBitVector::memory_size(batch_cnt);
col_expr.dyn_buf_header_offset_ = cur_total_size;
cur_total_size += sql::ObDynReserveBuf::supported(obj_meta.get_type()) ? batch_cnt * sizeof(sql::ObDynReserveBuf) : 0;
col_expr.res_buf_off_ = cur_total_size;
cur_total_size += col_expr.res_buf_len_ * batch_cnt;
char **frame_arr = (char **)frame_allocator.alloc(sizeof(char *));
char *frame = (char *)frame_allocator.alloc(cur_total_size);
if (nullptr == frame || nullptr == frame_arr) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "null frame", K(ret));
} else {
memset(frame, 0, cur_total_size);
eval_ctx.frames_ = frame_arr;
eval_ctx.frames_[0] = frame;
if (OB_FAIL(col_expr.init_vector(eval_ctx, format, batch_cnt))) {
STORAGE_LOG(WARN, "failed to init vector", K(ret));
} else {
if (is_uniform_format(format)) {
col_expr.reset_datums_ptr(frame, batch_cnt);
} else if (is_discrete_format(format)) {
col_expr.reset_discretes_ptr(frame, batch_cnt, col_expr.get_discrete_vector_ptrs(eval_ctx));
}
}
}
return ret;
}
bool VectorDecodeTestUtil::verify_vector_and_datum_match(
const ObIVector &vector,
const int64_t vec_idx,
const ObDatum &datum)
{
int bret = false;
ObDatum vec_datum;
if (datum.is_null()) {
bret = vector.is_null(vec_idx);
} else {
ObLength length = vector.get_length(vec_idx);
vec_datum.len_ = length;
vec_datum.ptr_ = vector.get_payload(vec_idx);
bret = ObDatum::binary_equal(vec_datum, datum);
}
if (!bret) {
LOG_INFO("datum not match with datum from vector", K(vec_idx), K(datum), K(vec_datum));
}
return bret;
}
bool VectorDecodeTestUtil::need_test_vec_with_type(
const VectorFormat &vector_format,
const VecValueTypeClass &vec_tc)
{
bool need_test_column = true;
if (vector_format == VEC_FIXED) {
VecValueTypeClass fixed_tc_arr[] = {VEC_TC_INTEGER, VEC_TC_UINTEGER, VEC_TC_FLOAT, VEC_TC_DOUBLE,
VEC_TC_FIXED_DOUBLE, VEC_TC_DATETIME, VEC_TC_DATE, VEC_TC_TIME, VEC_TC_YEAR, VEC_TC_UNKNOWN,
VEC_TC_BIT, VEC_TC_ENUM_SET, VEC_TC_TIMESTAMP_TZ, VEC_TC_TIMESTAMP_TINY, VEC_TC_INTERVAL_YM,
VEC_TC_INTERVAL_DS, VEC_TC_DEC_INT32, VEC_TC_DEC_INT64, VEC_TC_DEC_INT128, VEC_TC_DEC_INT256,
VEC_TC_DEC_INT512};
VecValueTypeClass *vec = std::find(std::begin(fixed_tc_arr), std::end(fixed_tc_arr), vec_tc);
if (vec == std::end(fixed_tc_arr)) {
need_test_column = false;
}
} else if (vector_format == VEC_DISCRETE) {
VecValueTypeClass var_tc_arr[] = {VEC_TC_NUMBER, VEC_TC_EXTEND, VEC_TC_STRING, VEC_TC_ENUM_SET_INNER,
VEC_TC_RAW, VEC_TC_ROWID, VEC_TC_LOB, VEC_TC_JSON, VEC_TC_GEO, VEC_TC_UDT};
VecValueTypeClass *vec = std::find(std::begin(var_tc_arr), std::end(var_tc_arr), vec_tc);
if (vec == std::end(var_tc_arr)) {
need_test_column = false;
}
} else if (vector_format == VEC_CONTINUOUS) {
// not support shallow copy to continuous vector for now
need_test_column = VEC_TC_NUMBER == vec_tc;
} else {
need_test_column = true;
}
return need_test_column;
}
int VectorDecodeTestUtil::test_batch_decode_perf(
ObMicroBlockDecoder &decoder,
const int64_t col_idx,
const ObObjMeta col_meta,
const int64_t decode_cnt,
const VectorFormat vector_format)
{
int ret = OB_SUCCESS;
ObArenaAllocator test_allocator;
sql::ObExecContext exec_context(test_allocator);
sql::ObEvalCtx eval_ctx(exec_context);
sql::ObExpr col_expr;
const int16_t precision = col_meta.is_decimal_int() ? col_meta.get_stored_precision() : PRECISION_UNKNOWN_YET;
VecValueTypeClass vec_tc = common::get_vec_value_tc(col_meta.get_type(), col_meta.get_scale(), precision);
int64_t row_cnt = 0;
void *datum_buf = nullptr;
if (OB_UNLIKELY(!VectorDecodeTestUtil::need_test_vec_with_type(vector_format, vec_tc))) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("not supported test vector type with column meta", K(ret),
K(vector_format), K(col_meta), K(vec_tc), K(precision));
} else if (OB_FAIL(decoder.get_row_count(row_cnt))) {
LOG_WARN("failed to get row cnt", K(ret));
} else if (OB_FAIL(VectorDecodeTestUtil::generate_column_output_expr(
row_cnt, col_meta, vector_format, eval_ctx, col_expr, test_allocator))) {
LOG_WARN("failed to generate column output expr", K(ret));
} else if (OB_ISNULL(datum_buf = test_allocator.alloc(sizeof(int8_t) * 128 * row_cnt))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate memory for datum buffer", K(ret));
} else {
// decode to vector
const char *ptr_arr[row_cnt];
uint32_t len_arr[row_cnt];
int64_t row_ids[row_cnt];
for (int64_t datum_idx = 0; datum_idx < row_cnt; ++datum_idx) {
row_ids[datum_idx] = datum_idx;
}
ObVectorDecodeCtx decode_ctx(ptr_arr, len_arr, row_ids, row_cnt, 0, col_expr.get_vector_header(eval_ctx));
const int64_t vector_start_ts = ObTimeUtility::current_time();
for (int64_t decode_round = 0; decode_round < decode_cnt; ++decode_round) {
if (OB_FAIL(decoder.decoders_[col_idx].decode_vector(decoder.row_index_, decode_ctx))) {
LOG_WARN("failed to decode vector", K(ret), K(decode_ctx));
}
}
const int64_t vector_end_ts = ObTimeUtility::current_time();
// decode to datums
ObDatum datums[row_cnt];
for (int64_t j = 0; j < row_cnt; ++j) {
datums[j].ptr_ = reinterpret_cast<char *>(datum_buf) + j * 128;
row_ids[j] = j;
}
const int64_t datum_start_ts = ObTimeUtility::current_time();
for (int64_t decode_round = 0; decode_round < decode_cnt; ++decode_round) {
if (OB_FAIL(decoder.decoders_[col_idx].batch_decode(decoder.row_index_, row_ids, ptr_arr, row_cnt, datums))) {
LOG_WARN("failed to decode vector", K(ret), K(decode_ctx));
}
}
const int64_t datum_end_ts = ObTimeUtility::current_time();
FLOG_INFO("finish one batch decode perf test: ", K(col_idx), K(col_meta), K(vector_format),
K(decode_cnt), K(vector_start_ts), K(vector_end_ts), K(datum_start_ts), K(datum_end_ts),
"vector decode time", vector_end_ts - vector_start_ts,
"batch decode time", datum_end_ts - datum_start_ts,
KPC(decoder.decoders_[col_idx].ctx_));
}
return ret;
}
} // end of namespace blocksstable