hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Do not report ERROR when sample rate equals 100

Browse Source
This commit is contained in:
ZenoWang
2024-02-06 14:49:31 +00:00
committed by ob-robot
parent 1f1d5c08ae
commit c8ef409bf3
3710 changed files with 486984 additions and 3083329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

230
unittest/storage/blocksstable/encoding/test_raw_decoder.cpp
View File

@ -27,7 +27,7 @@
#include "lib/string/ob_sql_string.h"
#include "../ob_row_generate.h"
#include "common/rowkey/ob_rowkey.h"
#include "test_column_decoder.h"
#include "unittest/storage/mock_ob_table_read_info.h"
namespace oceanbase
@ -53,13 +53,11 @@ public:
int ObMicroBlockRawEncoder::build_block(char *&buf, int64_t &size)
{
int ret = OB_SUCCESS;
int64_t encoders_need_size = 0;
const int64_t col_header_size = ctx_.column_cnt_ * (sizeof(ObColumnHeader));
char *encoding_meta_buf = nullptr;
if (IS_NOT_INIT) {
int64_t need_size = 0;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_UNLIKELY(datum_rows_.empty())) {
} else if (datum_rows_.empty()) {
ret = OB_INNER_STAT_ERROR;
LOG_WARN("empty micro block", K(ret));
} else if (OB_FAIL(set_datum_rows_ptr())) {
@ -68,11 +66,11 @@ int ObMicroBlockRawEncoder::build_block(char *&buf, int64_t &size)
LOG_WARN("pivot rows to columns failed", K(ret));
} else if (OB_FAIL(row_indexs_.reserve(datum_rows_.count()))) {
LOG_WARN("array reserve failed", K(ret), "count", datum_rows_.count());
} else if (OB_FAIL(encoder_detection(encoders_need_size))) {
} else if (OB_FAIL(encoder_detection(need_size))) {
LOG_WARN("detect column encoding failed", K(ret));
} else {
encoders_need_size = 0;
for (int64_t i = 0; OB_SUCC(ret) && i < ctx_.column_cnt_; ++i) {
for (int64_t i = 0; i < ctx_.column_cnt_; ++i) {
const bool force_var_store = false;
if (NULL != encoders_[i]) {
free_encoder(encoders_[i]);
@ -89,36 +87,12 @@ int ObMicroBlockRawEncoder::build_block(char *&buf, int64_t &size)
encoders_[i] = e;
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < encoders_.count(); i++) {
int64_t need_size = 0;
if (OB_FAIL(encoders_.at(i)->get_encoding_store_meta_need_space(need_size))) {
STORAGE_LOG(WARN, "fail to get_encoding_store_meta_need_space", K(ret), K(i), K(encoders_));
} else {
need_size += encoders_.at(i)->calc_encoding_fix_data_need_space();
encoders_need_size += need_size;
}
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(data_buffer_.ensure_space(col_header_size + encoders_need_size))) {
STORAGE_LOG(WARN, "fail to ensure space", K(ret), K(data_buffer_));
} else if (OB_ISNULL(encoding_meta_buf = static_cast<char *>(encoding_meta_allocator_.alloc(encoders_need_size)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "fail to alloc fix header buf", K(ret), K(encoders_need_size));
} else {
STORAGE_LOG(DEBUG, "[debug] build micro block", K_(estimate_size), K_(header_size), K_(expand_pct),
K(datum_rows_.count()), K(ctx_));
// <1> store encoding metas and fix cols data in encoding_meta_buffer
// <1> store encoding metas and fix cols data
int64_t encoding_meta_offset = 0;
int64_t encoding_meta_size = 0;
ObBufferWriter meta_buf_writer(encoding_meta_buf, encoders_need_size, 0);
if (OB_FAIL(store_encoding_meta_and_fix_cols(meta_buf_writer, encoding_meta_offset))) {
if (OB_FAIL(store_encoding_meta_and_fix_cols(encoding_meta_offset))) {
LOG_WARN("failed to store encoding meta and fixed col data", K(ret));
} else if (FALSE_IT(encoding_meta_size = meta_buf_writer.length())) {
} else if (OB_FAIL(data_buffer_.write_nop(encoding_meta_size))) {
STORAGE_LOG(WARN, "failed to write nop", K(ret), K(meta_buf_writer), K(data_buffer_));
}
// <2> set row data store offset
@ -127,7 +101,7 @@ int ObMicroBlockRawEncoder::build_block(char *&buf, int64_t &size)
if (OB_FAIL(set_row_data_pos(fix_data_size))) {
LOG_WARN("set row data position failed", K(ret));
} else {
get_header(data_buffer_)->var_column_count_ = static_cast<uint16_t>(var_data_encoders_.count());
get_header(data_buffer_)->var_column_count_ = static_cast<int16_t>(var_data_encoders_.count());
}
}
@ -149,12 +123,10 @@ int ObMicroBlockRawEncoder::build_block(char *&buf, int64_t &size)
}
ObIntegerArrayGenerator gen;
const int64_t row_index_size = row_indexs_.count() * get_header(data_buffer_)->row_index_byte_;
if (OB_FAIL(data_buffer_.ensure_space(row_index_size))) {
STORAGE_LOG(WARN, "fail to ensure space", K(ret), K(row_index_size), K(data_buffer_));
} else if (OB_FAIL(gen.init(data_buffer_.data() + data_buffer_.length(), get_header(data_buffer_)->row_index_byte_))) {
if (OB_FAIL(gen.init(data_buffer_.data() + data_buffer_.length(), get_header(data_buffer_)->row_index_byte_))) {
LOG_WARN("init integer array generator failed",
K(ret), "byte", get_header(data_buffer_)->row_index_byte_);
} else if (OB_FAIL(data_buffer_.write_nop(row_index_size))) {
} else if (OB_FAIL(data_buffer_.write_nop(row_index_size, true))) {
LOG_WARN("advance data buffer failed", K(ret), K(row_index_size));
} else {
for (int64_t idx = 0; idx < row_indexs_.count(); ++idx) {
@ -164,20 +136,19 @@ int ObMicroBlockRawEncoder::build_block(char *&buf, int64_t &size)
}
}
// <5> fill header, encoding_meta and fix cols data
// <5> fill header
if (OB_SUCC(ret)) {
get_header(data_buffer_)->row_count_ = static_cast<uint32_t>(datum_rows_.count());
get_header(data_buffer_)->row_count_ = static_cast<int16_t>(datum_rows_.count());
get_header(data_buffer_)->has_string_out_row_ = has_string_out_row_;
get_header(data_buffer_)->all_lob_in_row_ = !has_lob_out_row_;
get_header(data_buffer_)->max_merged_trans_version_ = max_merged_trans_version_;
const int64_t header_size = get_header(data_buffer_)->header_size_;
char *data = data_buffer_.data() + header_size;
FOREACH(e, encoders_) {
MEMCPY(data, &(*e)->get_column_header(), sizeof(ObColumnHeader));
data += sizeof(ObColumnHeader);
}
// fill encoding meta and fix cols data
MEMCPY(data_buffer_.data() + encoding_meta_offset, encoding_meta_buf, encoding_meta_size);
}
if (OB_SUCC(ret)) {
@ -190,7 +161,7 @@ int ObMicroBlockRawEncoder::build_block(char *&buf, int64_t &size)
ObIColumnEncoder *e = encoders_.at(idx);
pe.type_ = static_cast<ObColumnHeader::Type>(e->get_column_header().type_);
if (ObColumnHeader::is_inter_column_encoder(pe.type_)) {
pe.ref_col_idx_ = static_cast<ObSpanColumnEncoder *>(e)->get_ref_col_idx();
pe.ref_col_idx_ = static_cast<ObColumnEqualEncoder *>(e)->get_ref_col_idx();
} else {
pe.ref_col_idx_ = 0;
}
@ -243,7 +214,6 @@ public:
decode_res_pool_ = new(allocator_.alloc(sizeof(ObDecodeResourcePool))) ObDecodeResourcePool;
tenant_ctx_.set(decode_res_pool_);
share::ObTenantEnv::set_tenant(&tenant_ctx_);
encoder_.encoding_meta_allocator_.set_tenant_id(OB_SERVER_TENANT_ID);
encoder_.data_buffer_.allocator_.set_tenant_id(OB_SERVER_TENANT_ID);
encoder_.row_buf_holder_.allocator_.set_tenant_id(OB_SERVER_TENANT_ID);
decode_res_pool_->init();
@ -265,12 +235,6 @@ public:
sql::ObPushdownWhiteFilterNode &filter_node,
common::ObBitmap &result_bitmap,
common::ObFixedArray<ObObj, ObIAllocator> &objs);
void test_batch_decode_to_vector(
const bool is_condensed,
const bool has_null,
const bool align_row_id,
const VectorFormat vector_format);
protected:
ObRowGenerate row_generate_;
ObMicroBlockEncodingCtx ctx_;
@ -545,147 +509,6 @@ int TestRawDecoder::test_filter_pushdown_with_pd_info(
return ret;
}
void TestRawDecoder::test_batch_decode_to_vector(
const bool is_condensed,
const bool has_null,
const bool align_row_id,
const VectorFormat vector_format)
{
FLOG_INFO("start one batch decode to vector test", K(is_condensed), K(has_null), K(align_row_id), K(vector_format));
ObArenaAllocator test_allocator;
encoder_.reuse();
// Generate data and encode
void *row_buf = test_allocator.alloc(sizeof(ObDatumRow) * ROW_CNT);
ASSERT_TRUE(nullptr != row_buf);
ObDatumRow *rows = new (row_buf) ObDatumRow[ROW_CNT];
for (int64_t i = 0; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, rows[i].init(test_allocator, full_column_cnt_));
}
ObDatumRow row;
ASSERT_EQ(OB_SUCCESS, row.init(test_allocator, full_column_cnt_));
for (int64_t i = 0; i < ROW_CNT - 35; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
if (has_null) {
for (int64_t j = 0; j < full_column_cnt_; ++j) {
row.storage_datums_[j].set_null();
}
for (int64_t i = ROW_CNT - 35; i < 40; ++i) {
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
} else {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(row));
for (int64_t i = ROW_CNT - 35; i < 40; ++i) {
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
}
for (int64_t i = 40; i < 60; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
for (int64_t i = 60; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(0 - i, row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
if (is_condensed) {
encoder_.ctx_.encoder_opt_.enable_bit_packing_ = false;
} else {
encoder_.ctx_.encoder_opt_.enable_bit_packing_ = true;
}
char *buf = NULL;
int64_t size = 0;
ASSERT_EQ(OB_SUCCESS, encoder_.build_block(buf, size));
ObMicroBlockDecoder decoder;
ObMicroBlockData data(encoder_.data_buffer_.data(), encoder_.data_buffer_.length());
ASSERT_EQ(OB_SUCCESS, decoder.init(data, read_info_));
ObArenaAllocator frame_allocator;
sql::ObExecContext exec_context(test_allocator);
sql::ObEvalCtx eval_ctx(exec_context);
const char *ptr_arr[ROW_CNT];
uint32_t len_arr[ROW_CNT];
for (int64_t i = 0; i < full_column_cnt_; ++i) {
bool need_test_column = true;
ObObjMeta col_meta = col_descs_.at(i).col_type_;
const int16_t precision = col_meta.is_decimal_int() ? col_meta.get_stored_precision() : PRECISION_UNKNOWN_YET;
VecValueTypeClass vec_tc = common::get_vec_value_tc(
col_meta.get_type(),
col_meta.get_scale(),
precision);
if (i >= ROWKEY_CNT && i < read_info_.get_rowkey_count()) {
need_test_column = false;
} else if (vector_format == VEC_FIXED) {
VecValueTypeClass fixed_tc_arr[] = {VEC_TC_INTEGER, VEC_TC_UINTEGER, VEC_TC_FLOAT, VEC_TC_DOUBLE,
VEC_TC_FIXED_DOUBLE, VEC_TC_DATETIME, VEC_TC_DATE, VEC_TC_TIME, VEC_TC_YEAR, VEC_TC_UNKNOWN,
VEC_TC_BIT, VEC_TC_ENUM_SET, VEC_TC_TIMESTAMP_TZ, VEC_TC_TIMESTAMP_TINY, VEC_TC_INTERVAL_YM,
VEC_TC_INTERVAL_DS, VEC_TC_DEC_INT32, VEC_TC_DEC_INT64, VEC_TC_DEC_INT128, VEC_TC_DEC_INT256,
VEC_TC_DEC_INT512};
VecValueTypeClass *vec = std::find(std::begin(fixed_tc_arr), std::end(fixed_tc_arr), vec_tc);
if (vec == std::end(fixed_tc_arr)) {
need_test_column = false;
}
} else if (vector_format == VEC_DISCRETE) {
VecValueTypeClass var_tc_arr[] = {VEC_TC_NUMBER, VEC_TC_EXTEND, VEC_TC_STRING, VEC_TC_ENUM_SET_INNER,
VEC_TC_RAW, VEC_TC_ROWID, VEC_TC_LOB, VEC_TC_JSON, VEC_TC_GEO, VEC_TC_UDT};
VecValueTypeClass *vec = std::find(std::begin(var_tc_arr), std::end(var_tc_arr), vec_tc);
if (vec == std::end(var_tc_arr)) {
need_test_column = false;
}
} else if (vector_format == VEC_CONTINUOUS) {
// not support shallow copy to continuous vector for now
need_test_column = VEC_TC_NUMBER == vec_tc;
} else {
need_test_column = true;
}
if (!need_test_column) {
continue;
}
sql::ObExpr col_expr;
int64_t test_row_cnt = align_row_id ? ROW_CNT : ROW_CNT / 2;
ASSERT_EQ(OB_SUCCESS, VectorDecodeTestUtil::generate_column_output_expr(
ROW_CNT, col_meta, vector_format, eval_ctx, col_expr, frame_allocator));
int32_t col_offset = i;
LOG_INFO("Current col: ", K(i), K(col_meta), K(*decoder.decoders_[col_offset].ctx_),
K(precision), K(vec_tc), K(need_test_column));
int64_t row_ids[test_row_cnt];
int64_t row_id_idx = 0;
for (int64_t datum_idx = 0; datum_idx < ROW_CNT; ++datum_idx) {
if (!align_row_id && 0 == datum_idx % 2) {
// skip
} else if (row_id_idx == test_row_cnt) {
// skip
} else {
row_ids[row_id_idx] = datum_idx;
++row_id_idx;
}
}
ObVectorDecodeCtx vector_ctx(ptr_arr, len_arr, row_ids, test_row_cnt, 0, col_expr.get_vector_header(eval_ctx));
ASSERT_EQ(OB_SUCCESS, decoder.decoders_[col_offset].decode_vector(decoder.row_index_, vector_ctx));
for (int64_t vec_idx = 0; vec_idx < test_row_cnt; ++vec_idx) {
ASSERT_TRUE(VectorDecodeTestUtil::verify_vector_and_datum_match(*(col_expr.get_vector_header(eval_ctx).get_vector()),
vec_idx, rows[row_ids[vec_idx]].storage_datums_[col_offset]));
}
// ASSERT_EQ(OB_SUCCESS, VectorDecodeTestUtil::test_batch_decode_perf(decoder, col_offset, col_meta, 100000, vector_format));
decoder.decoder_allocator_.reuse();
}
}
TEST_F(TestRawDecoder, filter_pushdown_all_eq_ne)
{
// Generate data and encode
@ -1166,25 +989,6 @@ TEST_F(TestRawDecoder, batch_decode_to_datum)
}
}
TEST_F(TestRawDecoder, batch_decode_to_vector)
{
#define TEST_ONE_WITH_ALIGN(row_aligned, vec_format) \
test_batch_decode_to_vector(false, true, row_aligned, vec_format); \
test_batch_decode_to_vector(false, false, row_aligned, vec_format); \
test_batch_decode_to_vector(true, true, row_aligned, vec_format); \
test_batch_decode_to_vector(true, false, row_aligned, vec_format);
#define TEST_ONE(vec_format) \
TEST_ONE_WITH_ALIGN(true, vec_format) \
TEST_ONE_WITH_ALIGN(false, vec_format)
TEST_ONE(VEC_UNIFORM);
TEST_ONE(VEC_FIXED);
TEST_ONE(VEC_DISCRETE);
TEST_ONE(VEC_CONTINUOUS);
#undef TEST_ONE
#undef TEST_ONE_WITH_ALIGN
}
TEST_F(TestRawDecoder, opt_batch_decode_to_datum)
{