hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] impl vectorization 2.0

Browse Source 
Co-authored-by: Naynahs <cfzy002@126.com>
Co-authored-by: hwx65 <1780011298@qq.com>
Co-authored-by: oceanoverflow <oceanoverflow@gmail.com>
This commit is contained in:
obdev
2023-12-22 03:43:19 +00:00
committed by ob-robot
parent 1178245448
commit b6773084c6
592 changed files with 358124 additions and 303288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
unittest/CMakeLists.txt
View File

@ -17,6 +17,8 @@ file(COPY . DESTINATION . FILES_MATCHING PATTERN "*.test")
file(COPY . DESTINATION . FILES_MATCHING PATTERN "*.sql")
file(COPY . DESTINATION . FILES_MATCHING PATTERN "*.result")
file(COPY . DESTINATION . FILES_MATCHING PATTERN "*.schema")
file(COPY . DESTINATION . FILES_MATCHING PATTERN "*.cfg")
file(COPY . DESTINATION . FILES_MATCHING PATTERN "*.sh")
file(COPY run_tests.sh DESTINATION .)
add_subdirectory(sql)

1
unittest/share/CMakeLists.txt
View File

@ -96,4 +96,5 @@ add_subdirectory(interrupt)
add_subdirectory(tablet)
add_subdirectory(location_cache)
add_subdirectory(detect)
add_subdirectory(vector)
add_subdirectory(index_usage)

1
unittest/share/vector/CMakeLists.txt Normal file
View File

@ -0,0 +1 @@
ob_unittest(test_vector_op)

358
unittest/share/vector/test_vector_op.cpp Normal file
View File

@ -0,0 +1,358 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#include <random>
#include <time.h>
#include <vector>
#include "share/vector/vector_basic_op.h"
#include "share/vector/ob_fixed_length_vector.h"
#include "share/vector/ob_discrete_vector.h"
#include "share/vector/vector_op_util.h"
#include "lib/timezone/ob_timezone_info.h"
#include "share/datum/ob_datum_funcs.h"
#include "share/vector/expr_cmp_func.h"
#include "unittest/share/vector/util.h"
namespace oceanbase
{
namespace common
{
void prepare_datums(std::vector<ObDatum> &datums, const int64_t datum_len, const int64_t datum_cnt)
{
void *data = std::malloc(datum_cnt * datum_len);
ASSERT_TRUE(data != nullptr);
for (int i = 0; i < datum_cnt; i++) {
char *ptr = (char *)data + (i * datum_len);
ObDatum d(ptr, datum_len, false);
datums.push_back(d);
}
}
sql::ObBitVector *mock_skip(const int64_t cases)
{
int size = cases / 8 + 1;
void *data = std::malloc(size);
// ASSERT_TRUE(data != nullptr);
std::memset(data, 0, size);
return reinterpret_cast<sql::ObBitVector *>(data);
}
class ObTestVectorBasicOp: public ::testing::Test
{
private:
DISALLOW_COPY_AND_ASSIGN(ObTestVectorBasicOp);
};
std::ostream& operator<<(std::ostream &out, const ObIntervalDSValue &v)
{
out << "<NSEC: " << v.nsecond_ << ", FSEC: " << v.fractional_second_ << ">";
return out;
}
std::ostream& operator<<(std::ostream &out, const ObOTimestampData &v)
{
out << "<DESC: " << v.time_ctx_.desc_ << ", US: " << v.time_us_ << ">";
return out;
}
template<typename T, typename P>
void cmp(std::vector<T> &l, std::vector<T> &r, std::vector<P> &input, std::string name)
{
if (l != r) {
std::cout << "COMPARE " << name << '\n';
ASSERT_EQ(l.size(), r.size());
for (int i = 0; i < l.size(); i++) {
if (l[i] != r[i]) {
std::cout << "case " << i << ": " << input[i] << '\n';
ASSERT_EQ(l[i], r[i]);
}
}
}
}
static const int test_cases = 50000;
template<typename ValueType, VecValueTypeClass tc>
void test_fixed_length_hash(const ObObjMeta &meta, std::string case_name)
{
std::vector<ValueType> data;
for (int i = 0; i < test_cases; i++) {
data.push_back(RandomData<ValueType>::rand());
}
std::vector<uint64_t> seeds;
for (int i = 0; i < test_cases; i++) {
seeds.push_back(RandomData<int64_t>::rand());
}
sql::ObBitVector *skip = mock_skip(test_cases);
sql::ObExpr mock_expr;
mock_expr.obj_meta_ = meta;
sql::EvalBound bound(static_cast<uint16_t>(test_cases), true);
std::vector<uint64_t> vec_hash(test_cases, 0);
std::vector<uint64_t> datum_hash(test_cases, 0);
auto vec = new ObFixedLengthVector<ValueType, VectorBasicOp<tc>>((char *)data.data(), skip);
sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(
meta.get_type(), meta.get_collation_type(), meta.get_scale(), false, meta.has_lob_header());
std::vector<ObDatum> datums;
prepare_datums(datums, sizeof(ValueType), test_cases);
for (int i = 0; i < test_cases; i++) {
std::memcpy(const_cast<char *>(datums[i].ptr_), &(data[i]), sizeof(ValueType));
}
int ret = vec->murmur_hash(mock_expr, vec_hash.data(), *skip, bound, seeds.data(), true);
ASSERT_EQ(ret, 0);
basic_funcs->murmur_hash_batch_(datum_hash.data(), datums.data(), true, *skip, test_cases,
seeds.data(), true);
cmp(datum_hash, vec_hash, data, case_name + " murmurhash");
ret = vec->murmur_hash_v3(mock_expr, vec_hash.data(), *skip, bound, seeds.data(), true);
ASSERT_EQ(ret, 0);
basic_funcs->murmur_hash_v2_batch_(datum_hash.data(), datums.data(), true, *skip, test_cases,
seeds.data(), true);
cmp(datum_hash, vec_hash, data, case_name + " murmurhash_v2");
ret = vec->default_hash(mock_expr, vec_hash.data(), *skip, bound, seeds.data(), true);
ASSERT_EQ(ret, 0);
basic_funcs->default_hash_batch_(datum_hash.data(), datums.data(), true, *skip, test_cases,
seeds.data(), true);
cmp(datum_hash, vec_hash, data, case_name + " default_hash");
}
std::ostream& operator <<(std::ostream &out, const Item &v)
{
out << "<LEN: " << v.data_len << ", DATA:";
for (int i = 0; i < v.data_len; i++) {
out << " " << int64_t(v.data[i]);
}
out << ">";
return out;
}
template<VecValueTypeClass tc>
void test_discrete_hash(const ObObjMeta &meta, std::string case_name)
{
std::vector<Item> items;
for (int i = 0; i < test_cases; i++) {
items.push_back(rand_item<tc>());
}
std::vector<int32_t> lens(test_cases, 0);
std::vector<char *> ptrs(test_cases, nullptr);
for (int i = 0; i < test_cases; i++) {
lens[i] = items[i].data_len;
ptrs[i] = items[i].data;
}
std::vector<uint64_t> seeds;
for (int i = 0; i < test_cases; i++) {
seeds.push_back(RandomData<int64_t>::rand());
}
std::vector<ObDatum> datums;
for (int i = 0; i < test_cases; i++) {
ObDatum d;
d.ptr_ = items[i].data;
d.len_ = items[i].data_len;
d.null_ = false;
datums.push_back(d);
}
sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(
meta.get_type(), meta.get_collation_type(), meta.get_scale(), false, meta.has_lob_header());
sql::ObBitVector *skip = mock_skip(test_cases);
std::vector<uint64_t> vec_hash(test_cases, 0);
std::vector<uint64_t> datum_hash(test_cases, 0);
sql::ObExpr mock_expr;
mock_expr.obj_meta_ = meta;
sql::EvalBound bound(test_cases, true);
auto vec = new ObDiscreteVector<VectorBasicOp<tc>>(lens.data(), ptrs.data(), skip);
int ret = vec->murmur_hash(mock_expr, vec_hash.data(), *skip, bound, seeds.data(), true);
ASSERT_EQ(ret, 0);
basic_funcs->murmur_hash_batch_(datum_hash.data(), datums.data(), true, *skip, test_cases,
seeds.data(), true);
cmp(vec_hash, datum_hash, items, "murmurhash");
ret = vec->murmur_hash_v3(mock_expr, vec_hash.data(), *skip, bound, seeds.data(), true);
ASSERT_EQ(ret, 0);
basic_funcs->murmur_hash_v2_batch_(datum_hash.data(), datums.data(), true, *skip, test_cases,
seeds.data(), true);
cmp(vec_hash, datum_hash, items, "murmurhash_v2");
ret = vec->default_hash(mock_expr, vec_hash.data(), *skip, bound, seeds.data(), true);
ASSERT_EQ(ret, 0);
basic_funcs->default_hash_batch_(datum_hash.data(), datums.data(), true, *skip, test_cases,
seeds.data(), true);
cmp(vec_hash, datum_hash, items, "default_hash");
}
struct CmpItem
{
void *l_data;
void *r_data;
int32_t l_len;
int32_t r_len;
bool l_null;
bool r_null;
};
template<typename T>
ObDatum get_datum(const T &v, const bool null_v)
{
ObDatum d;
d.ptr_ = (const char *)&v;
d.len_ = sizeof(T);
d.null_ = null_v;
return d;
}
template<VecValueTypeClass l_tc, VecValueTypeClass r_tc, typename LType, typename RType>
void test_fixed_length_cmp(const ObObjMeta &l_meta, const ObObjMeta &r_meta, std::string case_name)
{
std::vector<LType> l_datas(test_cases, LType());
std::vector<RType> r_datas(test_cases, RType());
std::vector<CmpItem> items;
ObDatumCmpFuncType null_first_datum_cmp = ObDatumFuncs::get_nullsafe_cmp_func(
l_meta.get_type(), r_meta.get_type(), NULL_FIRST, l_meta.get_collation_type(),
l_meta.get_scale(), false, false);
ObDatumCmpFuncType null_last_datum_cmp = ObDatumFuncs::get_nullsafe_cmp_func(
l_meta.get_type(), r_meta.get_type(), NULL_LAST, l_meta.get_collation_type(),
l_meta.get_scale(), false, false);
sql::NullSafeRowCmpFunc null_first_row_cmp = nullptr;
sql::NullSafeRowCmpFunc null_last_row_cmp = nullptr;
const sql::ObDatumMeta l_d_meta(l_meta.get_type(), l_meta.get_collation_type(),
l_meta.get_scale());
const sql::ObDatumMeta r_d_meta(r_meta.get_type(), r_meta.get_collation_type(),
r_meta.get_scale());
VectorCmpExprFuncsHelper::get_cmp_set(l_d_meta, r_d_meta, null_first_row_cmp, null_last_row_cmp);
int datum_cmp_ret = 0, row_cmp_ret = 0;
for (int i = 0; i < test_cases; i++) {
LType l_v = RandomData<LType>::rand();
LType r_v = RandomData<RType>::rand();
bool l_null = RandomData<bool>::rand();
bool r_null = RandomData<bool>::rand();
ObDatum l_datum = get_datum(l_v, l_null);
ObDatum r_datum = get_datum(r_v, r_null);
int ret = null_first_datum_cmp(l_datum, r_datum, datum_cmp_ret);
ASSERT_EQ(ret, 0);
ret = null_first_row_cmp(l_meta, r_meta, &l_v, sizeof(LType), l_null, &r_v, sizeof(RType),
r_null, row_cmp_ret);
ASSERT_EQ(ret, 0);
if (datum_cmp_ret != row_cmp_ret) {
std::cout << case_name << "NULL FIRST CMP\n";
std::cout << "L: " << l_v << ", R: " << r_v << ", L_NULL: " << l_null
<< ", R_NULL: " << r_null << '\n';
ASSERT_EQ(datum_cmp_ret, row_cmp_ret);
}
ret = null_last_datum_cmp(l_datum, r_datum, datum_cmp_ret);
ASSERT_EQ(ret, 0);
ret = null_last_row_cmp(l_meta, r_meta, &l_v, sizeof(LType), l_null, &r_v, sizeof(RType),
r_null, row_cmp_ret);
ASSERT_EQ(ret, 0);
if (datum_cmp_ret != row_cmp_ret) {
std::cout << case_name << "NULL LAST CMP\n";
std::cout << "L: " << l_v << ", R: " << r_v << ", L_NULL: " << l_null
<< ", R_NULL: " << r_null << '\n';
ASSERT_EQ(datum_cmp_ret, row_cmp_ret);
}
}
}
TEST(ObTestVectorBasicOp, hash_op)
{
ObObjMeta meta;
meta.set_int();
test_fixed_length_hash<int64_t, VEC_TC_INTEGER>(meta, "integer");
meta.set_uint64();
test_fixed_length_hash<uint64_t, VEC_TC_UINTEGER>(meta, "uinteger");
meta.set_float();
test_fixed_length_hash<float, VEC_TC_FLOAT>(meta, "float");
meta.set_double();
test_fixed_length_hash<double, VEC_TC_DOUBLE>(meta, "double");
meta.set_double();
for (int scale = 0; scale <= OB_MAX_DOUBLE_FLOAT_SCALE; scale++) {
meta.set_scale(scale);
test_fixed_length_hash<double, VEC_TC_FIXED_DOUBLE>(meta, "fixed_double");
}
meta.set_timestamp_tz();
test_fixed_length_hash<ObOTimestampData, VEC_TC_TIMESTAMP_TZ>(meta, "otimestamp");
meta.set_interval_ds();
test_fixed_length_hash<ObIntervalDSValue, VEC_TC_INTERVAL_DS>(meta, "intervalds");
meta.set_decimal_int(0);
test_fixed_length_hash<int32_t, VEC_TC_DEC_INT32>(meta, "decint_32");
test_fixed_length_hash<int64_t, VEC_TC_DEC_INT64>(meta, "decint_64");
test_fixed_length_hash<int128_t, VEC_TC_DEC_INT128>(meta, "decint_128");
test_fixed_length_hash<int256_t, VEC_TC_DEC_INT256>(meta, "decint_256");
test_fixed_length_hash<int512_t, VEC_TC_DEC_INT512>(meta, "decint_512");
meta.set_number();
test_discrete_hash<VEC_TC_NUMBER>(meta, "number");
meta.set_collation_type(CS_TYPE_UTF8MB4_BIN);
meta.set_varchar();
test_discrete_hash<VEC_TC_STRING>(meta, "string");
meta.set_urowid();
test_discrete_hash<VEC_TC_ROWID>(meta, "rowid");
meta.set_raw();
test_discrete_hash<VEC_TC_RAW>(meta, "raw");
}
TEST(ObTestVectorBasicOp, cmp_op)
{
ObObjMeta l_meta, r_meta;
l_meta.set_int();
r_meta.set_int();
test_fixed_length_cmp<VEC_TC_INTEGER, VEC_TC_INTEGER, int64_t, int64_t>(l_meta, r_meta,
"int-int");
r_meta.set_uint64();
test_fixed_length_cmp<VEC_TC_INTEGER, VEC_TC_UINTEGER, int64_t, uint64_t>(l_meta, r_meta,
"int-uint");
l_meta.set_uint64();
test_fixed_length_cmp<VEC_TC_UINTEGER, VEC_TC_UINTEGER, uint64_t, uint64_t>(l_meta, r_meta,
"uint-uint");
l_meta.set_float(), r_meta.set_float();
test_fixed_length_cmp<VEC_TC_FLOAT, VEC_TC_FLOAT, float, float>(l_meta, r_meta, "float-float");
l_meta.set_double(), r_meta.set_double();
test_fixed_length_cmp<VEC_TC_DOUBLE, VEC_TC_DOUBLE, double, double>(l_meta, r_meta,
"double-double");
l_meta.set_timestamp_tz(), r_meta.set_timestamp_tz();
test_fixed_length_cmp<VEC_TC_TIMESTAMP_TZ, VEC_TC_TIMESTAMP_TZ, ObOTimestampData,
ObOTimestampData>(l_meta, r_meta, "otimestamp-otimestamp");
l_meta.set_interval_ds(), r_meta.set_interval_ds();
test_fixed_length_cmp<VEC_TC_INTERVAL_DS, VEC_TC_INTERVAL_DS, ObIntervalDSValue,
ObIntervalDSValue>(l_meta, r_meta, "interval-interval");
l_meta.set_decimal_int(0), r_meta.set_decimal_int(0);
test_fixed_length_cmp<VEC_TC_DEC_INT32, VEC_TC_DEC_INT32, int32_t, int32_t>(l_meta, r_meta,
"dec32-dec32");
test_fixed_length_cmp<VEC_TC_DEC_INT64, VEC_TC_DEC_INT64, int64_t, int64_t>(l_meta, r_meta,
"dec64-dec64");
test_fixed_length_cmp<VEC_TC_DEC_INT128, VEC_TC_DEC_INT128, int128_t, int128_t>(l_meta, r_meta,
"dec128-dec128");
test_fixed_length_cmp<VEC_TC_DEC_INT256, VEC_TC_DEC_INT256, int256_t, int256_t>(l_meta, r_meta,
"dec256-dec256");
test_fixed_length_cmp<VEC_TC_DEC_INT512, VEC_TC_DEC_INT512, int512_t, int512_t>(l_meta, r_meta,
"dec512-dec512");
}
} // end namespace share
} // end namespace oceanbase
int main(int argc, char **argv)
{
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

231
unittest/share/vector/util.h Normal file
View File

@ -0,0 +1,231 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <vector>
#include <random>
#include <time.h>
#include "share/vector/type_traits.h"
#include "lib/allocator/ob_allocator.h"
#include "lib/number/ob_number_v2.h"
#include "lib/wide_integer/ob_wide_integer.h"
#include "lib/timezone/ob_timezone_info.h"
#include "sql/engine/expr/ob_expr.h"
namespace oceanbase
{
namespace common
{
struct MockAllocator: public ObIAllocator
{
void* alloc(const int64_t size) override
{
return malloc(size);
}
void* alloc(const int64_t num, const lib::ObMemAttr &attr) override {
return malloc(num);
}
void free(void *ptr) override {
std::free(ptr);
}
};
struct Item
{
char *data;
int32_t data_len;
};
template<VecValueTypeClass>
Item rand_item()
{
return Item();
}
template<>
Item rand_item<VEC_TC_NUMBER>()
{
std::default_random_engine rd(time(NULL));
std::uniform_int_distribution<int32_t> digit_len(1, 30);
std::uniform_int_distribution<int32_t> digits(0, 9);
std::uniform_int_distribution<int32_t> neg(0, 10);
std::string num_str;
int32_t int_len = digit_len(rd), dec_len = digit_len(rd);
if (neg(rd) <= 5) {
num_str.push_back('-');
}
for (int i = 0; i < int_len; i++) {
num_str.push_back(digits(rd) + '0');
}
num_str.push_back('.');
for (int i = 0; i < dec_len; i++) {
num_str.push_back(digits(rd) + '0');
}
MockAllocator alloc;
number::ObNumber v;
int ret = v.from(num_str.c_str(), num_str.size(), alloc);
int32_t data_len = sizeof(ObNumberDesc) + v.get_length() * sizeof(uint32_t);
char *data = (char *)alloc.alloc(data_len);
*reinterpret_cast<ObNumberDesc *>(data) = v.get_desc();
std::memcpy(data + sizeof(ObNumberDesc), v.get_digits(), sizeof(uint32_t) * v.get_length());
return Item{data, data_len};
}
template<>
Item rand_item<VEC_TC_STRING>()
{
static const std::string chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
std::default_random_engine rd(time(NULL));
std::uniform_int_distribution<int32_t> idx(0, chars.size() - 1);
std::string str;
std::uniform_int_distribution<int32_t> data_len(0, 1000);
int32_t len = data_len(rd);
for (int i = 0; i < len; i++) {
str.push_back(chars[idx(rd)]);
}
void *data = std::malloc(len);
std::memcpy(data, str.data(), len);
return Item{(char *)data, len};
}
template<typename T>
struct RandomData
{
static T rand()
{
return T();
}
};
template<>
struct RandomData<int64_t>
{
static int64_t rand()
{
std::default_random_engine random(time(NULL));
std::uniform_int_distribution<int64_t> dist(INT64_MIN, INT64_MAX);
return dist(random);
}
};
template <typename T>
struct WideIntegerRandomData
{
static T rand()
{
std::default_random_engine random(time(NULL));
std::uniform_int_distribution<int64_t> dist(0, UINT64_MAX);
T res = 0;
for (int i = 0; i < T::ITEM_COUNT; i++) {
res = (res << 64) + dist(random);
}
return res;
}
};
template<>
struct RandomData<int128_t>: WideIntegerRandomData<int128_t> {};
template<>
struct RandomData<int256_t>: WideIntegerRandomData<int256_t> {};
template<>
struct RandomData<int512_t>: WideIntegerRandomData<int512_t> {};
template<>
struct RandomData<uint64_t>
{
static uint64_t rand()
{
std::default_random_engine random(time(NULL));
std::uniform_int_distribution<uint64_t> dist(0, UINT64_MAX);
return dist(random);
}
};
template<>
struct RandomData<double>
{
static double rand()
{
std::default_random_engine random(time(NULL));
std::uniform_real_distribution<double> dist(-9999999999.0, 9999999999.0);
return dist(random);
}
};
template<>
struct RandomData<float>
{
static float rand()
{
std::default_random_engine random(time(NULL));
std::uniform_real_distribution<float> dist(-9999999999.0, 9999999999.0);
return dist(random);
}
};
template<>
struct RandomData<bool>
{
static bool rand()
{
std::default_random_engine random(time(NULL));
std::uniform_real_distribution<float> dist(-10.0, 10.0);
return dist(random) >= 0;
}
};
template<>
struct RandomData<ObOTimestampData>
{
static ObOTimestampData rand()
{
std::default_random_engine random(time(NULL));
std::uniform_int_distribution<int64_t> us(1, 100000000);
std::uniform_int_distribution<int16_t> tz(1, 2047);
std::uniform_int_distribution<int16_t> tran_type(0, 31);
std::uniform_int_distribution<int16_t> tail_nsec(0,999);
ObOTimestampData::UnionTZCtx rand_tz_ctx;
rand_tz_ctx.tail_nsec_ = tail_nsec(random);
rand_tz_ctx.version_ = 0;
rand_tz_ctx.store_tz_id_ = true;
rand_tz_ctx.is_null_ = false;
rand_tz_ctx.time_reserved_ = 0;
rand_tz_ctx.tz_id_ = tz(random);
rand_tz_ctx.tran_type_id_ = tran_type(random);
int64_t time_us = us(random);
return ObOTimestampData(time_us, rand_tz_ctx);
}
};
template<>
struct RandomData<ObIntervalDSValue>
{
static ObIntervalDSValue rand()
{
std::default_random_engine random(time(NULL));
std::uniform_int_distribution<int32_t> frac_sec(INT32_MIN, INT32_MAX);
std::uniform_int_distribution<int64_t> nsec(INT64_MIN, INT64_MAX);
return ObIntervalDSValue(nsec(random), frac_sec(random));
}
};
// sql::ObExpr *rand_data_expr(VecValueTypeClass data_tc, sql::ObEvalCtx)
} // end common
} // oceanbase

2
unittest/sql/CMakeLists.txt
View File

@ -5,7 +5,7 @@ add_library(sql_ut_base OBJECT
optimizer/test_optimizer_utils.h
optimizer/test_optimizer_utils.cpp
)
target_link_libraries(sql_ut_base PUBLIC oceanbase mockcontainer)
target_link_libraries(sql_ut_base PUBLIC mockcontainer)
function(sql_unittest case)
ob_unittest(${ARGV})

1
unittest/sql/engine/CMakeLists.txt
View File

@ -1,6 +1,7 @@
#sql_unittest(test_exec_context)
sql_unittest(test_physical_plan)
sql_unittest(test_sql_fixed_array)
sql_unittest(test_bit_vector)
add_subdirectory(aggregate)
add_subdirectory(dml)

5
unittest/sql/engine/aggregate/CMakeLists.txt
View File

@ -7,3 +7,8 @@
#aggr_unittest(test_merge_groupby)
#aggr_unittest(test_scalar_aggregate)
#aggr_unittest(test_merge_distinct)
function(aggr_unittest2 case)
sql_unittest(${ARGV})
target_sources(${case} PRIVATE ../test_op_engine.cpp ../ob_fake_table_scan_vec_op.cpp)
endfunction()
aggr_unittest2(test_hash_groupby2)

8
unittest/sql/engine/aggregate/test_hash_groupby2.cfg Normal file
View File

@ -0,0 +1,8 @@
digit_data_format=4
string_data_format=4
data_range_level=0
skips_probability=10
nulls_probability=30
round=10
batch_size=256
output_result_to_file=1

135
unittest/sql/engine/aggregate/test_hash_groupby2.cpp Normal file
View File

@ -0,0 +1,135 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
// #define USING_LOG_PREFIX SQL_ENGINE
#define USING_LOG_PREFIX COMMON
#include <iterator>
#include <gtest/gtest.h>
#include "../test_op_engine.h"
#include "../ob_test_config.h"
#include <vector>
#include <string>
using namespace ::oceanbase::sql;
namespace test
{
class TestHashGroupByVec : public TestOpEngine
{
public:
TestHashGroupByVec();
virtual ~TestHashGroupByVec();
virtual void SetUp();
virtual void TearDown();
private:
// disallow copy
DISALLOW_COPY_AND_ASSIGN(TestHashGroupByVec);
protected:
// function members
protected:
// data members
};
TestHashGroupByVec::TestHashGroupByVec()
{
std::string schema_filename = ObTestOpConfig::get_instance().test_filename_prefix_ + ".schema";
strcpy(schema_file_path_, schema_filename.c_str());
}
TestHashGroupByVec::~TestHashGroupByVec()
{}
void TestHashGroupByVec::SetUp()
{
TestOpEngine::SetUp();
}
void TestHashGroupByVec::TearDown()
{
destroy();
}
TEST_F(TestHashGroupByVec, basic_test)
{
std::string test_file_path = ObTestOpConfig::get_instance().test_filename_prefix_ + ".test";
int ret = basic_random_test(test_file_path);
EXPECT_EQ(ret, 0);
}
// TEST_F(TestHashGroupByVec, basic_test2)
// {
// int ret = OB_SUCCESS;
// std::string test_file_path = ObTestOpConfig::get_instance().test_filename_prefix_ + ".test";
// if(OB_FAIL(basic_random_test_output_to_file(test_file_path, true))) {
// LOG_ERROR("Some error occur in running vectorization 2.0 operator", K(ret));
// } else if (OB_FAIL(basic_random_test_output_to_file(test_file_path, false))) {
// LOG_ERROR("Some error occur in running original operator", K(ret));
// }
// EXPECT_EQ(ret, 0);
// }
// TEST_F(TestHashGroupByVec, your_own_test)
// {
// std::string test_file_path = ObTestOpConfig::get_instance().test_filename_prefix_ + ".test";
// std::ifstream if_tests(test_file_path);
// if (if_tests.is_open() == false) { return; }
// std::string line;
// while (std::getline(if_tests, line)) {
// // handle query
// if (line.size() <= 0) continue;
// if (line.at(0) == '#') continue;
// ObOperator *root = NULL;
// ObExecutor exector;
// if (OB_FAIL(get_tested_op_from_string(line, false, root, exector))) {
// LOG_WARN("generate tested op fail, sql: ", K(line.data()));
// } else {
// int round = 1;
// const int64_t max_row_cnt = 256;
// const ObBatchRows *child_brs = nullptr;
// LOG_INFO("============== Final output ===============", K(round));
// while (!root->brs_.end_) {
// if (OB_FAIL(root->get_next_batch(max_row_cnt, child_brs))) {
// LOG_ERROR("root op fail to get_next_batch data", K(original_root));
// break;
// }
// }
// }
// }
// }
} // namespace test
int main(int argc, char **argv)
{
ObTestOpConfig::get_instance().test_filename_prefix_ = "test_hash_groupby2";
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-bg") == 0) {
ObTestOpConfig::get_instance().test_filename_prefix_ += "_bg";
ObTestOpConfig::get_instance().run_in_background_ = true;
}
}
ObTestOpConfig::get_instance().init();
system(("rm -f " + ObTestOpConfig::get_instance().test_filename_prefix_ + ".log").data());
system(("rm -f " + ObTestOpConfig::get_instance().test_filename_prefix_ + ".log.*").data());
oceanbase::common::ObClockGenerator::init();
observer::ObReqTimeGuard req_timeinfo_guard;
OB_LOGGER.set_log_level("INFO");
OB_LOGGER.set_file_name((ObTestOpConfig::get_instance().test_filename_prefix_ + ".log").data(), true);
init_sql_factories();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

5
unittest/sql/engine/aggregate/test_hash_groupby2.schema Normal file
View File

@ -0,0 +1,5 @@
#create database opt;
#use opt;
create table t1(c1 int, c2 int);
create table t2(c1 int, c2 int);
create table t3(c1 int, c2 int, c3 double, c4 char(20), c5 varchar(40));

98
unittest/sql/engine/aggregate/test_hash_groupby2.sh Executable file
View File

@ -0,0 +1,98 @@
#!/bin/sh
# data format
# enum VectorFormat: uint8_t
#{
# VEC_INVALID = 0,
# VEC_FIXED,
# VEC_DISCRETE,
# VEC_CONTINUOUS,
# VEC_UNIFORM,
# VEC_UNIFORM_CONST,
# VEC_MAX_FORMAT
#};
#
# batch size
batch_size_round=(1 50 150 256)
# 4 rounds test cases
round_array=(10 100 1000 10000)
# 3 data range test cases
data_range_level_array=(0 1 2)
# 3 skips_probability test cases
skips_probability_array=(0 30 80)
# 3 nulls_probability test cases
nulls_probability_array=(0 30 80)
# 6 combined data format test cases
# VEC_UNIFORM VEC_FIX
fix_data_format_array=("fix_data_format=4" "fix_data_format=1")
# VEC_UNIFORM VEC_DISCRETE VEC_CONTINUOUS
#string_data_format_array=("string_data_format=4" "string_data_format=2" "string_data_format=3")
string_data_format_array=("string_data_format=4")
test_file_prefix="./test_hash_groupby2_bg"
cfg_file="./test_hash_groupby2_bg.cfg"
origin_result_file="./origin_result_bg.data"
vec_result_file="./vec_result_bg.data"
test_case_round=1
for batch_size in ${batch_size_round[@]}
do
for round in ${round_array[@]}
do
for data_range_level in ${data_range_level_array[@]}
do
for skips_probability in ${skips_probability_array[@]}
do
for nulls_probability in ${nulls_probability_array[@]}
do
for fix_data_format in ${fix_data_format_array[@]}
do
for string_data_format in ${string_data_format_array[@]}
do
> ${cfg_file}
echo "batch_size="${batch_size} >> ${cfg_file}
echo "output_result_to_file=1" >> ${cfg_file}
echo "round="${round} >> ${cfg_file}
echo "data_range_level="${data_range_level} >> ${cfg_file}
echo "skips_probability="${skips_probability} >> ${cfg_file}
echo "nulls_probability="${nulls_probability} >> ${cfg_file}
echo ${fix_data_format} >> ${cfg_file}
echo ${string_data_format} >> ${cfg_file}
echo "###################"
echo "Test Case Round: "${test_case_round}
echo "{"
echo "round: "$round
echo "data_range_level: "${data_range_level}
echo "skips_probability: "${skips_probability}
echo "nulls_probability: "${nulls_probability}
echo "fix_data_format: "${fix_data_format}
echo "string_data_format: "${string_data_format}
echo "}"
echo "###################"
./test_hash_groupby2_bg -bg
sort $origin_result_file -o $origin_result_file
sort $vec_result_file -o $vec_result_file
diff $origin_result_file $vec_result_file > /dev/null
if [ $? == 0 ]; then
echo "Both result file are the same!"
else
echo "Get Incorrect Result! Exit!"
exit
fi
test_case_round=$((test_case_round+1))
done
done
done
done
done
done
done
echo "Done"

8
unittest/sql/engine/aggregate/test_hash_groupby2.test Normal file
View File

@ -0,0 +1,8 @@
#select c2, sum(c1), min(c1), max(c1) from t1 group by c2;
#select/*+USE_HASH_AGGREGATION*/ c2, count(c1), sum(c1 + 1), min(c1*2), max(c1 - 1) from t1 group by c2;
#select /*+leading(t1, t2) USE_HASH(t1, t2)*/* from t1, t2 where t1.c1 = t2.c1;
#select /*+leading(t1, t2) USE_HASH(t1, t2)*/* from t1, t2 where t1.c2 = t2.c2;
#select /*+ use_hash(t1 t2)*/ * from t1 a left outer join t2 b on a.c1=b.c1 order by a.c1, a.c2;
#select /*+ use_hash(t1 t2)*/ * from t1 a right outer join t2 b on t1.c1=t2.c1;
#select /*+ use_hash(t1 t2)*/ * from t1 a full outer join t2 b on t1.c1=t2.c1;
#select * from t1 order by c1, c2;

212
unittest/sql/engine/data_generator.h Normal file
View File

@ -0,0 +1,212 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <random>
#include <chrono>
#include <vector>
#include <unordered_map>
#include <type_traits>
#include "src/sql/engine/expr/ob_expr.h"
#include "src/sql/engine/ob_bit_vector.h"
#include "ob_test_config.h"
namespace oceanbase
{
namespace sql
{
class ObDataGenerator
{
public:
static ObDataGenerator &get_instance()
{
static ObDataGenerator data_generator;
return data_generator;
}
template <typename T>
int generate_data(const int64_t op_id, const int expr_i, const int expr_count, const int round, const int batch_size,
const int len, bool &is_duplicate)
{
int ret = OB_SUCCESS;
LOG_DEBUG("[DG]Start generate data for: ", K(op_id), K(round), K(expr_i), K(expr_count));
// if it's a exist generated round data and corresponding expr_i have data
if (op_2_round_2_temp_store_.count(op_id) != 0 && op_2_round_2_temp_store_[op_id].count(round) != 0
&& expr_i < op_2_round_2_temp_store_[op_id][round].size()) {
return ret;
}
// generate new random data
std::string generate_data;
std::vector<TempDataStore> &temp_store = op_2_round_2_temp_store_[op_id][round];
temp_store.push_back(TempDataStore());
for (int i = 0; i < batch_size; i++) {
if (std::is_same<T, int>::value) {
int random_data;
// if (expr_i == expr_count - 1) {
// random_data = 1;
// } else {
// random_data = u_int32_dis_(e_);
// }
random_data = u_int32_dis_->operator()(e_);
temp_store[expr_i].temp_int32_vector_.push_back(random_data);
generate_data += std::to_string(random_data) + " ";
} else if (std::is_same<T, int64_t>::value) {
int64_t random_data;
random_data = u_int64_dis_->operator()(e_);
temp_store[expr_i].temp_int64_vector_.push_back(random_data);
generate_data += std::to_string(random_data) + " ";
} else if (std::is_same<T, double>::value) {
double random_data;
random_data = u_r_dis_->operator()(e_);
temp_store[expr_i].temp_double_vector_.push_back(random_data);
generate_data += std::to_string(random_data) + " ";
} else if (std::is_same<T, std::string>::value) {
std::string random_data;
random_data = str_rand(len);
temp_store[expr_i].temp_string_vector_.push_back(random_data);
generate_data += random_data + " ";
} else {
LOG_INFO("Can not generate random value so far for: ");
assert(false);
}
}
LOG_DEBUG("Generate data: ", K(generate_data.data()));
set_random_null(op_id, expr_i, expr_count, round, batch_size);
return ret;
}
void set_random_null(const int64_t op_id, const int expr_i, const int expr_count, const int round,
const int batch_size)
{
std::string generate_data_nulls;
bool is_null;
for (int i = 0; i < batch_size; i++) {
is_null = zero_one_rand_by_probability(ObTestOpConfig::get_instance().nulls_probability_);
generate_data_nulls += std::to_string(!is_null) + " ";
op_2_round_2_temp_store_[op_id][round][expr_i].null_.push_back(is_null);
}
LOG_INFO("nulls : ", K(generate_data_nulls.data()));
}
void reset_temp_store(const uint64_t op_id, const int round)
{
if (op_2_round_2_temp_store_.count(op_id)) { op_2_round_2_temp_store_[op_id].erase(round); }
}
// Todo: replace this struct with ObTempColumnStore when br finish that
struct TempDataStore
{
bool empty_{true};
// temp store
std::vector<int> temp_int32_vector_;
std::vector<int64_t> temp_int64_vector_;
std::vector<double> temp_double_vector_;
std::vector<std::string> temp_string_vector_;
std::vector<bool> null_;
};
using TempDataStores = std::vector<TempDataStore>;
void register_op(const ObOperator *op)
{
LOG_INFO("id is ", K(op->get_spec().get_id()));
intereseting_op_count_++;
}
private:
ObDataGenerator()
{
e_ = std::default_random_engine(std::chrono::system_clock::now().time_since_epoch().count());
switch (ObTestOpConfig::get_instance().data_range_level_) {
case 0:
u_int32_dis_ = new std::uniform_int_distribution<int32_t>(-100, 100);
u_int64_dis_ = new std::uniform_int_distribution<int64_t>(-100, 100);
u_r_dis_ = new std::uniform_real_distribution<double>(-100, 100);
break;
case 1:
u_int32_dis_ = new std::uniform_int_distribution<int32_t>(-10000, 10000);
u_int64_dis_ = new std::uniform_int_distribution<int64_t>(-50000, 50000);
u_r_dis_ = new std::uniform_real_distribution<double>(-10000, 10000);
break;
case 2:
u_int32_dis_ = new std::uniform_int_distribution<int32_t>(INT32_MIN, INT32_MAX);
u_int64_dis_ = new std::uniform_int_distribution<int64_t>(INT64_MIN, INT64_MAX);
u_r_dis_ = new std::uniform_real_distribution<double>(-100000, 100000);
break;
default:
u_int32_dis_ = new std::uniform_int_distribution<int32_t>(-100, 100);
u_int64_dis_ = new std::uniform_int_distribution<int64_t>(-100, 100);
u_r_dis_ = new std::uniform_real_distribution<double>(-100, 100);
break;
}
}
~ObDataGenerator()
{
delete u_int32_dis_;
delete u_int64_dis_;
delete u_r_dis_;
}
std::string str_rand(int length)
{
std::uniform_int_distribution<int> u_l(0, length);
std::uniform_int_distribution<int> u_data(0, 10000);
int final_len = u_l(e_);
char tmp;
std::string buffer;
for (int i = 0; i < final_len; i++) {
tmp = u_data(e_) % 36;
if (tmp < 10) {
tmp += '0';
} else {
tmp -= 10;
tmp += 'A';
}
buffer += tmp;
}
return buffer;
}
// probability means the probability of generating result of 1
int zero_one_rand_by_probability(int probability)
{
OB_ASSERT(probability >= 0 && probability <= 100);
std::uniform_int_distribution<int> u_l(0, 100);
if (u_l(e_) < probability) { return 1; }
return 0;
}
private:
std::uniform_int_distribution<int32_t> *u_int32_dis_{nullptr};
std::uniform_int_distribution<int64_t> *u_int64_dis_{nullptr};
std::uniform_real_distribution<double> *u_r_dis_{nullptr};
std::default_random_engine e_;
int last_round_{-1};
int intereseting_op_count_{0};
bool inited_{false};
// temp store
std::unordered_map<int, std::unordered_map<int, TempDataStores>> op_2_round_2_temp_store_;
std::unordered_map<int, std::unordered_map<int, std::vector<bool>>> op_2_round_2_skips_;
};
} // end namespace sql
} // end namespace oceanbase

2
unittest/sql/engine/monitoring_dump/test_monitoring_dump.cpp
View File

@ -269,7 +269,7 @@ TEST_F(ObMonitoringDumpTest, test_get_next_row)
ObExecContext ctx;
ObTableScanFake table_scan;
ObMonitoringDump root(ctx.get_allocator());
root.set_flags(ObMonitorHint::OB_MONITOR_STAT | ObMonitorHint::OB_MONITOR_TRACING);
root.set_flags(ObAllocOpHint::OB_MONITOR_STAT | ObAllocOpHint::OB_MONITOR_TRACING);
ObPhysicalPlan physical_plan;
int64_t op_size = 2;
ObSQLSessionInfo origin_session;

299
unittest/sql/engine/ob_fake_table_scan_vec_op.cpp Normal file
View File

@ -0,0 +1,299 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX COMMON
#include "deps/oblib/src/common/object/ob_obj_type.h"
#include "test_op_engine.h"
#include "ob_fake_table_scan_vec_op.h"
#include "data_generator.h"
namespace oceanbase
{
namespace sql
{
int ObFakeTableScanVecOp::inner_open()
{
ObDataGenerator::get_instance().register_op(this);
std::string round;
ObTestOpConfig::get_instance().get_config("round", round);
if (!round.empty()) { max_round_ = current_round_ + std::stoi(round); }
return OB_SUCCESS;
}
int ObFakeTableScanVecOp::inner_get_next_batch(const int64_t max_row_cnt)
{
int ret = OB_SUCCESS;
clear_evaluated_flag();
uint64_t op_id = get_spec().get_id();
int64_t generate_random_value = 0;
bool is_duplicate = false;
const ObPushdownExprSpec &pd_expr_spec =
reinterpret_cast<const ObTableScanSpec *>(&spec_)->tsc_ctdef_.scan_ctdef_.pd_expr_spec_;
for (int j = 0; j < pd_expr_spec.access_exprs_.count(); j++) {
ObExpr *expr = pd_expr_spec.access_exprs_.at(j);
OB_FAIL(fill_random_data_into_expr_datum_frame(j, pd_expr_spec.access_exprs_.count(), expr,
ObTestOpConfig::get_instance().batch_size_, is_duplicate));
}
// random set skip
set_random_skip(current_round_, ObTestOpConfig::get_instance().batch_size_);
if (is_duplicate) { ObDataGenerator::get_instance().reset_temp_store(op_id, current_round_); }
current_round_++;
brs_.size_ = ObTestOpConfig::get_instance().batch_size_;
if (current_round_ == max_round_) { brs_.end_ = true; }
// print generate data
LOG_INFO("[DG] data generated by DataGenerator in ", K(current_round_ - 1));
if (op_id_2_output_streams_.count(op_id) == 0) {
std::string output_file_name = "generate_data_" + std::to_string(op_id) + ".data";
op_id_2_output_streams_[op_id].open(output_file_name.data(), std::ios::out | std::ios::trunc);
}
test::TestOpEngine::print_to_file(&brs_, this, pd_expr_spec.access_exprs_, false, &op_id_2_output_streams_[op_id]);
return ret;
}
int ObFakeTableScanVecOp::fill_random_data_into_expr_datum_frame(int expr_i, int expr_count, const ObExpr *expr,
const int output_row_count, bool &is_duplicate)
{
int ret = OB_SUCCESS;
LOG_DEBUG("generate_random_value for expr: ", K(*expr));
ObIVector *i_vec = NULL;
ObDatum *datums = NULL;
if (expr->enable_rich_format()) {
// vectorization 2.0 new operator
i_vec = expr->get_vector(eval_ctx_);
if (expr->is_fixed_length_data_) {
// for VEC_FIXED and VEC_UNIFORM
// we must first call init_vector because we need to use i_vec->set_xxx() which need use vector's meta data
OB_ASSERT(ObTestOpConfig::get_instance().digit_data_format_ == VEC_UNIFORM
|| ObTestOpConfig::get_instance().digit_data_format_ == VEC_FIXED);
expr->init_vector(eval_ctx_, ObTestOpConfig::get_instance().digit_data_format_, output_row_count, true);
} else {
// while for VEC_DISCRETE and VEC_CONTINUOUS it doesn't matter
OB_ASSERT(ObTestOpConfig::get_instance().string_data_format_ == VEC_UNIFORM
|| ObTestOpConfig::get_instance().string_data_format_ == VEC_DISCRETE
|| ObTestOpConfig::get_instance().string_data_format_ == VEC_CONTINUOUS);
expr->init_vector(eval_ctx_, ObTestOpConfig::get_instance().string_data_format_, output_row_count, true);
}
} else {
// vectorization 1.0 old operator
datums = expr->locate_datums_for_update(eval_ctx_, output_row_count);
}
// generate random data
get_random_data(expr_i, expr_count, expr, current_round_, output_row_count, expr->max_length_, is_duplicate);
int vec_continuous_offset = 0; // only used in VEC_CONTINUOUS
std::string vec_continuous_data; // use to store data of VEC_CONTINUOUS temporarily
ObDataGenerator::TempDataStore &data_store =
ObDataGenerator::get_instance().op_2_round_2_temp_store_[get_spec().get_id()][current_round_][expr_i];
for (int row = 0; row < output_row_count; row++) {
bool is_null = data_store.null_[row];
switch (expr->datum_meta_.get_type()) {
// why expr->datum_meta_.get_type() == ObInt32Type while expr->res_buf_len_ == 8 ?????
case ObInt32Type: {
int data = data_store.temp_int32_vector_[row];
if (i_vec != NULL) {
i_vec->set_int(row, static_cast<int64_t>(data));
if (is_null) { i_vec->set_null(row); }
} else {
datums[row].set_int32(data);
if (is_null) { datums[row].set_null(); }
}
break;
}
case ObIntType: {
int64_t data = data_store.temp_int64_vector_[row];
if (i_vec != NULL) {
i_vec->set_int(row, data);
if (is_null) { i_vec->set_null(row); }
} else {
datums[row].set_int(data);
if (is_null) { datums[row].set_null(); }
}
break;
}
case ObDoubleType: {
double data = data_store.temp_double_vector_[row];
if (i_vec != NULL) {
i_vec->set_double(row, data);
if (is_null) { i_vec->set_null(row); }
} else {
datums[row].set_double(data);
if (is_null) { datums[row].set_null(); }
}
break;
}
case ObNumberType: {
break;
}
case ObDecimalIntType: {
// have bug
// if decimal(20, 10)
// expr->res_buf_len_ = 16 is *not* equal to datums[x].len_ == 8
// where datums[x].len_ is set?
// i_vec->set_int(row,
// ObDataGenerator::get_instance().round_2_temp_store_[round_].first[expr_i].temp_int64_vector_[row]);
break;
}
case ObVarcharType:
case ObCharType: {
std::string tmp_str = data_store.temp_string_vector_[row];
int str_len = static_cast<int>(tmp_str.size());
if (i_vec != NULL) {
// vectorization 1.0 new operator
if (expr->get_format(eval_ctx_) == VEC_UNIFORM) {
if (is_null) {
i_vec->set_null(row);
} else {
i_vec->set_payload(row, tmp_str.data(), str_len);
}
} else {
if (is_null) { i_vec->set_null(row); }
if (expr->get_format(eval_ctx_) == VEC_DISCRETE) {
char **ptrs = expr->get_discrete_vector_ptrs(eval_ctx_);
int32_t *lens = expr->get_discrete_vector_lens(eval_ctx_);
ptrs[row] = expr->get_res_buf(eval_ctx_) + row * (expr->res_buf_len_);
lens[row] = str_len;
i_vec->set_payload(row, tmp_str.data(), str_len);
} else {
// VEC_CONTINUOUS
// offset[0] == 0 which is set in cg phase
vec_continuous_offset += str_len;
uint32_t *offset = expr->get_continuous_vector_offsets(eval_ctx_);
offset[row + 1] = vec_continuous_offset;
vec_continuous_data += tmp_str; // temporarily store data here
}
}
} else {
// vectorization 1.0 old operator
if (is_null) {
datums[row].set_null();
} else {
datums[row].set_string(tmp_str.data(), str_len);
}
}
break;
}
default: LOG_INFO("Can not generate random value so far for: ", K(expr->datum_meta_.get_type()));
}
}
if (expr->enable_rich_format() && expr->get_format(eval_ctx_) == VEC_CONTINUOUS && !vec_continuous_data.empty()) {
ObDynReserveBuf *drb = reinterpret_cast<ObDynReserveBuf *>(expr->get_continuous_vector_data(eval_ctx_));
// Todo: replace below logic when shengle handle continuous format memory
// in ObExpr::get_str_res_mem()
char *mem = NULL;
const int64_t alloc_size = next_pow2(vec_continuous_data.size());
if (OB_UNLIKELY(alloc_size > UINT32_MAX)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(vec_continuous_data.size()), K(alloc_size), K(ret));
} else if (OB_ISNULL(mem = static_cast<char *>(eval_ctx_.alloc_expr_res(alloc_size)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate memory failed", K(ret), K(ret));
} else {
// When extend memory, the old memory can not free, because the old memory may
// still be referenced. see: ob_datum_cast.cpp::common_copy_string
if (0 == drb->len_) { drb->magic_ = ObDynReserveBuf::MAGIC_NUM; }
drb->len_ = alloc_size;
drb->mem_ = mem;
MEMCPY(drb->mem_, vec_continuous_data.data(), vec_continuous_data.size());
ObContinuousBase *cont_vec = static_cast<ObContinuousBase *>(expr->get_vector(eval_ctx_));
cont_vec->set_data(drb->mem_);
}
LOG_DEBUG("extend expr result memory", K(ret), K(vec_continuous_data.size()), K(alloc_size), KP(this), KP(mem));
}
return ret;
}
int ObFakeTableScanVecOp::get_random_data(int expr_i, int expr_count, const ObExpr *expr, const int round,
const int batch_size, const int len, bool &is_duplicate)
{
int ret = OB_SUCCESS;
uint64_t op_id = get_spec().get_id();
switch (expr->datum_meta_.get_type()) {
// why expr->datum_meta_.get_type() == ObInt32Type while expr->res_buf_len_ == 8 ?????
case ObInt32Type: {
ObDataGenerator::get_instance().generate_data<int>(op_id, expr_i, expr_count, round, batch_size, len, is_duplicate);
break;
}
case ObIntType: {
ObDataGenerator::get_instance().generate_data<int64_t>(op_id, expr_i, expr_count, round, batch_size, len,
is_duplicate);
break;
}
case ObDoubleType: {
ObDataGenerator::get_instance().generate_data<double>(op_id, expr_i, expr_count, round, batch_size, len,
is_duplicate);
break;
}
case ObNumberType: {
break;
}
case ObDecimalIntType: {
// have bug
// if decimal(20, 10)
// expr->res_buf_len_ = 16 is *not* equal to datums[x].len_ == 8
// where datums[x].len_ is set?
ObDataGenerator::get_instance().generate_data<int64_t>(op_id, expr_i, expr_count, round, batch_size, len,
is_duplicate);
break;
}
case ObVarcharType:
case ObCharType: {
ObDataGenerator::get_instance().generate_data<std::string>(op_id, expr_i, expr_count, round, batch_size, len,
is_duplicate);
break;
}
default: LOG_INFO("Can not generate random value so far for: ", K(expr->datum_meta_.get_type()));
}
return ret;
}
void ObFakeTableScanVecOp::set_random_skip(const int round, const int batch_size)
{
std::string generate_data_skips;
uint64_t op_id = get_spec().get_id();
if (ObDataGenerator::get_instance().op_2_round_2_skips_[op_id].count(round) == 0) {
std::uniform_int_distribution<int> u_i(0, 1);
bool if_skip;
for (int i = 0; i < batch_size; i++) {
if_skip =
ObDataGenerator::get_instance().zero_one_rand_by_probability(ObTestOpConfig::get_instance().skips_probability_);
generate_data_skips += std::to_string(!if_skip) + " ";
ObDataGenerator::get_instance().op_2_round_2_skips_[op_id][round].push_back(if_skip);
}
}
for (int i = 0; i < batch_size; i++) {
if (ObDataGenerator::get_instance().op_2_round_2_skips_[op_id][round][i] == true) { brs_.skip_->set(i); }
}
LOG_INFO("skips : ", K(generate_data_skips.data()));
return;
}
} // namespace sql
} // namespace oceanbase

55
unittest/sql/engine/ob_fake_table_scan_vec_op.h Normal file
View File

@ -0,0 +1,55 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <random>
#include <chrono>
#include <fstream>
#include "../test_sql_utils.h"
#include "sql/engine/table/ob_table_scan_op.h"
#include "sql/engine/ob_operator.h"
#include "sql/engine/ob_operator_reg.h"
namespace oceanbase
{
namespace sql
{
class ObFakeTableScanVecOp : public ObTableScanOp
{
friend class ObDASScanOp;
friend class ObGlobalIndexLookupOpImpl;
public:
static constexpr int64_t CHECK_STATUS_ROWS_INTERVAL = 1 << 13;
ObFakeTableScanVecOp(ObExecContext &exec_ctx, const ObOpSpec &spec, ObOpInput *input) :
ObTableScanOp(exec_ctx, spec, input)
{}
~ObFakeTableScanVecOp() = default;
int inner_open() override;
int inner_get_next_batch(const int64_t max_row_cnt) override;
int fill_random_data_into_expr_datum_frame(int expr_i, int expr_count, const ObExpr *expr, const int output_max_count,
bool &is_duplicate);
int get_random_data(int expr_i, int expr_count, const ObExpr *expr, const int round, const int batch_size,
const int len, bool &is_duplicate);
void set_random_skip(const int round, const int batch_size);
public:
int max_round_{2};
int current_round_{1};
// io
std::unordered_map<uint64_t, std::ofstream> op_id_2_output_streams_;
};
} // end namespace sql
} // end namespace oceanbase

154
unittest/sql/engine/ob_test_config.h Normal file
View File

@ -0,0 +1,154 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <string>
#include <random>
#include <chrono>
#include <vector>
#include <unordered_map>
#include <type_traits>
#include <fstream>
#include "lib/ob_define.h"
// class ObOperator;
namespace oceanbase
{
namespace sql
{
class ObTestOpConfig
{
public:
static ObTestOpConfig &get_instance()
{
static ObTestOpConfig test_op_config;
return test_op_config;
}
private:
ObTestOpConfig()
{}
~ObTestOpConfig()
{}
void init()
{
std::string nulls_probability;
std::string test_config_file = test_filename_prefix_ + ".cfg";
test_filename_origin_output_file_ = test_filename_prefix_ + "_origin_result.data";
test_filename_vec_output_file_ = test_filename_prefix_ + "_vec_result.data";
std::ifstream if_tests_config(test_config_file);
OB_ASSERT(if_tests_config.is_open() == true);
std::string line;
while (std::getline(if_tests_config, line)) {
if (line.size() <= 0) continue;
if (line.at(0) == '#') continue;
std::vector<std::string> out;
const char delim = '=';
tokenize(line, delim, out);
OB_ASSERT(out.size() == 2);
configs_map_[out[0]] = out[1];
}
set_configs();
LOG_INFO("Testing config is: ", K(*this));
}
void tokenize(std::string const &str, const char delim, std::vector<std::string> &out)
{
size_t start;
size_t end = 0;
while ((start = str.find_first_not_of(delim, end)) != std::string::npos) {
end = str.find(delim, start);
out.push_back(str.substr(start, end - start));
}
}
int get_config(std::string key, std::string &value)
{
int ret = OB_SUCCESS;
if (configs_map_.count(key) == 0) {
ret = OB_INVALID_ARGUMENT;
} else {
value = configs_map_[key];
}
return ret;
}
void set_configs()
{
std::string digit_data_format;
std::string string_data_format;
std::string data_range_level;
std::string nulls_probability;
std::string skips_probability;
std::string round;
std::string batch_size;
std::string output_result_to_file;
get_config("digit_data_format", digit_data_format);
get_config("string_data_format", string_data_format);
get_config("data_range_level", data_range_level);
get_config("nulls_probability", nulls_probability);
get_config("skips_probability", skips_probability);
get_config("round", round);
get_config("batch_size", batch_size);
get_config("output_result_to_file", output_result_to_file);
if (!digit_data_format.empty()) { digit_data_format_ = static_cast<VectorFormat>(std::stoi(digit_data_format)); }
if (!string_data_format.empty()) { string_data_format_ = static_cast<VectorFormat>(std::stoi(string_data_format)); }
if (!data_range_level.empty()) { data_range_level_ = std::stoi(data_range_level); }
if (!nulls_probability.empty()) { nulls_probability_ = std::stoi(nulls_probability); }
if (!skips_probability.empty()) { skips_probability_ = std::stoi(skips_probability); }
if (!round.empty()) { round_ = std::stoi(round); }
if (!batch_size.empty()) { batch_size_ = std::stoi(batch_size); }
if (!output_result_to_file.empty()) { output_result_to_file_ = std::stoi(output_result_to_file); }
}
int64_t to_string(char *buf, const int64_t buf_len) const
{
int64_t pos = 0;
J_KV(K(digit_data_format_), K(string_data_format_), K(data_range_level_), K(nulls_probability_),
K(skips_probability_), K(round_), K(batch_size_), K(output_result_to_file_));
return pos;
}
private:
std::unordered_map<std::string, std::string> configs_map_;
bool run_in_background_{false};
std::string test_filename_prefix_;
std::string test_filename_origin_output_file_;
std::string test_filename_vec_output_file_;
bool output_result_to_file_{false};
// data foramt
VectorFormat digit_data_format_{VEC_UNIFORM};
VectorFormat string_data_format_{VEC_UNIFORM};
// distribution and data range
int data_range_level_{0};
// nulls and skips
int nulls_probability_{0};
int skips_probability_{0};
// round and batch_size
int round_{10};
int batch_size_{256};
};
} // end namespace sql
} // end namespace oceanbase

120
unittest/sql/engine/test_bit_vector.cpp Normal file
View File

@ -0,0 +1,120 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#include <cstring>
#include "lib/allocator/page_arena.h"
#include "common/object/ob_object.h"
#include "lib/container/ob_se_array.h"
#include "src/sql/engine/ob_bit_vector.h"
namespace oceanbase
{
namespace sql
{
class ObTestBitVector: public ::testing::Test
{
public:
ObTestBitVector() {}
~ObTestBitVector() {}
virtual void SetUp() {}
virtual void TearDown() {}
private:
DISALLOW_COPY_AND_ASSIGN(ObTestBitVector);
};
void expect_range(ObBitVector *dest_bit_vector, int64_t start, int64_t middle, int64_t end) {
for (int64_t i = 0; i < start; i++) {
EXPECT_EQ(0, dest_bit_vector->at(i));
}
for (int64_t i = start; i < middle; i++) {
EXPECT_EQ(1, dest_bit_vector->at(i));
dest_bit_vector->unset(i);
}
EXPECT_EQ(0, dest_bit_vector->at(middle));
for (int64_t i = middle + 1; i < end; i++) {
EXPECT_EQ(1, dest_bit_vector->at(i));
dest_bit_vector->unset(i);
}
for (int64_t i = end; i < end + 100; i++) {
EXPECT_EQ(0, dest_bit_vector->at(i));
}
}
void test_range(ObBitVector *dest_bit_vector, ObBitVector *src_bit_vector, int64_t start,
int64_t end)
{
for (int i = 0; i < 2000; i++) {
src_bit_vector->set(i);
}
int64_t middle = (start + end) / 2;
dest_bit_vector->set_all(start, end);
dest_bit_vector->unset(middle);
expect_range(dest_bit_vector, start, middle, end);
src_bit_vector->unset(middle);
dest_bit_vector->deep_copy(*src_bit_vector, start, end);
expect_range(dest_bit_vector, start, middle, end);
dest_bit_vector->bit_or(*src_bit_vector, start, end);
expect_range(dest_bit_vector, start, middle, end);
src_bit_vector->set(middle);
for (int64_t i = start; i < end; i++) {
dest_bit_vector->set(i);
}
EXPECT_EQ(1, dest_bit_vector->is_all_true(start, end));
if (start > 0) {
EXPECT_EQ(0, dest_bit_vector->is_all_true(start - 1, end));
}
EXPECT_EQ(0, dest_bit_vector->is_all_true(start, end + 1));
for (int64_t i = start; i < end; i++) {
dest_bit_vector->unset(i);
}
}
TEST(ObTestBitVector, bit_or_range)
{
char src_buf[1024];
char dest_buf[1024];
MEMSET(src_buf, 0, 1024);
MEMSET(dest_buf, 0, 1024);
ObBitVector *src_bit_vector = new (src_buf) ObBitVector;
ObBitVector *dest_bit_vector = new (dest_buf) ObBitVector;
test_range(dest_bit_vector, src_bit_vector, 13, 40);
test_range(dest_bit_vector, src_bit_vector, 13, 63);
test_range(dest_bit_vector, src_bit_vector, 13, 64);
test_range(dest_bit_vector, src_bit_vector, 13, 127);
test_range(dest_bit_vector, src_bit_vector, 13, 128);
test_range(dest_bit_vector, src_bit_vector, 13, 258);
test_range(dest_bit_vector, src_bit_vector, 0, 50);
test_range(dest_bit_vector, src_bit_vector, 0, 100);
test_range(dest_bit_vector, src_bit_vector, 0, 63);
test_range(dest_bit_vector, src_bit_vector, 0, 64);
test_range(dest_bit_vector, src_bit_vector, 0, 0);
test_range(dest_bit_vector, src_bit_vector, 64, 64);
test_range(dest_bit_vector, src_bit_vector, 64, 127);
}
}
}
int main(int argc, char **argv)
{
::testing::InitGoogleTest(&argc, argv);
int ret = RUN_ALL_TESTS();
return ret;
}

46
unittest/sql/engine/test_engine.schema
View File

@ -1,24 +1,26 @@
create database opt;
use opt;
#create database opt;
#use opt;
#create table t1(c1 int primary key, c2 int) partition by hash (c1) partitions 5
create table t2(c1 int primary key, c2 int, c3 varchar(32)) partition by hash (c1) partitions 3
create table t3(c1 int primary key, c2 int, c3 varchar(32)) partition by hash (c1) partitions 2
#create table t2(c1 int primary key, c2 int, c3 varchar(32)) partition by hash (c1) partitions 3
#create table t3(c1 int primary key, c2 int, c3 varchar(32)) partition by hash (c1) partitions 2
#create index idx_t1_c2 on t1(c2) LOCAL
create table t4(c1 int, c2 int, c3 int, primary key(c1, c2))
create index idx_t4_c2_c3 on t4(c2, c3)
create index idx_t4_c3 on t4(c3)
create index idx_t4_c2 on t4(c2)
create table t5(c1 int, c2 int, c3 int, primary key(c2,c3)) partition by key(c2, c3) partitions 3
create index idx_t5_c2 on t5(c2) LOCAL
create index idx_t5_c3 on t5(c3) LOCAL
create table t6(c1 int primary key, c2 tinyint) partition by key(c1) partitions 3
create table t7(c1 int primary key, c2 int)
create table t8(c1 int primary key, c2 int)
create table t2_no_part(c1 int primary key, c2 int, c3 int);
create table t9(c1 int primary key, c2 int not null default 1, c3 int default 2);
create table t10(c1 int, c2 int, c3 int, primary key(c2,c1)) partition by hash (c2) partitions 2;
create table t11(c1 int, c2 int, c3 int, primary key(c2,c1)) partition by hash (c2) partitions 2;
create table t12 (a bigint primary key, b char(20), c bigint);
create table t13 (b char(20), c bigint, a bigint primary key);
create table te1(c1 int primary key);
create table tab1(id int, parent_id int);
#create table t4(c1 int, c2 int, c3 int, primary key(c1, c2))
#create index idx_t4_c2_c3 on t4(c2, c3)
#create index idx_t4_c3 on t4(c3)
#create index idx_t4_c2 on t4(c2)
#create table t5(c1 int, c2 int, c3 int, primary key(c2,c3)) partition by key(c2, c3) partitions 3
#create index idx_t5_c2 on t5(c2) LOCAL
#create index idx_t5_c3 on t5(c3) LOCAL
#create table t6(c1 int primary key, c2 tinyint) partition by key(c1) partitions 3
create table t7(c1 int, c2 int)
#create table t8(c1 int primary key, c2 int)
#create table t2_no_part(c1 int primary key, c2 int, c3 int);
#create table t9(c1 int primary key, c2 int not null default 1, c3 int default 2);
#create table t10(c1 int, c2 int, c3 int, primary key(c2,c1)) partition by hash (c2) partitions 2;
#create table t11(c1 int, c2 int, c3 int, primary key(c2,c1)) partition by hash (c2) partitions 2;
#create table t12 (a bigint primary key, b char(20), c bigint);
#create table t13 (b char(20), c bigint, a bigint primary key);
#create table te1(c1 int primary key);
#create table tab1(id int, parent_id int);
#create table t1(c1 int, c2 double, c3 decimal(20,10), c4 char(20), c5 varchar(40));
create table t1(c1 int, c2 double, c4 char(20), c5 varchar(40));

7
unittest/sql/engine/test_engine.test
View File

@ -3,3 +3,10 @@
#insert into te1 values (0)
#update te1 set c1 = 1 where c1 = 2
#delete from te1 where c1 = 1
#select avg(c1) from t7 group by c2;
#select c2, sum(c1), min(c1), max(c1) from t7 group by c2;
select/*+USE_HASH_AGGREGATION*/ c2, count(c1), sum(c1 + 1), min(c1*2), max(c1 - 1) from t7 group by c2;
#select c2 from t7 order by c2;
#select /*+blocking('all')*/* from t7;
#select /*+blocking('all')*/* from t1;
#select distinct(c2) from t7;

810
unittest/sql/engine/test_op_engine.cpp Normal file
View File

@ -0,0 +1,810 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
// #define USING_LOG_PREFIX SQL_ENGINE
#define USING_LOG_PREFIX COMMON
#include <stdlib.h>
#include <sys/wait.h>
#include <iterator>
#include <gtest/gtest.h>
#include "test_op_engine.h"
#include "src/observer/omt/ob_tenant_config_mgr.h"
#include "sql/test_sql_utils.h"
#include "lib/container/ob_array.h"
#include "sql/ob_sql_init.h"
#include "sql/plan_cache/ob_cache_object_factory.h"
#include "observer/ob_req_time_service.h"
#include "ob_fake_table_scan_vec_op.h"
#include "share/ob_simple_mem_limit_getter.h"
#include "src/share/ob_local_device.h"
#include "src/share/ob_device_manager.h"
#include "src/storage/blocksstable/ob_storage_cache_suite.h"
#include "ob_test_config.h"
#include <vector>
#include <string>
using namespace oceanbase::sql;
namespace test
{
TestOpEngine::TestOpEngine() : tbase_{sys_tenant_id_}, vec_2_exec_ctx_(vec_2_alloc_)
{
vec_2_exec_ctx_.set_sql_ctx(&sql_ctx_);
}
TestOpEngine::~TestOpEngine()
{
destory();
}
void TestOpEngine::SetUp()
{
TestOptimizerUtils::SetUp();
addr_.set_ip_addr("1.1.1.1", 8888);
vec_2_exec_ctx_.set_my_session(&session_info_);
vec_2_exec_ctx_.create_physical_plan_ctx();
ASSERT_EQ(prepare_io(ObTestOpConfig::get_instance().test_filename_prefix_), OB_SUCCESS);
// init mock location service, used in optimizer compute table property
GCTX.location_service_ = &mock_location_service_;
// init MTL, used in ObTableScanOp::ObTableScanOp constructor
static ObDataAccessService instance;
tbase_.inner_set(&instance);
ASSERT_EQ(tbase_.init(), 0);
ObTenantEnv::set_tenant(&tbase_);
out_origin_result_stream_.open(ObTestOpConfig::get_instance().test_filename_origin_output_file_, std::ios::out | std::ios::trunc);
out_vec_result_stream_.open(ObTestOpConfig::get_instance().test_filename_vec_output_file_, std::ios::out | std::ios::trunc);
}
void TestOpEngine::TearDown()
{
destroy();
}
void TestOpEngine::destory()
{
OB_SERVER_BLOCK_MGR.stop();
OB_SERVER_BLOCK_MGR.wait();
OB_SERVER_BLOCK_MGR.destroy();
OB_STORE_CACHE.destroy();
ObIOManager::get_instance().destroy();
ObKVGlobalCache::get_instance().destroy();
ObClusterVersion::get_instance().destroy();
ObTmpFileManager::get_instance().destroy();
// THE_IO_DEVICE->destroy();
}
common::ObIODevice *TestOpEngine::get_device_inner()
{
int ret = OB_SUCCESS;
common::ObIODevice *device = NULL;
common::ObString storage_info(OB_LOCAL_PREFIX);
// for the local and nfs, storage_prefix and storage info are same
if (OB_FAIL(common::ObDeviceManager::get_instance().get_device(storage_info, storage_info, device))) {
LOG_WARN("get_device_inner", K(ret));
}
return device;
}
// copy from mittest/mtlenv/mock_tenant_module_env.h and unittest/storage/blocksstable/ob_data_file_prepare.h
// refine some code
// call prepare_io() for testing operators that needs to dump intermediate data
int TestOpEngine::prepare_io(const string & test_data_name_suffix)
{
int ret = OB_SUCCESS;
ObIODOpt iod_opt_array[5];
ObIODOpts iod_opts;
iod_opts.opts_ = iod_opt_array;
int64_t macro_block_count = 5 * 1024;
int64_t macro_block_size = 64 * 1024;
char cur_dir[OB_MAX_FILE_NAME_LENGTH];
char test_data_name[OB_MAX_FILE_NAME_LENGTH];
char data_dir[OB_MAX_FILE_NAME_LENGTH];
char file_dir[OB_MAX_FILE_NAME_LENGTH];
char clog_dir[OB_MAX_FILE_NAME_LENGTH];
char slog_dir[OB_MAX_FILE_NAME_LENGTH];
if (NULL == getcwd(cur_dir, OB_MAX_FILE_NAME_LENGTH)) {
ret = OB_BUF_NOT_ENOUGH;
STORAGE_LOG(WARN, "cannot get cur dir", K(ret));
} else if (OB_FAIL(databuff_printf(test_data_name, OB_MAX_FILE_NAME_LENGTH, "%s", test_data_name_suffix.data()))) {
STORAGE_LOG(WARN, "failed to gen test name", K(ret));
} else if (OB_FAIL(databuff_printf(data_dir, OB_MAX_FILE_NAME_LENGTH, "%s/data_%s", cur_dir, test_data_name))) {
STORAGE_LOG(WARN, "failed to gen data dir", K(ret));
} else if (OB_FAIL(databuff_printf(file_dir, OB_MAX_FILE_NAME_LENGTH, "%s/sstable/", data_dir))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
} else if (OB_FAIL(databuff_printf(slog_dir, OB_MAX_FILE_NAME_LENGTH, "%s/slog/", data_dir))) {
STORAGE_LOG(WARN, "failed to gen slog dir", K(ret));
} else if (OB_FAIL(databuff_printf(clog_dir, OB_MAX_FILE_NAME_LENGTH, "%s/clog/", data_dir))) {
STORAGE_LOG(WARN, "failed to gen clog dir", K(ret));
}
storage_env_.data_dir_ = data_dir;
storage_env_.sstable_dir_ = file_dir;
storage_env_.clog_dir_ = clog_dir;
storage_env_.default_block_size_ = common::OB_DEFAULT_MACRO_BLOCK_SIZE;
storage_env_.data_disk_size_ = macro_block_count * common::OB_DEFAULT_MACRO_BLOCK_SIZE;
storage_env_.data_disk_percentage_ = 0;
storage_env_.log_disk_size_ = 20 * 1024 * 1024 * 1024ll;
share::ObLocalDevice *local_device = static_cast<share::ObLocalDevice *>(get_device_inner());
THE_IO_DEVICE = local_device;
iod_opt_array[0].set("data_dir", storage_env_.data_dir_);
iod_opt_array[1].set("sstable_dir", storage_env_.sstable_dir_);
iod_opt_array[2].set("block_size", storage_env_.default_block_size_);
iod_opt_array[3].set("datafile_disk_percentage", storage_env_.data_disk_percentage_);
iod_opt_array[4].set("datafile_size", storage_env_.data_disk_size_);
iod_opts.opt_cnt_ = 5;
ObTenantIOConfig io_config = ObTenantIOConfig::default_instance();
const int64_t async_io_thread_count = 8;
const int64_t sync_io_thread_count = 2;
const int64_t max_io_depth = 256;
const int64_t bucket_num = 1024L;
const int64_t block_size = common::OB_MALLOC_BIG_BLOCK_SIZE;
char cmd[OB_MAX_FILE_NAME_LENGTH];
if (OB_FAIL(ret)) {
// do nothing
} else if (OB_FAIL(databuff_printf(cmd, OB_MAX_FILE_NAME_LENGTH, "rm -rf %s", data_dir))) {
LOG_WARN("failed to gen cmd", K(ret));
} else if (0 != system(cmd)) {
ret = OB_ERR_SYS;
LOG_WARN("failed to exec cmd", K(ret), K(cmd), K(errno), KERRMSG);
} else if (OB_FAIL(THE_IO_DEVICE->init(iod_opts))) {
LOG_WARN("fail to init io device", K(ret), K_(storage_env));
} else if (OB_FAIL(ObIOManager::get_instance().init())) {
LOG_WARN("fail to init io manager", K(ret));
} else if (OB_FAIL(ObIOManager::get_instance().add_device_channel(THE_IO_DEVICE, async_io_thread_count,
sync_io_thread_count, max_io_depth))) {
LOG_WARN("add device channel failed", K(ret));
} else if (OB_FAIL(ObIOManager::get_instance().start())) {
LOG_WARN("fail to start io manager", K(ret));
} else if (OB_FAIL(ObIOManager::get_instance().add_tenant_io_manager(OB_SYS_TENANT_ID, io_config))) {
LOG_WARN("add tenant io config failed", K(ret));
} else if (OB_FAIL(OB_SERVER_BLOCK_MGR.init(THE_IO_DEVICE, storage_env_.default_block_size_))) {
STORAGE_LOG(WARN, "init block manager fail", K(ret));
} else if (OB_FAIL(FileDirectoryUtils::create_full_path(file_dir))) {
STORAGE_LOG(WARN, "failed to create file dir", K(ret), K(file_dir));
} else if (OB_FAIL(OB_SERVER_BLOCK_MGR.start(0 /*reserver_size*/))) {
STORAGE_LOG(WARN, "Fail to start server block mgr", K(ret));
} else if (OB_FAIL(OB_SERVER_BLOCK_MGR.first_mark_device())) {
STORAGE_LOG(WARN, "Fail to start first mark device", K(ret));
} else if (OB_FAIL(OB_STORE_CACHE.init(10, 1, 1, 1, 1, 10000, 10))) {
LOG_WARN("fail to init OB_STORE_CACHE, ", K(ret));
} else {
}
FILE_MANAGER_INSTANCE_V2.init();
return ret;
}
int TestOpEngine::do_rewrite(ObStmt *&stmt, ObPhysicalPlan *phy_plan)
{
int ret = OB_SUCCESS;
ObSchemaChecker schema_checker;
if (OB_FAIL(schema_checker.init(sql_schema_guard_))) {
LOG_WARN("fail to init schema_checker", K(ret));
} else {
ObTransformerCtx transformer_ctx;
transformer_ctx.allocator_ = &allocator_;
transformer_ctx.session_info_ = &session_info_;
transformer_ctx.schema_checker_ = &schema_checker;
transformer_ctx.exec_ctx_ = &exec_ctx_;
transformer_ctx.expr_factory_ = &expr_factory_;
transformer_ctx.stmt_factory_ = &stmt_factory_;
// ctx.stat_mgr_ = &stat_manager_;
transformer_ctx.sql_schema_guard_ = &sql_schema_guard_;
transformer_ctx.self_addr_ = &addr_;
transformer_ctx.merged_version_ = OB_MERGED_VERSION_INIT;
transformer_ctx.phy_plan_ = phy_plan;
ObTransformerImpl transformer(&transformer_ctx);
ObDMLStmt *sql_stmt = dynamic_cast<ObDMLStmt *>(stmt);
if (stmt->is_explain_stmt()) { sql_stmt = static_cast<ObExplainStmt *>(stmt)->get_explain_query_stmt(); }
if (OB_SUCC(ret)) {
if (OB_FAIL(transformer.transform(sql_stmt))) {
LOG_WARN("Failed to transform statement", K(ret));
} else {
stmt = sql_stmt;
}
}
}
return ret;
}
int TestOpEngine::do_optimize(ObStmt *stmt, ObLogPlan *&plan, ObPhyPlanType distr, ObArenaAllocator &allocator,
ObExecContext &exec_ctx)
{
int ret = OB_SUCCESS;
ObDMLStmt *dml_stmt = dynamic_cast<ObDMLStmt *>(stmt);
ObOptimizerContext *ctx_ptr = static_cast<ObOptimizerContext *>(allocator.alloc(sizeof(ObOptimizerContext)));
exec_ctx.get_sql_ctx()->session_info_ = &session_info_;
ObOptimizerContext *opt_ctx = new (ctx_ptr) ObOptimizerContext(
&session_info_, &exec_ctx,
// schema_mgr_, // schema manager
&sql_schema_guard_,
//&stat_manager_, // statistics manager
NULL, // statistics manager
static_cast<ObIAllocator &>(allocator_), &param_store_, addr_, NULL, dml_stmt->get_query_ctx()->get_global_hint(),
expr_factory_, dml_stmt, false, stmt_factory_.get_query_ctx());
opt_ctx->set_opt_stat_manager(&opt_stat_manager_);
opt_ctx->disable_batch_rpc();
opt_ctx->set_local_server_addr("1.1.1.1", 8888);
opt_ctx->set_use_default_stat();
ObTableLocation table_location;
ret = opt_ctx->get_table_location_list().push_back(table_location);
ObOptimizer optimizer(*opt_ctx);
if (OB_FAIL(optimizer.optimize(*dml_stmt, plan))) { LOG_WARN("failed to optimize", "SQL", *dml_stmt); }
return ret;
}
int TestOpEngine::do_code_generate(const ObLogPlan &log_plan, ObCodeGenerator &code_gen, ObPhysicalPlan &phy_plan)
{
int ret = OB_SUCCESS;
const uint64_t cur_cluster_version = CLUSTER_CURRENT_VERSION;
// WARN: may have bug here
log_plan.get_optimizer_context().set_batch_size(ObTestOpConfig::get_instance().batch_size_);
phy_plan.set_batch_size(ObTestOpConfig::get_instance().batch_size_);
if (OB_FAIL(ret)) {
} else if (OB_FAIL(code_gen.generate_exprs(log_plan, phy_plan, cur_cluster_version))) {
LOG_WARN("fail to get all raw exprs", K(ret));
} else if (OB_FAIL(code_gen.generate_operators(log_plan, phy_plan, cur_cluster_version))) {
LOG_WARN("fail to generate plan", K(ret));
}
return ret;
}
int TestOpEngine::test_phy_plan(ObPhysicalPlan &plan)
{
UNUSED(plan);
int ret = OB_SUCCESS;
return ret;
}
ObOperator *TestOpEngine::subtitude_table_scan_to_fake(ObOperator *root)
{
for (uint32 i = 0; i < root->get_child_cnt(); i++) {
root->children_[i] = subtitude_table_scan_to_fake(root->children_[i]);
if (i == 0) {
root->left_ = root->children_[i];
root->left_->parent_ = root;
}
if (i == 1) {
root->right_ = root->children_[i];
root->right_->parent_ = root;
}
}
if (root->get_spec().get_type() == PHY_TABLE_SCAN) {
root = new oceanbase::sql::ObFakeTableScanVecOp(root->get_exec_ctx(), root->get_spec(), root->get_input());
}
return root;
}
int TestOpEngine::get_tested_op_from_string(const std::string &sql, bool vector_2, ObOperator *&op,
ObExecutor &executor)
{
int ret = OB_SUCCESS;
ObStmt *stmt = NULL;
ObLogPlan *log_plan = NULL;
ObPhysicalPlan *phy_plan = NULL;
ObArenaAllocator *p_alloc = NULL;
ObExecContext *p_exec_ctx = NULL;
if (vector_2) {
p_alloc = &vec_2_alloc_;
p_exec_ctx = &vec_2_exec_ctx_;
} else {
p_alloc = &allocator_;
p_exec_ctx = &exec_ctx_;
}
// 1.resolve
// 2.rewrite
// 3.optimize
do_resolve(sql.c_str(), stmt, true, JSON_FORMAT, OB_SUCCESS, false);
if (OB_FAIL(do_rewrite(stmt, phy_plan))) {
LOG_ERROR("rewrite failed", K(ret));
} else if (OB_FAIL(do_optimize(stmt, log_plan, OB_PHY_PLAN_LOCAL, *p_alloc, *p_exec_ctx))) {
LOG_ERROR("optimize failed", K(ret));
} else if (NULL == log_plan) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("log_plan is null");
} else if (OB_FAIL(ObCacheObjectFactory::alloc(phy_plan))) {
LOG_ERROR("fail to allocate mem to phy_plan", K(ret));
} else if (NULL == phy_plan) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("phy_plan is null");
} else {
// display explain plan
oceanbase::sql::ObExplainDisplayOpt option;
option.with_tree_line_ = true;
ObSqlPlan sql_plan(log_plan->get_allocator());
ObSEArray<common::ObString, 64> plan_strs;
if (OB_FAIL(sql_plan.print_sql_plan(log_plan, EXPLAIN_EXTENDED_NOADDR, option, plan_strs))) {
LOG_WARN("failed to store sql plan", K(ret));
} else {
LOG_INFO("Generate Logical plan:");
_OB_LOG(INFO, "%*s", plan_strs.at(0).length(), plan_strs.at(0).ptr());
}
// 4.generate physical plan
if (OB_FAIL(generate_physical_plan(log_plan, *phy_plan, *p_exec_ctx, vector_2))) {
LOG_ERROR("generate physical plan failed", K(ret));
}
// 5.open and get runtime op
// ObFakeTableScanOp will replace bottom ObTableScanOp here
if (OB_FAIL(open_and_get_op(*p_exec_ctx, executor, *phy_plan, op))) { LOG_ERROR("open operators failed", K(ret)); }
}
return ret;
}
std::string TestOpEngine::get_decimal_result_from_datum(ObExpr *expr, const ObDatum &datum)
{
int ret = OB_SUCCESS;
std::string result;
switch (get_decimalint_type(expr->datum_meta_.precision_)) {
case common::DECIMAL_INT_32: result = std::to_string(datum.get_decimal_int32()); break;
case common::DECIMAL_INT_64: result = std::to_string(datum.get_decimal_int64()); break;
case common::DECIMAL_INT_128: result = std::to_string(datum.get_decimal_int128()); break;
case common::DECIMAL_INT_256: result = std::to_string(datum.get_decimal_int256()); break;
case common::DECIMAL_INT_512: result = std::to_string(datum.get_decimal_int512()); break;
default: LOG_WARN("unexpected precision", K(expr->datum_meta_));
}
return result;
}
std::string TestOpEngine::get_decimal_result_from_payload(ObExpr *expr, const char *payload)
{
int ret = OB_SUCCESS;
std::string result;
switch (get_decimalint_type(expr->datum_meta_.precision_)) {
case common::DECIMAL_INT_32: result = std::to_string(*(int32_t *)payload); break;
case common::DECIMAL_INT_64: result = std::to_string(*(int64_t *)payload); break;
case common::DECIMAL_INT_128: result = std::to_string(*(int128_t *)payload); break;
case common::DECIMAL_INT_256: result = std::to_string(*(int256_t *)payload); break;
case common::DECIMAL_INT_512: result = std::to_string(*(int512_t *)payload); break;
default: LOG_WARN("unexpected precision", K(expr->datum_meta_));
}
return result;
}
std::string TestOpEngine::get_data_by_datum_type(const ObOperator *op, ObExpr *expr, ObEvalCtx &eval_ctx, int row)
{
std::string result_str;
ObIVector *i_vector = NULL;
ObDatum *datums = NULL;
if (expr->enable_rich_format() && op->spec_.get_type() != PHY_EXPR_VALUES) {
i_vector = expr->get_vector(eval_ctx);
if (i_vector->is_null(row)) { result_str = "null"; }
} else {
datums = expr->locate_batch_datums(eval_ctx);
if (datums[row].is_null()) { result_str = "null"; }
}
if (result_str != "null") {
switch (expr->datum_meta_.get_type()) {
// why expr->datum_meta_.get_type() == ObInt32Type while expr->res_buf_len_ == 8 ?????
case ObInt32Type:
if (i_vector != NULL) {
result_str = std::to_string(i_vector->get_int32(row));
// str_result = std::to_string(i_vector->get_int(row));
} else {
result_str = std::to_string(datums[row].get_int32());
}
break;
case ObIntType: {
if (i_vector != NULL) {
result_str = std::to_string(i_vector->get_int(row));
} else {
result_str = std::to_string(datums[row].get_int());
}
break;
}
case ObDoubleType: {
if (i_vector != NULL) {
result_str = std::to_string(i_vector->get_double(row));
} else {
result_str = std::to_string(datums[row].get_double());
}
break;
}
case ObNumberType: {
// result type of avg()
if (i_vector != NULL) {
ObNumber ob_num(i_vector->get_number(row));
char buf[ob_num.get_length()];
ob_num.to_string(buf, ob_num.get_length());
result_str = std::string(buf, ob_num.get_length());
} else {
ObNumber ob_num(datums[row].get_number());
char buf[ob_num.get_length()];
ob_num.to_string(buf, ob_num.get_length());
result_str = std::string(buf, ob_num.get_length());
}
break;
}
case ObDecimalIntType: {
// result type of sum()
if (i_vector != NULL) {
// TODO: replace this when i_vector have get_decimalXXX() api
result_str = get_decimal_result_from_payload(expr, i_vector->get_payload(row));
} else {
result_str = get_decimal_result_from_datum(expr, datums[row]);
}
break;
}
case ObVarcharType:
case ObCharType: {
if (i_vector != NULL) {
result_str = std::string(i_vector->get_payload(row), i_vector->get_length(row));
} else {
// str_result = std::to_string(datums[row].get_int32());
result_str = std::string(datums[row].ptr_, datums[row].len_);
}
break;
}
default: LOG_INFO("Can not display value so far for: ", K(expr->datum_meta_.get_type()));
}
}
return result_str;
}
int TestOpEngine::generate_physical_plan(ObLogPlan *log_plan, ObPhysicalPlan &phy_plan, ObExecContext &exec_ctx,
bool enable_rich_format)
{
int ret = OB_SUCCESS;
// begin generate phsical plan
ObPhysicalPlanCtx *pctx = exec_ctx.get_physical_plan_ctx();
if (NULL == pctx) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("pctx is null");
} else {
/*
bool ObStaticEngineExprCG::enable_rich_format() const {
//TODO shengle change the version
return cur_cluster_version_ >= CLUSTER_VERSION_4_1_0_0
&& op_cg_ctx_.session_->enable_rich_format();
}
*/
// So we need set here to support rich_format
ObCodeGenerator code_gen(false /*use_jit*/, CLUSTER_VERSION_4_3_0_0, &(pctx->get_datum_param_store()));
log_plan->get_optimizer_context().get_session_info()->sys_vars_cache_.set_enable_rich_vector_format(enable_rich_format);
phy_plan.set_use_rich_format(enable_rich_format);
if (OB_FAIL(do_code_generate(*log_plan, code_gen, phy_plan))) {
LOG_ERROR("Can not generate physical plan ", K(ret));
} else {
pctx->set_phy_plan(&phy_plan);
}
}
return ret;
}
int TestOpEngine::open_and_get_op(ObExecContext &exec_ctx, ObExecutor &ob_exe, ObPhysicalPlan &phy_plan,
ObOperator *&root)
{
int ret = OB_SUCCESS;
ObTaskExecutorCtx &task_exec_ctx = exec_ctx.get_task_exec_ctx();
ObExecuteResult &exe_result = task_exec_ctx.get_execute_result();
// ObExecutor ob_exe;
if (OB_FAIL(ob_exe.init(&phy_plan))) {
LOG_WARN("fail init exec ObExecutor", K(ret));
} else if (OB_FAIL(exec_ctx.init_phy_op(phy_plan.get_phy_operator_size()))) {
LOG_WARN("fail init exec phy op ctx", K(ret));
} else if (OB_FAIL(exec_ctx.init_expr_op(phy_plan.get_expr_operator_size()))) {
LOG_WARN("fail init exec expr op ctx", K(ret));
} else if (OB_FAIL(ob_exe.execute_plan(exec_ctx))) {
LOG_ERROR("execute plan fail ", K(ret));
} else {
// ob_exe.execute_plan(exec_ctx_) will use ObArenaAllocator to allocate memory for original ObTableScanOp
// while *subtitude_table_scan_to_fake()* will directly use *new* to new a ObFakeTableScanVecOp, for simple
exe_result.static_engine_root_ = subtitude_table_scan_to_fake(exe_result.static_engine_root_);
if (OB_FAIL(exe_result.open(exec_ctx))) {
LOG_ERROR("open plan fail ", K(ret));
} else {
root = exe_result.static_engine_root_;
}
}
return ret;
}
int TestOpEngine::print_and_cmp_final_output(const ObBatchRows *brs, ObOperator *root, bool is_comparing)
{
int ret = OB_SUCCESS;
std::string output_line;
for (int i = 0; i < brs->size_; i++) {
if (!brs->skip_->exist(i)) {
for (int j = 0; j < root->get_spec().output_.count(); j++) {
ObExpr *output_expr = root->get_spec().output_.at(j);
string result_str = get_data_by_datum_type(root, output_expr, root->eval_ctx_, i);
output_line += result_str + " ";
// compare whether is the same output
// cmp output results
if (is_comparing) {
if (result_str.compare(temp_cmp_data_[i][j]) != 0) {
// different
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Got an different output result in :", K(i), K(j));
LOG_WARN("Original op output value is :", K(ObString(temp_cmp_data_[i][j].data())));
LOG_WARN(" New op output value is :", K(ObString(result_str.data())));
}
} else {
temp_cmp_data_[i].push_back(result_str);
}
}
LOG_INFO(output_line.data());
output_line.clear();
}
}
return ret;
}
//
int TestOpEngine::print_to_file(const ObBatchRows *brs, ObOperator *root, const ExprFixedArray &exprs, bool is_result,
std::ofstream *out_data_stream)
{
int ret = OB_SUCCESS;
std::string output_line;
for (int i = 0; i < brs->size_; i++) {
if (!brs->skip_->exist(i)) {
for (int j = 0; j < exprs.count(); j++) {
ObExpr *expr = exprs.at(j);
string result_str = get_data_by_datum_type(root, expr, root->eval_ctx_, i);
output_line += result_str;
if (j != exprs.count() - 1) { output_line += ", "; }
}
if (is_result) {
*out_data_stream << output_line << std::endl;
} else {
*out_data_stream << "insert into " << reinterpret_cast<ObFakeTableScanVecOp *>(root)->get_tsc_spec().table_name_
<< " values(" << output_line << ");" << std::endl;
}
output_line.clear();
}
}
return ret;
}
int TestOpEngine::basic_random_test(const std::string &test_file)
{
// run tests
std::ifstream if_tests(test_file);
if (if_tests.is_open() != true) { return -1; }
int ret = OB_SUCCESS;
std::string line;
while (std::getline(if_tests, line)) {
// handle query
if (line.size() <= 0) continue;
if (line.at(0) == '#') continue;
if (ObTestOpConfig::get_instance().output_result_to_file_) {
system(("rm -f " + ObTestOpConfig::get_instance().test_filename_prefix_ + ".log").data());
system(("rm -f " + ObTestOpConfig::get_instance().test_filename_prefix_ + ".log.*").data());
system(("cat /dev/null > " + ObTestOpConfig::get_instance().test_filename_origin_output_file_).c_str());
system(("cat /dev/null > " + ObTestOpConfig::get_instance().test_filename_vec_output_file_).c_str());
}
ret = OB_SUCCESS;
ObOperator *original_root = NULL;
ObOperator *vec_2_root = NULL;
ObExecutor original_exector;
ObExecutor vec_2_exector;
if (OB_FAIL(get_tested_op_from_string(line, false, original_root, original_exector))) {
LOG_WARN("generate original tested op fail, sql: ", K(line.data()));
} else if (OB_FAIL(get_tested_op_from_string(line, true, vec_2_root, vec_2_exector))) {
LOG_WARN("generate vectorization 2.0 tested op fail, sql: ", K(line.data()));
} else {
// begin get_next_batch()
// 5.compare two op outputs
int round = 1;
const int64_t max_row_cnt = 256;
const ObBatchRows *original_child_brs = nullptr;
const ObBatchRows *vec_2_child_brs = nullptr;
LOG_INFO("============== Final output ===============", K(round));
while (!original_root->brs_.end_ || !vec_2_root->brs_.end_) {
if (OB_FAIL(original_root->get_next_batch(max_row_cnt, original_child_brs))) {
LOG_ERROR("root op fail to get_next_batch data", K(original_root));
break;
}
temp_cmp_data_.resize(original_child_brs->size_);
LOG_INFO("============== Original ===============", K(round));
if (ObTestOpConfig::get_instance().output_result_to_file_) {
if (OB_FAIL(print_to_file(original_child_brs, original_root, original_root->spec_.output_, true,
&out_origin_result_stream_))) {
// break, other error log has already print in inner function
break;
}
} else {
if (OB_FAIL(print_and_cmp_final_output(original_child_brs, original_root, false))) {
// break, other error log has already print in inner function
break;
}
}
if (OB_FAIL(vec_2_root->get_next_batch(max_row_cnt, vec_2_child_brs))) {
LOG_ERROR("root op fail to get_next_batch data", K(vec_2_root));
break;
}
LOG_INFO("============== Vectorization 2.0 ===============", K(round));
if (ObTestOpConfig::get_instance().output_result_to_file_) {
if (OB_FAIL(
print_to_file(vec_2_child_brs, vec_2_root, vec_2_root->spec_.output_, true, &out_vec_result_stream_))) {
// break, other error log has already print in inner function
break;
}
} else {
if (OB_FAIL(print_and_cmp_final_output(vec_2_child_brs, vec_2_root, true))) {
// break, other error log has already print in inner function
break;
}
}
if (!ObTestOpConfig::get_instance().output_result_to_file_) {
if (original_child_brs->size_ != vec_2_child_brs->size_) {
LOG_ERROR("Two operator output different [batch size] in", K(round), K(original_child_brs->size_),
K(vec_2_child_brs->size_));
ret = OB_ERR_UNEXPECTED;
break;
}
if (MEMCMP(original_child_brs->skip_->data_, vec_2_child_brs->skip_->data_,
original_child_brs->skip_->memory_size(original_child_brs->size_))
!= 0) {
LOG_ERROR("Two operator output different [skip size] in", K(round), K(original_child_brs->size_),
K(vec_2_child_brs->size_));
ret = OB_ERR_UNEXPECTED;
break;
}
}
round++;
temp_cmp_data_.clear();
}
//if output to file, compare data in file at last
if (ObTestOpConfig::get_instance().output_result_to_file_) {
if (original_root->get_spec().get_type() == PHY_HASH_JOIN && original_root->get_spec().get_type() == PHY_VEC_HASH_JOIN) {
system(("sort " + ObTestOpConfig::get_instance().test_filename_origin_output_file_ + " -o "
+ ObTestOpConfig::get_instance().test_filename_origin_output_file_)
.c_str());
system(("sort " + ObTestOpConfig::get_instance().test_filename_vec_output_file_ + " -o "
+ ObTestOpConfig::get_instance().test_filename_vec_output_file_)
.c_str());
}
int system_ret = system(("diff " + ObTestOpConfig::get_instance().test_filename_origin_output_file_ + " " + ObTestOpConfig::get_instance().test_filename_vec_output_file_ + " > /dev/null").c_str());
LOG_ERROR("CODE: ", K(system_ret));
LOG_ERROR("CODE: ", K(WEXITSTATUS(system_ret)));
if (WEXITSTATUS(system_ret) != 0) {
LOG_ERROR("Two operator output different!");
// Preserve the site
uint64_t ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
std::fstream error_output_stream;
std::string error_file_name = "mismatch." + std::to_string(ms);
LOG_ERROR("error_file_name: ", K(error_file_name.data()));
error_output_stream.open(error_file_name, std::fstream::in | std::fstream::out | std::fstream::app);
error_output_stream << line << std::endl;
system(("cat " + ObTestOpConfig::get_instance().test_filename_prefix_ + ".cfg" + " >> " + error_file_name)
.c_str());
system(("mv " + ObTestOpConfig::get_instance().test_filename_origin_output_file_ + " "
+ ObTestOpConfig::get_instance().test_filename_origin_output_file_ + std::to_string(ms))
.c_str());
system(("mv " + ObTestOpConfig::get_instance().test_filename_vec_output_file_ + " "
+ ObTestOpConfig::get_instance().test_filename_vec_output_file_ + std::to_string(ms))
.c_str());
}
}
}
exec_ctx_.~ObExecContext();
new (&exec_ctx_) ObExecContext(allocator_);
exec_ctx_.set_sql_ctx(&sql_ctx_);
exec_ctx_.set_my_session(&session_info_);
exec_ctx_.create_physical_plan_ctx();
vec_2_exec_ctx_.~ObExecContext();
new (&vec_2_exec_ctx_) ObExecContext(allocator_);
vec_2_exec_ctx_.set_sql_ctx(&sql_ctx_);
vec_2_exec_ctx_.set_my_session(&session_info_);
vec_2_exec_ctx_.create_physical_plan_ctx();
}
EXPECT_EQ(ret, 0);
if (ret == OB_SUCCESS) {
LOG_INFO(" ======================= ");
LOG_INFO("Test Pass!");
LOG_INFO("All new operator output is equal to the original one.");
LOG_INFO(" ======================= ");
} else {
LOG_INFO(" ======================= ");
LOG_INFO("Test Fail!");
LOG_INFO(" ======================= ");
}
return ret;
}
int TestOpEngine::basic_random_test_output_to_file(const std::string &test_file, bool vector_2)
{
// run tests
std::ifstream if_tests(test_file);
if (if_tests.is_open() != true) { return -1; }
int ret = OB_SUCCESS;
std::string line;
while (std::getline(if_tests, line)) {
// handle query
if (line.size() <= 0) continue;
if (line.at(0) == '#') continue;
ObOperator *root = NULL;
ObExecutor exector;
if (OB_FAIL(get_tested_op_from_string(line, vector_2, root, exector))) {
LOG_WARN("generate tested op fail, sql: ", K(line.data()));
} else {
int round = 1;
const int64_t max_row_cnt = 256;
const ObBatchRows *child_brs = nullptr;
LOG_INFO("============== Final output ===============", K(round));
while (!root->brs_.end_) {
if (OB_FAIL(root->get_next_batch(max_row_cnt, child_brs))) {
LOG_ERROR("root op fail to get_next_batch data", K(root));
break;
}
if (OB_FAIL(print_to_file(child_brs, root, root->spec_.output_, true,
vector_2 ? &out_vec_result_stream_ : &out_origin_result_stream_))) {
// break, other error log has already print in inner function
break;
}
round++;
}
}
exec_ctx_.~ObExecContext();
new (&exec_ctx_) ObExecContext(allocator_);
exec_ctx_.set_sql_ctx(&sql_ctx_);
exec_ctx_.set_my_session(&session_info_);
exec_ctx_.create_physical_plan_ctx();
}
EXPECT_EQ(ret, 0);
return ret;
}
} // namespace test

115
unittest/sql/engine/test_op_engine.h Normal file
View File

@ -0,0 +1,115 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
// #define USING_LOG_PREFIX SQL_ENGINE
#include <gtest/gtest.h>
#include <fstream>
#include "../optimizer/test_optimizer_utils.h"
#include "sql/code_generator/ob_code_generator.h"
#include "storage/blocksstable/ob_block_sstable_struct.h"
namespace test
{
class MockLocationService : public share::ObLocationService
{
public:
MockLocationService()
{}
virtual ~MockLocationService()
{}
virtual int nonblock_get(const uint64_t tenant_id, const ObTabletID &tablet_id, ObLSID &ls_id)
{
ls_id = ObLSID::SYS_LS_ID;
return OB_SUCCESS;
}
virtual int nonblock_get(const int64_t cluster_id, const uint64_t tenant_id, const ObLSID &ls_id,
ObLSLocation &location)
{
int ret = OB_SUCCESS;
ObAddr add;
ObReplicaProperty relica_pro;
ObLSRestoreStatus ls_restore_sta;
add.set_ip_addr("1.1.1.1", 8888);
ObLSReplicaLocation ls_replica_loc;
if (OB_FAIL(location.init(1, 1, ls_id, 1))) {
} else if (OB_FAIL(ls_replica_loc.init(add, LEADER, 100, REPLICA_TYPE_FULL, relica_pro, ls_restore_sta, 100))) {
} else {
location.add_replica_location(ls_replica_loc);
}
return ret;
}
};
class TestOpEngine : public TestOptimizerUtils
{
public:
TestOpEngine();
virtual ~TestOpEngine();
virtual void SetUp();
virtual void TearDown();
virtual void destory();
int basic_random_test(const std::string &line);
int basic_random_test_output_to_file(const std::string &test_file, bool vector_2);
private:
// disallow copy
DISALLOW_COPY_AND_ASSIGN(TestOpEngine);
protected:
// function members
common::ObIODevice *get_device_inner();
int prepare_io(const std::string & test_data_name_suffix);
int do_optimize(ObStmt *stmt, ObLogPlan *&plan, ObPhyPlanType distr, ObArenaAllocator &allocator,
ObExecContext &exec_ctx);
int do_code_generate(const ObLogPlan &log_plan, ObCodeGenerator &code_gen, ObPhysicalPlan &phy_plan);
int do_rewrite(ObStmt *&stmt, ObPhysicalPlan *phy_plan);
int test_phy_plan(ObPhysicalPlan &plan);
ObOperator *subtitude_table_scan_to_fake(ObOperator *root);
int get_tested_op_from_string(const std::string &sql, bool vector_2, ObOperator *&op, ObExecutor &executor);
int generate_physical_plan(ObLogPlan *log_plan, ObPhysicalPlan &phy_plan, ObExecContext &exec_ctx,
bool enable_rich_format);
int open_and_get_op(ObExecContext &exec_ctx, ObExecutor &ob_exe, ObPhysicalPlan &phy_plan, ObOperator *&root);
int print_and_cmp_final_output(const ObBatchRows *brs, ObOperator *root, bool is_comparing);
static std::string get_decimal_result_from_datum(ObExpr *expr, const ObDatum &datum);
static std::string get_decimal_result_from_payload(ObExpr *expr, const char *payload);
static std::string get_data_by_datum_type(const ObOperator *op, ObExpr *expr, ObEvalCtx &eval_ctx, int row);
static int print_to_file(const ObBatchRows *brs, ObOperator *root, const ExprFixedArray &exprs, bool is_result,
std::ofstream *out_data_stream);
protected:
// data members
std::vector<std::vector<std::string>> temp_cmp_data_;
ObTenantBase tbase_;
MockLocationService mock_location_service_;
//
std::string test_config_file_;
std::string env_dir_;
blocksstable::ObStorageEnv storage_env_;
std::ofstream out_origin_result_stream_;
std::ofstream out_vec_result_stream_;
ParamStore param_store_;
ObAddr addr_;
ObArenaAllocator vec_2_alloc_;
ObExecContext vec_2_exec_ctx_; // vec_2_exec_ctx_ for vectorization 2.0, there is a exec_ctx_ in father class which is
// used in vectorization 1.0
};
} // namespace test

1
unittest/sql/optimizer/test_optimizer_utils.h
View File

@ -71,6 +71,7 @@ class TestOptimizerUtils : public TestSqlUtils, public ::testing::Test {
virtual void init();
virtual void SetUp();
virtual void TearDown() {destroy();}
virtual void destroy() {}
// function members
int generate_logical_plan(ObResultSet &result,//ObIAllocator &allocator,
ObString &stmt,

134
unittest/sql/rewrite/test_query_range.cpp
View File

@ -975,80 +975,80 @@ TEST_F(ObQueryRangeTest, range_column_with_triple_key)
//_OB_LOG(INFO, "ranges: %s", to_cstring(ranges));
}
TEST_F(ObQueryRangeTest, simple_row_in)
{
_OB_LOG(INFO, "start test: (a, d) in ((1 , 1.5), (2, 2), (1, 3), (2, 3), (0, 0), ('3', 3))");
ParamStore &params = exec_ctx_.get_physical_plan_ctx()->get_param_store_for_update();
params.reset();
ObObjParam param;
param.set_int(1);
param.set_param_meta();
OK(params.push_back(param));
param.set_double(1.5);
param.set_param_meta();
OK(params.push_back(param));
// param.set_int(2);
// OK(params.push_back(param));
// param.set_int(2);
// OK(params.push_back(param));
//TEST_F(ObQueryRangeTest, simple_row_in)
//{
// _OB_LOG(INFO, "start test: (a, d) in ((1 , 1.5), (2, 2), (1, 3), (2, 3), (0, 0), ('3', 3))");
// ParamStore &params = exec_ctx_.get_physical_plan_ctx()->get_param_store_for_update();
// params.reset();
// ObObjParam param;
// param.set_int(1);
// param.set_param_meta();
// OK(params.push_back(param));
// param.set_double(1.5);
// param.set_param_meta();
// OK(params.push_back(param));
//// param.set_int(2);
//// OK(params.push_back(param));
//// param.set_int(2);
//// OK(params.push_back(param));
//// param.set_int(1);
//// OK(params.push_back(param));
//// param.set_int(3);
//// OK(params.push_back(param));
//// param.set_int(2);
//// OK(params.push_back(param));
//// param.set_int(3);
//// OK(params.push_back(param));
//// param.set_int(0);
//// OK(params.push_back(param));
//// param.set_int(0);
//// OK(params.push_back(param));
// param.set_varchar("3");
// param.set_collation_type(CS_TYPE_UTF8MB4_BIN);
// param.set_param_meta();
// OK(params.push_back(param));
// param.set_int(3);
// param.set_param_meta();
// OK(params.push_back(param));
// param.set_int(2);
// OK(params.push_back(param));
// param.set_int(3);
// OK(params.push_back(param));
// param.set_int(0);
// OK(params.push_back(param));
// param.set_int(0);
// OK(params.push_back(param));
param.set_varchar("3");
param.set_collation_type(CS_TYPE_UTF8MB4_BIN);
param.set_param_meta();
OK(params.push_back(param));
param.set_int(3);
param.set_param_meta();
OK(params.push_back(param));
ObRawExpr *condition = NULL;
const ObDataTypeCastParams dtc_params;
resolve_condition(triple_range_columns_, "(a, d) in ((? , ?)) or (a, d) = (?, ?)", condition, &params);
OK(query_range.preliminary_extract_query_range(triple_range_columns_, condition, dtc_params, &exec_ctx_));
_OB_LOG(INFO, "XXXX %s", to_cstring(query_range));
_OB_LOG(INFO, "XXXX params: %s", to_cstring(params));
OK(query_range.final_extract_query_range(exec_ctx_, dtc_params));
_OB_LOG(INFO, "XXXX final: %s", to_cstring(query_range));
ObQueryRangeArray ranges;
bool all_single_value_ranges = true;
OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params));
_OB_LOG(INFO, "ranges: %s", to_cstring(ranges));
// ASSERT_EQ(1, ranges.count());
// int64_t value = 0;
// ObRawExpr *condition = NULL;
// const ObDataTypeCastParams dtc_params;
//
// EXPECT_TRUE(get_methods.at(0));
// resolve_condition(triple_range_columns_, "(a, d) in ((? , ?)) or (a, d) = (?, ?)", condition, &params);
// OK(query_range.preliminary_extract_query_range(triple_range_columns_, condition, dtc_params, &exec_ctx_));
//
// // expect: ((1, 1, 1), (1, 1, 2))
// EXPECT_FALSE(ranges.at(0)->border_flag_.inclusive_start());
// EXPECT_FALSE(ranges.at(0)->border_flag_.inclusive_end());
// ASSERT_EQ(1, ranges.count());
// ASSERT_EQ(ranges.at(0)->start_key_.get_obj_ptr()[0].get_int(value), OB_SUCCESS);
// ASSERT_EQ(1, value);
// ASSERT_EQ(ranges.at(0)->start_key_.get_obj_ptr()[1].get_int(value), OB_SUCCESS);
// ASSERT_EQ(1, value);
// ASSERT_EQ(ranges.at(0)->start_key_.get_obj_ptr()[2].get_int(value), OB_SUCCESS);
// ASSERT_EQ(1, value);
// _OB_LOG(INFO, "XXXX %s", to_cstring(query_range));
// _OB_LOG(INFO, "XXXX params: %s", to_cstring(params));
// OK(query_range.final_extract_query_range(exec_ctx_, dtc_params));
// _OB_LOG(INFO, "XXXX final: %s", to_cstring(query_range));
//
// ASSERT_EQ(ranges.at(0)->end_key_.get_obj_ptr()[0].get_int(value), OB_SUCCESS);
// ASSERT_EQ(1, value);
// ASSERT_EQ(ranges.at(0)->end_key_.get_obj_ptr()[1].get_int(value), OB_SUCCESS);
// ASSERT_EQ(1, value);
// ASSERT_EQ(ranges.at(0)->end_key_.get_obj_ptr()[2].get_int(value), OB_SUCCESS);
// ASSERT_EQ(2, value);
}
// ObQueryRangeArray ranges;
// bool all_single_value_ranges = true;
//
// OK(query_range.get_tablet_ranges(ranges, all_single_value_ranges, dtc_params));
// _OB_LOG(INFO, "ranges: %s", to_cstring(ranges));
//// ASSERT_EQ(1, ranges.count());
//// int64_t value = 0;
////
//// EXPECT_TRUE(get_methods.at(0));
////
//// // expect: ((1, 1, 1), (1, 1, 2))
//// EXPECT_FALSE(ranges.at(0)->border_flag_.inclusive_start());
//// EXPECT_FALSE(ranges.at(0)->border_flag_.inclusive_end());
//// ASSERT_EQ(1, ranges.count());
//// ASSERT_EQ(ranges.at(0)->start_key_.get_obj_ptr()[0].get_int(value), OB_SUCCESS);
//// ASSERT_EQ(1, value);
//// ASSERT_EQ(ranges.at(0)->start_key_.get_obj_ptr()[1].get_int(value), OB_SUCCESS);
//// ASSERT_EQ(1, value);
//// ASSERT_EQ(ranges.at(0)->start_key_.get_obj_ptr()[2].get_int(value), OB_SUCCESS);
//// ASSERT_EQ(1, value);
////
//// ASSERT_EQ(ranges.at(0)->end_key_.get_obj_ptr()[0].get_int(value), OB_SUCCESS);
//// ASSERT_EQ(1, value);
//// ASSERT_EQ(ranges.at(0)->end_key_.get_obj_ptr()[1].get_int(value), OB_SUCCESS);
//// ASSERT_EQ(1, value);
//// ASSERT_EQ(ranges.at(0)->end_key_.get_obj_ptr()[2].get_int(value), OB_SUCCESS);
//// ASSERT_EQ(2, value);
//}
TEST_F(ObQueryRangeTest, basic_test)
{

22
unittest/sql/test_sql_utils.cpp
View File

@ -23,6 +23,7 @@
#include "optimizer/ob_mock_opt_stat_manager.h"
#include "sql/plan_cache/ob_plan_cache.h"
#include "sql/plan_cache/ob_ps_cache.h"
#include "share/ob_simple_mem_limit_getter.h"
#define CLUSTER_VERSION_2100 (oceanbase::common::cal_version(2, 1, 0, 0))
#define CLUSTER_VERSION_2200 (oceanbase::common::cal_version(2, 2, 0, 0))
using namespace oceanbase::observer;
@ -172,7 +173,15 @@ TestSqlUtils::TestSqlUtils()
{
memset(schema_file_path_, '\0', 128);
exec_ctx_.set_sql_ctx(&sql_ctx_);
(oceanbase::common::ObClusterVersion::get_instance().init(CLUSTER_VERSION_2200));
static ObTenantBase tenant_ctx(sys_tenant_id_);
ObTenantEnv::set_tenant(&tenant_ctx);
auto& cluster_version = ObClusterVersion::get_instance();
cluster_version.init(&common::ObServerConfig::get_instance(), &oceanbase::omt::ObTenantConfigMgr::get_instance());
oceanbase::omt::ObTenantConfigMgr::get_instance().add_tenant_config(sys_tenant_id_);
cluster_version.refresh_cluster_version("4.2.0.0");
ObServer &observer = ObServer::get_instance();
int ret = OB_SUCCESS;
if (OB_FAIL(observer.init_tz_info_mgr())) {
@ -192,6 +201,16 @@ void TestSqlUtils::init()
ObVirtualTenantManager::get_instance().init();
ObVirtualTenantManager::get_instance().add_tenant(sys_tenant_id_);
ObVirtualTenantManager::get_instance().set_tenant_mem_limit(sys_tenant_id_, 1024L * 1024L * 1024L, 1024L * 1024L * 1024L);
GCTX.schema_service_ = schema_service_;
oceanbase::transaction::ObBLService::get_instance().init();
const int64_t max_cache_size = 1024L * 1024L * 512;
static ObSimpleMemLimitGetter mem_limit_getter;
mem_limit_getter.add_tenant(OB_SYS_TENANT_ID, 0, max_cache_size);
mem_limit_getter.add_tenant(OB_SERVER_TENANT_ID, 0, INT64_MAX);
ObKVGlobalCache::get_instance().init(&mem_limit_getter);
if (OB_SUCCESS != (ret = ObPreProcessSysVars::init_sys_var())) {
_OB_LOG(WARN, "PreProcessing system value init failed, ret=%ld", ret);
ASSERT_TRUE(0);
@ -203,6 +222,7 @@ void TestSqlUtils::init()
ASSERT_TRUE(0);
} else {
sql_schema_guard_.set_schema_guard(&schema_guard_);
sql_ctx_.schema_guard_ = &schema_guard_;
ObString tenant("sql_test");
ASSERT_TRUE(OB_SUCCESS == session_info_.init_tenant(tenant, sys_tenant_id_));

105
unittest/storage/blocksstable/cs_encoding/ob_cs_encoding_test_base.h
View File

@ -15,6 +15,7 @@
#define OCEANBASE_CS_ENCODING_OB_CS_ENCODING_TEST_BASE_H_
#define USING_LOG_PREFIX STORAGE
#include "unittest/storage/blocksstable/encoding/test_column_decoder.h"
#include "../ob_row_generate.h"
#include "common/rowkey/ob_rowkey.h"
#include "lib/string/ob_sql_string.h"
@ -61,6 +62,10 @@ protected:
ObMicroBlockData &full_transformed_data,
ObMicroBlockCSDecoder &decoder);
int build_micro_block_desc(ObMicroBlockCSEncoder &encoder, ObMicroBlockDesc &desc, ObMicroBlockHeader* &header);
int check_decode_vector(ObMicroBlockCSDecoder &decoder,
const ObDatumRow *row_arr,
const int64_t row_cnt,
const VectorFormat vector_format);
int full_transform_check_row(const ObMicroBlockHeader *header,
const ObMicroBlockDesc &desc,
const ObDatumRow *row_arr,
@ -269,6 +274,16 @@ int ObCSEncodingTestBase::full_transform_check_row(const ObMicroBlockHeader *hea
}
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(check_decode_vector(decoder, row_arr, row_cnt, VEC_UNIFORM))) {
LOG_WARN("fail to check_decode_vector for VEC_UNIFORM", K(ret));
} else if (OB_FAIL(check_decode_vector(decoder, row_arr, row_cnt, VEC_FIXED))) {
LOG_WARN("fail to check_decode_vector for VEC_FIXED", K(ret));
} else if (OB_FAIL(check_decode_vector(decoder, row_arr, row_cnt, VEC_DISCRETE))) {
LOG_WARN("fail to check_decode_vector for VEC_DISCRETE", K(ret));
} else if (OB_FAIL(check_decode_vector(decoder, row_arr, row_cnt, VEC_CONTINUOUS))) {
LOG_WARN("fail to check_decode_vector for VEC_CONTINUOUS", K(ret));
}
if (OB_SUCC(ret) && check_by_get) {
ObCSEncodeBlockGetReader get_reader;
@ -291,6 +306,96 @@ int ObCSEncodingTestBase::full_transform_check_row(const ObMicroBlockHeader *hea
return ret;
}
int ObCSEncodingTestBase::check_decode_vector(ObMicroBlockCSDecoder &decoder,
const ObDatumRow *row_arr,
const int64_t row_cnt,
const VectorFormat vector_format)
{
int ret = OB_SUCCESS;
ObArenaAllocator frame_allocator;
sql::ObExecContext exec_context(allocator_);
sql::ObEvalCtx eval_ctx(exec_context);
char *buf = nullptr;
if (OB_ISNULL(buf = reinterpret_cast<char*>(allocator_.alloc(
row_cnt * (sizeof(char*)/*ptr_arr*/ + sizeof(uint32_t)/*len_arr*/ + sizeof(int64_t)/*row_ids*/))))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to allocate", K(ret));
} else {
const char** ptr_arr = reinterpret_cast<const char**>(buf);
buf += row_cnt * sizeof(char*);
uint32_t *len_arr = reinterpret_cast<uint32_t*>(buf) ;
buf += row_cnt * sizeof(uint32_t);
int64_t *row_ids = reinterpret_cast<int64_t*>(buf);
bool need_test_column = true;
for (int col_idx = 0; OB_SUCC(ret) && col_idx < ctx_.column_cnt_; col_idx++) {
sql::ObExpr col_expr;
ObObjMeta col_meta = col_descs_.at(col_idx).col_type_;
const int16_t precision = col_meta.is_decimal_int() ? col_meta.get_stored_precision() : PRECISION_UNKNOWN_YET;
VecValueTypeClass vec_tc = common::get_vec_value_tc(col_meta.get_type(), col_meta.get_scale(), precision);
need_test_column = true;
if (vector_format == VEC_FIXED) {
VecValueTypeClass fixed_tc_arr[] = {VEC_TC_INTEGER, VEC_TC_UINTEGER, VEC_TC_FLOAT, VEC_TC_DOUBLE,
VEC_TC_FIXED_DOUBLE, VEC_TC_DATETIME, VEC_TC_DATE, VEC_TC_TIME, VEC_TC_YEAR, VEC_TC_UNKNOWN,
VEC_TC_BIT, VEC_TC_ENUM_SET, VEC_TC_TIMESTAMP_TZ, VEC_TC_TIMESTAMP_TINY, VEC_TC_INTERVAL_YM,
VEC_TC_INTERVAL_DS, VEC_TC_DEC_INT32, VEC_TC_DEC_INT64, VEC_TC_DEC_INT128, VEC_TC_DEC_INT256,
VEC_TC_DEC_INT512};
VecValueTypeClass *vec = std::find(std::begin(fixed_tc_arr), std::end(fixed_tc_arr), vec_tc);
if (vec == std::end(fixed_tc_arr)) {
need_test_column = false;
}
} else if (vector_format == VEC_DISCRETE || vector_format == VEC_UNIFORM) {
VecValueTypeClass var_tc_arr[] = {VEC_TC_NUMBER, VEC_TC_EXTEND, VEC_TC_STRING, VEC_TC_ENUM_SET_INNER,
VEC_TC_RAW, VEC_TC_ROWID, VEC_TC_LOB, VEC_TC_JSON, VEC_TC_GEO, VEC_TC_UDT};
VecValueTypeClass *vec = std::find(std::begin(var_tc_arr), std::end(var_tc_arr), vec_tc);
if (vec == std::end(var_tc_arr)) {
need_test_column = false;
}
} else if (vector_format == VEC_CONTINUOUS) {
/* can't test now
VecValueTypeClass var_tc_arr[] = {VEC_TC_NUMBER, VEC_TC_EXTEND, VEC_TC_STRING, VEC_TC_ENUM_SET_INNER,
VEC_TC_RAW, VEC_TC_ROWID, VEC_TC_LOB, VEC_TC_JSON, VEC_TC_GEO, VEC_TC_UDT};
VecValueTypeClass *vec = std::find(std::begin(var_tc_arr), std::end(var_tc_arr), vec_tc);
if (vec == std::end(var_tc_arr)) {
need_test_column = false;
}
*/
need_test_column = false;
}
if (!need_test_column) {
continue;
}
LOG_INFO("Current col: ", K(col_idx), K(col_meta), K(vector_format), K(precision), K(vec_tc));
if (OB_FAIL(VectorDecodeTestUtil::generate_column_output_expr(
row_cnt, col_meta, vector_format, eval_ctx, col_expr, frame_allocator))) {
LOG_WARN("fail to generate_column_output_expr", K(ret), K(vec_tc), K(col_meta), K(vector_format));
} else {
for (int64_t row_idx = 0; row_idx < row_cnt; ++row_idx) {
row_ids[row_idx] = row_idx;
}
ObVectorDecodeCtx vector_ctx(ptr_arr, len_arr, row_ids, row_cnt, 0, col_expr.get_vector_header(eval_ctx));
if (OB_FAIL(decoder.get_col_data(col_idx, vector_ctx))) {
LOG_WARN("fail to get_col_dagta", K(col_idx), K(vector_ctx));
}
for (int64_t vec_idx = 0; OB_SUCC(ret) && vec_idx < row_cnt; ++vec_idx) {
if (false == VectorDecodeTestUtil::verify_vector_and_datum_match(
*vector_ctx.get_vector(), vec_idx, row_arr[vec_idx].storage_datums_[col_idx])) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("decode vector result mismatch",
K(ret), K(vec_idx), K(col_idx), K(col_meta), K(vec_tc), K(vector_format), K(precision));
}
}
}
}
}
return ret;
}
int ObCSEncodingTestBase::part_transform_check_row(const ObMicroBlockHeader *header,
const ObMicroBlockDesc &desc,
const ObDatumRow *row_arr,

55
unittest/storage/blocksstable/cs_encoding/ob_pd_filter_test_base.h
View File

@ -19,7 +19,6 @@
#include "ob_cs_encoding_test_base.h"
#include "storage/blocksstable/cs_encoding/ob_cs_encoding_util.h"
#include "storage/blocksstable/cs_encoding/ob_micro_block_cs_decoder.h"
#include "storage/blocksstable/cs_encoding/ob_micro_block_cs_encoder.h"
namespace oceanbase
{
@ -39,14 +38,14 @@ class ObPdFilterTestBase : public ObCSEncodingTestBase, public ::testing::Test
{
public:
ObPdFilterTestBase()
: enable_abnormal_filter_type_(false)
: abnormal_filter_type_(AbnormalFilterType::NOT_ABNORMAL)
{}
virtual ~ObPdFilterTestBase() {}
virtual void SetUp() {}
virtual void TearDown()
{
reuse();
enable_abnormal_filter_type_ = false;
abnormal_filter_type_ = AbnormalFilterType::NOT_ABNORMAL;
}
void set_obj_collation(ObObj &obj, const ObObjType &column_type);
@ -75,14 +74,19 @@ public:
const int64_t row_cnt,
const int64_t col_cnt,
const int64_t col_offset,
const ObObjType &col_type,
const ObObjMeta &col_meta,
const ObIArray<ObObj> &ref_objs,
ObMicroBlockCSDecoder &decoder,
const int64_t res_count);
public:
bool enable_abnormal_filter_type_;
enum AbnormalFilterType
{
NOT_ABNORMAL = 0,
WIDER_WIDTH,
OPPOSITE_SIGN
};
AbnormalFilterType abnormal_filter_type_;
};
void ObPdFilterTestBase::set_obj_collation(ObObj &obj, const ObObjType &column_type)
@ -126,11 +130,19 @@ void ObPdFilterTestBase::setup_obj(ObObj& obj, int64_t column_idx)
ObObjMeta column_meta = row_generate_.column_list_.at(column_idx).col_type_;
ObObjType column_type = column_meta.get_type();
bool is_integer = (ob_obj_type_class(column_type) == ObObjTypeClass::ObIntTC) || (ob_obj_type_class(column_type) == ObObjTypeClass::ObUIntTC);
if (enable_abnormal_filter_type_ && is_integer) {
if (ob_obj_type_class(column_type) == ObObjTypeClass::ObIntTC) {
column_meta.set_int();
} else {
column_meta.set_uint64();
if (is_integer) {
if (abnormal_filter_type_ == AbnormalFilterType::WIDER_WIDTH) {
if (ob_obj_type_class(column_type) == ObObjTypeClass::ObIntTC) {
column_meta.set_int();
} else {
column_meta.set_uint64();
}
} else if (abnormal_filter_type_ == AbnormalFilterType::OPPOSITE_SIGN) {
if (ob_obj_type_class(column_type) == ObObjTypeClass::ObIntTC) {
column_meta.set_uint64();
} else {
column_meta.set_int();
}
}
}
column_type = column_meta.get_type();
@ -273,7 +285,7 @@ int ObPdFilterTestBase::check_column_store_white_filter(
const int64_t row_cnt,
const int64_t col_cnt,
const int64_t col_offset,
const ObObjType &col_type,
const ObObjMeta &col_meta,
const ObIArray<ObObj> &ref_objs,
ObMicroBlockCSDecoder &decoder,
const int64_t res_count)
@ -303,13 +315,7 @@ int ObPdFilterTestBase::check_column_store_white_filter(
white_filter->col_offsets_.init(1);
white_filter->col_params_.init(1);
ObColumnParam col_param(allocator_);
ObObjMeta col_meta;
if (ref_objs.count() < 1) {
set_obj_meta_collation(col_meta, col_type);
col_param.set_meta_type(col_meta);
} else {
col_param.set_meta_type(ref_objs.at(0).meta_);
}
col_param.set_meta_type(col_meta);
white_filter->col_params_.push_back(&col_param);
white_filter->col_offsets_.push_back(col_offset);
white_filter->n_cols_ = 1;
@ -358,13 +364,14 @@ int ObPdFilterTestBase::check_column_store_white_filter(
}
} else {
white_filter->filter_.expr_->args_[i]->type_ = T_REF_COLUMN;
white_filter->filter_.expr_->args_[arg_cnt-1]->obj_meta_ = col_meta;
}
}
if (OB_UNLIKELY(2 > white_filter->filter_.expr_->arg_cnt_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected filter expr", K(ret), K(white_filter->filter_.expr_->arg_cnt_));
} else {
white_filter->cmp_func_ = get_datum_cmp_func(white_filter->filter_.expr_->args_[0]->obj_meta_, white_filter->filter_.expr_->args_[0]->obj_meta_);
white_filter->cmp_func_ = get_datum_cmp_func(white_filter->filter_.expr_->args_[arg_cnt-1]->obj_meta_, white_filter->filter_.expr_->args_[0]->obj_meta_);
}
if (OB_SUCC(ret) ) {
@ -408,7 +415,7 @@ int ObPdFilterTestBase::check_column_store_white_filter(
ASSERT_EQ(OB_SUCCESS, build_integer_filter_ref(ref_cnt, tmp_ref_arr, col_offset, ref_objs, false)); \
} \
ASSERT_EQ(OB_SUCCESS, check_column_store_white_filter(op_type, row_cnt, col_cnt, \
col_offset, col_types[col_offset], ref_objs, decoder, res_arr[i])) << "round: " << i << std::endl; \
col_offset, col_descs_[col_offset].col_type_, ref_objs, decoder, res_arr[i])) << "round: " << i << std::endl; \
} \
} \
@ -427,7 +434,7 @@ int ObPdFilterTestBase::check_column_store_white_filter(
ASSERT_EQ(OB_SUCCESS, build_decimal_filter_ref(ref_cnt, tmp_ref_arr, col_offset, ref_objs)); \
} \
ASSERT_EQ(OB_SUCCESS, check_column_store_white_filter(op_type, row_cnt, col_cnt, \
col_offset, col_types[col_offset], ref_objs, decoder, res_arr[i])) << "round: " << i << std::endl; \
col_offset, col_descs_[col_offset].col_type_, ref_objs, decoder, res_arr[i])) << "round: " << i << std::endl; \
} \
} \
@ -447,7 +454,7 @@ int ObPdFilterTestBase::check_column_store_white_filter(
ASSERT_EQ(OB_SUCCESS, build_integer_filter_ref(ref_cnt, tmp_ref_seed_arr, col_offset, ref_objs, true)); \
} \
ASSERT_EQ(OB_SUCCESS, check_column_store_white_filter(op_type, row_cnt, col_cnt, \
col_offset, col_types[col_offset], ref_objs, decoder, res_arr[i])) << "round: " << i << std::endl; \
col_offset, col_descs_[col_offset].col_type_, ref_objs, decoder, res_arr[i])) << "round: " << i << std::endl; \
} \
} \
@ -467,7 +474,7 @@ int ObPdFilterTestBase::check_column_store_white_filter(
} \
} \
ASSERT_EQ(OB_SUCCESS, check_column_store_white_filter(op_type, row_cnt, col_cnt, \
col_offset, col_types[col_offset], ref_objs, decoder, res_arr[i])); \
col_offset, col_descs_[col_offset].col_type_, ref_objs, decoder, res_arr[i])); \
} \
} \

8
unittest/storage/blocksstable/cs_encoding/test_decoder_filter_perf.h
View File

@ -1132,13 +1132,17 @@ void TestDecoderFilterPerf::init_encoding_ctx(
char *datum_ptr_buf = reinterpret_cast<char *>(cur_allocator.alloc(SIMPLE_ROW_CNT * cur_column_cnt * sizeof(char) * single_ptr_buf_len)); \
ObDatum *datum_buf = new ObDatum[SIMPLE_ROW_CNT * cur_column_cnt]; \
ObSEArray<ObSqlDatumInfo, 16> datum_arr; \
void *expr_arr = cur_allocator.alloc(sizeof(sql::ObExpr) * cur_column_cnt); \
sql::ObExpr *exprs = reinterpret_cast<sql::ObExpr *>(expr_arr); \
for (int64_t i = 0; i < cur_column_cnt; ++i) { \
ASSERT_EQ(OB_SUCCESS, datum_arr.push_back(ObSqlDatumInfo())); \
datum_arr.at(i).datum_ptr_ = datum_buf + SIMPLE_ROW_CNT * i; \
if (i == cur_column_cnt - 1) { \
datum_arr.at(i).map_type_ = ObObjDatumMapType::OBJ_DATUM_STRING; \
exprs[i].obj_datum_map_ = ObObjDatumMapType::OBJ_DATUM_STRING; \
datum_arr.at(i).expr_ = exprs + i; \
} else { \
datum_arr.at(i).map_type_ = ObObjDatumMapType::OBJ_DATUM_NUMBER; \
exprs[i].obj_datum_map_ = ObObjDatumMapType::OBJ_DATUM_NUMBER; \
datum_arr.at(i).expr_ = exprs + i; \
} \
cols.push_back(i); \
col_params.push_back(param); \

83
unittest/storage/blocksstable/cs_encoding/test_int_dict_pd_filter.cpp
View File

@ -376,7 +376,7 @@ TEST_F(TestIntDictPdFilter, test_int_dict_const_without_null_decoder)
// NOTICE:
// In this case, we will use 'abnormal' filter value, that means, although the column type is smallint,
// we will use some value larger than INT16_MAX or less than INT16_MIN to check the correctness of filter.
enable_abnormal_filter_type_ = true;
abnormal_filter_type_ = AbnormalFilterType::WIDER_WIDTH;
// check NU/NN
{
@ -539,6 +539,87 @@ TEST_F(TestIntDictPdFilter, test_exceed_range_compare_filter)
LOG_INFO(">>>>>>>>>>FINISH PD FILTER<<<<<<<<<<<");
}
//
// but actually int dict encoding does not trigger this problem, because int dict are always sorted
// and take other paths. only integer encoding may trigger this problem.
TEST_F(TestIntDictPdFilter, test_singed_and_unsigned_compare_filter)
{
const int64_t rowkey_cnt = 1;
const int64_t col_cnt = 3;
const bool enable_check = ENABLE_CASE_CHECK;
abnormal_filter_type_ = AbnormalFilterType::OPPOSITE_SIGN;
ObObjType col_types[col_cnt] = {ObInt32Type, ObUSmallIntType, ObIntType};
ASSERT_EQ(OB_SUCCESS, prepare(col_types, rowkey_cnt, col_cnt));
ctx_.column_encodings_[0] = ObCSColumnHeader::Type::INT_DICT;
ctx_.column_encodings_[1] = ObCSColumnHeader::Type::INT_DICT;
ctx_.column_encodings_[2] = ObCSColumnHeader::Type::INT_DICT;
const int64_t row_cnt = 2;
ObMicroBlockCSEncoder encoder;
ASSERT_EQ(OB_SUCCESS, encoder.init(ctx_));
ObDatumRow row_arr[row_cnt];
for (int64_t i = 0; i < row_cnt; ++i) {
ASSERT_EQ(OB_SUCCESS, row_arr[i].init(allocator_, col_cnt));
}
row_arr[0].storage_datums_[0].set_int32(0);
row_arr[0].storage_datums_[1].set_uint(0);
row_arr[0].storage_datums_[2].set_int(-1);
ASSERT_EQ(OB_SUCCESS, encoder.append_row(row_arr[0]));
row_arr[1].storage_datums_[0].set_int32(1);
row_arr[1].storage_datums_[1].set_uint(1);
row_arr[1].storage_datums_[2].set_int(0);
ASSERT_EQ(OB_SUCCESS, encoder.append_row(row_arr[1]));
HANDLE_TRANSFORM();
int64_t col_offset = 1;
bool need_check = true;
// check EQ NE
{
int64_t ref_arr[4] = {0, 1, 100, 1000};
int64_t res_arr_eq[4] = {1, 1, 0, 0};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_EQ, 4, 1, res_arr_eq);
int64_t res_arr_ne[4] = {1, 1, 2, 2};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_NE, 4, 1, res_arr_ne);
}
// check LT/LE/GT/GE
{
int64_t ref_arr[4] = {-1, 0, 1, 100};
int64_t res_arr_lt[4] = {0, 0, 1, 2};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_LT, 4, 1, res_arr_lt);
int64_t res_arr_le[4] = {0, 1, 2, 2};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_LE, 4, 1, res_arr_le);
}
{
int64_t ref_arr[4] = {-1, 0, 1, 100};
int64_t res_arr_gt[4] = {2, 1, 0, 0};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_GT, 4, 1, res_arr_gt);
int64_t res_arr_ge[4] = {2, 2, 1, 0};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_GE, 4, 1, res_arr_ge);
}
LOG_INFO(">>>>>>>>>>FINISH PD FILTER<<<<<<<<<<<");
col_offset = 2;
// check LT/LE/GT/GE
{
int64_t ref_arr[4] = {0, 1, 10, 100};
int64_t res_arr_lt[4] = {1, 2, 2, 2};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_LT, 4, 1, res_arr_lt);
int64_t res_arr_le[4] = {2, 2, 2, 2};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_LE, 4, 1, res_arr_le);
}
{
int64_t ref_arr[4] = {0, 1, 10, 100};
int64_t res_arr_gt[4] = {0, 0, 0, 0};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_GT, 4, 1, res_arr_gt);
int64_t res_arr_ge[4] = {1, 0, 0, 0};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_GE, 4, 1, res_arr_ge);
}
LOG_INFO(">>>>>>>>>>FINISH PD FILTER<<<<<<<<<<<");
}
}
}

84
unittest/storage/blocksstable/cs_encoding/test_integer_pd_filter.cpp
View File

@ -195,7 +195,7 @@ TEST_F(TestIntegerPdFilter, test_integer_decoder_nullbitmap_type)
const bool enable_check = ENABLE_CASE_CHECK;
const bool has_null = true;
const bool is_force_raw = true;
enable_abnormal_filter_type_ = true;
abnormal_filter_type_ = AbnormalFilterType::WIDER_WIDTH;
ObObjType col_types[col_cnt] = {ObInt32Type, ObTinyIntType};
ASSERT_EQ(OB_SUCCESS, prepare(col_types, rowkey_cnt, col_cnt));
ctx_.column_encodings_[0] = ObCSColumnHeader::Type::INTEGER;
@ -341,7 +341,7 @@ TEST_F(TestIntegerPdFilter, test_integer_abnormal_filter)
{
const int64_t rowkey_cnt = 1;
const int64_t col_cnt = 2;
enable_abnormal_filter_type_ = true;
abnormal_filter_type_ = AbnormalFilterType::WIDER_WIDTH;
const bool enable_check = ENABLE_CASE_CHECK;
ObObjType col_types[col_cnt] = {ObInt32Type, ObSmallIntType};
ASSERT_EQ(OB_SUCCESS, prepare(col_types, rowkey_cnt, col_cnt));
@ -554,6 +554,86 @@ TEST_F(TestIntegerPdFilter, test_exceed_range_compare_filter)
LOG_INFO(">>>>>>>>>>FINISH PD FILTER<<<<<<<<<<<");
}
//
TEST_F(TestIntegerPdFilter, test_singed_and_unsigned_compare_filter)
{
const int64_t rowkey_cnt = 1;
const int64_t col_cnt = 3;
const bool enable_check = ENABLE_CASE_CHECK;
abnormal_filter_type_ = AbnormalFilterType::OPPOSITE_SIGN;
ObObjType col_types[col_cnt] = {ObInt32Type, ObUSmallIntType, ObIntType};
ASSERT_EQ(OB_SUCCESS, prepare(col_types, rowkey_cnt, col_cnt));
ctx_.column_encodings_[0] = ObCSColumnHeader::Type::INTEGER; // integer
ctx_.column_encodings_[1] = ObCSColumnHeader::Type::INTEGER; // integer
ctx_.column_encodings_[2] = ObCSColumnHeader::Type::INTEGER; // integer
const int64_t row_cnt = 2;
ObMicroBlockCSEncoder encoder;
ASSERT_EQ(OB_SUCCESS, encoder.init(ctx_));
ObDatumRow row_arr[row_cnt];
for (int64_t i = 0; i < row_cnt; ++i) {
ASSERT_EQ(OB_SUCCESS, row_arr[i].init(allocator_, col_cnt));
}
row_arr[0].storage_datums_[0].set_int32(0);
row_arr[0].storage_datums_[1].set_uint(0);
row_arr[0].storage_datums_[2].set_int(-1);
ASSERT_EQ(OB_SUCCESS, encoder.append_row(row_arr[0]));
row_arr[1].storage_datums_[0].set_int32(1);
row_arr[1].storage_datums_[1].set_uint(1);
row_arr[1].storage_datums_[2].set_int(0);
ASSERT_EQ(OB_SUCCESS, encoder.append_row(row_arr[1]));
HANDLE_TRANSFORM();
int64_t col_offset = 1;
bool need_check = true;
// check EQ NE
{
int64_t ref_arr[4] = {0, 1, 100, 1000};
int64_t res_arr_eq[4] = {1, 1, 0, 0};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_EQ, 4, 1, res_arr_eq);
int64_t res_arr_ne[4] = {1, 1, 2, 2};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_NE, 4, 1, res_arr_ne);
}
// check LT/LE/GT/GE
{
int64_t ref_arr[4] = {-1, 0, 1, 100};
int64_t res_arr_lt[4] = {0, 0, 1, 2};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_LT, 4, 1, res_arr_lt);
int64_t res_arr_le[4] = {0, 1, 2, 2};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_LE, 4, 1, res_arr_le);
}
{
int64_t ref_arr[4] = {-1, 0, 1, 100};
int64_t res_arr_gt[4] = {2, 1, 0, 0};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_GT, 4, 1, res_arr_gt);
int64_t res_arr_ge[4] = {2, 2, 1, 0};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_GE, 4, 1, res_arr_ge);
}
LOG_INFO(">>>>>>>>>>FINISH PD FILTER<<<<<<<<<<<");
col_offset = 2;
// check LT/LE/GT/GE
{
int64_t ref_arr[4] = {0, 1, 10, 100};
int64_t res_arr_lt[4] = {1, 2, 2, 2};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_LT, 4, 1, res_arr_lt);
int64_t res_arr_le[4] = {2, 2, 2, 2};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_LE, 4, 1, res_arr_le);
}
{
int64_t ref_arr[4] = {0, 1, 10, 100};
int64_t res_arr_gt[4] = {0, 0, 0, 0};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_GT, 4, 1, res_arr_gt);
int64_t res_arr_ge[4] = {1, 0, 0, 0};
integer_type_filter_normal_check(true, ObWhiteFilterOperatorType::WHITE_OP_GE, 4, 1, res_arr_ge);
}
LOG_INFO(">>>>>>>>>>FINISH PD FILTER<<<<<<<<<<<");
}
}
}

581
unittest/storage/blocksstable/encoding/test_column_decoder.h
View File

@ -39,6 +39,32 @@ using namespace common;
using namespace storage;
using namespace share::schema;
class VectorDecodeTestUtil
{
public:
static int generate_column_output_expr(
const int64_t batch_cnt,
const ObObjMeta &obj_meta,
const VectorFormat &format,
sql::ObEvalCtx &eval_ctx,
sql::ObExpr &col_expr,
ObIAllocator &frame_allocator);
static bool verify_vector_and_datum_match(
const ObIVector &vector,
const int64_t vec_idx,
const ObDatum &datum);
static bool need_test_vec_with_type(
const VectorFormat &format,
const VecValueTypeClass &vec_tc);
static int test_batch_decode_perf(
ObMicroBlockDecoder &decoder,
const int64_t col_idx,
const ObObjMeta obj_meta,
const int64_t decode_cnt,
const VectorFormat vector_format);
};
class TestColumnDecoder : public ::testing::Test
{
public:
@ -113,7 +139,13 @@ public:
void filter_pushdown_comaprison_neg_test();
void batch_decode_to_datum_test(bool is_condensed = false);
void batch_decode_to_vector_test(
const bool is_condensed,
const bool has_null,
const bool align_row_id,
const VectorFormat vector_format);
void col_equal_batch_decode_to_vector_test(const VectorFormat vector_format);
void col_substr_batch_decode_to_vector_test(const VectorFormat vector_format);
void cell_decode_to_datum_test();
void cell_decode_to_datum_test_without_hex();
@ -1691,6 +1723,553 @@ void TestColumnDecoder::cell_inter_column_substring_to_datum_test()
// std::cout << "Batch decode by column cost time: " << batch_end_time - batch_start_time << std::endl;
// }
void TestColumnDecoder::batch_decode_to_vector_test(
const bool is_condensed,
const bool has_null,
const bool align_row_id,
const VectorFormat vector_format)
{
FLOG_INFO("start one batch decode to vector test", K(is_condensed), K(has_null), K(vector_format));
ObArenaAllocator test_allocator;
encoder_.reuse();
void *row_buf = allocator_.alloc(sizeof(ObDatumRow) * ROW_CNT);
ObDatumRow *rows = new (row_buf) ObDatumRow[ROW_CNT];
for (int64_t i = 0; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, rows[i].init(test_allocator, full_column_cnt_));
}
ObDatumRow row;
ASSERT_EQ(OB_SUCCESS, row.init(test_allocator, full_column_cnt_));
int64_t seed0 = 10000;
int64_t seed1 = 10001;
for (int64_t i = 0; i < ROW_CNT - 35; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
for (int64_t i = ROW_CNT - 35; i < ROW_CNT - 32; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed1, row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
if (has_null) {
for (int64_t j = 0; j < full_column_cnt_; ++j) {
row.storage_datums_[j].set_null();
}
for (int64_t i = ROW_CNT - 32; i < ROW_CNT - 30; ++i) {
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
} else {
for (int64_t i = ROW_CNT - 32; i < ROW_CNT - 30; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed1, row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
}
for (int64_t i = ROW_CNT - 30; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
if (is_condensed) {
const_cast<bool &>(encoder_.ctx_.encoder_opt_.enable_bit_packing_) = false;
} else {
const_cast<bool &>(encoder_.ctx_.encoder_opt_.enable_bit_packing_) = true;
}
char *buf = NULL;
int64_t size = 0;
ASSERT_EQ(OB_SUCCESS, encoder_.build_block(buf, size));
ObMicroBlockDecoder decoder;
ObMicroBlockData data(encoder_.data_buffer_.data(), encoder_.data_buffer_.length());
ASSERT_EQ(OB_SUCCESS, decoder.init(data, read_info_));
ObArenaAllocator frame_allocator;
sql::ObExecContext exec_context(test_allocator);
sql::ObEvalCtx eval_ctx(exec_context);
const char *ptr_arr[ROW_CNT];
uint32_t len_arr[ROW_CNT];
for (int64_t i = 0; i < full_column_cnt_; ++i) {
bool need_test_column = true;
ObObjMeta col_meta = col_descs_.at(i).col_type_;
const int16_t precision = col_meta.is_decimal_int() ? col_meta.get_stored_precision() : PRECISION_UNKNOWN_YET;
VecValueTypeClass vec_tc = common::get_vec_value_tc(
col_meta.get_type(),
col_meta.get_scale(),
precision);
if (i >= ROWKEY_CNT && i < read_info_.get_rowkey_count()) {
need_test_column = false;
} else {
need_test_column = VectorDecodeTestUtil::need_test_vec_with_type(vector_format, vec_tc);
}
if (!need_test_column) {
continue;
}
sql::ObExpr col_expr;
int64_t test_row_cnt = align_row_id ? ROW_CNT : ROW_CNT / 2;
ASSERT_EQ(OB_SUCCESS, VectorDecodeTestUtil::generate_column_output_expr(
ROW_CNT, col_meta, vector_format, eval_ctx, col_expr, frame_allocator));
int32_t col_offset = i;
LOG_INFO("Current col: ", K(i), K(col_meta), K(*decoder.decoders_[col_offset].ctx_), K(precision), K(vec_tc));
int64_t row_ids[test_row_cnt];
int64_t row_id_idx = 0;
for (int64_t datum_idx = 0; datum_idx < ROW_CNT; ++datum_idx) {
if (!align_row_id && 0 == datum_idx % 2) {
// skip
} else if (row_id_idx == test_row_cnt) {
// skip
} else {
row_ids[row_id_idx] = datum_idx;
++row_id_idx;
}
}
ObVectorDecodeCtx vector_ctx(ptr_arr, len_arr, row_ids, test_row_cnt, 0, col_expr.get_vector_header(eval_ctx));
ASSERT_EQ(OB_SUCCESS, decoder.decoders_[col_offset].decode_vector(decoder.row_index_, vector_ctx));
for (int64_t vec_idx = 0; vec_idx < test_row_cnt; ++vec_idx) {
ASSERT_TRUE(VectorDecodeTestUtil::verify_vector_and_datum_match(*(col_expr.get_vector_header(eval_ctx).get_vector()),
vec_idx, rows[row_ids[vec_idx]].storage_datums_[col_offset]));
}
// ASSERT_EQ(OB_SUCCESS, VectorDecodeTestUtil::test_batch_decode_perf(decoder, col_offset, col_meta, 100000, vector_format));
decoder.decoder_allocator_.reuse();
}
}
void TestColumnDecoder::col_equal_batch_decode_to_vector_test(const VectorFormat vector_format)
{
FLOG_INFO("start one column equal batch decode to vector test", K(vector_format));
ObArenaAllocator test_allocator;
encoder_.reuse();
void *row_buf = allocator_.alloc(sizeof(ObDatumRow) * ROW_CNT);
ObDatumRow *rows = new (row_buf) ObDatumRow[ROW_CNT];
for (int64_t i = 0; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, rows[i].init(allocator_, full_column_cnt_));
}
ObDatumRow row;
ASSERT_EQ(OB_SUCCESS, row.init(allocator_, full_column_cnt_));
int64_t seed0 = 10000;
int64_t seed1 = 10001;
for (int64_t i = 0; i < ROW_CNT - 35; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
for (int64_t j = read_info_.get_rowkey_count() + 1; j < full_column_cnt_; j += 2) {
row.storage_datums_[j] = row.storage_datums_[j - 1];
}
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
for (int64_t i = ROW_CNT - 35; i < ROW_CNT - 32; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed1, row));
for (int64_t j = read_info_.get_rowkey_count() + 1; j < full_column_cnt_; j += 2) {
row.storage_datums_[j] = row.storage_datums_[j - 1];
}
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
for (int64_t j = 0; j < full_column_cnt_; ++j) {
row.storage_datums_[j].set_null();
}
for (int64_t i = ROW_CNT - 32; i < ROW_CNT - 30; ++i) {
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
for (int64_t i = ROW_CNT - 30; i < ROW_CNT - 2; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
for (int64_t j = read_info_.get_rowkey_count() + 1; j < full_column_cnt_; j += 2) {
row.storage_datums_[j] = row.storage_datums_[j - 1];
}
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
// generate exception data
for (int64_t i = ROW_CNT - 2; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
char *buf = NULL;
int64_t size = 0;
ASSERT_EQ(OB_SUCCESS, encoder_.build_block(buf, size));
ObMicroBlockDecoder decoder;
ObMicroBlockData data(encoder_.data_buffer_.data(), encoder_.data_buffer_.length());
ASSERT_EQ(OB_SUCCESS, decoder.init(data, read_info_));
ObArenaAllocator frame_allocator;
sql::ObExecContext exec_context(test_allocator);
sql::ObEvalCtx eval_ctx(exec_context);
const char *ptr_arr[ROW_CNT];
uint32_t len_arr[ROW_CNT];
for (int64_t i = 0; i < full_column_cnt_; ++i) {
bool need_test_column = true;
ObObjMeta col_meta = col_descs_.at(i).col_type_;
const int16_t precision = col_meta.is_decimal_int() ? col_meta.get_stored_precision() : PRECISION_UNKNOWN_YET;
VecValueTypeClass vec_tc = common::get_vec_value_tc(
col_meta.get_type(),
col_meta.get_scale(),
precision);
if (i >= ROWKEY_CNT && i < read_info_.get_rowkey_count()) {
need_test_column = false;
} else {
need_test_column = VectorDecodeTestUtil::need_test_vec_with_type(vector_format, vec_tc);
}
if (!need_test_column) {
continue;
}
sql::ObExpr col_expr;
ASSERT_EQ(OB_SUCCESS, VectorDecodeTestUtil::generate_column_output_expr(
ROW_CNT, col_meta, vector_format, eval_ctx, col_expr, frame_allocator));
int32_t col_offset = i;
LOG_INFO("Current col: ", K(i), K(col_meta), K(*decoder.decoders_[col_offset].ctx_), K(precision), K(vec_tc));
int64_t row_ids[ROW_CNT];
for (int64_t datum_idx = 0; datum_idx < ROW_CNT; ++datum_idx) {
row_ids[datum_idx] = datum_idx;
}
ObVectorDecodeCtx vector_ctx(ptr_arr, len_arr, row_ids, ROW_CNT, 0, col_expr.get_vector_header(eval_ctx));
ASSERT_EQ(OB_SUCCESS, decoder.decoders_[col_offset].decode_vector(decoder.row_index_, vector_ctx));
for (int64_t vec_idx = 0; vec_idx < ROW_CNT; ++vec_idx) {
ASSERT_TRUE(VectorDecodeTestUtil::verify_vector_and_datum_match(*(col_expr.get_vector_header(eval_ctx).get_vector()),
vec_idx, rows[vec_idx].storage_datums_[col_offset]));
}
}
}
void TestColumnDecoder::col_substr_batch_decode_to_vector_test(const VectorFormat vector_format)
{
FLOG_INFO("start one column substring batch decode to vector test", K(vector_format));
ObArenaAllocator test_allocator;
encoder_.reuse();
void *row_buf = allocator_.alloc(sizeof(ObDatumRow) * ROW_CNT);
ObDatumRow *rows = new (row_buf) ObDatumRow[ROW_CNT];
for (int64_t i = 0; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, rows[i].init(allocator_, full_column_cnt_));
}
ObDatumRow row;
ASSERT_EQ(OB_SUCCESS, row.init(allocator_, full_column_cnt_));
int64_t seed0 = 10000;
int64_t seed1 = 10001;
for (int64_t i = 0; i < ROW_CNT - 35; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
// generate different start point data
for (int64_t j = read_info_.get_rowkey_count() + 1; j < full_column_cnt_ - 2; j += 2) {
ObString str = row.storage_datums_[j].get_string();
ObString sub_str;
ob_sub_str(allocator_, str, i / 5, sub_str);
row.storage_datums_[j - 1].set_string(sub_str);
}
// generate same start point data
ObString str = row.storage_datums_[full_column_cnt_ - 1].get_string();
ObString sub_str;
ob_sub_str(allocator_,str,1,sub_str);
row.storage_datums_[full_column_cnt_ - 2].set_string(sub_str);
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
for (int64_t i = ROW_CNT - 35; i < ROW_CNT - 32; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed1, row));
for (int64_t j = read_info_.get_rowkey_count() + 1; j < full_column_cnt_ - 2; j += 2) {
ObString str = row.storage_datums_[j].get_string();
ObString sub_str;
ob_sub_str(allocator_, str, i / 5, sub_str);
row.storage_datums_[j - 1].set_string(sub_str);
}
ObString str = row.storage_datums_[full_column_cnt_ - 1].get_string();
ObString sub_str;
ob_sub_str(allocator_,str,1,sub_str);
row.storage_datums_[full_column_cnt_ - 2].set_string(sub_str);
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
for (int64_t j = 0; j < full_column_cnt_; ++j) {
row.storage_datums_[j].set_null();
}
for (int64_t i = ROW_CNT - 32; i < ROW_CNT - 30; ++i) {
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
for (int64_t i = ROW_CNT - 30; i < ROW_CNT - 2; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
for (int64_t j = read_info_.get_rowkey_count() + 1; j < full_column_cnt_ - 2; j += 2) {
ObString str = row.storage_datums_[j].get_string();
ObString sub_str;
ob_sub_str(allocator_, str, i / 5, sub_str);
row.storage_datums_[j - 1].set_string(sub_str);
}
ObString str = row.storage_datums_[full_column_cnt_ - 1].get_string();
ObString sub_str;
ob_sub_str(allocator_,str,1,sub_str);
row.storage_datums_[full_column_cnt_ - 2].set_string(sub_str);
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
// generate exception data
for (int64_t i = ROW_CNT - 2; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, allocator_);
}
char *buf = NULL;
int64_t size = 0;
ASSERT_EQ(OB_SUCCESS, encoder_.build_block(buf, size));
ObMicroBlockDecoder decoder;
ObMicroBlockData data(encoder_.data_buffer_.data(), encoder_.data_buffer_.length());
ASSERT_EQ(OB_SUCCESS, decoder.init(data, read_info_));
ObArenaAllocator frame_allocator;
sql::ObExecContext exec_context(test_allocator);
sql::ObEvalCtx eval_ctx(exec_context);
const char *ptr_arr[ROW_CNT];
uint32_t len_arr[ROW_CNT];
for (int64_t i = 0; i < full_column_cnt_; ++i) {
bool need_test_column = true;
ObObjMeta col_meta = col_descs_.at(i).col_type_;
const int16_t precision = col_meta.is_decimal_int() ? col_meta.get_stored_precision() : PRECISION_UNKNOWN_YET;
VecValueTypeClass vec_tc = common::get_vec_value_tc(
col_meta.get_type(),
col_meta.get_scale(),
precision);
if (i >= ROWKEY_CNT && i < read_info_.get_rowkey_count()) {
need_test_column = false;
} else {
need_test_column = VectorDecodeTestUtil::need_test_vec_with_type(vector_format, vec_tc);
}
if (!need_test_column) {
continue;
}
sql::ObExpr col_expr;
ASSERT_EQ(OB_SUCCESS, VectorDecodeTestUtil::generate_column_output_expr(
ROW_CNT, col_meta, vector_format, eval_ctx, col_expr, frame_allocator));
int32_t col_offset = i;
LOG_INFO("Current col: ", K(i), K(col_meta), K(*decoder.decoders_[col_offset].ctx_), K(precision), K(vec_tc));
int64_t row_ids[ROW_CNT];
for (int64_t datum_idx = 0; datum_idx < ROW_CNT; ++datum_idx) {
row_ids[datum_idx] = datum_idx;
}
ObVectorDecodeCtx vector_ctx(ptr_arr, len_arr, row_ids, ROW_CNT, 0, col_expr.get_vector_header(eval_ctx));
ASSERT_EQ(OB_SUCCESS, decoder.decoders_[col_offset].decode_vector(decoder.row_index_, vector_ctx));
for (int64_t vec_idx = 0; vec_idx < ROW_CNT; ++vec_idx) {
ASSERT_TRUE(VectorDecodeTestUtil::verify_vector_and_datum_match(*(col_expr.get_vector_header(eval_ctx).get_vector()),
vec_idx, rows[vec_idx].storage_datums_[col_offset]));
}
}
}
int VectorDecodeTestUtil::generate_column_output_expr(
const int64_t batch_cnt,
const ObObjMeta &obj_meta,
const VectorFormat &format,
sql::ObEvalCtx &eval_ctx,
sql::ObExpr &col_expr,
ObIAllocator &frame_allocator)
{
// int ObStaticEngineExprCG::arrange_datums_data
int ret = OB_SUCCESS;
int64_t cur_total_size = 0;
col_expr.reset();
col_expr.batch_result_ = 1;
// hard coded
col_expr.frame_idx_ = 0;
col_expr.res_buf_len_ = 128;
const int16_t precision = obj_meta.is_decimal_int() ? obj_meta.get_stored_precision() : PRECISION_UNKNOWN_YET;
col_expr.vec_value_tc_ = get_vec_value_tc(obj_meta.get_type(), obj_meta.get_scale(), precision);
col_expr.is_fixed_length_data_ = common::is_fixed_length_vec(col_expr.vec_value_tc_);
col_expr.datum_off_ = cur_total_size;
cur_total_size += sizeof(ObDatum) * batch_cnt;
col_expr.pvt_skip_off_ = cur_total_size;
cur_total_size += sql::ObBitVector::memory_size(batch_cnt);
if (!col_expr.is_fixed_length_data_) {
col_expr.len_arr_off_ = cur_total_size;
cur_total_size += sizeof(uint32_t) * (batch_cnt + 1); // offsets size
col_expr.offset_off_ = cur_total_size;
cur_total_size += sizeof(char *) * (batch_cnt); // ptrs size
}
col_expr.vector_header_off_ = cur_total_size;
cur_total_size += sizeof(sql::VectorHeader);
col_expr.null_bitmap_off_ = cur_total_size;
cur_total_size += sql::ObBitVector::memory_size(batch_cnt);
if (!col_expr.is_fixed_length_data_) {
col_expr.cont_buf_off_ = cur_total_size;
cur_total_size += sizeof(sql::ObDynReserveBuf);
}
col_expr.eval_info_off_ = cur_total_size;
cur_total_size += sizeof(sql::ObEvalInfo);
col_expr.eval_flags_off_ = cur_total_size;
cur_total_size += sql::ObBitVector::memory_size(batch_cnt);
col_expr.dyn_buf_header_offset_ = cur_total_size;
cur_total_size += sql::ObDynReserveBuf::supported(obj_meta.get_type()) ? batch_cnt * sizeof(sql::ObDynReserveBuf) : 0;
col_expr.res_buf_off_ = cur_total_size;
cur_total_size += col_expr.res_buf_len_ * batch_cnt;
char **frame_arr = (char **)frame_allocator.alloc(sizeof(char *));
char *frame = (char *)frame_allocator.alloc(cur_total_size);
if (nullptr == frame || nullptr == frame_arr) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "null frame", K(ret));
} else {
memset(frame, 0, cur_total_size);
eval_ctx.frames_ = frame_arr;
eval_ctx.frames_[0] = frame;
if (OB_FAIL(col_expr.init_vector(eval_ctx, format, batch_cnt))) {
STORAGE_LOG(WARN, "failed to init vector", K(ret));
} else {
if (is_uniform_format(format)) {
col_expr.reset_datums_ptr(frame, batch_cnt);
} else if (is_discrete_format(format)) {
col_expr.reset_discretes_ptr(frame, batch_cnt, col_expr.get_discrete_vector_ptrs(eval_ctx));
}
}
}
return ret;
}
bool VectorDecodeTestUtil::verify_vector_and_datum_match(
const ObIVector &vector,
const int64_t vec_idx,
const ObDatum &datum)
{
int bret = false;
ObDatum vec_datum;
if (datum.is_null()) {
bret = vector.is_null(vec_idx);
} else {
ObLength length = vector.get_length(vec_idx);
vec_datum.len_ = length;
vec_datum.ptr_ = vector.get_payload(vec_idx);
bret = ObDatum::binary_equal(vec_datum, datum);
}
if (!bret) {
LOG_INFO("datum not match with datum from vector", K(vec_idx), K(datum), K(vec_datum));
}
return bret;
}
bool VectorDecodeTestUtil::need_test_vec_with_type(
const VectorFormat &vector_format,
const VecValueTypeClass &vec_tc)
{
bool need_test_column = true;
if (vector_format == VEC_FIXED) {
VecValueTypeClass fixed_tc_arr[] = {VEC_TC_INTEGER, VEC_TC_UINTEGER, VEC_TC_FLOAT, VEC_TC_DOUBLE,
VEC_TC_FIXED_DOUBLE, VEC_TC_DATETIME, VEC_TC_DATE, VEC_TC_TIME, VEC_TC_YEAR, VEC_TC_UNKNOWN,
VEC_TC_BIT, VEC_TC_ENUM_SET, VEC_TC_TIMESTAMP_TZ, VEC_TC_TIMESTAMP_TINY, VEC_TC_INTERVAL_YM,
VEC_TC_INTERVAL_DS, VEC_TC_DEC_INT32, VEC_TC_DEC_INT64, VEC_TC_DEC_INT128, VEC_TC_DEC_INT256,
VEC_TC_DEC_INT512};
VecValueTypeClass *vec = std::find(std::begin(fixed_tc_arr), std::end(fixed_tc_arr), vec_tc);
if (vec == std::end(fixed_tc_arr)) {
need_test_column = false;
}
} else if (vector_format == VEC_DISCRETE) {
VecValueTypeClass var_tc_arr[] = {VEC_TC_NUMBER, VEC_TC_EXTEND, VEC_TC_STRING, VEC_TC_ENUM_SET_INNER,
VEC_TC_RAW, VEC_TC_ROWID, VEC_TC_LOB, VEC_TC_JSON, VEC_TC_GEO, VEC_TC_UDT};
VecValueTypeClass *vec = std::find(std::begin(var_tc_arr), std::end(var_tc_arr), vec_tc);
if (vec == std::end(var_tc_arr)) {
need_test_column = false;
}
} else if (vector_format == VEC_CONTINUOUS) {
// not support shallow copy to continuous vector for now
need_test_column = VEC_TC_NUMBER == vec_tc;
} else {
need_test_column = true;
}
return need_test_column;
}
int VectorDecodeTestUtil::test_batch_decode_perf(
ObMicroBlockDecoder &decoder,
const int64_t col_idx,
const ObObjMeta col_meta,
const int64_t decode_cnt,
const VectorFormat vector_format)
{
int ret = OB_SUCCESS;
ObArenaAllocator test_allocator;
sql::ObExecContext exec_context(test_allocator);
sql::ObEvalCtx eval_ctx(exec_context);
sql::ObExpr col_expr;
const int16_t precision = col_meta.is_decimal_int() ? col_meta.get_stored_precision() : PRECISION_UNKNOWN_YET;
VecValueTypeClass vec_tc = common::get_vec_value_tc(col_meta.get_type(), col_meta.get_scale(), precision);
int64_t row_cnt = 0;
void *datum_buf = nullptr;
if (OB_UNLIKELY(!VectorDecodeTestUtil::need_test_vec_with_type(vector_format, vec_tc))) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("not supported test vector type with column meta", K(ret),
K(vector_format), K(col_meta), K(vec_tc), K(precision));
} else if (OB_FAIL(decoder.get_row_count(row_cnt))) {
LOG_WARN("failed to get row cnt", K(ret));
} else if (OB_FAIL(VectorDecodeTestUtil::generate_column_output_expr(
row_cnt, col_meta, vector_format, eval_ctx, col_expr, test_allocator))) {
LOG_WARN("failed to generate column output expr", K(ret));
} else if (OB_ISNULL(datum_buf = test_allocator.alloc(sizeof(int8_t) * 128 * row_cnt))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate memory for datum buffer", K(ret));
} else {
// decode to vector
const char *ptr_arr[row_cnt];
uint32_t len_arr[row_cnt];
int64_t row_ids[row_cnt];
for (int64_t datum_idx = 0; datum_idx < row_cnt; ++datum_idx) {
row_ids[datum_idx] = datum_idx;
}
ObVectorDecodeCtx decode_ctx(ptr_arr, len_arr, row_ids, row_cnt, 0, col_expr.get_vector_header(eval_ctx));
const int64_t vector_start_ts = ObTimeUtility::current_time();
for (int64_t decode_round = 0; decode_round < decode_cnt; ++decode_round) {
if (OB_FAIL(decoder.decoders_[col_idx].decode_vector(decoder.row_index_, decode_ctx))) {
LOG_WARN("failed to decode vector", K(ret), K(decode_ctx));
}
}
const int64_t vector_end_ts = ObTimeUtility::current_time();
// decode to datums
ObDatum datums[row_cnt];
for (int64_t j = 0; j < row_cnt; ++j) {
datums[j].ptr_ = reinterpret_cast<char *>(datum_buf) + j * 128;
row_ids[j] = j;
}
const int64_t datum_start_ts = ObTimeUtility::current_time();
for (int64_t decode_round = 0; decode_round < decode_cnt; ++decode_round) {
if (OB_FAIL(decoder.decoders_[col_idx].batch_decode(decoder.row_index_, row_ids, ptr_arr, row_cnt, datums))) {
LOG_WARN("failed to decode vector", K(ret), K(decode_ctx));
}
}
const int64_t datum_end_ts = ObTimeUtility::current_time();
FLOG_INFO("finish one batch decode perf test: ", K(col_idx), K(col_meta), K(vector_format),
K(decode_cnt), K(vector_start_ts), K(vector_end_ts), K(datum_start_ts), K(datum_end_ts),
"vector decode time", vector_end_ts - vector_start_ts,
"batch decode time", datum_end_ts - datum_start_ts,
KPC(decoder.decoders_[col_idx].ctx_));
}
return ret;
}
} // end of namespace blocksstable

99
unittest/storage/blocksstable/encoding/test_const_decoder.cpp
View File

@ -31,8 +31,80 @@ class TestConstDecoder : public TestColumnDecoder
public:
TestConstDecoder() : TestColumnDecoder(ObColumnHeader::Type::CONST) {}
virtual ~TestConstDecoder() {}
private:
void batch_decode_to_vector_no_exc_test(const VectorFormat vector_format);
};
void TestConstDecoder::batch_decode_to_vector_no_exc_test(const VectorFormat vector_format)
{
FLOG_INFO("start one batch decode to vector no exception test", K(vector_format));
ObArenaAllocator test_allocator;
encoder_.reuse();
void *row_buf = allocator_.alloc(sizeof(ObDatumRow) * ROW_CNT);
ObDatumRow *rows = new (row_buf) ObDatumRow[ROW_CNT];
for (int64_t i = 0; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, rows[i].init(test_allocator, full_column_cnt_));
}
ObDatumRow row;
ASSERT_EQ(OB_SUCCESS, row.init(test_allocator, full_column_cnt_));
int64_t seed0 = 10000;
for (int64_t i = 0; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed0, row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
char *buf = NULL;
int64_t size = 0;
ASSERT_EQ(OB_SUCCESS, encoder_.build_block(buf, size));
ObMicroBlockDecoder decoder;
ObMicroBlockData data(encoder_.data_buffer_.data(), encoder_.data_buffer_.length());
ASSERT_EQ(OB_SUCCESS, decoder.init(data, read_info_));
ObArenaAllocator frame_allocator;
sql::ObExecContext exec_context(test_allocator);
sql::ObEvalCtx eval_ctx(exec_context);
const char *ptr_arr[ROW_CNT];
uint32_t len_arr[ROW_CNT];
for (int64_t i = 0; i < full_column_cnt_; ++i) {
bool need_test_column = true;
ObObjMeta col_meta = col_descs_.at(i).col_type_;
const int16_t precision = col_meta.is_decimal_int() ? col_meta.get_stored_precision() : PRECISION_UNKNOWN_YET;
VecValueTypeClass vec_tc = common::get_vec_value_tc(
col_meta.get_type(),
col_meta.get_scale(),
precision);
if (i >= ROWKEY_CNT && i < read_info_.get_rowkey_count()) {
need_test_column = false;
} else {
need_test_column = VectorDecodeTestUtil::need_test_vec_with_type(vector_format, vec_tc);
}
if (!need_test_column) {
continue;
}
sql::ObExpr col_expr;
ASSERT_EQ(OB_SUCCESS, VectorDecodeTestUtil::generate_column_output_expr(
ROW_CNT, col_meta, vector_format, eval_ctx, col_expr, frame_allocator));
int32_t col_offset = i;
LOG_INFO("Current col: ", K(i), K(col_meta), K(*decoder.decoders_[col_offset].ctx_), K(precision), K(vec_tc));
int64_t row_ids[ROW_CNT];
for (int64_t datum_idx = 0; datum_idx < ROW_CNT; ++datum_idx) {
row_ids[datum_idx] = datum_idx;
}
ObVectorDecodeCtx vector_ctx(ptr_arr, len_arr, row_ids, ROW_CNT, 0, col_expr.get_vector_header(eval_ctx));
ASSERT_EQ(OB_SUCCESS, decoder.decoders_[col_offset].decode_vector(decoder.row_index_, vector_ctx));
for (int64_t vec_idx = 0; vec_idx < ROW_CNT; ++vec_idx) {
ASSERT_TRUE(VectorDecodeTestUtil::verify_vector_and_datum_match(*(col_expr.get_vector_header(eval_ctx).get_vector()),
vec_idx, rows[vec_idx].storage_datums_[col_offset]));
}
}
}
TEST_F(TestConstDecoder, no_exception_nu_nn)
{
ObDatumRow row;
@ -730,6 +802,33 @@ TEST_F(TestConstDecoder, batch_decode_to_datum_test_with_expection)
batch_decode_to_datum_test();
}
TEST_F(TestConstDecoder, batch_decode_to_vector_test)
{
#define TEST_ONE_WITH_ALIGN(row_aligned, vec_format) \
batch_decode_to_vector_test(false, true, row_aligned, vec_format); \
batch_decode_to_vector_test(false, false, row_aligned, vec_format); \
batch_decode_to_vector_test(true, true, row_aligned, vec_format); \
batch_decode_to_vector_test(true, false, row_aligned, vec_format);
#define TEST_ONE(vec_format) \
TEST_ONE_WITH_ALIGN(true, vec_format) \
TEST_ONE_WITH_ALIGN(false, vec_format)
TEST_ONE(VEC_UNIFORM);
TEST_ONE(VEC_FIXED);
TEST_ONE(VEC_DISCRETE);
TEST_ONE(VEC_CONTINUOUS);
#undef TEST_ONE
#undef TEST_ONE_WITH_ALIGN
}
TEST_F(TestConstDecoder, batch_decode_to_vector_no_exc_test)
{
batch_decode_to_vector_no_exc_test(VEC_UNIFORM);
batch_decode_to_vector_no_exc_test(VEC_FIXED);
batch_decode_to_vector_no_exc_test(VEC_DISCRETE);
batch_decode_to_vector_no_exc_test(VEC_CONTINUOUS);
}
TEST_F(TestConstDecoder, cell_decode_to_datum_test_without_expection)
{

132
unittest/storage/blocksstable/encoding/test_general_column_decoder.cpp
View File

@ -125,6 +125,26 @@ TEST_F(TestDictDecoder, cell_decode_to_datum_test)
cell_decode_to_datum_test();
}
TEST_F(TestDictDecoder, batch_decode_to_vector_test)
{
#define TEST_ONE_WITH_ALIGN(row_aligned, vec_format) \
batch_decode_to_vector_test(false, true, row_aligned, vec_format); \
batch_decode_to_vector_test(false, false, row_aligned, vec_format); \
batch_decode_to_vector_test(true, true, row_aligned, vec_format); \
batch_decode_to_vector_test(true, false, row_aligned, vec_format);
#define TEST_ONE(vec_format) \
TEST_ONE_WITH_ALIGN(true, vec_format) \
TEST_ONE_WITH_ALIGN(false, vec_format)
TEST_ONE(VEC_UNIFORM);
TEST_ONE(VEC_FIXED);
TEST_ONE(VEC_DISCRETE);
TEST_ONE(VEC_CONTINUOUS);
#undef TEST_ONE
#undef TEST_ONE_WITH_ALIGN
}
TEST_F(TestDictDecoder, batch_decode_single_var_len_dict) {
const int64_t string_len = UINT16_MAX + 3;
char *string_buf = nullptr;
@ -175,6 +195,26 @@ TEST_F(TestRLEDecoder, cell_decode_to_datum_test)
cell_decode_to_datum_test();
}
TEST_F(TestRLEDecoder, batch_decode_to_vector_test)
{
#define TEST_ONE_WITH_ALIGN(row_aligned, vec_format) \
batch_decode_to_vector_test(false, true, row_aligned, vec_format); \
batch_decode_to_vector_test(false, false, row_aligned, vec_format); \
batch_decode_to_vector_test(true, true, row_aligned, vec_format); \
batch_decode_to_vector_test(true, false, row_aligned, vec_format);
#define TEST_ONE(vec_format) \
TEST_ONE_WITH_ALIGN(true, vec_format) \
TEST_ONE_WITH_ALIGN(false, vec_format)
TEST_ONE(VEC_UNIFORM);
TEST_ONE(VEC_FIXED);
TEST_ONE(VEC_DISCRETE);
TEST_ONE(VEC_CONTINUOUS);
#undef TEST_ONE
#undef TEST_ONE_WITH_ALIGN
}
TEST_F(TestIntBaseDiffDecoder, batch_decode_to_datum_test)
{
batch_decode_to_datum_test();
@ -185,6 +225,24 @@ TEST_F(TestIntBaseDiffDecoder, cell_decode_to_datum_test)
cell_decode_to_datum_test();
}
TEST_F(TestIntBaseDiffDecoder, batch_decode_to_vector_test)
{
#define TEST_ONE_WITH_ALIGN(row_aligned, vec_format) \
batch_decode_to_vector_test(false, true, row_aligned, vec_format); \
batch_decode_to_vector_test(false, false, row_aligned, vec_format); \
batch_decode_to_vector_test(true, true, row_aligned, vec_format); \
batch_decode_to_vector_test(true, false, row_aligned, vec_format);
#define TEST_ONE(vec_format) \
TEST_ONE_WITH_ALIGN(true, vec_format) \
TEST_ONE_WITH_ALIGN(false, vec_format)
TEST_ONE(VEC_UNIFORM);
TEST_ONE(VEC_FIXED);
#undef TEST_ONE
#undef TEST_ONE_WITH_ALIGN
}
TEST_F(TestHexDecoder, batch_decode_to_datum_test)
{
batch_decode_to_datum_test();
@ -195,6 +253,25 @@ TEST_F(TestHexDecoder, cell_decode_to_datum_test)
cell_decode_to_datum_test();
}
TEST_F(TestHexDecoder, batch_decode_to_vector_test)
{
#define TEST_ONE_WITH_ALIGN(row_aligned, vec_format) \
batch_decode_to_vector_test(false, true, row_aligned, vec_format); \
batch_decode_to_vector_test(false, false, row_aligned, vec_format); \
batch_decode_to_vector_test(true, true, row_aligned, vec_format); \
batch_decode_to_vector_test(true, false, row_aligned, vec_format);
#define TEST_ONE(vec_format) \
TEST_ONE_WITH_ALIGN(true, vec_format) \
TEST_ONE_WITH_ALIGN(false, vec_format)
TEST_ONE(VEC_UNIFORM);
TEST_ONE(VEC_DISCRETE);
TEST_ONE(VEC_CONTINUOUS);
#undef TEST_ONE
#undef TEST_ONE_WITH_ALIGN
}
TEST_F(TestStringDiffDecoder, batch_decode_to_datum_test)
{
batch_decode_to_datum_test();
@ -205,6 +282,25 @@ TEST_F(TestStringDiffDecoder, cell_decode_to_datum_test)
cell_decode_to_datum_test();
}
TEST_F(TestStringDiffDecoder, batch_decode_to_vector_test)
{
#define TEST_ONE_WITH_ALIGN(row_aligned, vec_format) \
batch_decode_to_vector_test(false, true, row_aligned, vec_format); \
batch_decode_to_vector_test(false, false, row_aligned, vec_format); \
batch_decode_to_vector_test(true, true, row_aligned, vec_format); \
batch_decode_to_vector_test(true, false, row_aligned, vec_format);
#define TEST_ONE(vec_format) \
TEST_ONE_WITH_ALIGN(true, vec_format) \
TEST_ONE_WITH_ALIGN(false, vec_format)
TEST_ONE(VEC_UNIFORM);
TEST_ONE(VEC_DISCRETE);
TEST_ONE(VEC_CONTINUOUS);
#undef TEST_ONE
#undef TEST_ONE_WITH_ALIGN
}
TEST_F(TestStringPrefixDecoder, batch_decode_to_datum_test)
{
batch_decode_to_datum_test();
@ -220,15 +316,51 @@ TEST_F(TestStringPrefixDecoder, cell_decode_to_datum_test_without_hex)
cell_decode_to_datum_test_without_hex();
}
TEST_F(TestStringPrefixDecoder, batch_decode_to_vector_test)
{
#define TEST_ONE_WITH_ALIGN(row_aligned, vec_format) \
batch_decode_to_vector_test(false, true, row_aligned, vec_format); \
batch_decode_to_vector_test(false, false, row_aligned, vec_format); \
batch_decode_to_vector_test(true, true, row_aligned, vec_format); \
batch_decode_to_vector_test(true, false, row_aligned, vec_format);
#define TEST_ONE(vec_format) \
TEST_ONE_WITH_ALIGN(true, vec_format) \
TEST_ONE_WITH_ALIGN(false, vec_format)
TEST_ONE(VEC_UNIFORM);
TEST_ONE(VEC_DISCRETE);
TEST_ONE(VEC_CONTINUOUS);
#undef TEST_ONE
#undef TEST_ONE_WITH_ALIGN
}
TEST_F(TestColumnEqualDecoder, cell_decode_to_datum_test)
{
cell_column_equal_decode_to_datum_test();
}
TEST_F(TestColumnEqualDecoder, col_equal_batch_decode_to_vector_test)
{
col_equal_batch_decode_to_vector_test(VEC_FIXED);
col_equal_batch_decode_to_vector_test(VEC_UNIFORM);
col_equal_batch_decode_to_vector_test(VEC_DISCRETE);
col_equal_batch_decode_to_vector_test(VEC_CONTINUOUS);
}
TEST_F(TestInterColumnSubstringDecoder, cell_decode_to_datum_test)
{
cell_inter_column_substring_to_datum_test();
}
TEST_F(TestInterColumnSubstringDecoder, col_substr_batch_decode_to_vector_test)
{
col_substr_batch_decode_to_vector_test(VEC_UNIFORM);
col_substr_batch_decode_to_vector_test(VEC_DISCRETE);
col_substr_batch_decode_to_vector_test(VEC_CONTINUOUS);
}
// TEST_F(TestDictDecoder, batch_decode_perf_test)
// {
// batch_get_row_perf_test();

168
unittest/storage/blocksstable/encoding/test_raw_decoder.cpp
View File

@ -27,7 +27,7 @@
#include "lib/string/ob_sql_string.h"
#include "../ob_row_generate.h"
#include "common/rowkey/ob_rowkey.h"
#include "unittest/storage/mock_ob_table_read_info.h"
#include "test_column_decoder.h"
namespace oceanbase
@ -265,6 +265,12 @@ public:
sql::ObPushdownWhiteFilterNode &filter_node,
common::ObBitmap &result_bitmap,
common::ObFixedArray<ObObj, ObIAllocator> &objs);
void test_batch_decode_to_vector(
const bool is_condensed,
const bool has_null,
const bool align_row_id,
const VectorFormat vector_format);
protected:
ObRowGenerate row_generate_;
ObMicroBlockEncodingCtx ctx_;
@ -539,6 +545,147 @@ int TestRawDecoder::test_filter_pushdown_with_pd_info(
return ret;
}
void TestRawDecoder::test_batch_decode_to_vector(
const bool is_condensed,
const bool has_null,
const bool align_row_id,
const VectorFormat vector_format)
{
FLOG_INFO("start one batch decode to vector test", K(is_condensed), K(has_null), K(align_row_id), K(vector_format));
ObArenaAllocator test_allocator;
encoder_.reuse();
// Generate data and encode
void *row_buf = test_allocator.alloc(sizeof(ObDatumRow) * ROW_CNT);
ASSERT_TRUE(nullptr != row_buf);
ObDatumRow *rows = new (row_buf) ObDatumRow[ROW_CNT];
for (int64_t i = 0; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, rows[i].init(test_allocator, full_column_cnt_));
}
ObDatumRow row;
ASSERT_EQ(OB_SUCCESS, row.init(test_allocator, full_column_cnt_));
for (int64_t i = 0; i < ROW_CNT - 35; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
if (has_null) {
for (int64_t j = 0; j < full_column_cnt_; ++j) {
row.storage_datums_[j].set_null();
}
for (int64_t i = ROW_CNT - 35; i < 40; ++i) {
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
} else {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(row));
for (int64_t i = ROW_CNT - 35; i < 40; ++i) {
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
}
for (int64_t i = 40; i < 60; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
for (int64_t i = 60; i < ROW_CNT; ++i) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(0 - i, row));
ASSERT_EQ(OB_SUCCESS, encoder_.append_row(row)) << "i: " << i << std::endl;
rows[i].deep_copy(row, test_allocator);
}
if (is_condensed) {
encoder_.ctx_.encoder_opt_.enable_bit_packing_ = false;
} else {
encoder_.ctx_.encoder_opt_.enable_bit_packing_ = true;
}
char *buf = NULL;
int64_t size = 0;
ASSERT_EQ(OB_SUCCESS, encoder_.build_block(buf, size));
ObMicroBlockDecoder decoder;
ObMicroBlockData data(encoder_.data_buffer_.data(), encoder_.data_buffer_.length());
ASSERT_EQ(OB_SUCCESS, decoder.init(data, read_info_));
ObArenaAllocator frame_allocator;
sql::ObExecContext exec_context(test_allocator);
sql::ObEvalCtx eval_ctx(exec_context);
const char *ptr_arr[ROW_CNT];
uint32_t len_arr[ROW_CNT];
for (int64_t i = 0; i < full_column_cnt_; ++i) {
bool need_test_column = true;
ObObjMeta col_meta = col_descs_.at(i).col_type_;
const int16_t precision = col_meta.is_decimal_int() ? col_meta.get_stored_precision() : PRECISION_UNKNOWN_YET;
VecValueTypeClass vec_tc = common::get_vec_value_tc(
col_meta.get_type(),
col_meta.get_scale(),
precision);
if (i >= ROWKEY_CNT && i < read_info_.get_rowkey_count()) {
need_test_column = false;
} else if (vector_format == VEC_FIXED) {
VecValueTypeClass fixed_tc_arr[] = {VEC_TC_INTEGER, VEC_TC_UINTEGER, VEC_TC_FLOAT, VEC_TC_DOUBLE,
VEC_TC_FIXED_DOUBLE, VEC_TC_DATETIME, VEC_TC_DATE, VEC_TC_TIME, VEC_TC_YEAR, VEC_TC_UNKNOWN,
VEC_TC_BIT, VEC_TC_ENUM_SET, VEC_TC_TIMESTAMP_TZ, VEC_TC_TIMESTAMP_TINY, VEC_TC_INTERVAL_YM,
VEC_TC_INTERVAL_DS, VEC_TC_DEC_INT32, VEC_TC_DEC_INT64, VEC_TC_DEC_INT128, VEC_TC_DEC_INT256,
VEC_TC_DEC_INT512};
VecValueTypeClass *vec = std::find(std::begin(fixed_tc_arr), std::end(fixed_tc_arr), vec_tc);
if (vec == std::end(fixed_tc_arr)) {
need_test_column = false;
}
} else if (vector_format == VEC_DISCRETE) {
VecValueTypeClass var_tc_arr[] = {VEC_TC_NUMBER, VEC_TC_EXTEND, VEC_TC_STRING, VEC_TC_ENUM_SET_INNER,
VEC_TC_RAW, VEC_TC_ROWID, VEC_TC_LOB, VEC_TC_JSON, VEC_TC_GEO, VEC_TC_UDT};
VecValueTypeClass *vec = std::find(std::begin(var_tc_arr), std::end(var_tc_arr), vec_tc);
if (vec == std::end(var_tc_arr)) {
need_test_column = false;
}
} else if (vector_format == VEC_CONTINUOUS) {
// not support shallow copy to continuous vector for now
need_test_column = VEC_TC_NUMBER == vec_tc;
} else {
need_test_column = true;
}
if (!need_test_column) {
continue;
}
sql::ObExpr col_expr;
int64_t test_row_cnt = align_row_id ? ROW_CNT : ROW_CNT / 2;
ASSERT_EQ(OB_SUCCESS, VectorDecodeTestUtil::generate_column_output_expr(
ROW_CNT, col_meta, vector_format, eval_ctx, col_expr, frame_allocator));
int32_t col_offset = i;
LOG_INFO("Current col: ", K(i), K(col_meta), K(*decoder.decoders_[col_offset].ctx_),
K(precision), K(vec_tc), K(need_test_column));
int64_t row_ids[test_row_cnt];
int64_t row_id_idx = 0;
for (int64_t datum_idx = 0; datum_idx < ROW_CNT; ++datum_idx) {
if (!align_row_id && 0 == datum_idx % 2) {
// skip
} else if (row_id_idx == test_row_cnt) {
// skip
} else {
row_ids[row_id_idx] = datum_idx;
++row_id_idx;
}
}
ObVectorDecodeCtx vector_ctx(ptr_arr, len_arr, row_ids, test_row_cnt, 0, col_expr.get_vector_header(eval_ctx));
ASSERT_EQ(OB_SUCCESS, decoder.decoders_[col_offset].decode_vector(decoder.row_index_, vector_ctx));
for (int64_t vec_idx = 0; vec_idx < test_row_cnt; ++vec_idx) {
ASSERT_TRUE(VectorDecodeTestUtil::verify_vector_and_datum_match(*(col_expr.get_vector_header(eval_ctx).get_vector()),
vec_idx, rows[row_ids[vec_idx]].storage_datums_[col_offset]));
}
// ASSERT_EQ(OB_SUCCESS, VectorDecodeTestUtil::test_batch_decode_perf(decoder, col_offset, col_meta, 100000, vector_format));
decoder.decoder_allocator_.reuse();
}
}
TEST_F(TestRawDecoder, filter_pushdown_all_eq_ne)
{
// Generate data and encode
@ -1019,6 +1166,25 @@ TEST_F(TestRawDecoder, batch_decode_to_datum)
}
}
TEST_F(TestRawDecoder, batch_decode_to_vector)
{
#define TEST_ONE_WITH_ALIGN(row_aligned, vec_format) \
test_batch_decode_to_vector(false, true, row_aligned, vec_format); \
test_batch_decode_to_vector(false, false, row_aligned, vec_format); \
test_batch_decode_to_vector(true, true, row_aligned, vec_format); \
test_batch_decode_to_vector(true, false, row_aligned, vec_format);
#define TEST_ONE(vec_format) \
TEST_ONE_WITH_ALIGN(true, vec_format) \
TEST_ONE_WITH_ALIGN(false, vec_format)
TEST_ONE(VEC_UNIFORM);
TEST_ONE(VEC_FIXED);
TEST_ONE(VEC_DISCRETE);
TEST_ONE(VEC_CONTINUOUS);
#undef TEST_ONE
#undef TEST_ONE_WITH_ALIGN
}
TEST_F(TestRawDecoder, opt_batch_decode_to_datum)
{

190
unittest/storage/blocksstable/test_index_block_aggregator.cpp
View File

@ -37,17 +37,19 @@ public:
virtual void TearDown() {}
void init_schema(const int64_t col_count, const ObObjType *col_obj_types);
void init_skip_index_meta(const int64_t idx_col_count, const int64_t *min_max_col_idxs);
void init_sum_meta(const int64_t idx_col_count, const int64_t *sum_col_idxs);
void generate_row_by_seed(const int64_t seed, ObDatumRow &datum_row);
void reset_min_max_row();
void update_min_max_row(const ObDatumRow &row);
void update_sum_row(const ObDatumRow &row, ObObj *sum_res, ObObj *data);
void validate_sum_agg_row(const ObDatumRow &agg_row, const ObObj *sum_res, int64_t nop_col_cnt= 0, int64_t *nop_col_idxs = nullptr);
void validate_agg_row(const ObDatumRow &row, int64_t nop_col_cnt = 0, int64_t *nop_col_idxs = nullptr, ObSkipIndexColType *nop_col_types = nullptr);
void set_nop_cols(ObDatumRow &row, int64_t nop_col_cnt = 0, int64_t *nop_col_idxs = nullptr, ObSkipIndexColType *nop_col_types = nullptr);
bool is_col_in_nop_col_arr(const int64_t col_idx, const int64_t nop_col_cnt, int64_t *nop_col_idxs, int64_t &index);
void serialize_agg_row(const ObDatumRow &agg_row, const char *&row_buf, int64_t &row_size);
void get_cmp_func(const ObColDesc &col_desc, ObStorageDatumCmpFunc &cmp_func);
private:
ObArenaAllocator allocator_;
ObRowGenerate row_generate_;
ObArray<ObColDesc> col_descs_;
@ -121,6 +123,19 @@ void TestIndexBlockAggregator::init_skip_index_meta(
}
}
void TestIndexBlockAggregator::init_sum_meta(
const int64_t idx_col_count, const int64_t *sum_col_idxs)
{
for (int64_t i = 0; i < idx_col_count; ++i) {
ObSkipIndexColMeta meta;
ObSkipIndexColMeta max;
ObSkipIndexColMeta null_count;
meta.col_idx_ = sum_col_idxs[i];
meta.col_type_ = SK_IDX_SUM;
ASSERT_EQ(OB_SUCCESS, full_agg_metas_.push_back(meta));
}
}
void TestIndexBlockAggregator::generate_row_by_seed(const int64_t seed, ObDatumRow &datum_row)
{
// if (0 == seed) {
@ -181,6 +196,21 @@ void TestIndexBlockAggregator::update_min_max_row(const ObDatumRow &row)
}
}
void TestIndexBlockAggregator::update_sum_row(const ObDatumRow &row, ObObj *sum_res, ObObj *data)
{
for (int64_t col_id = 0; col_id < row.get_column_count(); ++col_id) {
const ObObjMeta col_type = col_descs_[col_id].col_type_;
if (!col_type.is_numeric_type()|| col_type.get_type_class() == ObObjTypeClass::ObBitTC || row.storage_datums_[col_id].is_null()) {
} else if (sum_res[col_id].is_null()) {
row.storage_datums_[col_id].to_obj(sum_res[col_id], col_type);
} else {
row.storage_datums_[col_id].to_obj(data[col_id], col_type);
ASSERT_EQ(OB_SUCCESS, sql::ObExprAdd::calc(sum_res[col_id], data[col_id], sum_res[col_id],
&allocator_, col_type.get_scale()));
}
}
}
void TestIndexBlockAggregator::validate_agg_row(
const ObDatumRow &datum_row, int64_t nop_col_cnt, int64_t *nop_col_idxs, ObSkipIndexColType *nop_col_types)
{
@ -223,6 +253,54 @@ void TestIndexBlockAggregator::validate_agg_row(
}
}
void TestIndexBlockAggregator::validate_sum_agg_row(const ObDatumRow &agg_row, const ObObj *sum_res,
int64_t nop_col_cnt, int64_t *nop_col_idxs)
{
for (int64_t i = 0; i < full_agg_metas_.count(); ++i) {
ObSkipIndexColMeta idx_meta = full_agg_metas_.at(i);
const int64_t col_idx = idx_meta.col_idx_;
const ObObjMeta col_type = col_descs_[col_idx].col_type_;
int64_t index = 0;
bool is_nop_column = is_col_in_nop_col_arr(col_idx, nop_col_cnt, nop_col_idxs, index);
if (is_nop_column || !col_type.is_numeric_type()|| col_type.get_type_class() == ObObjTypeClass::ObBitTC) {
ASSERT_TRUE(agg_row.storage_datums_[i].is_nop());
} else {
const ObObjTypeClass obj_tc = col_type.get_type_class();
switch (obj_tc) {
case ObObjTypeClass::ObIntTC:
case ObObjTypeClass::ObUIntTC:
case ObObjTypeClass::ObDecimalIntTC:
case ObObjTypeClass::ObNumberTC: {
int cmp = 0;
ObObj agg;
agg.set_number(agg_row.storage_datums_[i].get_number());
ASSERT_EQ(0, sum_res[i].compare(agg, cmp));
break;
}
case ObObjTypeClass::ObFloatTC: {
ObObj agg_obj;
agg_row.storage_datums_[i].to_obj(agg_obj, col_type);
int cmp = 0;
ASSERT_EQ(0, sum_res[col_idx].compare(agg_obj, cmp));
break;
}
case ObObjTypeClass::ObDoubleTC: {
ObObj agg_obj;
agg_row.storage_datums_[i].to_obj(agg_obj, col_type);
int cmp = 0;
ASSERT_EQ(0, sum_res[col_idx].compare(agg_obj, cmp));
break;
}
default: {
int ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "unexpect type", K(obj_tc));
break;
}
}
}
}
}
void TestIndexBlockAggregator::set_nop_cols(
ObDatumRow &row, int64_t nop_col_cnt, int64_t *nop_col_idxs, ObSkipIndexColType *nop_col_types)
{
@ -431,6 +509,116 @@ TEST_F(TestIndexBlockAggregator, basic_aggregate)
}
TEST_F(TestIndexBlockAggregator, test_sum)
{
static const int64_t test_column_cnt = 4;
const int64_t test_row_cnt = 10;
const int64_t extra_rowkey_cnt = ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt();
ObObjType col_obj_types[test_column_cnt];
col_obj_types[0] = ObIntType;
col_obj_types[1] = ObFloatType;
col_obj_types[2] = ObDoubleType;
col_obj_types[3] = ObCharType;
init_schema(test_column_cnt, col_obj_types);
int64_t sum_col_idxs[test_column_cnt];
for (int64_t i = 0; i < test_column_cnt; ++i) {
const int64_t agg_col_idx = i < rowkey_count_ ? i : i + extra_rowkey_cnt;
sum_col_idxs[i] = agg_col_idx;
}
ObObj data[test_column_cnt + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt()];
ObObj sum_res[test_column_cnt + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt()];
init_sum_meta(test_column_cnt, sum_col_idxs);
ObSkipIndexAggregator data_aggregator;
ObSkipIndexAggregator reuse_data_aggregator;
ObSkipIndexAggregator index_aggregator;
ObDatumRow data_agg_result;
ObDatumRow reuse_data_agg_result;
ObDatumRow index_agg_result;
ASSERT_EQ(OB_SUCCESS, data_agg_result.init(full_agg_metas_.count()));
ASSERT_EQ(OB_SUCCESS, reuse_data_agg_result.init(full_agg_metas_.count()));
ASSERT_EQ(OB_SUCCESS, index_agg_result.init(full_agg_metas_.count()));
ObArenaAllocator allocator;
for (int64_t test_round = 0; test_round < 7; ++test_round) {
allocator.reuse();
data_agg_result.reuse();
reuse_data_aggregator.reuse();
index_agg_result.reuse();
ASSERT_EQ(OB_SUCCESS, data_aggregator.init(full_agg_metas_, col_descs_, true, data_agg_result, allocator_));
ASSERT_EQ(OB_SUCCESS, reuse_data_aggregator.init(full_agg_metas_, col_descs_, true, reuse_data_agg_result, allocator_));
ASSERT_EQ(OB_SUCCESS, index_aggregator.init(full_agg_metas_, col_descs_, false, index_agg_result, allocator_));
const ObDatumRow *data_agg_row = nullptr;
const ObDatumRow *reuse_data_agg_row = nullptr;
const ObDatumRow *index_agg_row = nullptr;
ObDatumRow generate_row;
ASSERT_EQ(OB_SUCCESS, generate_row.init(full_column_count_));
for (int64_t i = 0; i < test_row_cnt; ++i) {
const int64_t seed = random() % test_row_cnt;
generate_row_by_seed(seed, generate_row);
update_sum_row(generate_row, sum_res, data);
ASSERT_EQ(OB_SUCCESS, data_aggregator.eval(generate_row));
ASSERT_EQ(OB_SUCCESS, data_aggregator.get_aggregated_row(data_agg_row));
ASSERT_TRUE(nullptr != data_agg_row);
const char *row_buf = nullptr;
int64_t row_size = 0;
serialize_agg_row(*data_agg_row, row_buf, row_size);
ASSERT_TRUE(nullptr != row_buf);
ASSERT_EQ(OB_SUCCESS, reuse_data_aggregator.eval(row_buf, row_size, i));
ASSERT_EQ(OB_SUCCESS, reuse_data_aggregator.get_aggregated_row(reuse_data_agg_row));
ASSERT_TRUE(nullptr != reuse_data_agg_row);
if (0 == i / 2) {
ASSERT_EQ(OB_SUCCESS, index_aggregator.eval(*data_agg_row));
} else {
ASSERT_EQ(OB_SUCCESS, index_aggregator.eval(row_buf, row_size, i));
}
ASSERT_EQ(OB_SUCCESS, index_aggregator.get_aggregated_row(index_agg_row));
ASSERT_TRUE(nullptr != index_agg_row);
validate_sum_agg_row(*data_agg_row, sum_res);
validate_sum_agg_row(*reuse_data_agg_row, sum_res);
validate_sum_agg_row(*index_agg_row, sum_res);
reuse_data_aggregator.reuse();
index_aggregator.reuse();
}
data_aggregator.reset();
reuse_data_aggregator.reset();
index_aggregator.reset();
}
// test nop agg
const int64_t nop_col_cnt = 1;
int64_t nop_col_idxs[nop_col_cnt] = {3};
for (int64_t col_id = 0; col_id < test_column_cnt + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); ++col_id) {
sum_res[col_id].set_null();
}
data_agg_result.reuse();
index_agg_result.reuse();
ASSERT_EQ(OB_SUCCESS, data_aggregator.init(full_agg_metas_, col_descs_, true, data_agg_result, allocator_));
ASSERT_EQ(OB_SUCCESS, index_aggregator.init(full_agg_metas_, col_descs_, false, index_agg_result, allocator_));
const ObDatumRow *data_agg_row = nullptr;
const ObDatumRow *index_agg_row = nullptr;
ObDatumRow generate_row;
ASSERT_EQ(OB_SUCCESS, generate_row.init(full_column_count_));
for (int64_t i = 0; i < test_row_cnt; ++i) {
const int64_t seed = random() % test_row_cnt;
generate_row_by_seed(seed, generate_row);
update_sum_row(generate_row, sum_res, data);
ASSERT_EQ(OB_SUCCESS, data_aggregator.eval(generate_row));
ASSERT_EQ(OB_SUCCESS, data_aggregator.get_aggregated_row(data_agg_row));
ASSERT_TRUE(nullptr != data_agg_row);
set_nop_cols(*const_cast<ObDatumRow *>(data_agg_row), nop_col_cnt, nop_col_idxs);
ASSERT_EQ(OB_SUCCESS, index_aggregator.eval(*data_agg_row));
ASSERT_EQ(OB_SUCCESS, index_aggregator.get_aggregated_row(index_agg_row));
ASSERT_TRUE(nullptr != index_agg_row);
validate_sum_agg_row(*data_agg_row, sum_res, nop_col_cnt, nop_col_idxs);
validate_sum_agg_row(*index_agg_row, sum_res, nop_col_cnt, nop_col_idxs);
index_aggregator.reuse();
}
}
}
}

8
unittest/storage/ddl/test_chunk_compact_store.cpp
View File

@ -284,7 +284,7 @@ TEST_F(TestCompactChunk, test_read_writer_compact)
cs_chunk.init(1, 1,
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, true, share::SORT_COMPACT_LEVEL);
RowMeta row_meta(allocator_);
ChunkRowMeta row_meta(allocator_);
row_meta.col_cnt_ = COLUMN_CNT;
row_meta.fixed_cnt_ = COLUMN_CNT;
row_meta.var_data_off_ = 8 * row_meta.fixed_cnt_;
@ -340,7 +340,7 @@ TEST_F(TestCompactChunk, test_read_writer_compact_vardata)
cs_chunk.init(1, 1,
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, true, share::SORT_COMPACT_LEVEL);
RowMeta row_meta(allocator_);
ChunkRowMeta row_meta(allocator_);
row_meta.col_cnt_ = COLUMN_CNT;
row_meta.fixed_cnt_ = 0;
row_meta.var_data_off_ = 0;
@ -653,7 +653,7 @@ TEST_F(TestCompactChunk, test_rescan_get_last_row_compact)
ObCompactStore cs_chunk;
cs_chunk.init(1, 1,
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, false/*disable trunc*/, share::SORT_COMPACT_LEVEL);
RowMeta row_meta(allocator_);
ChunkRowMeta row_meta(allocator_);
row_meta.col_cnt_ = COLUMN_CNT;
row_meta.fixed_cnt_ = 0;
row_meta.var_data_off_ = 0;
@ -722,7 +722,7 @@ TEST_F(TestCompactChunk, test_rescan_add_storagedatum)
ObCompactStore cs_chunk;
cs_chunk.init(1, 1,
ObCtxIds::DEFAULT_CTX_ID, "SORT_CACHE_CTX", true, 0, false/*disable trunc*/, share::SORT_COMPACT_LEVEL);
RowMeta row_meta(allocator_);
ChunkRowMeta row_meta(allocator_);
row_meta.col_cnt_ = COLUMN_CNT;
row_meta.fixed_cnt_ = 0;
row_meta.var_data_off_ = 0;

6
unittest/storage/mock_ob_table_read_info.h
View File

@ -9,6 +9,10 @@
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_STORAGE_MOCK_OB_TABLE_READ_INFO_H_
#define OCEANBASE_STORAGE_MOCK_OB_TABLE_READ_INFO_H_
#include <gtest/gtest.h>
#include <sys/stat.h>
#include <sys/types.h>
@ -77,3 +81,5 @@ int MockObTableReadInfo::init(common::ObIAllocator &allocator,
}
}
#endif