database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature-wip](array-type) Support array type which doesn't contain null (#9809)

Browse Source
This commit is contained in:
Adonis Ling
2022-06-12 23:35:28 +08:00
committed by GitHub
parent bf984c1e80
commit 415b6b8086
14 changed files with 283 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
be/test/runtime/array_test.cpp
View File

@ -52,9 +52,10 @@ namespace doris {
template <typename... Ts>
ColumnPB create_column_pb(const std::string& type, const Ts&... sub_column_types) {
ColumnPB column;
column.set_type(type);
auto prefix = "NOT_NULL_";
column.set_is_nullable(type.compare(0, strlen(prefix), prefix) != 0);
column.set_type(column.is_nullable() ? type : type.substr(strlen(prefix)));
column.set_aggregation("NONE");
column.set_is_nullable(true);
if (type == "ARRAY") {
column.set_length(OLAP_ARRAY_MAX_BYTES);
}
@ -326,7 +327,7 @@ private:
meta->set_encoding(item_encoding);
}
meta->set_compression(segment_v2::LZ4F);
meta->set_is_nullable(true);
meta->set_is_nullable(column.is_nullable());
for (uint32_t i = 0; i < column.get_subtype_count(); ++i) {
init_column_meta<array_encoding, item_encoding>(meta->add_children_columns(), column_id,
column.get_sub_column(i));
@ -465,6 +466,36 @@ TEST_F(ArrayTest, TestBoolean) {
test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb, literal_arrays);
}
TEST_F(ArrayTest, TestNotNullBoolean) {
// depth 1
auto column_pb = create_column_pb("ARRAY", "NOT_NULL_BOOLEAN");
std::vector<std::string> literal_arrays = {
"[]",
"[true, false, false]",
};
test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb, literal_arrays);
// depth 2
column_pb = create_column_pb("ARRAY", "ARRAY", "NOT_NULL_BOOLEAN");
literal_arrays = {
"[]",
"[[]]",
"[[false, true, false]]",
"[[false, true, false], [true, false, true]]",
};
test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb, literal_arrays);
// depth 3
column_pb = create_column_pb("ARRAY", "ARRAY", "ARRAY", "NOT_NULL_BOOLEAN");
literal_arrays = {
"[]",
"[[]]",
"[[[]]]",
"[[[]], [[false], [true, false]], [[false, true, false]]]",
};
test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb, literal_arrays);
}
void test_integer(const std::string& type, ArrayTest& test_suite) {
// depth 1
auto column_pb = create_column_pb("ARRAY", type);
@ -512,6 +543,44 @@ TEST_F(ArrayTest, TestInteger) {
test_integer("LARGEINT", *this);
}
void test_not_null_integer(const std::string& type, ArrayTest& test_suite) {
// depth 1
auto column_pb = create_column_pb("ARRAY", type);
std::vector<std::string> literal_arrays = {
"[]",
"[1, 2, 3]",
};
test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
literal_arrays);
// depth 2
column_pb = create_column_pb("ARRAY", "ARRAY", type);
literal_arrays = {
"[]",
"[[]]",
"[[1, 2, 3]]",
"[[1, 2, 3], [4, 5, 6]]",
};
test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
literal_arrays);
// depth 3
column_pb = create_column_pb("ARRAY", "ARRAY", "ARRAY", type);
literal_arrays = {
"[]", "[[]]", "[[[]]]", "[[[1, 2, 3]]]", "[[[]], [[1], [2, 3]], [[4, 5, 6]]]",
};
test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
literal_arrays);
}
TEST_F(ArrayTest, TestNotNullInteger) {
test_not_null_integer("NOT_NULL_TINYINT", *this);
test_not_null_integer("NOT_NULL_SMALLINT", *this);
test_not_null_integer("NOT_NULL_INT", *this);
test_not_null_integer("NOT_NULL_BIGINT", *this);
test_not_null_integer("NOT_NULL_LARGEINT", *this);
}
void test_float(const std::string& type, ArrayTest& test_suite) {
// depth 1
auto column_pb = create_column_pb("ARRAY", type);
@ -555,6 +624,40 @@ TEST_F(ArrayTest, TestFloat) {
test_float("DOUBLE", *this);
}
void test_not_null_float(const std::string& type, ArrayTest& test_suite) {
// depth 1
auto column_pb = create_column_pb("ARRAY", type);
std::vector<std::string> literal_arrays = {
"[]",
"[1.5, 2.5, 3.5]",
};
test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
literal_arrays);
// depth 2
column_pb = create_column_pb("ARRAY", "ARRAY", type);
literal_arrays = {
"[]",
"[[]]",
"[[1.5, 2.5, 3.5]]",
"[[1.5, 2.5, 3.5], [4.5, 5.5, 6.5]]",
};
test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
literal_arrays);
// depth 3
column_pb = create_column_pb("ARRAY", "ARRAY", "ARRAY", type);
literal_arrays = {
"[]", "[[]]", "[[[]]]", "[[[1.5]]]", "[[[]], [[1.5], [2.5, 3.5]], [[4.5, 5.5, 6.5]]]",
};
test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
literal_arrays);
}
TEST_F(ArrayTest, TestNotNullFloat) {
test_not_null_float("NOT_NULL_FLOAT", *this);
test_not_null_float("NOT_NULL_DOUBLE", *this);
}
void test_string(const std::string& type, ArrayTest& test_suite) {
// depth 1
auto column_pb = create_column_pb("ARRAY", type);
@ -590,6 +693,35 @@ TEST_F(ArrayTest, TestString) {
test_string("STRING", *this);
}
void test_not_null_string(const std::string& type, ArrayTest& test_suite) {
// depth 1
auto column_pb = create_column_pb("ARRAY", type);
std::vector<std::string> literal_arrays = {
"[]",
"[\"a\", \"b\", \"c\"]",
};
test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::DICT_ENCODING>(column_pb,
literal_arrays);
// more depths
column_pb = create_column_pb("ARRAY", "ARRAY", "ARRAY", type);
literal_arrays = {
"[]",
"[[]]",
"[[[]]]",
"[[[\"a\", \"b\", \"c\"]]]",
"[[[\"a\", \"c\"], [\"d\", \"e\", \"f\"]], [[\"g\"]]]",
};
test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::DICT_ENCODING>(column_pb,
literal_arrays);
}
TEST_F(ArrayTest, TestNotNullString) {
test_not_null_string("NOT_NULL_CHAR", *this);
test_not_null_string("NOT_NULL_VARCHAR", *this);
test_not_null_string("NOT_NULL_STRING", *this);
}
void test_datetime(const std::string& type, ArrayTest& test_suite) {
auto column_pb = create_column_pb("ARRAY", type);
std::vector<std::string> literal_arrays;
@ -681,9 +813,83 @@ TEST_F(ArrayTest, TestDateTime) {
test_datetime("DATETIME", *this);
}
void test_not_null_datetime(const std::string& type, ArrayTest& test_suite) {
auto column_pb = create_column_pb("ARRAY", type);
std::vector<std::string> literal_arrays;
if (type == "DATE") {
literal_arrays = {
"[]",
"[\"2022-04-01\", \"2022-04-02\", \"2022-04-03\"]",
};
} else {
literal_arrays = {
"[]",
"[\"2022-04-01 19:30:40\", \"2022-04-02 19:30:40 \", \"2022-04-03 19:30:40\"]",
};
}
test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
literal_arrays);
// depth 2
column_pb = create_column_pb("ARRAY", "ARRAY", type);
if (type == "DATE") {
literal_arrays = {
"[]",
"[[]]",
"[[\"2022-04-01\", \"2022-04-02\", \"2022-04-03\"], [\"2022-04-04\", "
"\"2022-04-05\", "
"\"2022-04-06\"]]",
};
} else {
literal_arrays = {
"[]",
"[[]]",
"[[\"2022-04-01 19:30:40\", \"2022-04-02 19:30:40\", \"2022-04-03 19:30:40\"], "
"[\"2022-04-04 19:30:40\", "
"\"2022-04-05\", "
"\"2022-04-06\"]]",
};
}
test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
literal_arrays);
// depth 3
column_pb = create_column_pb("ARRAY", "ARRAY", "ARRAY", type);
if (type == "DATE") {
literal_arrays = {
"[]",
"[[]]",
"[[[]]]",
"[[[\"2022-04-01\"]]]",
"[[[]], [[\"2022-04-01\"], [\"2022-04-02\", \"2022-04-03\"]], "
"[[\"2022-04-04\", "
"\"2022-04-05\", \"2022-04-06\"]]]",
};
} else {
literal_arrays = {
"[]",
"[[]]",
"[[[]]]",
"[[[\"2022-04-01 19:30:40\"]]]",
"[[[]], [[\"2022-04-01 19:30:40\"], [\"2022-04-02 19:30:40\", \"2022-04-03 "
"19:30:40\"]], "
"[[\"2022-04-04 19:30:40\", "
"\"2022-04-05 19:30:40\", \"2022-04-06 19:30:40\"]]]",
};
}
test_suite.test<segment_v2::DEFAULT_ENCODING, segment_v2::BIT_SHUFFLE>(column_pb,
literal_arrays);
}
TEST_F(ArrayTest, TestNotNullDateTime) {
test_not_null_datetime("NOT_NULL_DATE", *this);
test_not_null_datetime("NOT_NULL_DATETIME", *this);
}
TEST_F(ArrayTest, TestDecimal) {
test_integer("DECIMAL", *this);
test_not_null_integer("NOT_NULL_DECIMAL", *this);
test_float("DECIMAL", *this);
test_not_null_float("NOT_NULL_DECIMAL", *this);
}
} // namespace doris