database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

branch-2.1: [fix](schema-change) fix array/map/struct in schema-change not-null to null will make core #45305 (#45482)

Browse Source 
Cherry-picked from #45305

Co-authored-by: amory <wangqiannan@selectdb.com>
This commit is contained in:
github-actions[bot]
2024-12-17 17:05:17 +08:00
committed by GitHub
parent 8dc845671f
commit 191ef9b8b0
3 changed files with 177 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
be/src/olap/rowset/segment_v2/column_reader.cpp
View File

@ -777,8 +777,18 @@ Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr
size_t num_read = *n;
auto null_map_ptr =
static_cast<vectorized::ColumnNullable&>(*dst).get_null_map_column_ptr();
bool null_signs_has_null = false;
RETURN_IF_ERROR(_null_iterator->next_batch(&num_read, null_map_ptr, &null_signs_has_null));
// in not-null to null linked-schemachange mode,
// actually we do not change dat data include meta in footer,
// so may dst from changed meta which is nullable but old data is not nullable,
// if so, we should set null_map to all null by default
if (_null_iterator) {
bool null_signs_has_null = false;
RETURN_IF_ERROR(
_null_iterator->next_batch(&num_read, null_map_ptr, &null_signs_has_null));
} else {
auto& null_map = assert_cast<vectorized::ColumnUInt8&>(*null_map_ptr);
null_map.insert_many_vals(0, num_read);
}
DCHECK(num_read == *n);
}
return Status::OK();
@ -838,8 +848,18 @@ Status StructFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumn
size_t num_read = *n;
auto null_map_ptr =
static_cast<vectorized::ColumnNullable&>(*dst).get_null_map_column_ptr();
bool null_signs_has_null = false;
RETURN_IF_ERROR(_null_iterator->next_batch(&num_read, null_map_ptr, &null_signs_has_null));
// in not-null to null linked-schemachange mode,
// actually we do not change dat data include meta in footer,
// so may dst from changed meta which is nullable but old data is not nullable,
// if so, we should set null_map to all null by default
if (_null_iterator) {
bool null_signs_has_null = false;
RETURN_IF_ERROR(
_null_iterator->next_batch(&num_read, null_map_ptr, &null_signs_has_null));
} else {
auto& null_map = assert_cast<vectorized::ColumnUInt8&>(*null_map_ptr);
null_map.insert_many_vals(0, num_read);
}
DCHECK(num_read == *n);
}
@ -992,8 +1012,18 @@ Status ArrayFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnP
auto null_map_ptr =
static_cast<vectorized::ColumnNullable&>(*dst).get_null_map_column_ptr();
size_t num_read = *n;
bool null_signs_has_null = false;
RETURN_IF_ERROR(_null_iterator->next_batch(&num_read, null_map_ptr, &null_signs_has_null));
// in not-null to null linked-schemachange mode,
// actually we do not change dat data include meta in footer,
// so may dst from changed meta which is nullable but old data is not nullable,
// if so, we should set null_map to all null by default
if (_null_iterator) {
bool null_signs_has_null = false;
RETURN_IF_ERROR(
_null_iterator->next_batch(&num_read, null_map_ptr, &null_signs_has_null));
} else {
auto& null_map = assert_cast<vectorized::ColumnUInt8&>(*null_map_ptr);
null_map.insert_many_vals(0, num_read);
}
DCHECK(num_read == *n);
}

84
regression-test/data/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.out
View File

@ -44,3 +44,87 @@ col3 array<int> Yes false \N NONE
col4 map<int,int> Yes false \N NONE
col5 struct<f1:int> Yes false \N NONE
-- !sql_before --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
-- !sql --
col0 bigint No true \N
col2 int No false \N NONE
col3 array<int> Yes false \N NONE
col4 map<int,int> Yes false \N NONE
col5 struct<f1:int> Yes false \N NONE
col6 variant Yes false \N NONE
-- !sql_after --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
-- !sql_before --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
-- !sql --
col0 bigint No true \N
col2 int No false \N NONE
col3 array<int> Yes false \N NONE
col4 map<int,int> No false \N NONE
col5 struct<f1:int> No false \N NONE
col6 variant No false \N NONE
-- !sql_after --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
-- !sql_before --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
-- !sql --
col0 bigint No true \N
col2 int No false \N NONE
col3 array<int> Yes false \N NONE
col4 map<int,int> Yes false \N NONE
col5 struct<f1:int> Yes false \N NONE
col6 variant Yes false \N NONE
-- !sql_after --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
-- !sql_before --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
-- !sql --
col0 bigint No true \N
col2 int No false \N NONE
col3 array<int> No false \N NONE
col4 map<int,int> Yes false \N NONE
col5 struct<f1:int> No false \N NONE
col6 variant No false \N NONE
-- !sql_after --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
-- !sql_before --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
-- !sql --
col0 bigint No true \N
col2 int No false \N NONE
col3 array<int> Yes false \N NONE
col4 map<int,int> Yes false \N NONE
col5 struct<f1:int> Yes false \N NONE
col6 variant Yes false \N NONE
-- !sql_after --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
-- !sql_before --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
-- !sql --
col0 bigint No true \N
col2 int No false \N NONE
col3 array<int> No false \N NONE
col4 map<int,int> No false \N NONE
col5 struct<f1:int> Yes false \N NONE
col6 variant No false \N NONE
-- !sql_after --
1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}

57
regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.groovy
View File

@ -62,4 +62,61 @@ suite("create_nestedtypes_with_schemachange", "p0") {
// struct with other type
create_nested_table_and_schema_change.call("test_struct_schemachange_1", "STRUCT<f1: varchar(1)>", "col5", "errCode = 2");
def create_nested_table_and_schema_change_null_trans = {testTablex, nested_type, column_name, notNull2Null ->
def null_define = "NULL"
if (notNull2Null) {
null_define = "NOT NULL"
}
// create basic type
sql "DROP TABLE IF EXISTS $testTablex"
sql """ CREATE TABLE IF NOT EXISTS $testTablex (
col0 BIGINT NOT NULL,
col2 int NOT NULL,
col3 array<int> $null_define,
col4 map<int, int> $null_define,
col5 struct<f1: int> $null_define,
col6 variant $null_define
)
/* mow */
UNIQUE KEY(col0) DISTRIBUTED BY HASH(col0) BUCKETS 4 PROPERTIES (
"enable_unique_key_merge_on_write" = "true",
"replication_num" = "1",
'light_schema_change' = 'true', 'disable_auto_compaction'='true'
); """
// insert data
sql """ INSERT INTO $testTablex VALUES (1, 2, array(1, 2), map(1, 2), named_struct('f1', 1), '{"a": [1,2,3]}')"""
// select
qt_sql_before "select * from $testTablex"
if (notNull2Null) {
sql "ALTER TABLE $testTablex MODIFY COLUMN $column_name $nested_type NULL"
waitForSchemaChangeDone {
sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='$testTablex' ORDER BY createtime DESC LIMIT 1 """
time 600
}
} else {
// schema change from null to non-nullable is not supported
test {
sql "ALTER TABLE $testTablex MODIFY COLUMN $column_name $nested_type NOT NULL"
exception "Can not change from nullable to non-nullable"
}
}
// desc table
qt_sql "DESC $testTablex"
qt_sql_after "select * from $testTablex"
}
// array
create_nested_table_and_schema_change_null_trans.call("test_array_schemachange_null", "ARRAY<INT>", "col3", false)
create_nested_table_and_schema_change_null_trans.call("test_array_schemachange_null1", "ARRAY<INT>", "col3", true)
// map
create_nested_table_and_schema_change_null_trans.call("test_map_schemachange_null", "Map<INT, INT>", "col4", false)
create_nested_table_and_schema_change_null_trans.call("test_map_schemachange_null1", "Map<INT, INT>", "col4", true)
// struct
create_nested_table_and_schema_change_null_trans.call("test_struct_schemachange_null", "struct<f1: int>", "col5", false)
create_nested_table_and_schema_change_null_trans.call("test_struct_schemachange_null1", "struct<f1: int>", "col5", true)
// variant
// create_nested_table_and_schema_change_null_trans.call("test_v_schemachange_null", "variant", "col6", false)
// create_nested_table_and_schema_change_null_trans.call("test_v_schemachange_null1", "variant", "col6", true)
}