database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](light-schema-change) fix schema consistency check failed (#23283)

Browse Source
This commit is contained in:
Siyang Tang
2023-08-28 16:40:30 +08:00
committed by GitHub
parent 6ac694aede
commit 650cc25ea4
18 changed files with 25 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
be/src/service/internal_service.cpp
View File

@ -753,7 +753,7 @@ void PInternalServiceImpl::_get_column_ids_by_tablet_ids(
for (const auto& param : params) {
int64_t index_id = param.indexid();
auto tablet_ids = param.tablet_ids();
std::set<std::vector<int32_t>> filter_set;
std::set<std::set<int32_t>> filter_set;
for (const int64_t tablet_id : tablet_ids) {
TabletSharedPtr tablet = tablet_mgr->get_tablet(tablet_id);
if (tablet == nullptr) {
@ -766,9 +766,10 @@ void PInternalServiceImpl::_get_column_ids_by_tablet_ids(
}
// check schema consistency, column ids should be the same
const auto& columns = tablet->tablet_schema()->columns();
std::vector<int32_t> column_ids(columns.size());
std::transform(columns.begin(), columns.end(), column_ids.begin(),
[](const TabletColumn& c) { return c.unique_id(); });
std::set<int32_t> column_ids;
for (const auto& col : columns) {
column_ids.insert(col.unique_id());
}
filter_set.insert(column_ids);
}
if (filter_set.size() > 1) {

2
regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy
View File

@ -63,7 +63,7 @@ suite ("test_agg_keys_schema_change") {
`hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列",
`bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列")
AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 1
BUCKETS 8
PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" );
"""

2
regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy
View File

@ -80,7 +80,7 @@ suite ("test_agg_mv_schema_change") {
`hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列",
`bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列")
AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 1
BUCKETS 8
PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" );
"""

2
regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy
View File

@ -80,7 +80,7 @@ suite ("test_agg_rollup_schema_change") {
`hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列",
`bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列")
AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 1
BUCKETS 8
PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" );
"""

2
regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy
View File

@ -61,7 +61,7 @@ suite ("test_agg_vals_schema_change") {
`hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列",
`bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列")
AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 1
BUCKETS 8
PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" );
"""

2
regression-test/suites/schema_change_p0/test_alter_table_column.groovy
View File

@ -193,7 +193,7 @@ suite("test_alter_table_column") {
`v1` int(11) SUM NULL COMMENT ""
) ENGINE=OLAP
AGGREGATE KEY(`k1`, `k2`)
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
DISTRIBUTED BY HASH(`k1`) BUCKETS 5
PROPERTIES (
"storage_type" = "COLUMN",
"replication_num" = "1"

1
regression-test/suites/schema_change_p0/test_delete_schema_change.sql
View File

@ -13,6 +13,7 @@ CREATE TABLE IF NOT EXISTS schema_change_delete_regression_test (
`max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间",
`min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间")
DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 8
PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" );
INSERT INTO schema_change_delete_regression_test VALUES

2
regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy
View File

@ -62,7 +62,7 @@ suite ("test_dup_keys_schema_change") {
`max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间",
`min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间")
DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 1
BUCKETS 8
PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" );
"""

2
regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy
View File

@ -76,7 +76,7 @@ suite ("test_dup_mv_schema_change") {
`max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间",
`min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间")
DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 1
BUCKETS 8
PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" );
"""

2
regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy
View File

@ -80,7 +80,7 @@ suite ("test_dup_rollup_schema_change") {
`max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间",
`min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间")
DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 1
BUCKETS 8
PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" );
"""

2
regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy
View File

@ -57,7 +57,7 @@ suite ("test_dup_vals_schema_change") {
`max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间",
`min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间")
DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 1
BUCKETS 8
PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" );
"""

6
regression-test/suites/schema_change_p0/test_rename_column.groovy
View File

@ -40,6 +40,7 @@ suite ("test_rename_column") {
`max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间",
`min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间")
UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 8
PROPERTIES ( "replication_num" = "1" , "light_schema_change" = "false")
"""
qt_desc """ desc ${tableName} """
@ -120,6 +121,7 @@ suite ("test_rename_column") {
`max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间",
`min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间")
UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 8
PROPERTIES ( "replication_num" = "1" , "light_schema_change" = "false")
"""
test {
@ -142,7 +144,7 @@ suite ("test_rename_column") {
`hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列",
`bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列")
AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 1
BUCKETS 8
PROPERTIES ( "replication_num" = "1", "light_schema_change" = "true" );
"""
qt_desc """ desc ${tableName} """
@ -209,7 +211,7 @@ suite ("test_rename_column") {
`hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列",
`bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列")
AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 1
BUCKETS 8
PROPERTIES ( "replication_num" = "1", "light_schema_change" = "true" );
"""

2
regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy
View File

@ -57,7 +57,7 @@ suite ("test_uniq_keys_schema_change") {
`max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间",
`min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间")
UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 1
BUCKETS 8
PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false");
"""

2
regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy
View File

@ -75,7 +75,7 @@ suite ("test_uniq_mv_schema_change") {
`max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间",
`min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间")
UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 1
BUCKETS 8
PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false", 'enable_unique_key_merge_on_write' = 'false');
"""

2
regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy
View File

@ -79,7 +79,7 @@ suite ("test_uniq_rollup_schema_change") {
`max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间",
`min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间")
UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 1
BUCKETS 8
PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false", 'enable_unique_key_merge_on_write' = 'false');
"""

2
regression-test/suites/schema_change_p0/test_uniq_seq_col_schema_change.groovy
View File

@ -29,7 +29,7 @@ suite("test_uniq_seq_col_schema_change", "schema_change") {
value3 INT
)
UNIQUE KEY (k1)
DISTRIBUTED BY HASH(k1) BUCKETS 1
DISTRIBUTED BY HASH(k1) BUCKETS 8
properties("replication_num" = "1",
"light_schema_change" = "false",
"function_column.sequence_type" = 'INT');

2
regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy
View File

@ -59,7 +59,7 @@ suite ("test_uniq_vals_schema_change") {
`max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间",
`min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间")
UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 1
BUCKETS 8
PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" );
"""

1
regression-test/suites/schema_change_p0/test_update_schema_change.groovy
View File

@ -38,6 +38,7 @@ suite ("test_update_schema_change") {
`max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间",
`min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间")
UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
BUCKETS 8
PROPERTIES ( "replication_num" = "1" , "light_schema_change" = "false");
"""