From 650cc25ea46e122b66880e3058e39d0c636b64fc Mon Sep 17 00:00:00 2001 From: Siyang Tang <82279870+TangSiyang2001@users.noreply.github.com> Date: Mon, 28 Aug 2023 16:40:30 +0800 Subject: [PATCH] [fix](light-schema-change) fix schema consistency check failed (#23283) --- be/src/service/internal_service.cpp | 9 +++++---- .../schema_change_p0/test_agg_keys_schema_change.groovy | 2 +- .../schema_change_p0/test_agg_mv_schema_change.groovy | 2 +- .../test_agg_rollup_schema_change.groovy | 2 +- .../schema_change_p0/test_agg_vals_schema_change.groovy | 2 +- .../schema_change_p0/test_alter_table_column.groovy | 2 +- .../schema_change_p0/test_delete_schema_change.sql | 1 + .../schema_change_p0/test_dup_keys_schema_change.groovy | 2 +- .../schema_change_p0/test_dup_mv_schema_change.groovy | 2 +- .../test_dup_rollup_schema_change.groovy | 2 +- .../schema_change_p0/test_dup_vals_schema_change.groovy | 2 +- .../suites/schema_change_p0/test_rename_column.groovy | 6 ++++-- .../schema_change_p0/test_uniq_keys_schema_change.groovy | 2 +- .../schema_change_p0/test_uniq_mv_schema_change.groovy | 2 +- .../test_uniq_rollup_schema_change.groovy | 2 +- .../test_uniq_seq_col_schema_change.groovy | 2 +- .../schema_change_p0/test_uniq_vals_schema_change.groovy | 2 +- .../schema_change_p0/test_update_schema_change.groovy | 1 + 18 files changed, 25 insertions(+), 20 deletions(-) diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp index b458af6336..5aa5c95231 100644 --- a/be/src/service/internal_service.cpp +++ b/be/src/service/internal_service.cpp @@ -753,7 +753,7 @@ void PInternalServiceImpl::_get_column_ids_by_tablet_ids( for (const auto& param : params) { int64_t index_id = param.indexid(); auto tablet_ids = param.tablet_ids(); - std::set> filter_set; + std::set> filter_set; for (const int64_t tablet_id : tablet_ids) { TabletSharedPtr tablet = tablet_mgr->get_tablet(tablet_id); if (tablet == nullptr) { @@ -766,9 +766,10 @@ void PInternalServiceImpl::_get_column_ids_by_tablet_ids( } // check schema consistency, column ids should be the same const auto& columns = tablet->tablet_schema()->columns(); - std::vector column_ids(columns.size()); - std::transform(columns.begin(), columns.end(), column_ids.begin(), - [](const TabletColumn& c) { return c.unique_id(); }); + std::set column_ids; + for (const auto& col : columns) { + column_ids.insert(col.unique_id()); + } filter_set.insert(column_ids); } if (filter_set.size() > 1) { diff --git a/regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy b/regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy index e6e0572d18..afc5b784fb 100644 --- a/regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_agg_keys_schema_change.groovy @@ -63,7 +63,7 @@ suite ("test_agg_keys_schema_change") { `hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列", `bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列") AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - BUCKETS 1 + BUCKETS 8 PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" ); """ diff --git a/regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy index c34522084a..6d3f452967 100644 --- a/regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_agg_mv_schema_change.groovy @@ -80,7 +80,7 @@ suite ("test_agg_mv_schema_change") { `hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列", `bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列") AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - BUCKETS 1 + BUCKETS 8 PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" ); """ diff --git a/regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy b/regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy index bee550e430..ccd44b7b3d 100644 --- a/regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_agg_rollup_schema_change.groovy @@ -80,7 +80,7 @@ suite ("test_agg_rollup_schema_change") { `hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列", `bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列") AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - BUCKETS 1 + BUCKETS 8 PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" ); """ diff --git a/regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy b/regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy index c5605a9565..ebe882a543 100644 --- a/regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_agg_vals_schema_change.groovy @@ -61,7 +61,7 @@ suite ("test_agg_vals_schema_change") { `hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列", `bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列") AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - BUCKETS 1 + BUCKETS 8 PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" ); """ diff --git a/regression-test/suites/schema_change_p0/test_alter_table_column.groovy b/regression-test/suites/schema_change_p0/test_alter_table_column.groovy index 35807a5024..21f147283a 100644 --- a/regression-test/suites/schema_change_p0/test_alter_table_column.groovy +++ b/regression-test/suites/schema_change_p0/test_alter_table_column.groovy @@ -193,7 +193,7 @@ suite("test_alter_table_column") { `v1` int(11) SUM NULL COMMENT "" ) ENGINE=OLAP AGGREGATE KEY(`k1`, `k2`) - DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + DISTRIBUTED BY HASH(`k1`) BUCKETS 5 PROPERTIES ( "storage_type" = "COLUMN", "replication_num" = "1" diff --git a/regression-test/suites/schema_change_p0/test_delete_schema_change.sql b/regression-test/suites/schema_change_p0/test_delete_schema_change.sql index 782561ada4..d25cc58bce 100644 --- a/regression-test/suites/schema_change_p0/test_delete_schema_change.sql +++ b/regression-test/suites/schema_change_p0/test_delete_schema_change.sql @@ -13,6 +13,7 @@ CREATE TABLE IF NOT EXISTS schema_change_delete_regression_test ( `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + BUCKETS 8 PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" ); INSERT INTO schema_change_delete_regression_test VALUES diff --git a/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy index af3a4b45ae..dd2fa43f09 100644 --- a/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_keys_schema_change.groovy @@ -62,7 +62,7 @@ suite ("test_dup_keys_schema_change") { `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - BUCKETS 1 + BUCKETS 8 PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" ); """ diff --git a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy index ad510d698e..168c450ebe 100644 --- a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy @@ -76,7 +76,7 @@ suite ("test_dup_mv_schema_change") { `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - BUCKETS 1 + BUCKETS 8 PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" ); """ diff --git a/regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy index ebb78ec32b..1f1e0c5b61 100644 --- a/regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_rollup_schema_change.groovy @@ -80,7 +80,7 @@ suite ("test_dup_rollup_schema_change") { `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - BUCKETS 1 + BUCKETS 8 PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" ); """ diff --git a/regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy index 716eb6bb3a..6a3d9f61c9 100644 --- a/regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_vals_schema_change.groovy @@ -57,7 +57,7 @@ suite ("test_dup_vals_schema_change") { `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - BUCKETS 1 + BUCKETS 8 PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" ); """ diff --git a/regression-test/suites/schema_change_p0/test_rename_column.groovy b/regression-test/suites/schema_change_p0/test_rename_column.groovy index 9ac575ca97..558888954c 100644 --- a/regression-test/suites/schema_change_p0/test_rename_column.groovy +++ b/regression-test/suites/schema_change_p0/test_rename_column.groovy @@ -40,6 +40,7 @@ suite ("test_rename_column") { `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + BUCKETS 8 PROPERTIES ( "replication_num" = "1" , "light_schema_change" = "false") """ qt_desc """ desc ${tableName} """ @@ -120,6 +121,7 @@ suite ("test_rename_column") { `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + BUCKETS 8 PROPERTIES ( "replication_num" = "1" , "light_schema_change" = "false") """ test { @@ -142,7 +144,7 @@ suite ("test_rename_column") { `hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列", `bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列") AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - BUCKETS 1 + BUCKETS 8 PROPERTIES ( "replication_num" = "1", "light_schema_change" = "true" ); """ qt_desc """ desc ${tableName} """ @@ -209,7 +211,7 @@ suite ("test_rename_column") { `hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列", `bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列") AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - BUCKETS 1 + BUCKETS 8 PROPERTIES ( "replication_num" = "1", "light_schema_change" = "true" ); """ diff --git a/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy index 3c6778f1a6..d3eaec8cfa 100644 --- a/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_uniq_keys_schema_change.groovy @@ -57,7 +57,7 @@ suite ("test_uniq_keys_schema_change") { `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - BUCKETS 1 + BUCKETS 8 PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false"); """ diff --git a/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy index 7408e6c1d1..71c324a729 100644 --- a/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy @@ -75,7 +75,7 @@ suite ("test_uniq_mv_schema_change") { `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - BUCKETS 1 + BUCKETS 8 PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false", 'enable_unique_key_merge_on_write' = 'false'); """ diff --git a/regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy index 821a229487..52d4799807 100644 --- a/regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_uniq_rollup_schema_change.groovy @@ -79,7 +79,7 @@ suite ("test_uniq_rollup_schema_change") { `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - BUCKETS 1 + BUCKETS 8 PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false", 'enable_unique_key_merge_on_write' = 'false'); """ diff --git a/regression-test/suites/schema_change_p0/test_uniq_seq_col_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_seq_col_schema_change.groovy index d1adc9e0eb..be3bc10256 100644 --- a/regression-test/suites/schema_change_p0/test_uniq_seq_col_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_uniq_seq_col_schema_change.groovy @@ -29,7 +29,7 @@ suite("test_uniq_seq_col_schema_change", "schema_change") { value3 INT ) UNIQUE KEY (k1) - DISTRIBUTED BY HASH(k1) BUCKETS 1 + DISTRIBUTED BY HASH(k1) BUCKETS 8 properties("replication_num" = "1", "light_schema_change" = "false", "function_column.sequence_type" = 'INT'); diff --git a/regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy index a9ee1ddab4..43c08dc730 100644 --- a/regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_uniq_vals_schema_change.groovy @@ -59,7 +59,7 @@ suite ("test_uniq_vals_schema_change") { `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) - BUCKETS 1 + BUCKETS 8 PROPERTIES ( "replication_num" = "1", "light_schema_change" = "false" ); """ diff --git a/regression-test/suites/schema_change_p0/test_update_schema_change.groovy b/regression-test/suites/schema_change_p0/test_update_schema_change.groovy index 74c787f976..aa375b274c 100644 --- a/regression-test/suites/schema_change_p0/test_update_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_update_schema_change.groovy @@ -38,6 +38,7 @@ suite ("test_update_schema_change") { `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + BUCKETS 8 PROPERTIES ( "replication_num" = "1" , "light_schema_change" = "false"); """