From 1d6dc9a5f090ab26ad70111b981bc50519907fae Mon Sep 17 00:00:00 2001 From: meiyi Date: Tue, 20 Feb 2024 22:18:52 +0800 Subject: [PATCH] [fix](catalog recycle bin) Forbid recover partition if table schame is changed #31146 --- .../doris/catalog/CatalogRecycleBin.java | 11 +- .../recover_with_schema_change.out | 39 ++++++ .../doris/regression/action/TestAction.groovy | 1 + .../recover_with_schema_change.groovy | 127 ++++++++++++++++++ 4 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 regression-test/data/catalog_recycle_bin_p0/recover_with_schema_change.out create mode 100644 regression-test/suites/catalog_recycle_bin_p0/recover_with_schema_change.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java index 177d4641fc..3ae6307445 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java @@ -810,8 +810,17 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable { throw new DdlException("Can not recover partition[" + partitionName + "]. Partition item conflict."); } - // check if partition name exists + // check if schema change Partition recoverPartition = recoverPartitionInfo.getPartition(); + Set tableIndex = table.getIndexIdToMeta().keySet(); + Set partitionIndex = recoverPartition.getMaterializedIndices(IndexExtState.ALL).stream() + .map(i -> i.getId()).collect(Collectors.toSet()); + if (!tableIndex.equals(partitionIndex)) { + throw new DdlException("table's index not equal with partition's index. table's index=" + tableIndex + + ", partition's index=" + partitionIndex); + } + + // check if partition name exists Preconditions.checkState(recoverPartition.getName().equalsIgnoreCase(partitionName)); if (!Strings.isNullOrEmpty(newPartitionName)) { if (table.checkPartitionNameExist(newPartitionName)) { diff --git a/regression-test/data/catalog_recycle_bin_p0/recover_with_schema_change.out b/regression-test/data/catalog_recycle_bin_p0/recover_with_schema_change.out new file mode 100644 index 0000000000..11667bab84 --- /dev/null +++ b/regression-test/data/catalog_recycle_bin_p0/recover_with_schema_change.out @@ -0,0 +1,39 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 a \N 2022-01-02 +2 a \N 2023-01-02 +3 a \N 2024-01-02 +4 a 10 2022-01-02 + +-- !sql -- +a +a + +-- !sql -- +2 2023-01-02 a +3 2024-01-02 a + +-- !sql -- +a +a + +-- !sql -- +2 a 2023-01-02 +3 a 2024-01-02 + +-- !sql -- +a +a + +-- !sql -- +2 a 2023-01-02 +3 a 2024-01-02 + +-- !sql -- +a +a + +-- !sql -- +2 a 2023-01-02 +3 a 2024-01-02 + diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy index c6b19bccd2..684cb9d5be 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy @@ -68,6 +68,7 @@ class TestAction implements SuiteAction { } else { if (exception != null || result.exception != null) { def msg = result.exception?.toString() + log.info("Exception: ${msg}") Assert.assertTrue("Expect exception msg contains '${exception}', but meet '${msg}'", msg != null && exception != null && msg.contains(exception)) } diff --git a/regression-test/suites/catalog_recycle_bin_p0/recover_with_schema_change.groovy b/regression-test/suites/catalog_recycle_bin_p0/recover_with_schema_change.groovy new file mode 100644 index 0000000000..a23c449ab9 --- /dev/null +++ b/regression-test/suites/catalog_recycle_bin_p0/recover_with_schema_change.groovy @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("recover_with_schema_change") { + def tableName = "recover_with_schema_change" + + for (int i = 0; i <= 4; i++) { + def table = tableName + i + + // create table and insert data + sql """ drop table if exists ${table} """ + sql """ + create table ${table} ( + `id` int(11), + `name` varchar(128), + `da` date + ) + engine=olap + duplicate key(id) + partition by range(da)( + PARTITION p3 VALUES LESS THAN ('2023-01-01'), + PARTITION p4 VALUES LESS THAN ('2024-01-01'), + PARTITION p5 VALUES LESS THAN ('2025-01-01') + ) + distributed by hash(id) buckets 2 + properties( + "replication_num"="1", + "light_schema_change"="true" + ); + """ + + sql """ insert into ${table} values(1, 'a', '2022-01-02'); """ + sql """ insert into ${table} values(2, 'a', '2023-01-02'); """ + sql """ insert into ${table} values(3, 'a', '2024-01-02'); """ + + if (i == 3 || i == 4) { + // create mv + createMV """ create materialized view mv_${table} as select name from ${table}; """ + } + + // drop partition + sql """ ALTER TABLE ${table} DROP PARTITION p3; """ + + if (i == 0) { + // do light weight schema change + sql """ ALTER TABLE ${table} add column `age` int after name; """ + waitForSchemaChangeDone { + sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${table}' ORDER BY createtime DESC LIMIT 1 """ + time 60 + } + + // recover partition should success + sql """ recover partition p3 from regression_test_catalog_recycle_bin_p0.${table}; """ + } else if (i == 1) { + // do hard weight schema change + sql """ ALTER TABLE ${table} order by(`id`, `da`, `name`); """ + waitForSchemaChangeDone { + sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${table}' ORDER BY createtime DESC LIMIT 1 """ + time 60 + } + + // recover partition should fail + test { + sql """ recover partition p3 from regression_test_catalog_recycle_bin_p0.${table}; """ + exception "table's index not equal with partition's index" + } + } else if (i == 2) { + // create mv + createMV """ create materialized view mv_${table} as select name from ${table}; """ + + // recover partition should fail + test { + sql """ recover partition p3 from regression_test_catalog_recycle_bin_p0.${table}; """ + exception "table's index not equal with partition's index" + } + } else if (i == 3) { + // drop mv + sql """ drop materialized view mv_${table} on ${table}; """ + + // recover partition should fail + test { + sql """ recover partition p3 from regression_test_catalog_recycle_bin_p0.${table}; """ + exception "table's index not equal with partition's index" + } + } else if (i == 4) { + // drop mv + sql """ drop materialized view mv_${table} on ${table}; """ + + // create mv + createMV """ create materialized view mv_${table} as select name from ${table}; """ + + // recover partition should fail + test { + sql """ recover partition p3 from regression_test_catalog_recycle_bin_p0.${table}; """ + exception "table's index not equal with partition's index" + } + } + + // write data + if (i == 0) { + sql """ insert into ${table} values(4, 'a', 10, '2022-01-02'); """ + } else { + test { + sql """ insert into ${table} values(4, 'b', '2022-01-02'); """ + exception "Insert has filtered data in strict mode" + } + order_qt_sql """ select name from ${table}; """ + } + + // read data + order_qt_sql """ select * from ${table}; """ + } +}