diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java index b28a7fd08d..e328270ae1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; @@ -55,6 +56,7 @@ import java.util.stream.Stream; public class CatalogRecycleBin extends MasterDaemon implements Writable { private static final Logger LOG = LogManager.getLogger(CatalogRecycleBin.class); + private static final int DEFAULT_INTERVAL_SECONDS = 30; // 30 seconds // erase meta at least after minEraseLatency milliseconds // to avoid erase log ahead of drop log private static final long minEraseLatency = 10 * 60 * 1000; // 10 min @@ -66,7 +68,7 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable { private Map idToRecycleTime; public CatalogRecycleBin() { - super("recycle bin"); + super("recycle bin", FeConstants.runningUnitTest ? 10L : DEFAULT_INTERVAL_SECONDS * 1000L); idToDatabase = Maps.newHashMap(); idToTable = Maps.newHashMap(); idToPartition = Maps.newHashMap(); @@ -124,7 +126,7 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable { } public synchronized boolean recycleDatabase(Database db, Set tableNames, Set tableIds, - boolean isReplay, long replayRecycleTime) { + boolean isReplay, boolean isForceDrop, long replayRecycleTime) { long recycleTime = 0; if (idToDatabase.containsKey(db.getId())) { LOG.error("db[{}] already in recycle bin.", db.getId()); @@ -137,17 +139,21 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable { // recycle db RecycleDatabaseInfo databaseInfo = new RecycleDatabaseInfo(db, tableNames, tableIds); idToDatabase.put(db.getId(), databaseInfo); - if (!isReplay || replayRecycleTime == 0) { + if (isForceDrop) { + // The 'force drop' database should be recycle immediately. + recycleTime = 0; + } else if (!isReplay || replayRecycleTime == 0) { recycleTime = System.currentTimeMillis(); } else { recycleTime = replayRecycleTime; } idToRecycleTime.put(db.getId(), recycleTime); - LOG.info("recycle db[{}-{}]", db.getId(), db.getFullName()); + LOG.info("recycle db[{}-{}], is force drop: {}", db.getId(), db.getFullName(), isForceDrop); return true; } - public synchronized boolean recycleTable(long dbId, Table table, boolean isReplay, long replayRecycleTime) { + public synchronized boolean recycleTable(long dbId, Table table, boolean isReplay, + boolean isForceDrop, long replayRecycleTime) { long recycleTime = 0; if (idToTable.containsKey(table.getId())) { LOG.error("table[{}] already in recycle bin.", table.getId()); @@ -156,14 +162,17 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable { // recycle table RecycleTableInfo tableInfo = new RecycleTableInfo(dbId, table); - if (!isReplay || replayRecycleTime == 0) { + if (isForceDrop) { + // The 'force drop' table should be recycle immediately. + recycleTime = 0; + } else if (!isReplay || replayRecycleTime == 0) { recycleTime = System.currentTimeMillis(); } else { recycleTime = replayRecycleTime; } idToRecycleTime.put(table.getId(), recycleTime); idToTable.put(table.getId(), tableInfo); - LOG.info("recycle table[{}-{}]", table.getId(), table.getName()); + LOG.info("recycle table[{}-{}], is force drop: {}", table.getId(), table.getName(), isForceDrop); return true; } @@ -419,6 +428,11 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable { LOG.info("before replay erase table[{}]", tableId); RecycleTableInfo tableInfo = idToTable.remove(tableId); idToRecycleTime.remove(tableId); + if (tableInfo == null) { + // FIXME(walter): Sometimes `eraseTable` in 'DROP DB ... FORCE' may be executed earlier than + // finish drop db, especially in the case of drop db with many tables. + return; + } Table table = tableInfo.getTable(); if (table.getType() == TableType.OLAP) { Env.getCurrentEnv().onEraseOlapTable((OlapTable) table, true); @@ -537,7 +551,9 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable { public synchronized Database recoverDatabase(String dbName, long dbId) throws DdlException { RecycleDatabaseInfo dbInfo = null; - long recycleTime = -1; + // The recycle time of the force dropped tables and databases will be set to zero, use 1 here to + // skip these databases and tables. + long recycleTime = 1; Iterator> iterator = idToDatabase.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry entry = iterator.next(); @@ -616,7 +632,9 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable { String newTableName) throws DdlException { // make sure to get db lock Table table = null; - long recycleTime = -1; + // The recycle time of the force dropped tables and databases will be set to zero, use 1 here to + // skip these databases and tables. + long recycleTime = 1; long dbId = db.getId(); Iterator> iterator = idToTable.entrySet().iterator(); while (iterator.hasNext()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java index c1b7ca293b..c3ba8e625a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java @@ -822,22 +822,42 @@ public class TabletInvertedIndex { // just for ut public Table getReplicaMetaTable() { - return replicaMetaTable; + long stamp = readLock(); + try { + return HashBasedTable.create(replicaMetaTable); + } finally { + readUnlock(stamp); + } } // just for ut public Table getBackingReplicaMetaTable() { - return backingReplicaMetaTable; + long stamp = readLock(); + try { + return HashBasedTable.create(backingReplicaMetaTable); + } finally { + readUnlock(stamp); + } } // just for ut public Table getTabletMetaTable() { - return tabletMetaTable; + long stamp = readLock(); + try { + return HashBasedTable.create(tabletMetaTable); + } finally { + readUnlock(stamp); + } } // just for ut public Map getTabletMetaMap() { - return tabletMetaMap; + long stamp = readLock(); + try { + return new HashMap(tabletMetaMap); + } finally { + readUnlock(stamp); + } } private boolean isLocal(TStorageMedium storageMedium) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 9b8985a8c8..6b1e3d90e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -478,6 +478,7 @@ public class InternalCatalog implements CatalogIf { public void dropDb(DropDbStmt stmt) throws DdlException { String dbName = stmt.getDbName(); + LOG.info("begin drop database[{}], is force : {}", dbName, stmt.isForceDrop()); // 1. check if database exists if (!tryLock(false)) { @@ -536,12 +537,8 @@ public class InternalCatalog implements CatalogIf { MetaLockUtils.writeUnlockTables(tableList); } - if (!stmt.isForceDrop()) { - Env.getCurrentRecycleBin().recycleDatabase(db, tableNames, tableIds, false, 0); - recycleTime = Env.getCurrentRecycleBin().getRecycleTimeById(db.getId()); - } else { - Env.getCurrentEnv().eraseDatabase(db.getId(), false); - } + Env.getCurrentRecycleBin().recycleDatabase(db, tableNames, tableIds, false, stmt.isForceDrop(), 0); + recycleTime = Env.getCurrentRecycleBin().getRecycleTimeById(db.getId()); } finally { db.writeUnlock(); } @@ -588,11 +585,7 @@ public class InternalCatalog implements CatalogIf { } finally { MetaLockUtils.writeUnlockTables(tableList); } - if (!isForceDrop) { - Env.getCurrentRecycleBin().recycleDatabase(db, tableNames, tableIds, true, recycleTime); - } else { - Env.getCurrentEnv().eraseDatabase(db.getId(), false); - } + Env.getCurrentRecycleBin().recycleDatabase(db, tableNames, tableIds, true, isForceDrop, recycleTime); Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentEnv().getInternalCatalog().getId(), db.getId()); } finally { db.writeUnlock(); @@ -861,6 +854,7 @@ public class InternalCatalog implements CatalogIf { public void dropTable(DropTableStmt stmt) throws DdlException { String dbName = stmt.getDbName(); String tableName = stmt.getTableName(); + LOG.info("begin to drop table: {} from db: {}, is force: {}", tableName, dbName, stmt.isForceDrop()); // check database Database db = (Database) getDbOrDdlException(dbName); @@ -943,13 +937,7 @@ public class InternalCatalog implements CatalogIf { } db.dropTable(table.getName()); - if (!isForceDrop) { - Env.getCurrentRecycleBin().recycleTable(db.getId(), table, isReplay, recycleTime); - } else { - if (table.getType() == TableType.OLAP) { - Env.getCurrentEnv().onEraseOlapTable((OlapTable) table, isReplay); - } - } + Env.getCurrentRecycleBin().recycleTable(db.getId(), table, isReplay, isForceDrop, recycleTime); LOG.info("finished dropping table[{}] in db[{}]", table.getName(), db.getFullName()); return true; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DropDbTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DropDbTest.java index c262958c6b..4bc976b3c7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DropDbTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DropDbTest.java @@ -121,15 +121,18 @@ public class DropDbTest { @Test public void testForceDropDb() throws Exception { String dropDbSql = "drop database test2 force"; - Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("default_cluster:test2"); - OlapTable table = (OlapTable) db.getTableOrMetaException("tbl1"); - Partition partition = table.getAllPartitions().iterator().next(); - long tabletId = partition.getBaseIndex().getTablets().get(0).getId(); dropDb(dropDbSql); - db = Env.getCurrentInternalCatalog().getDbNullable("default_cluster:test2"); - List replicaList = Env.getCurrentEnv().getTabletInvertedIndex().getReplicasByTabletId(tabletId); + Database db = Env.getCurrentInternalCatalog().getDbNullable("default_cluster:test2"); Assert.assertNull(db); - Assert.assertTrue(replicaList.isEmpty()); + // After unify force and non-force drop db, the replicas will be recycled eventually. + // + // Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("default_cluster:test2"); + // OlapTable table = (OlapTable) db.getTableOrMetaException("tbl1"); + // Partition partition = table.getAllPartitions().iterator().next(); + // long tabletId = partition.getBaseIndex().getTablets().get(0).getId(); + // ... + // List replicaList = Env.getCurrentEnv().getTabletInvertedIndex().getReplicasByTabletId(tabletId); + // Assert.assertTrue(replicaList.isEmpty()); String recoverDbSql = "recover database test2"; RecoverDbStmt recoverDbStmt = (RecoverDbStmt) UtFrameUtils.parseAndAnalyzeStmt(recoverDbSql, connectContext); ExceptionChecker.expectThrowsWithMsg(DdlException.class, diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DropTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DropTableTest.java index accb1f7da8..97be554d63 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DropTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DropTableTest.java @@ -98,14 +98,17 @@ public class DropTableTest { @Test public void testForceDropTable() throws Exception { - Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("default_cluster:test"); - OlapTable table = (OlapTable) db.getTableOrMetaException("tbl2"); - Partition partition = table.getAllPartitions().iterator().next(); - long tabletId = partition.getBaseIndex().getTablets().get(0).getId(); String dropTableSql = "drop table test.tbl2 force"; dropTable(dropTableSql); - List replicaList = Env.getCurrentEnv().getTabletInvertedIndex().getReplicasByTabletId(tabletId); - Assert.assertTrue(replicaList.isEmpty()); + // After unify force and non-force drop table, the replicas will be recycled eventually. + // + // Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("default_cluster:test"); + // OlapTable table = (OlapTable) db.getTableOrMetaException("tbl2"); + // Partition partition = table.getAllPartitions().iterator().next(); + // long tabletId = partition.getBaseIndex().getTablets().get(0).getId(); + // ... + // List replicaList = Env.getCurrentEnv().getTabletInvertedIndex().getReplicasByTabletId(tabletId); + // Assert.assertTrue(replicaList.isEmpty()); String recoverDbSql = "recover table test.tbl2"; RecoverTableStmt recoverTableStmt = (RecoverTableStmt) UtFrameUtils.parseAndAnalyzeStmt(recoverDbSql, connectContext); ExceptionChecker.expectThrowsWithMsg(DdlException.class, diff --git a/fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java b/fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java index b520d73a8b..ec0779b452 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java @@ -417,7 +417,13 @@ public class TabletRepairAndBalanceTest { ExceptionChecker.expectThrowsNoException(() -> dropTable(dropStmt1)); ExceptionChecker.expectThrowsNoException(() -> dropTable(dropStmt2)); ExceptionChecker.expectThrowsNoException(() -> dropTable(dropStmt3)); - Assert.assertEquals(0, replicaMetaTable.size()); + Assert.assertNull(db.getTableNullable("tbl1")); + Assert.assertNull(db.getTableNullable("col_tbl1")); + Assert.assertNull(db.getTableNullable("col_tbl2")); + // After unify force and non-force drop table, the indexes will be erase eventually. + while (colocateTableIndex.getAllGroupIds().size() > 0) { + Thread.sleep(1000); + } // set all backends' tag to default for (int i = 0; i < backends.size(); ++i) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index 62e1e95ec1..b5ef93b97a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -746,6 +746,9 @@ public abstract class TestWithFeService { } Replica replica = cell.getValue(); TabletMeta tabletMeta = Env.getCurrentInvertedIndex().getTabletMeta(cell.getRowKey()); + if (tabletMeta == null) { + continue; + } ImmutableMap diskMap = be.getDisks(); for (DiskInfo diskInfo : diskMap.values()) { if (diskInfo.getStorageMedium() == tabletMeta.getStorageMedium()) { diff --git a/regression-test/suites/ddl_p0/test_drop_force.groovy b/regression-test/suites/ddl_p0/test_drop_force.groovy new file mode 100644 index 0000000000..565df0b768 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_drop_force.groovy @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// this suite is for creating table with timestamp datatype in defferent +// case. For example: 'year' and 'Year' datatype should also be valid in definition + +suite("sql_force_drop") { + def testTable = "test_force_drop" + + sql "CREATE DATABASE IF NOT EXISTS test_force_drop_database" + sql """ + CREATE TABLE IF NOT EXISTS test_force_drop_database.table1 ( + `actorid` varchar(128), + `gameid` varchar(128), + `eventtime` datetimev2(3) + ) + engine=olap + duplicate key(actorid, gameid, eventtime) + partition by range(eventtime)( + from ("2000-01-01") to ("2021-01-01") interval 1 year, + from ("2021-01-01") to ("2022-01-01") interval 1 MONth, + from ("2022-01-01") to ("2023-01-01") interval 1 WEEK, + from ("2023-01-01") TO ("2023-02-01") interval 1 DAY + ) + distributed by hash(actorid) buckets 1 + properties( + "replication_num"="1", + "light_schema_change"="true", + "compression"="zstd" + ); + """ + sql """ + CREATE TABLE IF NOT EXISTS test_force_drop_database.table2 ( + `actorid` varchar(128), + `gameid` varchar(128), + `eventtime` datetimev2(3) + ) + engine=olap + duplicate key(actorid, gameid, eventtime) + partition by range(eventtime)( + from ("2000-01-01") to ("2021-01-01") interval 1 year, + from ("2021-01-01") to ("2022-01-01") interval 1 MONth, + from ("2022-01-01") to ("2023-01-01") interval 1 WEEK, + from ("2023-01-01") TO ("2023-02-01") interval 1 DAY + ) + distributed by hash(actorid) buckets 1 + properties( + "replication_num"="1", + "light_schema_change"="true", + "compression"="zstd" + ); + """ + + sql " drop table test_force_drop_database.table2 " + sql " recover table test_force_drop_database.table2 " + sql " drop table test_force_drop_database.table2 FORCE" + sql " drop database test_force_drop_database " + sql " recover database test_force_drop_database " + sql " drop database test_force_drop_database FORCE" +} +