database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](dynamic_partition) fix dynamic partition not work when drop and recover olap table (#19031)

Browse Source 
when olap table is dynamic partition enable, if drop and recover olap table, the table should be added to DynamicPartitionScheduler again

---------

Co-authored-by: caiconghui1 <caiconghui1@jd.com>
This commit is contained in:
caiconghui
2023-05-29 13:02:10 +08:00
committed by GitHub
parent 8378ab5e41
commit f217e052d3
3 changed files with 58 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.DynamicPartitionUtil;
import org.apache.doris.common.util.MasterDaemon;
import org.apache.doris.common.util.RangeUtils;
import org.apache.doris.common.util.TimeUtils;
@ -696,6 +697,7 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable {
RecoverInfo recoverInfo = new RecoverInfo(db.getId(), table.getId(), -1L, "", newTableName, "");
Env.getCurrentEnv().getEditLog().logRecoverTable(recoverInfo);
}
DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), (OlapTable) table, isReplay);
} finally {
table.writeUnlock();
}

11
fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
View File

@ -109,6 +109,11 @@ public class DynamicPartitionScheduler extends MasterDaemon {
dynamicPartitionTableInfo.add(Pair.of(dbId, tableId));
}
// only for test
public boolean containsDynamicPartitionTable(Long dbId, Long tableId) {
return dynamicPartitionTableInfo.contains(Pair.of(dbId, tableId));
}
public void removeDynamicPartitionTable(Long dbId, Long tableId) {
dynamicPartitionTableInfo.remove(Pair.of(dbId, tableId));
}
@ -273,8 +278,10 @@ public class DynamicPartitionScheduler extends MasterDaemon {
} catch (Exception e) {
isPartitionExists = true;
if (addPartitionKeyRange.equals(partitionItem.getItems())) {
LOG.info("partition range {} exist in table {}, clear fail msg",
addPartitionKeyRange, olapTable.getName());
if (LOG.isDebugEnabled()) {
LOG.debug("partition range {} exist in table {}, clear fail msg",
addPartitionKeyRange, olapTable.getName());
}
clearCreatePartitionFailedMsg(olapTable.getId());
} else {
recordCreatePartitionFailedMsg(db.getFullName(), olapTable.getName(),

47
fe/fe-core/src/test/java/org/apache/doris/catalog/RecoverTest.java
View File

@ -135,6 +135,10 @@ public class RecoverTest {
.flatMap(db -> db.getTable(tblName)).isPresent();
}
private static boolean checkTableInDynamicScheduler(Long dbId, Long tableId) {
return Env.getCurrentEnv().getDynamicPartitionScheduler().containsDynamicPartitionTable(dbId, tableId);
}
private static boolean checkPartitionExist(String dbName, String tblName, String partName) {
return Env.getCurrentInternalCatalog()
.getDb(ClusterNamespace.getFullName(SystemInfoService.DEFAULT_CLUSTER, dbName))
@ -389,4 +393,47 @@ public class RecoverTest {
recoverPartition("test2", "table2", "p1", -1);
Assert.assertTrue(checkPartitionExist("test2", "table2", "p1"));
}
@Test
public void testDynamicTableRecover() throws Exception {
createDb("test3");
createTable("CREATE TABLE test3.`table3` (\n"
+ " `event_date` datetime(3) NOT NULL COMMENT \"\",\n"
+ " `app_name` varchar(64) NOT NULL COMMENT \"\",\n"
+ " `package_name` varchar(64) NOT NULL COMMENT \"\",\n"
+ " `age` varchar(32) NOT NULL COMMENT \"\",\n"
+ " `gender` varchar(32) NOT NULL COMMENT \"\",\n"
+ " `level` varchar(64) NOT NULL COMMENT \"\",\n"
+ " `city` varchar(64) NOT NULL COMMENT \"\",\n"
+ " `model` varchar(64) NOT NULL COMMENT \"\",\n"
+ " `brand` varchar(64) NOT NULL COMMENT \"\",\n"
+ " `hours` varchar(16) NOT NULL COMMENT \"\",\n"
+ " `use_num` int(11) SUM NOT NULL COMMENT \"\",\n"
+ " `use_time` double SUM NOT NULL COMMENT \"\",\n"
+ " `start_times` bigint(20) SUM NOT NULL COMMENT \"\"\n"
+ ") ENGINE=OLAP\n"
+ "AGGREGATE KEY(`event_date`, `app_name`, `package_name`, `age`, `gender`, `level`, `city`, \n"
+ " `model`, `brand`, `hours`) COMMENT \"OLAP\"\n"
+ "PARTITION BY RANGE(`event_date`)\n"
+ "(PARTITION p1 VALUES [('2020-02-27 00:00:00'), ('2020-03-02 00:00:00')),\n"
+ "PARTITION p2 VALUES [('2020-03-02 00:00:00'), ('2020-03-07 00:00:00')))\n"
+ "DISTRIBUTED BY HASH(`event_date`, `app_name`, `package_name`, `age`, `gender`, `level`, `city`, \n"
+ " `model`, `brand`, `hours`) BUCKETS 1 PROPERTIES (\n"
+ "\"replication_num\" = \"1\",\n"
+ "\"dynamic_partition.enable\" = \"true\",\n"
+ "\"dynamic_partition.time_unit\" = \"DAY\",\n"
+ "\"dynamic_partition.end\" = \"3\",\n"
+ "\"dynamic_partition.prefix\" = \"p\",\n"
+ "\"dynamic_partition.buckets\" = \"1\",\n"
+ "\"dynamic_partition.replication_num\" = \"1\",\n"
+ "\"dynamic_partition.create_history_partition\"=\"true\",\n"
+ "\"dynamic_partition.start\" = \"-3\"\n"
+ ");\n");
Long dbId = getDbId("test3");
Long tableId = getTableId("test3", "table3");
dropTable("test3", "table3");
recoverTable("test3", "table3", -1);
Assert.assertTrue(checkTableInDynamicScheduler(dbId, tableId));
}
}