database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[bugfix](DDL) Fix the bug of incorrect partition policy setting (#25021)

Browse Source
This commit is contained in:
AlexYue
2023-10-09 11:28:48 +08:00
committed by GitHub
parent 0f59f49768
commit e4100b4ebd
2 changed files with 47 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -2520,10 +2520,13 @@ public class InternalCatalog implements CatalogIf<Database> {
"Can not create UNIQUE KEY table that enables Merge-On-write"
+ " with storage policy(" + partionStoragePolicy + ")");
}
if (!partionStoragePolicy.equals("")) {
storagePolicy = partionStoragePolicy;
// The table's storage policy has higher priority than partition's policy,
// so we'll directly use table's policy when it's set. Otherwise we use the
// partition's policy
if (!storagePolicy.isEmpty()) {
partionStoragePolicy = storagePolicy;
}
Env.getCurrentEnv().getPolicyMgr().checkStoragePolicyExist(storagePolicy);
Env.getCurrentEnv().getPolicyMgr().checkStoragePolicyExist(partionStoragePolicy);
Partition partition = createPartitionWithIndices(db.getClusterName(), db.getId(),
olapTable.getId(), olapTable.getName(), olapTable.getBaseIndexId(), entry.getValue(),
@ -2531,8 +2534,8 @@ public class InternalCatalog implements CatalogIf<Database> {
dataProperty.getStorageMedium(), partitionInfo.getReplicaAllocation(entry.getValue()),
versionInfo, bfColumns, bfFpp, tabletIdSet, olapTable.getCopiedIndexes(), isInMemory,
storageFormat, partitionInfo.getTabletType(entry.getValue()), compressionType,
olapTable.getDataSortInfo(), olapTable.getEnableUniqueKeyMergeOnWrite(), storagePolicy,
idGeneratorBuffer, olapTable.disableAutoCompaction(),
olapTable.getDataSortInfo(), olapTable.getEnableUniqueKeyMergeOnWrite(),
partionStoragePolicy, idGeneratorBuffer, olapTable.disableAutoCompaction(),
olapTable.enableSingleReplicaCompaction(), skipWriteIndexOnLoad,
olapTable.getCompactionPolicy(), olapTable.getTimeSeriesCompactionGoalSizeMbytes(),
olapTable.getTimeSeriesCompactionFileCountThreshold(),
@ -2540,6 +2543,8 @@ public class InternalCatalog implements CatalogIf<Database> {
storeRowColumn, binlogConfigForTask,
dataProperty.isStorageMediumSpecified());
olapTable.addPartition(partition);
olapTable.getPartitionInfo().getDataProperty(partition.getId())
.setStoragePolicy(partionStoragePolicy);
}
} else {
throw new DdlException("Unsupported partition method: " + partitionInfo.getType().name());

37
fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
View File

@ -220,6 +220,14 @@ public class AlterTest {
"CREATE STORAGE POLICY testPolicyAnotherResource\n" + "PROPERTIES(\n" + " \"storage_resource\" = \"remote_s3_1\",\n"
+ " \"cooldown_ttl\" = \"1\"\n" + ");");
createTable("CREATE TABLE test.tbl_remote1\n" + "(\n" + " k1 date,\n" + " k2 int,\n" + " v1 int sum\n"
+ ")\n" + "PARTITION BY RANGE(k1)\n" + "(\n" + " PARTITION p1 values less than('2020-02-01'),\n"
+ " PARTITION p2 values less than('2020-03-01') ('storage_policy' = 'testPolicy'),\n"
+ " PARTITION p3 values less than('2020-04-01'),\n"
+ " PARTITION p4 values less than('2020-05-01')\n" + ")\n" + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n"
+ "PROPERTIES" + "(" + " 'replication_num' = '1',\n" + " 'in_memory' = 'false',\n"
+ " 'storage_medium' = 'SSD',\n" + " 'storage_cooldown_time' = '2100-05-09 00:00:00'\n" + ");");
createTable("CREATE TABLE test.tbl_remote\n" + "(\n" + " k1 date,\n" + " k2 int,\n" + " v1 int sum\n"
+ ")\n" + "PARTITION BY RANGE(k1)\n" + "(\n" + " PARTITION p1 values less than('2020-02-01'),\n"
+ " PARTITION p2 values less than('2020-03-01'),\n"
@ -705,6 +713,35 @@ public class AlterTest {
}
@Test
public void testAlterRemoteStorageTableDataPropertiesPolicy() throws Exception {
Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("default_cluster:test");
OlapTable tblRemote = (OlapTable) db.getTableOrMetaException("tbl_remote1");
Partition p1 = tblRemote.getPartition("p1");
Partition p2 = tblRemote.getPartition("p2");
Assert.assertEquals(tblRemote.getPartitionInfo().getStoragePolicy(p2.getId()), "testPolicy");
Partition p3 = tblRemote.getPartition("p3");
Partition p4 = tblRemote.getPartition("p4");
DateLiteral dateLiteral = new DateLiteral("2100-05-09 00:00:00", Type.DATETIME);
long cooldownTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone());
DataProperty oldDataPropertyWithPolicy = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "testPolicy");
List<Partition> partitionList = Lists.newArrayList(p1, p3, p4);
for (Partition partition : partitionList) {
Assert.assertEquals(tblRemote.getPartitionInfo().getStoragePolicy(partition.getId()), "");
}
Assert.assertEquals(oldDataPropertyWithPolicy, tblRemote.getPartitionInfo().getDataProperty(p2.getId()));
// alter recover to old state
String stmt = "alter table test.tbl_remote1 modify partition (p1, p3, p4) set ("
+ "'storage_policy' = 'testPolicy')";
alterTable(stmt, false);
for (Partition partition : partitionList) {
Assert.assertEquals(tblRemote.getPartitionInfo().getStoragePolicy(partition.getId()), "testPolicy");
}
}
@Test
public void testDynamicPartitionDropAndAdd() throws Exception {
// test day range