From e4100b4ebde664c6d8a32370852440e63390964d Mon Sep 17 00:00:00 2001 From: AlexYue Date: Mon, 9 Oct 2023 11:28:48 +0800 Subject: [PATCH] [bugfix](DDL) Fix the bug of incorrect partition policy setting (#25021) --- .../doris/datasource/InternalCatalog.java | 15 +++++--- .../org/apache/doris/alter/AlterTest.java | 37 +++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 83e23e571a..9ccba00c32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2520,10 +2520,13 @@ public class InternalCatalog implements CatalogIf { "Can not create UNIQUE KEY table that enables Merge-On-write" + " with storage policy(" + partionStoragePolicy + ")"); } - if (!partionStoragePolicy.equals("")) { - storagePolicy = partionStoragePolicy; + // The table's storage policy has higher priority than partition's policy, + // so we'll directly use table's policy when it's set. Otherwise we use the + // partition's policy + if (!storagePolicy.isEmpty()) { + partionStoragePolicy = storagePolicy; } - Env.getCurrentEnv().getPolicyMgr().checkStoragePolicyExist(storagePolicy); + Env.getCurrentEnv().getPolicyMgr().checkStoragePolicyExist(partionStoragePolicy); Partition partition = createPartitionWithIndices(db.getClusterName(), db.getId(), olapTable.getId(), olapTable.getName(), olapTable.getBaseIndexId(), entry.getValue(), @@ -2531,8 +2534,8 @@ public class InternalCatalog implements CatalogIf { dataProperty.getStorageMedium(), partitionInfo.getReplicaAllocation(entry.getValue()), versionInfo, bfColumns, bfFpp, tabletIdSet, olapTable.getCopiedIndexes(), isInMemory, storageFormat, partitionInfo.getTabletType(entry.getValue()), compressionType, - olapTable.getDataSortInfo(), olapTable.getEnableUniqueKeyMergeOnWrite(), storagePolicy, - idGeneratorBuffer, olapTable.disableAutoCompaction(), + olapTable.getDataSortInfo(), olapTable.getEnableUniqueKeyMergeOnWrite(), + partionStoragePolicy, idGeneratorBuffer, olapTable.disableAutoCompaction(), olapTable.enableSingleReplicaCompaction(), skipWriteIndexOnLoad, olapTable.getCompactionPolicy(), olapTable.getTimeSeriesCompactionGoalSizeMbytes(), olapTable.getTimeSeriesCompactionFileCountThreshold(), @@ -2540,6 +2543,8 @@ public class InternalCatalog implements CatalogIf { storeRowColumn, binlogConfigForTask, dataProperty.isStorageMediumSpecified()); olapTable.addPartition(partition); + olapTable.getPartitionInfo().getDataProperty(partition.getId()) + .setStoragePolicy(partionStoragePolicy); } } else { throw new DdlException("Unsupported partition method: " + partitionInfo.getType().name()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java index d681376d24..4a78ef4de2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java @@ -220,6 +220,14 @@ public class AlterTest { "CREATE STORAGE POLICY testPolicyAnotherResource\n" + "PROPERTIES(\n" + " \"storage_resource\" = \"remote_s3_1\",\n" + " \"cooldown_ttl\" = \"1\"\n" + ");"); + createTable("CREATE TABLE test.tbl_remote1\n" + "(\n" + " k1 date,\n" + " k2 int,\n" + " v1 int sum\n" + + ")\n" + "PARTITION BY RANGE(k1)\n" + "(\n" + " PARTITION p1 values less than('2020-02-01'),\n" + + " PARTITION p2 values less than('2020-03-01') ('storage_policy' = 'testPolicy'),\n" + + " PARTITION p3 values less than('2020-04-01'),\n" + + " PARTITION p4 values less than('2020-05-01')\n" + ")\n" + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES" + "(" + " 'replication_num' = '1',\n" + " 'in_memory' = 'false',\n" + + " 'storage_medium' = 'SSD',\n" + " 'storage_cooldown_time' = '2100-05-09 00:00:00'\n" + ");"); + createTable("CREATE TABLE test.tbl_remote\n" + "(\n" + " k1 date,\n" + " k2 int,\n" + " v1 int sum\n" + ")\n" + "PARTITION BY RANGE(k1)\n" + "(\n" + " PARTITION p1 values less than('2020-02-01'),\n" + " PARTITION p2 values less than('2020-03-01'),\n" @@ -705,6 +713,35 @@ public class AlterTest { } + @Test + public void testAlterRemoteStorageTableDataPropertiesPolicy() throws Exception { + Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("default_cluster:test"); + OlapTable tblRemote = (OlapTable) db.getTableOrMetaException("tbl_remote1"); + Partition p1 = tblRemote.getPartition("p1"); + Partition p2 = tblRemote.getPartition("p2"); + Assert.assertEquals(tblRemote.getPartitionInfo().getStoragePolicy(p2.getId()), "testPolicy"); + Partition p3 = tblRemote.getPartition("p3"); + Partition p4 = tblRemote.getPartition("p4"); + + DateLiteral dateLiteral = new DateLiteral("2100-05-09 00:00:00", Type.DATETIME); + long cooldownTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); + DataProperty oldDataPropertyWithPolicy = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "testPolicy"); + List partitionList = Lists.newArrayList(p1, p3, p4); + for (Partition partition : partitionList) { + Assert.assertEquals(tblRemote.getPartitionInfo().getStoragePolicy(partition.getId()), ""); + } + Assert.assertEquals(oldDataPropertyWithPolicy, tblRemote.getPartitionInfo().getDataProperty(p2.getId())); + + // alter recover to old state + String stmt = "alter table test.tbl_remote1 modify partition (p1, p3, p4) set (" + + "'storage_policy' = 'testPolicy')"; + alterTable(stmt, false); + for (Partition partition : partitionList) { + Assert.assertEquals(tblRemote.getPartitionInfo().getStoragePolicy(partition.getId()), "testPolicy"); + } + + } + @Test public void testDynamicPartitionDropAndAdd() throws Exception { // test day range