database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhance](ColdHeatSeparation) forbid change storage policy to another one with different storage resource (#22519)

Browse Source
This commit is contained in:
AlexYue
2023-08-10 16:32:09 +08:00
committed by GitHub
parent 50fbe31f93
commit fd0c161081
2 changed files with 35 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
View File

@ -190,7 +190,11 @@ public class PropertyAnalyzer {
TStorageMedium storageMedium = oldDataProperty.getStorageMedium();
long cooldownTimestamp = oldDataProperty.getCooldownTimeMs();
String newStoragePolicy = oldDataProperty.getStoragePolicy();
final String oldStoragePolicy = oldDataProperty.getStoragePolicy();
// When we create one table with table's property set storage policy,
// the properties wouldn't contain storage policy so the hasStoragePolicy would be false,
// then we would just set the partition's storage policy the same as the table's
String newStoragePolicy = oldStoragePolicy;
boolean hasStoragePolicy = false;
boolean storageMediumSpecified = false;
@ -252,6 +256,26 @@ public class PropertyAnalyzer {
}
StoragePolicy storagePolicy = (StoragePolicy) policy;
// Consider a scenario where if cold data has already been uploaded to resource A,
// and the user attempts to modify the policy to upload it to resource B,
// the data needs to be transferred from A to B.
// However, Doris currently does not support cross-bucket data transfer, therefore,
// changing the policy to a different policy with different resource is disabled.
// As for the case where the resource is the same, modifying the cooldown time is allowed,
// as this will not affect the already cooled data, but only the new data after modifying the policy.
if (null != oldStoragePolicy && !oldStoragePolicy.equals(newStoragePolicy)) {
// check remote storage policy
StoragePolicy oldPolicy = StoragePolicy.ofCheck(oldStoragePolicy);
Policy p = Env.getCurrentEnv().getPolicyMgr().getPolicy(oldPolicy);
if ((p instanceof StoragePolicy)) {
String newResource = storagePolicy.getStorageResource();
String oldResource = ((StoragePolicy) p).getStorageResource();
if (!newResource.equals(oldResource)) {
throw new AnalysisException("currently do not support change origin "
+ "storage policy to another one with different resource: ");
}
}
}
// check remote storage cool down timestamp
if (storagePolicy.getCooldownTimestampMs() != -1) {
if (storagePolicy.getCooldownTimestampMs() <= currentTimeMs) {

11
fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
View File

@ -160,6 +160,10 @@ public class AlterTest {
"CREATE STORAGE POLICY testPolicy2\n" + "PROPERTIES(\n" + " \"storage_resource\" = \"remote_s3\",\n"
+ " \"cooldown_ttl\" = \"1\"\n" + ");");
createRemoteStoragePolicy(
"CREATE STORAGE POLICY testPolicyAnotherResource\n" + "PROPERTIES(\n" + " \"storage_resource\" = \"remote_s3_1\",\n"
+ " \"cooldown_ttl\" = \"1\"\n" + ");");
createTable("CREATE TABLE test.tbl_remote\n" + "(\n" + " k1 date,\n" + " k2 int,\n" + " v1 int sum\n"
+ ")\n" + "PARTITION BY RANGE(k1)\n" + "(\n" + " PARTITION p1 values less than('2020-02-01'),\n"
+ " PARTITION p2 values less than('2020-03-01'),\n"
@ -597,11 +601,16 @@ public class AlterTest {
}
Assert.assertEquals(oldDataProperty, tblRemote.getPartitionInfo().getDataProperty(p1.getId()));
// alter remote_storage
// alter remote_storage to one not exist policy
stmt = "alter table test.tbl_remote modify partition (p2, p3, p4) set ('storage_policy' = 'testPolicy3')";
alterTable(stmt, true);
Assert.assertEquals(oldDataProperty, tblRemote.getPartitionInfo().getDataProperty(p1.getId()));
// alter remote_storage to one another one which points to another resource
stmt = "alter table test.tbl_remote modify partition (p2, p3, p4) set ('storage_policy' = 'testPolicyAnotherResource')";
alterTable(stmt, true);
Assert.assertEquals(oldDataProperty, tblRemote.getPartitionInfo().getDataProperty(p1.getId()));
// alter recover to old state
stmt = "alter table test.tbl_remote modify partition (p2, p3, p4) set ("
+ "'storage_medium' = 'SSD', "