database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Fix](storage) Fix bug that cooldown time is error (#15444)

Browse Source 
Cooldown time is wrong for data in SSD, because cooldown time for all `table/partitionis`
is only calculated once when class `DataProperty` loaded and that cannot be updated later.
This patch is to ensure that cooldown time for each table/partition can be calculated in real time
when table/partition is created.
Co-authored-by: weizuo <weizuo@xiaomi.com>
This commit is contained in:
weizuo93
2022-12-29 21:01:36 +08:00
committed by GitHub
parent e651a9bb11
commit 79113b0cd1
8 changed files with 17 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java
View File

@ -58,7 +58,7 @@ public class SinglePartitionDesc implements AllPartitionDesc {
this.partitionKeyDesc = partitionKeyDesc;
this.properties = properties;
this.partitionDataProperty = DataProperty.DEFAULT_DATA_PROPERTY;
this.partitionDataProperty = new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM);
this.replicaAlloc = ReplicaAllocation.DEFAULT_ALLOCATION;
this.storagePolicy = "";
}
@ -127,7 +127,7 @@ public class SinglePartitionDesc implements AllPartitionDesc {
// analyze data property
partitionDataProperty = PropertyAnalyzer.analyzeDataProperty(properties,
DataProperty.DEFAULT_DATA_PROPERTY);
new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
Preconditions.checkNotNull(partitionDataProperty);
// analyze replication num

5
fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
View File

@ -35,9 +35,8 @@ import java.io.IOException;
import java.util.Objects;
public class DataProperty implements Writable, GsonPostProcessable {
public static final DataProperty DEFAULT_DATA_PROPERTY = new DataProperty(
"SSD".equalsIgnoreCase(Config.default_storage_medium) ? TStorageMedium.SSD : TStorageMedium.HDD
);
public static final TStorageMedium DEFAULT_STORAGE_MEDIUM =
"SSD".equalsIgnoreCase(Config.default_storage_medium) ? TStorageMedium.SSD : TStorageMedium.HDD;
public static final long MAX_COOLDOWN_TIME_MS = 253402271999000L; // 9999-12-31 23:59:59
@SerializedName(value = "storageMedium")

5
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -1949,7 +1949,7 @@ public class InternalCatalog implements CatalogIf<Database> {
DataProperty dataProperty = null;
try {
dataProperty = PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(),
DataProperty.DEFAULT_DATA_PROPERTY);
new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());
}
@ -2106,7 +2106,8 @@ public class InternalCatalog implements CatalogIf<Database> {
try {
// just for remove entries in stmt.getProperties(),
// and then check if there still has unknown properties
PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(), DataProperty.DEFAULT_DATA_PROPERTY);
PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(),
new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
if (partitionInfo.getType() == PartitionType.RANGE) {
DynamicPartitionUtil.checkAndSetDynamicPartitionProperty(olapTable, properties, db);

4
fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
View File

@ -225,7 +225,7 @@ public class CatalogTestUtil {
// table
PartitionInfo partitionInfo = new SinglePartitionInfo();
partitionInfo.setDataProperty(partitionId, DataProperty.DEFAULT_DATA_PROPERTY);
partitionInfo.setDataProperty(partitionId, new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
partitionInfo.setReplicaAllocation(partitionId, new ReplicaAllocation((short) 3));
OlapTable table = new OlapTable(tableId, testTable1, columns, KeysType.AGG_KEYS, partitionInfo,
distributionInfo);
@ -284,7 +284,7 @@ public class CatalogTestUtil {
// table
PartitionInfo partitionInfo = new SinglePartitionInfo();
partitionInfo.setDataProperty(testPartitionId2, DataProperty.DEFAULT_DATA_PROPERTY);
partitionInfo.setDataProperty(testPartitionId2, new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
partitionInfo.setReplicaAllocation(testPartitionId2, new ReplicaAllocation((short) 1));
OlapTable table = new OlapTable(testTableId2, testTable2, columns, KeysType.DUP_KEYS, partitionInfo,
distributionInfo);

2
fe/fe-core/src/test/java/org/apache/doris/catalog/DataPropertyTest.java
View File

@ -28,7 +28,7 @@ public class DataPropertyTest {
@Test
public void testCooldownTimeMs() throws Exception {
Config.default_storage_medium = "ssd";
DataProperty dataProperty = DataProperty.DEFAULT_DATA_PROPERTY;
DataProperty dataProperty = new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM);
Assert.assertNotEquals(DataProperty.MAX_COOLDOWN_TIME_MS, dataProperty.getCooldownTimeMs());
dataProperty = new DataProperty(TStorageMedium.SSD);

2
fe/fe-core/src/test/java/org/apache/doris/common/util/UnitTestUtil.java
View File

@ -109,7 +109,7 @@ public class UnitTestUtil {
// table
PartitionInfo partitionInfo = new SinglePartitionInfo();
partitionInfo.setDataProperty(partitionId, DataProperty.DEFAULT_DATA_PROPERTY);
partitionInfo.setDataProperty(partitionId, new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
partitionInfo.setReplicaAllocation(partitionId, new ReplicaAllocation((short) 3));
partitionInfo.setIsInMemory(partitionId, false);
partitionInfo.setTabletType(partitionId, TTabletType.TABLET_TYPE_DISK);

5
fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java
View File

@ -171,7 +171,7 @@ public abstract class DorisHttpTestCase {
// table
PartitionInfo partitionInfo = new SinglePartitionInfo();
partitionInfo.setDataProperty(testPartitionId, DataProperty.DEFAULT_DATA_PROPERTY);
partitionInfo.setDataProperty(testPartitionId, new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
partitionInfo.setReplicaAllocation(testPartitionId, new ReplicaAllocation((short) 3));
OlapTable table = new OlapTable(testTableId, name, columns, KeysType.AGG_KEYS, partitionInfo,
distributionInfo);
@ -189,7 +189,8 @@ public abstract class DorisHttpTestCase {
columns.add(k1);
columns.add(k2);
PartitionInfo partitionInfo = new SinglePartitionInfo();
partitionInfo.setDataProperty(testPartitionId + 100, DataProperty.DEFAULT_DATA_PROPERTY);
partitionInfo.setDataProperty(testPartitionId + 100,
new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
partitionInfo.setReplicaAllocation(testPartitionId + 100, ReplicaAllocation.DEFAULT_ALLOCATION);
EsTable table = null;
Map<String, String> props = new HashMap<>();

5
fe/fe-core/src/test/java/org/apache/doris/persist/BatchModifyPartitionsInfoTest.java
View File

@ -60,8 +60,9 @@ public class BatchModifyPartitionsInfoTest {
List<Long> partitionIds = Lists.newArrayList(PARTITION_ID_1, PARTITION_ID_2, PARTITION_ID_3);
for (long partitionId : partitionIds) {
modifyInfos.add(new ModifyPartitionInfo(DB_ID, TB_ID, partitionId, DataProperty.DEFAULT_DATA_PROPERTY,
ReplicaAllocation.DEFAULT_ALLOCATION, true, "", Maps.newHashMap()));
modifyInfos.add(new ModifyPartitionInfo(DB_ID, TB_ID, partitionId,
new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM), ReplicaAllocation.DEFAULT_ALLOCATION,
true, "", Maps.newHashMap()));
}
BatchModifyPartitionsInfo batchModifyPartitionsInfo = new BatchModifyPartitionsInfo(modifyInfos);