diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index fcfea3b2b2..d4470a1029 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -1142,8 +1142,9 @@ public class RestoreJob extends AbstractJob { // replicas try { - Map> beIds = Env.getCurrentSystemInfo() + Pair>, TStorageMedium> beIdsAndMedium = Env.getCurrentSystemInfo() .selectBackendIdsForReplicaCreation(replicaAlloc, nextIndexs, null, false, false); + Map> beIds = beIdsAndMedium.first; for (Map.Entry> entry : beIds.entrySet()) { for (Long beId : entry.getValue()) { long newReplicaId = env.getNextId(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java index 731776384d..2974d33716 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java @@ -118,6 +118,10 @@ public class DataProperty implements Writable, GsonPostProcessable { storageMediumSpecified = isSpecified; } + public void setStorageMedium(TStorageMedium medium) { + this.storageMedium = medium; + } + public static DataProperty read(DataInput in) throws IOException { if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_108) { String json = Text.readString(in); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 6b0a218f30..dfb0d8bcd9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -637,9 +637,10 @@ public class OlapTable extends Table implements MTMVRelatedTableIf { // replicas try { - Map> tag2beIds = + Pair>, TStorageMedium> tag2beIdsAndMedium = Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation( replicaAlloc, nextIndexs, null, false, false); + Map> tag2beIds = tag2beIdsAndMedium.first; for (Map.Entry> entry3 : tag2beIds.entrySet()) { for (Long beId : entry3.getValue()) { long newReplicaId = env.getNextId(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 99e5b15f6e..d05f9cce48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1565,20 +1565,13 @@ public class InternalCatalog implements CatalogIf { } try { long partitionId = idGeneratorBuffer.getNextId(); - Partition partition = createPartitionWithIndices(db.getId(), olapTable.getId(), - olapTable.getName(), olapTable.getBaseIndexId(), partitionId, partitionName, indexIdToMeta, - distributionInfo, dataProperty.getStorageMedium(), singlePartitionDesc.getReplicaAlloc(), - singlePartitionDesc.getVersionInfo(), bfColumns, olapTable.getBfFpp(), tabletIdSet, - olapTable.getCopiedIndexes(), singlePartitionDesc.isInMemory(), olapTable.getStorageFormat(), - singlePartitionDesc.getTabletType(), olapTable.getCompressionType(), olapTable.getDataSortInfo(), - olapTable.getEnableUniqueKeyMergeOnWrite(), storagePolicy, idGeneratorBuffer, - olapTable.disableAutoCompaction(), olapTable.enableSingleReplicaCompaction(), - olapTable.skipWriteIndexOnLoad(), olapTable.getCompactionPolicy(), - olapTable.getTimeSeriesCompactionGoalSizeMbytes(), - olapTable.getTimeSeriesCompactionFileCountThreshold(), - olapTable.getTimeSeriesCompactionTimeThresholdSeconds(), - olapTable.getTimeSeriesCompactionEmptyRowsetsThreshold(), - olapTable.storeRowColumn(), + Partition partition = createPartitionWithIndices(db.getId(), olapTable, + partitionId, partitionName, indexIdToMeta, + distributionInfo, dataProperty, singlePartitionDesc.getReplicaAlloc(), + singlePartitionDesc.getVersionInfo(), bfColumns, tabletIdSet, + singlePartitionDesc.isInMemory(), + singlePartitionDesc.getTabletType(), + storagePolicy, idGeneratorBuffer, binlogConfig, dataProperty.isStorageMediumSpecified(), null); // TODO cluster key ids @@ -1827,33 +1820,33 @@ public class InternalCatalog implements CatalogIf { } } - private Partition createPartitionWithIndices(long dbId, long tableId, String tableName, - long baseIndexId, long partitionId, String partitionName, Map indexIdToMeta, - DistributionInfo distributionInfo, TStorageMedium storageMedium, ReplicaAllocation replicaAlloc, - Long versionInfo, Set bfColumns, double bfFpp, Set tabletIdSet, List indexes, - boolean isInMemory, TStorageFormat storageFormat, TTabletType tabletType, TCompressionType compressionType, - DataSortInfo dataSortInfo, boolean enableUniqueKeyMergeOnWrite, String storagePolicy, - IdGeneratorBuffer idGeneratorBuffer, boolean disableAutoCompaction, - boolean enableSingleReplicaCompaction, boolean skipWriteIndexOnLoad, - String compactionPolicy, Long timeSeriesCompactionGoalSizeMbytes, - Long timeSeriesCompactionFileCountThreshold, Long timeSeriesCompactionTimeThresholdSeconds, - Long timeSeriesCompactionEmptyRowsetsThreshold, - boolean storeRowColumn, BinlogConfig binlogConfig, - boolean isStorageMediumSpecified, List clusterKeyIndexes) throws DdlException { + protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long partitionId, + String partitionName, Map indexIdToMeta, + DistributionInfo distributionInfo, DataProperty dataProperty, + ReplicaAllocation replicaAlloc, + Long versionInfo, Set bfColumns, Set tabletIdSet, + boolean isInMemory, + TTabletType tabletType, + String storagePolicy, + IdGeneratorBuffer idGeneratorBuffer, + BinlogConfig binlogConfig, + boolean isStorageMediumSpecified, List clusterKeyIndexes) + throws DdlException { + // create base index first. - Preconditions.checkArgument(baseIndexId != -1); - MaterializedIndex baseIndex = new MaterializedIndex(baseIndexId, IndexState.NORMAL); + Preconditions.checkArgument(tbl.getBaseIndexId() != -1); + MaterializedIndex baseIndex = new MaterializedIndex(tbl.getBaseIndexId(), IndexState.NORMAL); // create partition with base index Partition partition = new Partition(partitionId, partitionName, baseIndex, distributionInfo); // add to index map Map indexMap = new HashMap<>(); - indexMap.put(baseIndexId, baseIndex); + indexMap.put(tbl.getBaseIndexId(), baseIndex); // create rollup index if has for (long indexId : indexIdToMeta.keySet()) { - if (indexId == baseIndexId) { + if (indexId == tbl.getBaseIndexId()) { continue; } @@ -1869,6 +1862,7 @@ public class InternalCatalog implements CatalogIf { long version = partition.getVisibleVersion(); short totalReplicaNum = replicaAlloc.getTotalReplicaNum(); + TStorageMedium realStorageMedium = null; for (Map.Entry entry : indexMap.entrySet()) { long indexId = entry.getKey(); MaterializedIndex index = entry.getValue(); @@ -1876,9 +1870,16 @@ public class InternalCatalog implements CatalogIf { // create tablets int schemaHash = indexMeta.getSchemaHash(); - TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, schemaHash, storageMedium); - createTablets(index, ReplicaState.NORMAL, distributionInfo, version, replicaAlloc, tabletMeta, - tabletIdSet, idGeneratorBuffer, isStorageMediumSpecified); + TabletMeta tabletMeta = new TabletMeta(dbId, tbl.getId(), partitionId, indexId, + schemaHash, dataProperty.getStorageMedium()); + realStorageMedium = createTablets(index, ReplicaState.NORMAL, distributionInfo, version, replicaAlloc, + tabletMeta, tabletIdSet, idGeneratorBuffer, dataProperty.isStorageMediumSpecified()); + if (realStorageMedium != null && !realStorageMedium.equals(dataProperty.getStorageMedium())) { + dataProperty.setStorageMedium(realStorageMedium); + LOG.info("real medium not eq default " + + "tableName={} tableId={} partitionName={} partitionId={} readMedium {}", + tbl.getName(), tbl.getId(), partitionName, partitionId, realStorageMedium); + } boolean ok = false; String errMsg = null; @@ -1897,17 +1898,20 @@ public class InternalCatalog implements CatalogIf { long backendId = replica.getBackendId(); long replicaId = replica.getId(); countDownLatch.addMark(backendId, tabletId); - CreateReplicaTask task = new CreateReplicaTask(backendId, dbId, tableId, partitionId, indexId, + CreateReplicaTask task = new CreateReplicaTask(backendId, dbId, tbl.getId(), partitionId, indexId, tabletId, replicaId, shortKeyColumnCount, schemaHash, version, keysType, storageType, - storageMedium, schema, bfColumns, bfFpp, countDownLatch, indexes, isInMemory, tabletType, - dataSortInfo, compressionType, enableUniqueKeyMergeOnWrite, storagePolicy, - disableAutoCompaction, enableSingleReplicaCompaction, skipWriteIndexOnLoad, - compactionPolicy, timeSeriesCompactionGoalSizeMbytes, - timeSeriesCompactionFileCountThreshold, timeSeriesCompactionTimeThresholdSeconds, - timeSeriesCompactionEmptyRowsetsThreshold, - storeRowColumn, binlogConfig); + realStorageMedium, schema, bfColumns, tbl.getBfFpp(), countDownLatch, + tbl.getCopiedIndexes(), tbl.isInMemory(), tabletType, + tbl.getDataSortInfo(), tbl.getCompressionType(), + tbl.getEnableUniqueKeyMergeOnWrite(), storagePolicy, tbl.disableAutoCompaction(), + tbl.enableSingleReplicaCompaction(), tbl.skipWriteIndexOnLoad(), + tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(), + tbl.getTimeSeriesCompactionFileCountThreshold(), + tbl.getTimeSeriesCompactionTimeThresholdSeconds(), + tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), + tbl.storeRowColumn(), binlogConfig); - task.setStorageFormat(storageFormat); + task.setStorageFormat(tbl.getStorageFormat()); task.setClusterKeyIndexes(clusterKeyIndexes); batchTask.addTask(task); // add to AgentTaskQueue for handling finish report. @@ -1971,14 +1975,14 @@ public class InternalCatalog implements CatalogIf { } } - if (index.getId() != baseIndexId) { + if (index.getId() != tbl.getBaseIndexId()) { // add rollup index to partition partition.createRollupIndex(index); } } // end for indexMap LOG.info("succeed in creating partition[{}-{}], table : [{}-{}]", partitionId, partitionName, - tableId, tableName); + tbl.getId(), tbl.getName()); return partition; } @@ -2566,20 +2570,14 @@ public class InternalCatalog implements CatalogIf { "Database " + db.getFullName() + " create unpartitioned table " + tableName + " increasing " + totalReplicaNum + " of replica exceeds quota[" + db.getReplicaQuota() + "]"); } - Partition partition = createPartitionWithIndices(db.getId(), olapTable.getId(), - olapTable.getName(), olapTable.getBaseIndexId(), partitionId, partitionName, + Partition partition = createPartitionWithIndices(db.getId(), olapTable, partitionId, partitionName, olapTable.getIndexIdToMeta(), partitionDistributionInfo, - partitionInfo.getDataProperty(partitionId).getStorageMedium(), - partitionInfo.getReplicaAllocation(partitionId), versionInfo, bfColumns, bfFpp, tabletIdSet, - olapTable.getCopiedIndexes(), isInMemory, storageFormat, tabletType, compressionType, - olapTable.getDataSortInfo(), olapTable.getEnableUniqueKeyMergeOnWrite(), storagePolicy, - idGeneratorBuffer, olapTable.disableAutoCompaction(), - olapTable.enableSingleReplicaCompaction(), skipWriteIndexOnLoad, - olapTable.getCompactionPolicy(), olapTable.getTimeSeriesCompactionGoalSizeMbytes(), - olapTable.getTimeSeriesCompactionFileCountThreshold(), - olapTable.getTimeSeriesCompactionTimeThresholdSeconds(), - olapTable.getTimeSeriesCompactionEmptyRowsetsThreshold(), - storeRowColumn, binlogConfigForTask, + partitionInfo.getDataProperty(partitionId), + partitionInfo.getReplicaAllocation(partitionId), versionInfo, bfColumns, tabletIdSet, + isInMemory, tabletType, + storagePolicy, + idGeneratorBuffer, + binlogConfigForTask, partitionInfo.getDataProperty(partitionId).isStorageMediumSpecified(), keysDesc.getClusterKeysColumnIds()); olapTable.addPartition(partition); @@ -2665,20 +2663,15 @@ public class InternalCatalog implements CatalogIf { Env.getCurrentEnv().getPolicyMgr().checkStoragePolicyExist(partionStoragePolicy); Partition partition = createPartitionWithIndices(db.getId(), - olapTable.getId(), olapTable.getName(), olapTable.getBaseIndexId(), entry.getValue(), + olapTable, entry.getValue(), entry.getKey(), olapTable.getIndexIdToMeta(), partitionDistributionInfo, - dataProperty.getStorageMedium(), partitionInfo.getReplicaAllocation(entry.getValue()), - versionInfo, bfColumns, bfFpp, tabletIdSet, olapTable.getCopiedIndexes(), isInMemory, - storageFormat, partitionInfo.getTabletType(entry.getValue()), compressionType, - olapTable.getDataSortInfo(), olapTable.getEnableUniqueKeyMergeOnWrite(), - partionStoragePolicy, idGeneratorBuffer, olapTable.disableAutoCompaction(), - olapTable.enableSingleReplicaCompaction(), skipWriteIndexOnLoad, - olapTable.getCompactionPolicy(), olapTable.getTimeSeriesCompactionGoalSizeMbytes(), - olapTable.getTimeSeriesCompactionFileCountThreshold(), - olapTable.getTimeSeriesCompactionTimeThresholdSeconds(), - olapTable.getTimeSeriesCompactionEmptyRowsetsThreshold(), - storeRowColumn, binlogConfigForTask, - dataProperty.isStorageMediumSpecified(), keysDesc.getClusterKeysColumnIds()); + dataProperty, partitionInfo.getReplicaAllocation(entry.getValue()), + versionInfo, bfColumns, tabletIdSet, isInMemory, + partitionInfo.getTabletType(entry.getValue()), + partionStoragePolicy, idGeneratorBuffer, + binlogConfigForTask, + dataProperty.isStorageMediumSpecified(), + keysDesc.getClusterKeysColumnIds()); olapTable.addPartition(partition); olapTable.getPartitionInfo().getDataProperty(partition.getId()) .setStoragePolicy(partionStoragePolicy); @@ -2872,7 +2865,7 @@ public class InternalCatalog implements CatalogIf { } @VisibleForTesting - public void createTablets(MaterializedIndex index, ReplicaState replicaState, + public TStorageMedium createTablets(MaterializedIndex index, ReplicaState replicaState, DistributionInfo distributionInfo, long version, ReplicaAllocation replicaAlloc, TabletMeta tabletMeta, Set tabletIdSet, IdGeneratorBuffer idGeneratorBuffer, boolean isStorageMediumSpecified) throws DdlException { @@ -2925,9 +2918,12 @@ public class InternalCatalog implements CatalogIf { if (chooseBackendsArbitrary) { // This is the first colocate table in the group, or just a normal table, // choose backends - chosenBackendIds = systemInfoService.selectBackendIdsForReplicaCreation(replicaAlloc, nextIndexs, + Pair>, TStorageMedium> chosenBackendIdsAndMedium + = systemInfoService.selectBackendIdsForReplicaCreation( + replicaAlloc, nextIndexs, storageMedium, isStorageMediumSpecified, false); - + chosenBackendIds = chosenBackendIdsAndMedium.first; + storageMedium = chosenBackendIdsAndMedium.second; for (Map.Entry> entry : chosenBackendIds.entrySet()) { backendsPerBucketSeq.putIfAbsent(entry.getKey(), Lists.newArrayList()); backendsPerBucketSeq.get(entry.getKey()).add(entry.getValue()); @@ -2959,6 +2955,7 @@ public class InternalCatalog implements CatalogIf { ColocatePersistInfo info = ColocatePersistInfo.createForBackendsPerBucketSeq(groupId, backendsPerBucketSeq); Env.getCurrentEnv().getEditLog().logColocateBackendsPerBucketSeq(info); } + return storageMedium; } /* @@ -3110,23 +3107,16 @@ public class InternalCatalog implements CatalogIf { // which is the right behavior. long oldPartitionId = entry.getValue(); long newPartitionId = idGeneratorBuffer.getNextId(); - Partition newPartition = createPartitionWithIndices(db.getId(), copiedTbl.getId(), - copiedTbl.getName(), copiedTbl.getBaseIndexId(), newPartitionId, entry.getKey(), + Partition newPartition = createPartitionWithIndices(db.getId(), copiedTbl, + newPartitionId, entry.getKey(), copiedTbl.getIndexIdToMeta(), partitionsDistributionInfo.get(oldPartitionId), - copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).getStorageMedium(), + copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId), copiedTbl.getPartitionInfo().getReplicaAllocation(oldPartitionId), null /* version info */, - copiedTbl.getCopiedBfColumns(), copiedTbl.getBfFpp(), tabletIdSet, copiedTbl.getCopiedIndexes(), - copiedTbl.isInMemory(), copiedTbl.getStorageFormat(), - copiedTbl.getPartitionInfo().getTabletType(oldPartitionId), copiedTbl.getCompressionType(), - copiedTbl.getDataSortInfo(), copiedTbl.getEnableUniqueKeyMergeOnWrite(), + copiedTbl.getCopiedBfColumns(), tabletIdSet, + copiedTbl.isInMemory(), + copiedTbl.getPartitionInfo().getTabletType(oldPartitionId), olapTable.getPartitionInfo().getDataProperty(oldPartitionId).getStoragePolicy(), - idGeneratorBuffer, olapTable.disableAutoCompaction(), - olapTable.enableSingleReplicaCompaction(), olapTable.skipWriteIndexOnLoad(), - olapTable.getCompactionPolicy(), olapTable.getTimeSeriesCompactionGoalSizeMbytes(), - olapTable.getTimeSeriesCompactionFileCountThreshold(), - olapTable.getTimeSeriesCompactionTimeThresholdSeconds(), - olapTable.getTimeSeriesCompactionEmptyRowsetsThreshold(), - olapTable.storeRowColumn(), binlogConfig, + idGeneratorBuffer, binlogConfig, copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).isStorageMediumSpecified(), clusterKeyIdxes); newPartitions.add(newPartition); diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java index 1a132b5449..0dc314240b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java @@ -485,7 +485,7 @@ public class SystemInfoService { * @return return the selected backend ids group by tag. * @throws DdlException */ - public Map> selectBackendIdsForReplicaCreation( + public Pair>, TStorageMedium> selectBackendIdsForReplicaCreation( ReplicaAllocation replicaAlloc, Map nextIndexs, TStorageMedium storageMedium, boolean isStorageMediumSpecified, boolean isOnlyForCheck) @@ -520,6 +520,7 @@ public class SystemInfoService { List beIds = selectBackendIdsByPolicy(policy, entry.getValue()); // first time empty, retry with different storage medium // if only for check, no need to retry different storage medium to get backend + TStorageMedium originalStorageMedium = storageMedium; if (beIds.isEmpty() && storageMedium != null && !isStorageMediumSpecified && !isOnlyForCheck) { storageMedium = (storageMedium == TStorageMedium.HDD) ? TStorageMedium.SSD : TStorageMedium.HDD; builder.setStorageMedium(storageMedium); @@ -534,10 +535,10 @@ public class SystemInfoService { } // after retry different storage medium, it's still empty if (beIds.isEmpty()) { - LOG.error("failed backend(s) for policy:" + policy); + LOG.error("failed backend(s) for policy: {} real medium {}", policy, originalStorageMedium); String errorReplication = "replication tag: " + entry.getKey() + ", replication num: " + entry.getValue() - + ", storage medium: " + storageMedium; + + ", storage medium: " + originalStorageMedium; failedEntries.add(errorReplication); } else { chosenBackendIds.put(entry.getKey(), beIds); @@ -554,7 +555,7 @@ public class SystemInfoService { } Preconditions.checkState(totalReplicaNum == replicaAlloc.getTotalReplicaNum()); - return chosenBackendIds; + return Pair.of(chosenBackendIds, storageMedium); } /** diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ModifyBackendTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ModifyBackendTest.java index 6d5380034d..ca4a658c4d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ModifyBackendTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ModifyBackendTest.java @@ -84,7 +84,7 @@ public class ModifyBackendTest { ExceptionChecker.expectThrowsWithMsg(DdlException.class, "Failed to find enough backend, please check the replication num,replication tag and storage medium and avail capacity of backends.\n" + "Create failed replications:\n" - + "replication tag: {\"location\" : \"default\"}, replication num: 1, storage medium: SSD", + + "replication tag: {\"location\" : \"default\"}, replication num: 1, storage medium: HDD", () -> DdlExecutor.execute(Env.getCurrentEnv(), createStmt)); createStr = "create table test.tbl1(\n" + "k1 int\n" + ") distributed by hash(k1)\n" + "buckets 3 properties(\n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ReplicaAllocationTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ReplicaAllocationTest.java index c53715cd81..971abe9b80 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ReplicaAllocationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ReplicaAllocationTest.java @@ -21,6 +21,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.Pair; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.meta.MetaContext; import org.apache.doris.resource.Tag; @@ -56,8 +57,8 @@ public class ReplicaAllocationTest { (TStorageMedium) any, false, true); minTimes = 0; result = new Delegate() { - Map> selectBackendIdsForReplicaCreation() { - return Maps.newHashMap(); + Pair>, TStorageMedium> selectBackendIdsForReplicaCreation() { + return Pair.of(Maps.newHashMap(), TStorageMedium.HDD); } }; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java index 22e12b37da..e933c0df17 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.ReplicaAllocation; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.Pair; import org.apache.doris.meta.MetaContext; import org.apache.doris.resource.Tag; import org.apache.doris.system.SystemInfoService.HostInfo; @@ -403,8 +404,9 @@ public class SystemInfoServiceTest { // also check if the random selection logic can evenly distribute the replica. Map beCounterMap = Maps.newHashMap(); for (int i = 0; i < 10000; ++i) { - Map> res = infoService.selectBackendIdsForReplicaCreation(replicaAlloc, + Pair>, TStorageMedium> ret = infoService.selectBackendIdsForReplicaCreation(replicaAlloc, Maps.newHashMap(), TStorageMedium.HDD, false, false); + Map> res = ret.first; Assert.assertEquals(3, res.get(Tag.DEFAULT_BACKEND_TAG).size()); for (Long beId : res.get(Tag.DEFAULT_BACKEND_TAG)) { beCounterMap.put(beId, beCounterMap.getOrDefault(beId, 0) + 1); diff --git a/regression-test/suites/storage_medium_p0/test_partition_default_medium.groovy b/regression-test/suites/storage_medium_p0/test_partition_default_medium.groovy new file mode 100644 index 0000000000..3543ce64ab --- /dev/null +++ b/regression-test/suites/storage_medium_p0/test_partition_default_medium.groovy @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.NodeType +import org.apache.doris.regression.suite.SuiteCluster + +suite("test_partition_default_medium") { + def options = new ClusterOptions() + options.feConfigs += [ + 'default_storage_medium=HDD', + ] + options.beDisks = ['SSD=4'] + + def checkCreateTablePartitionDefaultMediumEq = {tbl, sum -> + sleep 1000 + + def partitions = sql_return_maparray "SHOW PARTITIONS FROM $tbl;" + def partitionsMedium = [:] + partitions.each { + def num = partitionsMedium.get(it.StorageMedium) + if (partitionsMedium) { + partitionsMedium.put(it.StorageMedium, ++num) + } else { + partitionsMedium.put(it.StorageMedium, 1) + } + } + log.info("table ${tbl} partition mediums $partitionsMedium") + def count = partitionsMedium.values().stream().distinct().count() + assertEquals(count, 1) + assertEquals(partitionsMedium.get("SSD"), sum.toInteger()) + } + + docker(options) { + def single_partition_tbl = "single_partition_tbl" + def multi_partition_tbl = "multi_partition_tbl" + def dynamic_partition_tbl = "dynamic_partition_tbl" + sql """drop table if exists $single_partition_tbl""" + sql """drop table if exists $multi_partition_tbl""" + sql """drop table if exists $dynamic_partition_tbl""" + + sql """ + CREATE TABLE ${single_partition_tbl} + ( + k1 BIGINT, + k2 LARGEINT, + v1 VARCHAR(2048), + v2 SMALLINT DEFAULT "10" + ) + UNIQUE KEY(k1, k2) + DISTRIBUTED BY HASH (k1, k2) BUCKETS 32; + """ + + checkCreateTablePartitionDefaultMediumEq(single_partition_tbl, 1) + + + sql """ + CREATE TABLE $multi_partition_tbl + ( + k1 DATE, + k2 DECIMAL(10, 2) DEFAULT "10.5", + k3 CHAR(10) COMMENT "string column", + k4 INT NOT NULL DEFAULT "1" COMMENT "int column" + ) + DUPLICATE KEY(k1, k2) + COMMENT "my first table" + PARTITION BY RANGE(k1) + ( + PARTITION p1 VALUES LESS THAN ("2020-02-01"), + PARTITION p2 VALUES LESS THAN ("2020-03-01"), + PARTITION p3 VALUES LESS THAN ("2020-04-01") + ) + DISTRIBUTED BY HASH(k1) BUCKETS 32; + """ + checkCreateTablePartitionDefaultMediumEq(multi_partition_tbl, 3) + + sql """ + CREATE TABLE $dynamic_partition_tbl + ( + k1 DATE + ) + PARTITION BY RANGE(k1) () + DISTRIBUTED BY HASH(k1) + PROPERTIES + ( + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.start" = "-7", + "dynamic_partition.end" = "3", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "32" + ); + """ + checkCreateTablePartitionDefaultMediumEq(dynamic_partition_tbl, 4) + } +}