From 94c50012b2d60228861aaac0877decd550901ed2 Mon Sep 17 00:00:00 2001 From: caiconghui <55968745+caiconghui@users.noreply.github.com> Date: Mon, 19 Jul 2021 20:26:14 +0800 Subject: [PATCH] [Performance] Optimize the performance of tabletReport (#6200) 1. Use parallelStream to speed up tabletReport. 2. Add partitionIdInMemorySet to speed up tabletToInMemory check. 3. Add disable_storage_medium_check to disable storage medium check when user doesn't care what tablet's storage medium is, and remove enable_strict_storage_medium_check config to fix some potential migration task failures. Co-authored-by: caiconghui --- .../administrator-guide/config/fe_config.md | 10 +- .../administrator-guide/config/fe_config.md | 6 +- .../org/apache/doris/backup/RestoreJob.java | 3 +- .../org/apache/doris/catalog/Catalog.java | 29 ++- .../org/apache/doris/catalog/OlapTable.java | 15 +- .../apache/doris/catalog/PartitionInfo.java | 10 + .../apache/doris/catalog/TableProperty.java | 2 +- .../doris/catalog/TabletInvertedIndex.java | 96 +++++--- .../java/org/apache/doris/common/Config.java | 24 +- .../apache/doris/common/util/TimeUtils.java | 41 +--- .../PartitionInMemoryInfoCollector.java | 92 +++++++ .../apache/doris/master/ReportHandler.java | 228 ++++++++---------- ...java => DbUsedDataQuotaInfoCollector.java} | 8 +- .../org/apache/doris/alter/AlterTest.java | 1 + .../apache/doris/catalog/CreateTableTest.java | 6 +- .../catalog/DynamicPartitionTableTest.java | 1 + 16 files changed, 321 insertions(+), 251 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java rename fe/fe-core/src/main/java/org/apache/doris/transaction/{UpdateDbUsedDataQuotaDaemon.java => DbUsedDataQuotaInfoCollector.java} (88%) diff --git a/docs/en/administrator-guide/config/fe_config.md b/docs/en/administrator-guide/config/fe_config.md index cf5f66db11..fe301a613f 100644 --- a/docs/en/administrator-guide/config/fe_config.md +++ b/docs/en/administrator-guide/config/fe_config.md @@ -366,7 +366,7 @@ MasterOnly：true Whether to enable spark load temporarily, it is not enabled by default -### enable_strict_storage_medium_check +### disable_storage_medium_check Default：false @@ -374,12 +374,8 @@ IsMutable：true MasterOnly：true -This configuration indicates that when the table is being built, it checks for the presence of the appropriate storage medium in the cluster. For example, when the user specifies that the storage medium is' SSD 'when the table is built, but only' HDD 'disks exist in the cluster, - -If this parameter is' True ', the error 'Failed to find enough host in all Backends with storage medium with storage medium is SSD, need 3'. - -If this parameter is' False ', no error is reported when the table is built. Instead, the table is built on a disk with 'HDD' as the storage medium - +If disable_storage_medium_check is true, ReportHandler would not check tablet's storage medium and disable storage cool down function, the default value is false. You can set the value true when you don't care what the storage medium of the tablet is. + ### drop_backend_after_decommission Default：false diff --git a/docs/zh-CN/administrator-guide/config/fe_config.md b/docs/zh-CN/administrator-guide/config/fe_config.md index 5cb3d701a8..bc50413e8d 100644 --- a/docs/zh-CN/administrator-guide/config/fe_config.md +++ b/docs/zh-CN/administrator-guide/config/fe_config.md @@ -352,7 +352,7 @@ show data （其他用法：HELP SHOW DATA） 是否临时启用 spark load，默认不启用 -### enable_strict_storage_medium_check +### disable_storage_medium_check 默认值：false @@ -360,9 +360,7 @@ show data （其他用法：HELP SHOW DATA） 是否为 Master FE 节点独有的配置项：true -如果设置为 true，FE 将在创建表时通过存储介质检查后端可用容量 - -参数 `enable_strict_storage_medium_check` 为 `False` 该参数只是一个“尽力而为”的设置。即使集群内没有设置 SSD 存储介质，也不会报错，而是自动存储在可用的数据目录中。 同样，如果 SSD 介质不可访问、空间不足，都可能导致数据初始直接存储在其他可用介质上。而数据到期迁移到 HDD 时，如果 HDD 介质不可访问、空间不足，也可能迁移失败（但是会不断尝试）。 如果FE参数 `enable_strict_storage_medium_check` 为 `True` 则当集群内没有设置 SSD 存储介质时，会报错 `Failed to find enough host in all backends with storage medium is SSD` +如果 disable_storage_medium_check 为true， ReportHandler 将不会检查 tablet 的存储介质， 并使得存储冷却功能失效，默认值为false。当您不关心 tablet 的存储介质是什么时，可以将值设置为true 。 ### drop_backend_after_decommission diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index a88f3c19cb..4aba922e43 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -395,7 +395,7 @@ public class RestoreJob extends AbstractJob { * A. Table already exist * A1. Partition already exist, generate file mapping * A2. Partition does not exist, add restored partition to the table. - * Reset all index/tablet/replica id, and create replica on BE outside the db lock. + * Reset all index/tablet/replica id, and create replica on BE outside the table lock. * B. Table does not exist * B1. Add table to the db, reset all table/index/tablet/replica id, * and create replica on BE outside the db lock. @@ -1566,7 +1566,6 @@ public class RestoreJob extends AbstractJob { } finally { restoreTbl.writeUnlock(); } - } // remove restored resource diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index fcc58a25fd..3e2266e5e3 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -159,6 +159,7 @@ import org.apache.doris.load.routineload.RoutineLoadScheduler; import org.apache.doris.load.routineload.RoutineLoadTaskScheduler; import org.apache.doris.master.Checkpoint; import org.apache.doris.master.MetaHelper; +import org.apache.doris.master.PartitionInMemoryInfoCollector; import org.apache.doris.meta.MetaContext; import org.apache.doris.metric.MetricRepo; import org.apache.doris.mysql.privilege.PaloAuth; @@ -214,9 +215,9 @@ import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TStorageType; import org.apache.doris.thrift.TTabletType; import org.apache.doris.thrift.TTaskType; +import org.apache.doris.transaction.DbUsedDataQuotaInfoCollector; import org.apache.doris.transaction.GlobalTransactionMgr; import org.apache.doris.transaction.PublishVersionDaemon; -import org.apache.doris.transaction.UpdateDbUsedDataQuotaDaemon; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -306,7 +307,8 @@ public class Catalog { private BackupHandler backupHandler; private PublishVersionDaemon publishVersionDaemon; private DeleteHandler deleteHandler; - private UpdateDbUsedDataQuotaDaemon updateDbUsedDataQuotaDaemon; + private DbUsedDataQuotaInfoCollector dbUsedDataQuotaInfoCollector; + private PartitionInMemoryInfoCollector partitionInMemoryInfoCollector; private MasterDaemon labelCleaner; // To clean old LabelInfo, ExportJobInfos private MasterDaemon txnCleaner; // To clean aborted or timeout txns @@ -492,7 +494,8 @@ public class Catalog { this.metaDir = Config.meta_dir; this.publishVersionDaemon = new PublishVersionDaemon(); this.deleteHandler = new DeleteHandler(); - this.updateDbUsedDataQuotaDaemon = new UpdateDbUsedDataQuotaDaemon(); + this.dbUsedDataQuotaInfoCollector = new DbUsedDataQuotaInfoCollector(); + this.partitionInMemoryInfoCollector = new PartitionInMemoryInfoCollector(); this.replayedJournalId = new AtomicLong(0L); this.isElectable = false; @@ -1304,8 +1307,10 @@ public class Catalog { routineLoadTaskScheduler.start(); // start dynamic partition task dynamicPartitionScheduler.start(); - // start daemon thread to update db used data quota for db txn manager periodly - updateDbUsedDataQuotaDaemon.start(); + // start daemon thread to update db used data quota for db txn manager periodically + dbUsedDataQuotaInfoCollector.start(); + // start daemon thread to update global partition in memory information periodically + partitionInMemoryInfoCollector.start(); streamLoadRecordMgr.start(); } @@ -3407,7 +3412,7 @@ public class Catalog { } } - public void replayErasePartition(long partitionId) throws DdlException { + public void replayErasePartition(long partitionId) { Catalog.getCurrentRecycleBin().replayErasePartition(partitionId); } @@ -3449,7 +3454,7 @@ public class Catalog { Partition partition = new Partition(partitionId, partitionName, baseIndex, distributionInfo); // add to index map - Map indexMap = new HashMap(); + Map indexMap = new HashMap<>(); indexMap.put(baseIndexId, baseIndex); // create rollup index if has @@ -3771,8 +3776,7 @@ public class Catalog { // a set to record every new tablet created when create table // if failed in any step, use this set to do clear things - Set tabletIdSet = new HashSet(); - + Set tabletIdSet = new HashSet<>(); // create partition try { if (partitionInfo.getType() == PartitionType.UNPARTITIONED) { @@ -3866,7 +3870,6 @@ public class Catalog { for (Long tabletId : tabletIdSet) { Catalog.getCurrentInvertedIndex().deleteTablet(tabletId); } - // only remove from memory, because we have not persist it if (getColocateTableIndex().isColocateTable(tableId)) { getColocateTableIndex().removeTable(tableId); @@ -4355,7 +4358,7 @@ public class Catalog { if (chooseBackendsArbitrary) { // This is the first colocate table in the group, or just a normal table, // randomly choose backends - if (Config.enable_strict_storage_medium_check) { + if (!Config.disable_storage_medium_check) { chosenBackendIds = chosenBackendIdBySeq(replicationNum, clusterName, tabletMeta.getStorageMedium()); } else { chosenBackendIds = chosenBackendIdBySeq(replicationNum, clusterName); @@ -5543,7 +5546,7 @@ public class Catalog { // need to update partition info meta for(Partition partition: table.getPartitions()) { - table.getPartitionInfo().setIsInMemory(partition.getId(), tableProperty.IsInMemory()); + table.getPartitionInfo().setIsInMemory(partition.getId(), tableProperty.isInMemory()); } ModifyTablePropertyOperationLog info = new ModifyTablePropertyOperationLog(db.getId(), table.getId(), properties); @@ -5575,7 +5578,7 @@ public class Catalog { // need to replay partition info meta if (opCode == OperationType.OP_MODIFY_IN_MEMORY) { for(Partition partition: olapTable.getPartitions()) { - olapTable.getPartitionInfo().setIsInMemory(partition.getId(), tableProperty.IsInMemory()); + olapTable.getPartitionInfo().setIsInMemory(partition.getId(), tableProperty.isInMemory()); } } } finally { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index a2376272f0..844c2b7bd9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -442,23 +442,14 @@ public class OlapTable extends Table { if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) { for (Map.Entry entry : origPartNameToId.entrySet()) { long newPartId = catalog.getNextId(); - partitionInfo.idToDataProperty.put(newPartId, - partitionInfo.idToDataProperty.remove(entry.getValue())); - partitionInfo.idToReplicationNum.remove(entry.getValue()); - partitionInfo.idToReplicationNum.put(newPartId, (short) restoreReplicationNum); - partitionInfo.getIdToItem(false).put(newPartId, - partitionInfo.getIdToItem(false).remove(entry.getValue())); - partitionInfo.idToInMemory.put(newPartId, partitionInfo.idToInMemory.remove(entry.getValue())); + partitionInfo.resetPartitionIdForRestore(newPartId, entry.getValue(), (short) restoreReplicationNum, false); idToPartition.put(newPartId, idToPartition.remove(entry.getValue())); } } else { // Single partitioned long newPartId = catalog.getNextId(); for (Map.Entry entry : origPartNameToId.entrySet()) { - partitionInfo.idToDataProperty.put(newPartId, partitionInfo.idToDataProperty.remove(entry.getValue())); - partitionInfo.idToReplicationNum.remove(entry.getValue()); - partitionInfo.idToReplicationNum.put(newPartId, (short) restoreReplicationNum); - partitionInfo.idToInMemory.put(newPartId, partitionInfo.idToInMemory.remove(entry.getValue())); + partitionInfo.resetPartitionIdForRestore(newPartId, entry.getValue(), (short) restoreReplicationNum, true); idToPartition.put(newPartId, idToPartition.remove(entry.getValue())); } } @@ -1529,7 +1520,7 @@ public class OlapTable extends Table { public Boolean isInMemory() { if (tableProperty != null) { - return tableProperty.IsInMemory(); + return tableProperty.isInMemory(); } return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java index 1606bffccf..9451910ccf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java @@ -282,6 +282,16 @@ public class PartitionInfo implements Writable { } } + public void resetPartitionIdForRestore(long newPartitionId, long oldPartitionId, short restoreReplicationNum, boolean isSinglePartitioned) { + idToDataProperty.put(newPartitionId, idToDataProperty.remove(oldPartitionId)); + idToReplicationNum.remove(oldPartitionId); + idToReplicationNum.put(newPartitionId, restoreReplicationNum); + if (!isSinglePartitioned) { + idToItem.put(newPartitionId, idToItem.remove(oldPartitionId)); + } + idToInMemory.put(newPartitionId, idToInMemory.remove(oldPartitionId)); + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, type.name()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index e89d2d9490..8f1b2c9f41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -163,7 +163,7 @@ public class TableProperty implements Writable { return replicationNum; } - public boolean IsInMemory() { + public boolean isInMemory() { return isInMemory; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java index 3e424e79d2..4bf3b0f05d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java @@ -17,8 +17,11 @@ package org.apache.doris.catalog; +import com.google.common.collect.ImmutableSet; +import org.apache.commons.lang3.tuple.ImmutableTriple; +import org.apache.commons.lang3.tuple.Triple; import org.apache.doris.catalog.Replica.ReplicaState; -import org.apache.doris.common.Pair; +import org.apache.doris.common.Config; import org.apache.doris.thrift.TPartitionVersionInfo; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TTablet; @@ -89,6 +92,8 @@ public class TabletInvertedIndex { // backend id -> (tablet id -> replica) private Table backingReplicaMetaTable = HashBasedTable.create(); + private volatile ImmutableSet partitionIdInMemorySet = ImmutableSet.of(); + public TabletInvertedIndex() { } @@ -118,16 +123,7 @@ public class TabletInvertedIndex { Map> transactionsToPublish, ListMultimap transactionsToClear, ListMultimap tabletRecoveryMap, - Set> tabletWithoutPartitionId) { - - for (TTablet backendTablet : backendTablets.values()) { - for (TTabletInfo tabletInfo : backendTablet.tablet_infos) { - if (!tabletInfo.isSetPartitionId() || tabletInfo.getPartitionId() < 1) { - tabletWithoutPartitionId.add(new Pair<>(tabletInfo.getTabletId(), tabletInfo.getSchemaHash())); - } - } - } - + List> tabletToInMemory) { readLock(); long start = System.currentTimeMillis(); try { @@ -135,7 +131,7 @@ public class TabletInvertedIndex { Map replicaMetaWithBackend = backingReplicaMetaTable.row(backendId); if (replicaMetaWithBackend != null) { // traverse replicas in meta with this backend - for (Map.Entry entry : replicaMetaWithBackend.entrySet()) { + replicaMetaWithBackend.entrySet().parallelStream().forEach(entry -> { long tabletId = entry.getKey(); Preconditions.checkState(tabletMetaMap.containsKey(tabletId)); TabletMeta tabletMeta = tabletMetaMap.get(tabletId); @@ -146,10 +142,17 @@ public class TabletInvertedIndex { for (TTabletInfo backendTabletInfo : backendTablet.getTabletInfos()) { if (tabletMeta.containsSchemaHash(backendTabletInfo.getSchemaHash())) { foundTabletsWithValidSchema.add(tabletId); + if (partitionIdInMemorySet.contains(backendTabletInfo.getPartitionId()) != backendTabletInfo.isIsInMemory()) { + synchronized (tabletToInMemory) { + tabletToInMemory.add(new ImmutableTriple<>(tabletId, backendTabletInfo.getSchemaHash(), !backendTabletInfo.isIsInMemory())); + } + } // 1. (intersection) if (needSync(replica, backendTabletInfo)) { // need sync - tabletSyncMap.put(tabletMeta.getDbId(), tabletId); + synchronized (tabletSyncMap) { + tabletSyncMap.put(tabletMeta.getDbId(), tabletId); + } } // check and set path @@ -175,20 +178,27 @@ public class TabletInvertedIndex { backendTabletInfo.getSchemaHash(), backendTabletInfo.isSetUsed() ? backendTabletInfo.isUsed() : "unknown", backendTabletInfo.isSetVersionMiss() ? backendTabletInfo.isVersionMiss() : "unset"); - tabletRecoveryMap.put(tabletMeta.getDbId(), tabletId); + synchronized (tabletRecoveryMap) { + tabletRecoveryMap.put(tabletMeta.getDbId(), tabletId); + } } - // check if need migration long partitionId = tabletMeta.getPartitionId(); - TStorageMedium storageMedium = storageMediumMap.get(partitionId); - if (storageMedium != null && backendTabletInfo.isSetStorageMedium()) { - if (storageMedium != backendTabletInfo.getStorageMedium()) { - tabletMigrationMap.put(storageMedium, tabletId); - } - if (storageMedium != tabletMeta.getStorageMedium()) { - tabletMeta.setStorageMedium(storageMedium); + if (!Config.disable_storage_medium_check) { + // check if need migration + TStorageMedium storageMedium = storageMediumMap.get(partitionId); + if (storageMedium != null && backendTabletInfo.isSetStorageMedium()) { + if (storageMedium != backendTabletInfo.getStorageMedium()) { + synchronized (tabletMigrationMap) { + tabletMigrationMap.put(storageMedium, tabletId); + } + } + if (storageMedium != tabletMeta.getStorageMedium()) { + tabletMeta.setStorageMedium(storageMedium); + } } } + // check if should clear transactions if (backendTabletInfo.isSetTransactionIds()) { List transactionIds = backendTabletInfo.getTransactionIds(); @@ -196,7 +206,9 @@ public class TabletInvertedIndex { for (Long transactionId : transactionIds) { TransactionState transactionState = transactionMgr.getTransactionState(tabletMeta.getDbId(), transactionId); if (transactionState == null || transactionState.getTransactionStatus() == TransactionStatus.ABORTED) { - transactionsToClear.put(transactionId, tabletMeta.getPartitionId()); + synchronized (transactionsToClear) { + transactionsToClear.put(transactionId, tabletMeta.getPartitionId()); + } LOG.debug("transaction id [{}] is not valid any more, " + "clear it from backend [{}]", transactionId, backendId); } else if (transactionState.getTransactionStatus() == TransactionStatus.VISIBLE) { @@ -218,12 +230,14 @@ public class TabletInvertedIndex { TPartitionVersionInfo versionInfo = new TPartitionVersionInfo(tabletMeta.getPartitionId(), partitionCommitInfo.getVersion(), partitionCommitInfo.getVersionHash()); - ListMultimap map = transactionsToPublish.get(transactionState.getDbId()); - if (map == null) { - map = ArrayListMultimap.create(); - transactionsToPublish.put(transactionState.getDbId(), map); + synchronized (transactionsToPublish) { + ListMultimap map = transactionsToPublish.get(transactionState.getDbId()); + if (map == null) { + map = ArrayListMultimap.create(); + transactionsToPublish.put(transactionState.getDbId(), map); + } + map.put(transactionId, versionInfo); } - map.put(transactionId, versionInfo); } } } @@ -243,9 +257,11 @@ public class TabletInvertedIndex { // 2. (meta - be) // may need delete from meta LOG.debug("backend[{}] does not report tablet[{}-{}]", backendId, tabletId, tabletMeta); - tabletDeleteFromMeta.put(tabletMeta.getDbId(), tabletId); + synchronized (tabletDeleteFromMeta) { + tabletDeleteFromMeta.put(tabletMeta.getDbId(), tabletId); + } } - } // end for replicaMetaWithBackend + }); } } finally { readUnlock(); @@ -253,10 +269,10 @@ public class TabletInvertedIndex { long end = System.currentTimeMillis(); LOG.info("finished to do tablet diff with backend[{}]. sync: {}. metaDel: {}. foundValid: {}. foundInvalid: {}." - + " migration: {}. found invalid transactions {}. found republish transactions {} " + + " migration: {}. found invalid transactions {}. found republish transactions {}. tabletInMemorySync: {}." + " cost: {} ms", backendId, tabletSyncMap.size(), tabletDeleteFromMeta.size(), foundTabletsWithValidSchema.size(), foundTabletsWithInvalidSchema.size(), - tabletMigrationMap.size(), transactionsToClear.size(), transactionsToPublish.size(), (end - start)); + tabletMigrationMap.size(), transactionsToClear.size(), transactionsToPublish.size(), tabletToInMemory.size(), (end - start)); } public Long getTabletIdByReplica(long replicaId) { @@ -484,12 +500,12 @@ public class TabletInvertedIndex { if (Catalog.isCheckpointThread()) { return; } - writeLock(); + readLock(); try { Preconditions.checkState(tabletMetaTable.contains(partitionId, indexId)); tabletMetaTable.get(partitionId, indexId).setNewSchemaHash(newSchemaHash); } finally { - writeUnlock(); + readUnlock(); } } @@ -497,12 +513,12 @@ public class TabletInvertedIndex { if (Catalog.isCheckpointThread()) { return; } - writeLock(); + readLock(); try { Preconditions.checkState(tabletMetaTable.contains(partitionId, indexId)); tabletMetaTable.get(partitionId, indexId).updateToNewSchemaHash(); } finally { - writeUnlock(); + readUnlock(); } } @@ -510,14 +526,14 @@ public class TabletInvertedIndex { if (Catalog.isCheckpointThread()) { return; } - writeLock(); + readLock(); try { TabletMeta tabletMeta = tabletMetaTable.get(partitionId, indexId); if (tabletMeta != null) { tabletMeta.deleteNewSchemaHash(); } } finally { - writeUnlock(); + readUnlock(); } } @@ -601,6 +617,10 @@ public class TabletInvertedIndex { } } + public void setPartitionIdInMemorySet(ImmutableSet partitionIdInMemorySet) { + this.partitionIdInMemorySet = partitionIdInMemorySet; + } + public Map getReplicaToTabletMap() { return replicaToTabletMap; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index 988ea26929..b77a8ceaeb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -751,9 +751,17 @@ public class Config extends ConfigBase { */ @ConfField(mutable = true, masterOnly = true) public static int max_backend_down_time_second = 3600; // 1h + + /** + * If disable_storage_medium_check is true, ReportHandler would not check tablet's storage medium + * and disable storage cool down function, the default value is false. + * You can set the value true when you don't care what the storage medium of the tablet is. + */ + @ConfField(mutable = true, masterOnly = true) + public static boolean disable_storage_medium_check = false; /** * When create a table(or partition), you can specify its storage medium(HDD or SSD). - * If not set, this specifies the default medium when creat. + * If not set, this specifies the default medium when created. */ @ConfField public static String default_storage_medium = "HDD"; /** @@ -1020,7 +1028,7 @@ public class Config extends ConfigBase { /* * One master daemon thread will update database used data quota for db txn manager every db_used_data_quota_update_interval_secs */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = false, masterOnly = true) public static int db_used_data_quota_update_interval_secs = 300; /** @@ -1247,12 +1255,6 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static boolean drop_backend_after_decommission = true; - /** - * If set to true, FE will check backend available capacity by storage medium when create table - */ - @ConfField(mutable = true, masterOnly = true) - public static boolean enable_strict_storage_medium_check = false; - /** * enable spark load for temporary use */ @@ -1435,4 +1437,10 @@ public class Config extends ConfigBase { */ @ConfField(mutable = true) public static int default_max_query_instances = -1; + + /* + * One master daemon thread will update global partition in memory info every partition_in_memory_update_interval_secs + */ + @ConfField(mutable = false, masterOnly = true) + public static int partition_in_memory_update_interval_secs = 300; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java index 11fee36e1b..d90325fd30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java @@ -38,7 +38,6 @@ import java.text.ParsePosition; import java.text.SimpleDateFormat; import java.time.DateTimeException; import java.time.ZoneId; -import java.util.Calendar; import java.util.Date; import java.util.SimpleTimeZone; import java.util.TimeZone; @@ -81,6 +80,9 @@ public class TimeUtils { public static Date MIN_DATETIME = null; public static Date MAX_DATETIME = null; + private static ThreadLocal datetimeFormatThreadLocal = + ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")); + static { TIME_ZONE = new SimpleTimeZone(8 * 3600 * 1000, ""); @@ -148,11 +150,11 @@ public class TimeUtils { return dateFormat.format(new Date(timeStamp)); } - public static synchronized String longToTimeString(long timeStamp) { + public static String longToTimeString(long timeStamp) { + SimpleDateFormat datetimeFormatTimeZone = datetimeFormatThreadLocal.get(); TimeZone timeZone = getTimeZone(); - SimpleDateFormat dateFormatTimeZone = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - dateFormatTimeZone.setTimeZone(timeZone); - return longToTimeString(timeStamp, dateFormatTimeZone); + datetimeFormatTimeZone.setTimeZone(timeZone); + return longToTimeString(timeStamp, datetimeFormatTimeZone); } public static synchronized Date getTimeAsDate(String timeString) { @@ -208,32 +210,7 @@ public class TimeUtils { return format(date, type.getPrimitiveType()); } - /* - * only used for ETL - */ - public static long dateTransform(long time, PrimitiveType type) { - Calendar cal = Calendar.getInstance(TIME_ZONE); - cal.setTimeInMillis(time); - - int year = cal.get(Calendar.YEAR); - int month = cal.get(Calendar.MONTH) + 1; - int day = cal.get(Calendar.DAY_OF_MONTH); - - if (type == PrimitiveType.DATE) { - return year * 16 * 32L + month * 32 + day; - } else if (type == PrimitiveType.DATETIME) { - // datetime - int hour = cal.get(Calendar.HOUR_OF_DAY); - int minute = cal.get(Calendar.MINUTE); - int second = cal.get(Calendar.SECOND); - return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second; - } else { - Preconditions.checkState(false, "invalid date type: " + type); - return -1L; - } - } - - public static long timeStringToLong(String timeStr) { + public static synchronized long timeStringToLong(String timeStr) { Date d; try { d = DATETIME_FORMAT.parse(timeStr); @@ -244,7 +221,7 @@ public class TimeUtils { } public static long timeStringToLong(String timeStr, TimeZone timeZone) { - SimpleDateFormat dateFormatTimeZone = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + SimpleDateFormat dateFormatTimeZone = datetimeFormatThreadLocal.get(); dateFormatTimeZone.setTimeZone(timeZone); Date d; try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java new file mode 100644 index 0000000000..2181ff5166 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.master; + +import com.google.common.collect.ImmutableSet; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TabletInvertedIndex; +import org.apache.doris.common.Config; +import org.apache.doris.common.util.MasterDaemon; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; + +public class PartitionInMemoryInfoCollector extends MasterDaemon { + + private static final Logger LOG = LogManager.getLogger(PartitionInMemoryInfoCollector.class); + + public PartitionInMemoryInfoCollector() { + super("PartitionInMemoryInfoCollector", Config.partition_in_memory_update_interval_secs * 1000); + } + + @Override + protected void runAfterCatalogReady() { + updatePartitionInMemoryInfo(); + } + + private void updatePartitionInMemoryInfo() { + Catalog catalog = Catalog.getCurrentCatalog(); + TabletInvertedIndex tabletInvertedIndex = catalog.getTabletInvertedIndex(); + ImmutableSet.Builder builder = ImmutableSet.builder(); + List dbIdList = catalog.getDbIds(); + for (Long dbId : dbIdList) { + Database db = catalog.getDb(dbId); + if (db == null) { + LOG.warn("Database [" + dbId + "] does not exist, skip to update database used data quota"); + continue; + } + if (db.isInfoSchemaDb()) { + continue; + } + try { + int partitionInMemoryCount = 0; + for (Table table : db.getTables()) { + if (table.getType() != Table.TableType.OLAP) { + continue; + } + table.readLock(); + try { + OlapTable olapTable = (OlapTable) table; + for (Partition partition : olapTable.getAllPartitions()) { + if (olapTable.getPartitionInfo().getIsInMemory(partition.getId())) { + partitionInMemoryCount++; + builder.add(partition.getId()); + } + } + } finally { + table.readUnlock(); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("Update database[{}] partition in memory info, partitionInMemoryCount : {}.", db.getFullName(), partitionInMemoryCount); + } + } catch (Exception e) { + LOG.warn("Update database[" + db.getFullName() + "] partition in memory info failed", e); + } + } + ImmutableSet partitionIdInMemorySet = builder.build(); + tabletInvertedIndex.setPartitionIdInMemorySet(partitionIdInMemorySet); + } + + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index 31a2dc4ab8..fe94732409 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -17,6 +17,7 @@ package org.apache.doris.master; + import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.MaterializedIndex; @@ -69,7 +70,6 @@ import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TStorageType; import org.apache.doris.thrift.TTablet; import org.apache.doris.thrift.TTabletInfo; -import org.apache.doris.thrift.TTabletMetaType; import org.apache.doris.thrift.TTaskType; import com.google.common.collect.LinkedListMultimap; @@ -80,14 +80,12 @@ import com.google.common.collect.Queues; import com.google.common.collect.Sets; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang3.tuple.ImmutableTriple; import org.apache.commons.lang3.tuple.Triple; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.thrift.TException; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -98,6 +96,13 @@ public class ReportHandler extends Daemon { private BlockingQueue reportQueue = Queues.newLinkedBlockingQueue(); + private enum ReportType { + UNKNOWN, + TASK, + DISK, + TABLET + } + public ReportHandler() { GaugeMetric gaugeQueueSize = new GaugeMetric( "report_queue_size", MetricUnit.NOUNIT, "report queue size") { @@ -133,26 +138,27 @@ public class ReportHandler extends Daemon { Map tablets = null; long reportVersion = -1; - String reportType = ""; + ReportType reportType = ReportType.UNKNOWN; + if (request.isSetTasks()) { tasks = request.getTasks(); - reportType += "task"; + reportType = ReportType.TASK; } if (request.isSetDisks()) { disks = request.getDisks(); - reportType += "disk"; + reportType = ReportType.DISK; } if (request.isSetTablets()) { tablets = request.getTablets(); reportVersion = request.getReportVersion(); - reportType += "tablet"; + reportType = ReportType.TABLET; } else if (request.isSetTabletList()) { // the 'tablets' member will be deprecated in future. tablets = buildTabletMap(request.getTabletList()); reportVersion = request.getReportVersion(); - reportType += "tablet"; + reportType = ReportType.TABLET; } if (request.isSetTabletMaxCompactionScore()) { @@ -242,16 +248,17 @@ public class ReportHandler extends Daemon { backendId, backendTablets.size(), backendReportVersion); // storage medium map - HashMap storageMediumMap = Catalog.getCurrentCatalog().getPartitionIdToStorageMediumMap(); + HashMap storageMediumMap = Config.disable_storage_medium_check ? + Maps.newHashMap() : Catalog.getCurrentCatalog().getPartitionIdToStorageMediumMap(); // db id -> tablet id ListMultimap tabletSyncMap = LinkedListMultimap.create(); // db id -> tablet id ListMultimap tabletDeleteFromMeta = LinkedListMultimap.create(); // tablet ids which schema hash is valid - Set foundTabletsWithValidSchema = new HashSet(); + Set foundTabletsWithValidSchema = Sets.newConcurrentHashSet(); // tablet ids which schema hash is invalid - Map foundTabletsWithInvalidSchema = new HashMap(); + Map foundTabletsWithInvalidSchema = Maps.newConcurrentMap(); // storage medium -> tablet id ListMultimap tabletMigrationMap = LinkedListMultimap.create(); @@ -262,7 +269,7 @@ public class ReportHandler extends Daemon { // db id -> tablet id ListMultimap tabletRecoveryMap = LinkedListMultimap.create(); - Set> tabletWithoutPartitionId = Sets.newHashSet(); + List> tabletToInMemory = Lists.newArrayList(); // 1. do the diff. find out (intersection) / (be - meta) / (meta - be) Catalog.getCurrentInvertedIndex().tabletReport(backendId, backendTablets, storageMediumMap, @@ -274,35 +281,48 @@ public class ReportHandler extends Daemon { transactionsToPublish, transactionsToClear, tabletRecoveryMap, - tabletWithoutPartitionId); + tabletToInMemory); // 2. sync - sync(backendTablets, tabletSyncMap, backendId, backendReportVersion); + if (!tabletSyncMap.isEmpty()) { + sync(backendTablets, tabletSyncMap, backendId, backendReportVersion); + } // 3. delete (meta - be) // BE will automatically drop defective tablets. these tablets should also be dropped in catalog - deleteFromMeta(tabletDeleteFromMeta, backendId, backendReportVersion); + if (!tabletDeleteFromMeta.isEmpty()) { + deleteFromMeta(tabletDeleteFromMeta, backendId, backendReportVersion); + } // 4. handle (be - meta) - deleteFromBackend(backendTablets, foundTabletsWithValidSchema, foundTabletsWithInvalidSchema, backendId); + if (foundTabletsWithValidSchema.size() != backendTablets.size()) { + deleteFromBackend(backendTablets, foundTabletsWithValidSchema, foundTabletsWithInvalidSchema, backendId); + } // 5. migration (ssd <-> hdd) - handleMigration(tabletMigrationMap, backendId); + if (!Config.disable_storage_medium_check && !tabletMigrationMap.isEmpty()) { + handleMigration(tabletMigrationMap, backendId); + } // 6. send clear transactions to be - handleClearTransactions(transactionsToClear, backendId); + if (!transactionsToClear.isEmpty()) { + handleClearTransactions(transactionsToClear, backendId); + } // 7. send publish version request to be - handleRepublishVersionInfo(transactionsToPublish, backendId); + if (!transactionsToPublish.isEmpty()) { + handleRepublishVersionInfo(transactionsToPublish, backendId); + } // 8. send recover request to be - handleRecoverTablet(tabletRecoveryMap, backendTablets, backendId); + if (!tabletRecoveryMap.isEmpty()) { + handleRecoverTablet(tabletRecoveryMap, backendTablets, backendId); + } - // 9. send set tablet partition info to be - handleSetTabletPartitionId(backendId, tabletWithoutPartitionId); - - // 10. send set tablet in memory to be - handleSetTabletInMemory(backendId, backendTablets); + // 9. send set tablet in memory to be + if (!tabletToInMemory.isEmpty()) { + handleSetTabletInMemory(backendId, tabletToInMemory); + } final SystemInfoService currentSystemInfo = Catalog.getCurrentSystemInfo(); Backend reportBackend = currentSystemInfo.getBackend(backendId); @@ -312,7 +332,7 @@ public class ReportHandler extends Daemon { } long end = System.currentTimeMillis(); - LOG.info("tablet report from backend[{}] cost: {} ms", backendId, (end - start)); + LOG.info("finished to handle tablet report from backend[{}] cost: {} ms", backendId, (end - start)); } private static void taskReport(long backendId, Map> runningTasks) { @@ -660,42 +680,10 @@ public class ReportHandler extends Daemon { int deleteFromBackendCounter = 0; int addToMetaCounter = 0; AgentBatchTask batchTask = new AgentBatchTask(); - for (Long tabletId : backendTablets.keySet()) { - TTablet backendTablet = backendTablets.get(tabletId); - for (TTabletInfo backendTabletInfo : backendTablet.getTabletInfos()) { - boolean needDelete = false; - if (!foundTabletsWithValidSchema.contains(tabletId)) { - if (isBackendReplicaHealthy(backendTabletInfo)) { - // if this tablet is not in meta. try adding it. - // if add failed. delete this tablet from backend. - try { - addReplica(tabletId, backendTabletInfo, backendId); - // update counter - needDelete = false; - ++addToMetaCounter; - } catch (MetaNotFoundException e) { - LOG.warn("failed add to meta. tablet[{}], backend[{}]. {}", - tabletId, backendId, e.getMessage()); - needDelete = true; - } - } else { - needDelete = true; - } - } - - if (needDelete) { - // drop replica - DropReplicaTask task = new DropReplicaTask(backendId, tabletId, backendTabletInfo.getSchemaHash()); - batchTask.addTask(task); - LOG.warn("delete tablet[" + tabletId + " - " + backendTabletInfo.getSchemaHash() - + "] from backend[" + backendId + "] because not found in meta"); - ++deleteFromBackendCounter; - } - } // end for tabletInfos - - if (foundTabletsWithInvalidSchema.containsKey(tabletId)) { - // this tablet is found in meta but with invalid schema hash. - // delete it. + // This means that the meta of all backend tablets can be found in fe, we only need to process tablets with invalid Schema + if (foundTabletsWithValidSchema.size() + foundTabletsWithInvalidSchema.size() == backendTablets.size()) { + for (Long tabletId : foundTabletsWithInvalidSchema.keySet()) { + // this tablet is found in meta but with invalid schema hash. delete it. int schemaHash = foundTabletsWithInvalidSchema.get(tabletId).getSchemaHash(); DropReplicaTask task = new DropReplicaTask(backendId, tabletId, schemaHash); batchTask.addTask(task); @@ -703,8 +691,54 @@ public class ReportHandler extends Daemon { + "] because invalid schema hash"); ++deleteFromBackendCounter; } - } // end for backendTabletIds - AgentTaskExecutor.submit(batchTask); + } else { + for (Long tabletId : backendTablets.keySet()) { + if (foundTabletsWithInvalidSchema.containsKey(tabletId)) { + int schemaHash = foundTabletsWithInvalidSchema.get(tabletId).getSchemaHash(); + DropReplicaTask task = new DropReplicaTask(backendId, tabletId, schemaHash); + batchTask.addTask(task); + LOG.warn("delete tablet[" + tabletId + " - " + schemaHash + "] from backend[" + backendId + + "] because invalid schema hash"); + ++deleteFromBackendCounter; + continue; + } + TTablet backendTablet = backendTablets.get(tabletId); + for (TTabletInfo backendTabletInfo : backendTablet.getTabletInfos()) { + boolean needDelete = false; + if (!foundTabletsWithValidSchema.contains(tabletId)) { + if (isBackendReplicaHealthy(backendTabletInfo)) { + // if this tablet is not in meta. try adding it. + // if add failed. delete this tablet from backend. + try { + addReplica(tabletId, backendTabletInfo, backendId); + // update counter + needDelete = false; + ++addToMetaCounter; + } catch (MetaNotFoundException e) { + LOG.warn("failed add to meta. tablet[{}], backend[{}]. {}", + tabletId, backendId, e.getMessage()); + needDelete = true; + } + } else { + needDelete = true; + } + } + + if (needDelete) { + // drop replica + DropReplicaTask task = new DropReplicaTask(backendId, tabletId, backendTabletInfo.getSchemaHash()); + batchTask.addTask(task); + LOG.warn("delete tablet[" + tabletId + " - " + backendTabletInfo.getSchemaHash() + + "] from backend[" + backendId + "] because not found in meta"); + ++deleteFromBackendCounter; + } + } // end for tabletInfos + } // end for backendTabletIds + } + + if (batchTask.getTaskNum() != 0) { + AgentTaskExecutor.submit(batchTask); + } LOG.info("delete {} tablet(s) from backend[{}]", deleteFromBackendCounter, backendId); LOG.info("add {} replica(s) to meta. backend[{}]", addToMetaCounter, backendId); @@ -760,10 +794,6 @@ public class ReportHandler extends Daemon { private static void handleRecoverTablet(ListMultimap tabletRecoveryMap, Map backendTablets, long backendId) { - if (tabletRecoveryMap.isEmpty()) { - return; - } - // print a warn log here to indicate the exceptions on the backend LOG.warn("find {} tablets on backend {} which is bad or misses versions that need clone or force recovery", tabletRecoveryMap.size(), backendId); @@ -868,70 +898,13 @@ public class ReportHandler extends Daemon { } } - private static void handleSetTabletPartitionId(long backendId, Set> tabletWithoutPartitionId) { - LOG.info("find [{}] tablets without partition id, try to set them", tabletWithoutPartitionId.size()); - if (tabletWithoutPartitionId.size() < 1) { - return; - } + private static void handleSetTabletInMemory(long backendId, List> tabletToInMemory) { AgentBatchTask batchTask = new AgentBatchTask(); - UpdateTabletMetaInfoTask task = new UpdateTabletMetaInfoTask( - backendId, tabletWithoutPartitionId, TTabletMetaType.PARTITIONID); + UpdateTabletMetaInfoTask task = new UpdateTabletMetaInfoTask(backendId, tabletToInMemory); batchTask.addTask(task); AgentTaskExecutor.submit(batchTask); } - private static void handleSetTabletInMemory(long backendId, Map backendTablets) { - // - List> tabletToInMemory = Lists.newArrayList(); - - TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex(); - for (TTablet backendTablet : backendTablets.values()) { - for (TTabletInfo tabletInfo : backendTablet.tablet_infos) { - if (!tabletInfo.isSetIsInMemory()) { - continue; - } - long tabletId = tabletInfo.getTabletId(); - boolean beIsInMemory = tabletInfo.is_in_memory; - TabletMeta tabletMeta = invertedIndex.getTabletMeta(tabletId); - long dbId = tabletMeta != null ? tabletMeta.getDbId() : TabletInvertedIndex.NOT_EXIST_VALUE; - long tableId = tabletMeta != null ? tabletMeta.getTableId() : TabletInvertedIndex.NOT_EXIST_VALUE; - long partitionId = tabletMeta != null ? tabletMeta.getPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE; - - Database db = Catalog.getCurrentCatalog().getDb(dbId); - if (db == null) { - continue; - } - - OlapTable olapTable = (OlapTable) db.getTable(tableId); - if (olapTable == null) { - continue; - } - olapTable.readLock(); - try { - Partition partition = olapTable.getPartition(partitionId); - if (partition == null) { - continue; - } - boolean feIsInMemory = olapTable.getPartitionInfo().getIsInMemory(partitionId); - if (beIsInMemory != feIsInMemory) { - tabletToInMemory.add(new ImmutableTriple<>(tabletId, tabletInfo.getSchemaHash(), feIsInMemory)); - } - } finally { - olapTable.readUnlock(); - } - } - } - - LOG.info("find [{}] tablets need set in memory meta", tabletToInMemory.size()); - // When report, needn't synchronous - if (!tabletToInMemory.isEmpty()) { - AgentBatchTask batchTask = new AgentBatchTask(); - UpdateTabletMetaInfoTask task = new UpdateTabletMetaInfoTask(backendId, tabletToInMemory); - batchTask.addTask(task); - AgentTaskExecutor.submit(batchTask); - } - } - private static void handleClearTransactions(ListMultimap transactionsToClear, long backendId) { AgentBatchTask batchTask = new AgentBatchTask(); for (Long transactionId : transactionsToClear.keySet()) { @@ -939,7 +912,6 @@ public class ReportHandler extends Daemon { transactionId, transactionsToClear.get(transactionId)); batchTask.addTask(clearTransactionTask); } - AgentTaskExecutor.submit(batchTask); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/UpdateDbUsedDataQuotaDaemon.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java similarity index 88% rename from fe/fe-core/src/main/java/org/apache/doris/transaction/UpdateDbUsedDataQuotaDaemon.java rename to fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java index 67e4c5789a..f62770b772 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/UpdateDbUsedDataQuotaDaemon.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java @@ -27,11 +27,11 @@ import org.apache.logging.log4j.Logger; import java.util.List; -public class UpdateDbUsedDataQuotaDaemon extends MasterDaemon { - private static final Logger LOG = LogManager.getLogger(UpdateDbUsedDataQuotaDaemon.class); +public class DbUsedDataQuotaInfoCollector extends MasterDaemon { + private static final Logger LOG = LogManager.getLogger(DbUsedDataQuotaInfoCollector.class); - public UpdateDbUsedDataQuotaDaemon() { - super("UpdateDbUsedDataQuota", Config.db_used_data_quota_update_interval_secs * 1000); + public DbUsedDataQuotaInfoCollector() { + super("DbUsedDataQuotaInfoCollector", Config.db_used_data_quota_update_interval_secs * 1000); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java index 816a76204f..62b15613a7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java @@ -64,6 +64,7 @@ public class AlterTest { FeConstants.default_scheduler_interval_millisecond = 100; Config.dynamic_partition_enable = true; Config.dynamic_partition_check_interval_seconds = 1; + Config.disable_storage_medium_check = true; UtFrameUtils.createMinDorisCluster(runningDir); // create connect context diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java index 2ebbc40cb3..6af49d2510 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java @@ -20,6 +20,7 @@ package org.apache.doris.catalog; import org.apache.doris.analysis.CreateDbStmt; import org.apache.doris.analysis.CreateTableStmt; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.ConfigBase; import org.apache.doris.common.DdlException; import org.apache.doris.common.ExceptionChecker; @@ -43,6 +44,7 @@ public class CreateTableTest { @BeforeClass public static void beforeClass() throws Exception { + Config.disable_storage_medium_check = true; UtFrameUtils.createMinDorisCluster(runningDir); // create connect context @@ -131,7 +133,7 @@ public class CreateTableTest { + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1');")); - ConfigBase.setMutableConfig("enable_strict_storage_medium_check", "false"); + ConfigBase.setMutableConfig("disable_storage_medium_check", "true"); ExceptionChecker .expectThrowsNoException(() -> createTable("create table test.tb7(key1 int, key2 varchar(10)) \n" + "distributed by hash(key1) buckets 1 properties('replication_num' = '1', 'storage_medium' = 'ssd');")); @@ -254,7 +256,7 @@ public class CreateTableTest { + "duplicate key(k1, k2, k3)\n" + "distributed by hash(k1) buckets 1\n" + "properties('replication_num' = '1');")); - ConfigBase.setMutableConfig("enable_strict_storage_medium_check", "true"); + ConfigBase.setMutableConfig("disable_storage_medium_check", "false"); ExceptionChecker .expectThrowsWithMsg(DdlException.class, "Failed to find enough host with storage medium is SSD in all backends. need: 1", () -> createTable("create table test.tb7(key1 int, key2 varchar(10)) distributed by hash(key1) \n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index 72723e6638..754a44eac4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -61,6 +61,7 @@ public class DynamicPartitionTableTest { public static void beforeClass() throws Exception { FeConstants.default_scheduler_interval_millisecond = 1000; FeConstants.runningUnitTest = true; + Config.disable_storage_medium_check = true; UtFrameUtils.createMinDorisCluster(runningDir);