database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

branch-2.1-pick: [Fix](log) correct tablet diff log in TabletInvertedIndex.tabletReport (#49390) (#49445)

Browse Source 
pick https://github.com/apache/doris/pull/49390
This commit is contained in:
bobhan1
2025-03-28 11:50:31 +08:00
committed by GitHub
parent 57f04a7b9b
commit 7d64a80959
2 changed files with 16 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
View File

@ -134,7 +134,7 @@ public class TabletInvertedIndex {
ListMultimap<TStorageMedium, Long> tabletMigrationMap,
Map<Long, Long> partitionVersionSyncMap,
Map<Long, SetMultimap<Long, TPartitionVersionInfo>> transactionsToPublish,
ListMultimap<Long, Long> transactionsToClear,
SetMultimap<Long, Long> transactionsToClear,
ListMultimap<Long, Long> tabletRecoveryMap,
List<TTabletMetaInfo> tabletToUpdate,
List<CooldownConf> cooldownConfToPush,
@ -375,15 +375,21 @@ public class TabletInvertedIndex {
cooldownTablets.forEach(p -> handleCooldownConf(p.first, p.second, cooldownConfToPush, cooldownConfToUpdate));
long end = System.currentTimeMillis();
long toClearTransactionsNum = transactionsToClear.keySet().size();
long toClearTransactionsPartitions = transactionsToClear.values().size();
long toPublishTransactionsNum = transactionsToPublish.values().stream()
.mapToLong(m -> m.keySet().size()).sum();
long toPublishTransactionsPartitions = transactionsToPublish.values().stream()
.mapToLong(m -> m.values().size()).sum();
LOG.info("finished to do tablet diff with backend[{}]. fe tablet num: {}, backend tablet num: {}. sync: {}."
+ " metaDel: {}. foundInMeta: {}. migration: {}. backend partition num: {}, backend need "
+ "update: {}. found invalid transactions {}. found republish "
+ "transactions {}. tabletToUpdate: {}. need recovery: {}. cost: {} ms",
+ "update: {}. found invalid transactions {}(partitions: {}). found republish "
+ "transactions {}(partitions: {}). tabletToUpdate: {}. need recovery: {}. cost: {} ms",
backendId, feTabletNum, backendTablets.size(), tabletSyncMap.size(),
tabletDeleteFromMeta.size(), tabletFoundInMeta.size(), tabletMigrationMap.size(),
backendPartitionsVersion.size(), partitionVersionSyncMap.size(),
transactionsToClear.size(), transactionsToPublish.size(), tabletToUpdate.size(),
tabletRecoveryMap.size(), (end - start));
backendPartitionsVersion.size(), partitionVersionSyncMap.size(), toClearTransactionsNum,
toClearTransactionsPartitions, toPublishTransactionsNum, toPublishTransactionsPartitions,
tabletToUpdate.size(), tabletRecoveryMap.size(), (end - start));
}
public Long getTabletIdByReplica(long replicaId) {

7
fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
View File

@ -89,6 +89,7 @@ import org.apache.doris.thrift.TTabletMetaInfo;
import org.apache.doris.thrift.TTaskType;
import com.google.common.base.Preconditions;
import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
@ -494,7 +495,7 @@ public class ReportHandler extends Daemon {
// dbid -> txn id -> [partition info]
Map<Long, SetMultimap<Long, TPartitionVersionInfo>> transactionsToPublish = Maps.newHashMap();
ListMultimap<Long, Long> transactionsToClear = LinkedListMultimap.create();
SetMultimap<Long, Long> transactionsToClear = LinkedHashMultimap.create();
// db id -> tablet id
ListMultimap<Long, Long> tabletRecoveryMap = LinkedListMultimap.create();
@ -1237,11 +1238,11 @@ public class ReportHandler extends Daemon {
AgentTaskExecutor.submit(batchTask);
}
private static void handleClearTransactions(ListMultimap<Long, Long> transactionsToClear, long backendId) {
private static void handleClearTransactions(SetMultimap<Long, Long> transactionsToClear, long backendId) {
AgentBatchTask batchTask = new AgentBatchTask();
for (Long transactionId : transactionsToClear.keySet()) {
ClearTransactionTask clearTransactionTask = new ClearTransactionTask(backendId,
transactionId, transactionsToClear.get(transactionId));
transactionId, Lists.newArrayList(transactionsToClear.get(transactionId)));
batchTask.addTask(clearTransactionTask);
}
AgentTaskExecutor.submit(batchTask);