database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](statistics) collect table level loaded rows on BE to make RPC light weight (#24609)

Browse Source
This commit is contained in:
Siyang Tang
2023-09-28 10:51:50 +08:00
committed by GitHub
parent 42207df89f
commit 188d9ab94e
9 changed files with 60 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
View File

@ -491,8 +491,8 @@ public class MasterImpl {
// not remove the task from queue and be will retry
return;
}
if (request.isSetTabletIdToDeltaNumRows()) {
publishVersionTask.setTabletIdToDeltaNumRows(request.getTabletIdToDeltaNumRows());
if (request.isSetTableIdToDeltaNumRows()) {
publishVersionTask.setTableIdToDeltaNumRows(request.getTableIdToDeltaNumRows());
}
AgentTaskQueue.removeTask(publishVersionTask.getBackendId(),
publishVersionTask.getTaskType(),

12
fe/fe-core/src/main/java/org/apache/doris/task/PublishVersionTask.java
View File

@ -40,9 +40,9 @@ public class PublishVersionTask extends AgentTask {
private Map<Long, Long> succTablets;
/**
* To collect loaded rows for each tablet from each BE
* To collect loaded rows for each table from each BE
*/
private final Map<Long, Long> tabletIdToDeltaNumRows = Maps.newHashMap();
private final Map<Long, Long> tableIdToDeltaNumRows = Maps.newHashMap();
public PublishVersionTask(long backendId, long transactionId, long dbId,
List<TPartitionVersionInfo> partitionVersionInfos, long createTime) {
@ -88,11 +88,11 @@ public class PublishVersionTask extends AgentTask {
this.errorTablets.addAll(errorTablets);
}
public void setTabletIdToDeltaNumRows(Map<Long, Long> tabletIdToDeltaNumRows) {
this.tabletIdToDeltaNumRows.putAll(tabletIdToDeltaNumRows);
public void setTableIdToDeltaNumRows(Map<Long, Long> tabletIdToDeltaNumRows) {
this.tableIdToDeltaNumRows.putAll(tabletIdToDeltaNumRows);
}
public Map<Long, Long> getTabletIdToDeltaNumRows() {
return tabletIdToDeltaNumRows;
public Map<Long, Long> getTableIdToDeltaNumRows() {
return tableIdToDeltaNumRows;
}
}

13
fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
View File

@ -1799,8 +1799,17 @@ public class DatabaseTransactionMgr {
}
}
AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
LOG.debug("table id to loaded rows:{}", transactionState.getTableIdToNumDeltaRows());
transactionState.getTableIdToNumDeltaRows().forEach(analysisManager::updateUpdatedRows);
Map<Long, Long> tableIdToTotalNumDeltaRows = transactionState.getTableIdToTotalNumDeltaRows();
LOG.debug("table id to loaded rows:{}", tableIdToTotalNumDeltaRows);
Map<Long, Long> tableIdToNumDeltaRows = Maps.newHashMap();
tableIdToTotalNumDeltaRows
.forEach((tableId, numRows) -> {
OlapTable table = (OlapTable) db.getTableNullable(tableId);
if (table != null) {
tableIdToNumDeltaRows.put(tableId, numRows / table.getReplicaCount());
}
});
tableIdToNumDeltaRows.forEach(analysisManager::updateUpdatedRows);
return true;
}

24
fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java
View File

@ -18,8 +18,6 @@
package org.apache.doris.transaction;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TabletInvertedIndex;
import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.common.Config;
import org.apache.doris.common.util.MasterDaemon;
import org.apache.doris.metric.MetricRepo;
@ -127,8 +125,6 @@ public class PublishVersionDaemon extends MasterDaemon {
AgentTaskExecutor.submit(batchTask);
}
TabletInvertedIndex tabletInvertedIndex = Env.getCurrentEnv().getTabletInvertedIndex();
Set<Long> tabletIdFilter = Sets.newHashSet();
Map<Long, Long> tableIdToNumDeltaRows = Maps.newHashMap();
// try to finish the transaction, if failed just retry in next loop
for (TransactionState transactionState : readyTransactionStates) {
@ -138,26 +134,18 @@ public class PublishVersionDaemon extends MasterDaemon {
.stream()
.peek(task -> {
if (task.isFinished() && CollectionUtils.isEmpty(task.getErrorTablets())) {
Map<Long, Long> tabletIdToDeltaNumRows =
task.getTabletIdToDeltaNumRows();
tabletIdToDeltaNumRows.forEach((tabletId, numRows) -> {
if (!tabletIdFilter.add(tabletId)) {
// means the delta num rows for this tablet id has been collected
return;
}
TabletMeta tabletMeta = tabletInvertedIndex.getTabletMeta(tabletId);
if (tabletMeta == null) {
// for delete, drop, schema change etc. here may be a null value
return;
}
long tableId = tabletMeta.getTableId();
tableIdToNumDeltaRows.computeIfPresent(tableId, (tblId, orgNum) -> orgNum + numRows);
Map<Long, Long> tableIdToDeltaNumRows =
task.getTableIdToDeltaNumRows();
tableIdToDeltaNumRows.forEach((tableId, numRows) -> {
tableIdToDeltaNumRows
.computeIfPresent(tableId, (id, orgNumRows) -> orgNumRows + numRows);
tableIdToNumDeltaRows.putIfAbsent(tableId, numRows);
});
}
});
boolean hasBackendAliveAndUnfinishedTask = publishVersionTaskStream
.anyMatch(task -> !task.isFinished() && infoService.checkBackendAlive(task.getBackendId()));
transactionState.setTableIdToTotalNumDeltaRows(tableIdToNumDeltaRows);
boolean shouldFinishTxn = !hasBackendAliveAndUnfinishedTask || transactionState.isPublishTimeout();
if (shouldFinishTxn) {

13
fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionState.java
View File

@ -253,7 +253,10 @@ public class TransactionState implements Writable {
// tbl id -> (index ids)
private Map<Long, Set<Long>> loadedTblIndexes = Maps.newHashMap();
private Map<Long, Long> tableIdToNumDeltaRows = Maps.newHashMap();
/**
* the value is the num delta rows of all replicas in each table
*/
private final Map<Long, Long> tableIdToTotalNumDeltaRows = Maps.newHashMap();
private String errorLogUrl = null;
@ -703,12 +706,12 @@ public class TransactionState implements Writable {
}
}
public Map<Long, Long> getTableIdToNumDeltaRows() {
return tableIdToNumDeltaRows;
public Map<Long, Long> getTableIdToTotalNumDeltaRows() {
return tableIdToTotalNumDeltaRows;
}
public void setTableIdToNumDeltaRows(Map<Long, Long> tableIdToNumDeltaRows) {
this.tableIdToNumDeltaRows.putAll(tableIdToNumDeltaRows);
public void setTableIdToTotalNumDeltaRows(Map<Long, Long> tableIdToTotalNumDeltaRows) {
this.tableIdToTotalNumDeltaRows.putAll(tableIdToTotalNumDeltaRows);
}
public void setErrorMsg(String errMsg) {