database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](statistics)Remove retry load when load stats cache fail (#28904)

Browse Source 
Remove retry load when load stats cache fail. This case usually happens when BE is down or BE OOM, retry doesn't work in these cases and may increase BE work load.
This commit is contained in:
Jibing-Li
2023-12-27 09:59:34 +08:00
committed by GitHub
parent f6850f8cdb
commit a4e69f7272
2 changed files with 2 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java
View File

@ -17,7 +17,6 @@
package org.apache.doris.statistics;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.qe.InternalQueryExecutionException;
@ -64,7 +63,8 @@ public class ColumnStatisticsCacheLoader extends StatisticsCacheLoader<Optional<
try {
columnResults = StatisticsRepository.loadColStats(key.tableId, key.idxId, key.colName);
} catch (InternalQueryExecutionException e) {
retryLoad(key);
LOG.info("Failed to load stats for table {} column {}. Reason:{}",
key.tableId, key.colName, e.getMessage());
return Optional.empty();
}
ColumnStatistic columnStatistics;
@ -80,42 +80,4 @@ public class ColumnStatisticsCacheLoader extends StatisticsCacheLoader<Optional<
return Optional.of(columnStatistics);
}
}
private void retryLoad(StatisticsCacheKey key) {
singleThreadPool.submit(new RetryTask(key, 1));
}
private static class RetryTask implements Runnable {
StatisticsCacheKey key;
int retryTimes;
public RetryTask(StatisticsCacheKey key, int retryTimes) {
this.key = key;
this.retryTimes = retryTimes;
}
@Override
public void run() {
List<ResultRow> columnResults = null;
try {
columnResults = StatisticsRepository.loadColStats(key.tableId, key.idxId, key.colName);
} catch (InternalQueryExecutionException e) {
if (this.retryTimes < StatisticConstants.LOAD_RETRY_TIMES) {
retryTimes++;
singleThreadPool.submit(this);
}
return;
}
ColumnStatistic columnStatistics;
try {
columnStatistics = StatisticsUtil.deserializeToColumnStatistics(columnResults);
} catch (Exception e) {
LOG.warn("Exception to deserialize column statistics", e);
return;
}
if (columnStatistics != null) {
Env.getCurrentEnv().getStatisticsCache().putCache(key, columnStatistics);
}
}
}
}

4
fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
View File

@ -39,9 +39,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.time.Duration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ThreadPoolExecutor;
@ -201,7 +199,6 @@ public class StatisticsCache {
if (CollectionUtils.isEmpty(recentStatsUpdatedCols)) {
return;
}
Map<StatisticsCacheKey, ColumnStatistic> keyToColStats = new HashMap<>();
for (ResultRow r : recentStatsUpdatedCols) {
try {
StatsId statsId = new StatsId(r);
@ -211,7 +208,6 @@ public class StatisticsCache {
final StatisticsCacheKey k =
new StatisticsCacheKey(tblId, idxId, colId);
final ColumnStatistic c = ColumnStatistic.fromResultRow(r);
keyToColStats.put(k, c);
putCache(k, c);
} catch (Throwable t) {
LOG.warn("Error when preheating stats cache", t);