database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](statistics)Fix stats cleaner delete partition stats bug (#30648)

Browse Source 
When a partition in OlapTable is removed, we should use partition id to delete the related stats record in column_statistics. Before, it was using id, which may cause delete useful stats of other partition.
This commit is contained in:
Jibing-Li
2024-01-31 20:50:33 +08:00
committed by yiguolei
parent eb2aafcd49
commit 9310dbf3fd
1 changed files with 8 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
View File

@ -169,12 +169,15 @@ public class StatisticsCleaner extends MasterDaemon {
doDelete("idx_id", expiredStats.expiredIdxId.stream()
.map(String::valueOf).collect(Collectors.toList()),
FeConstants.INTERNAL_DB_NAME + "." + tblName, false);
doDelete("part_id", expiredStats.expiredPartitionId.stream()
.map(String::valueOf).collect(Collectors.toList()),
FeConstants.INTERNAL_DB_NAME + "." + tblName, false);
doDelete("id", expiredStats.ids.stream()
.map(String::valueOf).collect(Collectors.toList()),
FeConstants.INTERNAL_DB_NAME + "." + tblName, false);
}
private void doDelete(String/*col name*/ colName, List<String> pred, String tblName, boolean taskOnly) {
private void doDelete(String colName, List<String> pred, String tblName, boolean taskOnly) {
String deleteTemplate = "DELETE FROM " + tblName + " WHERE ${left} IN (${right})";
if (CollectionUtils.isEmpty(pred)) {
return;
@ -241,7 +244,7 @@ public class StatisticsCleaner extends MasterDaemon {
continue;
}
if (!olapTable.getPartitionIds().contains(Long.parseLong(partId))) {
expiredStats.ids.add(id);
expiredStats.expiredPartitionId.add(Long.parseLong(partId));
}
} catch (Exception e) {
LOG.warn("Error occurred when retrieving expired stats", e);
@ -256,9 +259,8 @@ public class StatisticsCleaner extends MasterDaemon {
Set<Long> expiredCatalog = new HashSet<>();
Set<Long> expiredDatabase = new HashSet<>();
Set<Long> expiredTable = new HashSet<>();
Set<Long> expiredIdxId = new HashSet<>();
Set<Long> expiredPartitionId = new HashSet<>();
Set<String> ids = new HashSet<>();
public boolean isFull() {
@ -266,6 +268,7 @@ public class StatisticsCleaner extends MasterDaemon {
|| expiredDatabase.size() >= Config.max_allowed_in_element_num_of_delete
|| expiredTable.size() >= Config.max_allowed_in_element_num_of_delete
|| expiredIdxId.size() >= Config.max_allowed_in_element_num_of_delete
|| expiredPartitionId.size() >= Config.max_allowed_in_element_num_of_delete
|| ids.size() >= Config.max_allowed_in_element_num_of_delete;
}
@ -274,6 +277,7 @@ public class StatisticsCleaner extends MasterDaemon {
&& expiredDatabase.isEmpty()
&& expiredTable.isEmpty()
&& expiredIdxId.isEmpty()
&& expiredPartitionId.isEmpty()
&& ids.size() < Config.max_allowed_in_element_num_of_delete / 10;
}
}