database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](agg) streaming agg should not take too much memory when spilling enabled (#32426)

Browse Source
This commit is contained in:
Jerry Hu
2024-03-20 08:20:18 +08:00
committed by yiguolei
parent 7484a7ba5f
commit e892774c9a
4 changed files with 84 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -481,6 +481,7 @@ public class SessionVariable implements Serializable, Writable {
public static final String EXTERNAL_SORT_BYTES_THRESHOLD = "external_sort_bytes_threshold";
public static final String EXTERNAL_AGG_BYTES_THRESHOLD = "external_agg_bytes_threshold";
public static final String EXTERNAL_AGG_PARTITION_BITS = "external_agg_partition_bits";
public static final String SPILL_STREAMING_AGG_MEM_LIMIT = "spill_streaming_agg_mem_limit";
public static final String MIN_REVOCABLE_MEM = "min_revocable_mem";
public static final String ENABLE_JOIN_SPILL = "enable_join_spill";
public static final String ENABLE_SORT_SPILL = "enable_sort_spill";
@ -1688,9 +1689,14 @@ public class SessionVariable implements Serializable, Writable {
// Set to 0 to disable; min: 128M
public static final long MIN_EXTERNAL_AGG_BYTES_THRESHOLD = 134217728;
@VariableMgr.VarAttr(name = EXTERNAL_AGG_BYTES_THRESHOLD,
checker = "checkExternalAggBytesThreshold", fuzzy = true)
checker = "checkExternalAggBytesThreshold", fuzzy = true, varType = VariableAnnotation.DEPRECATED)
public long externalAggBytesThreshold = 0;
// The memory limit of streaming agg when spilling is enabled
// NOTE: streaming agg operator will not spill to disk.
@VariableMgr.VarAttr(name = SPILL_STREAMING_AGG_MEM_LIMIT)
public long spillStreamingAggMemLimit = 268435456; //256MB
public static final int MIN_EXTERNAL_AGG_PARTITION_BITS = 4;
public static final int MAX_EXTERNAL_AGG_PARTITION_BITS = 20;
@VariableMgr.VarAttr(name = EXTERNAL_AGG_PARTITION_BITS,
@ -2988,6 +2994,8 @@ public class SessionVariable implements Serializable, Writable {
tResult.setExternalAggBytesThreshold(0); // disable for now
tResult.setSpillStreamingAggMemLimit(spillStreamingAggMemLimit);
tResult.setExternalAggPartitionBits(externalAggPartitionBits);
tResult.setEnableFileCache(enableFileCache);