database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](scan) Support pushdown execute expr ctx (#15917)

Browse Source 
In the past, only simple predicates (slot=const), and, like, or (only bitmap index) could be pushed down to the storage layer. scan process:

Read part of the column first, and calculate the row ids with a simple push-down predicate.
Use row ids to read the remaining columns and pass them to the scanner, and the scanner filters the remaining predicates.
This pr will also push-down the remaining predicates (functions, nested predicates...) in the scanner to the storage layer for filtering. scan process:

Read part of the column first, and use the push-down simple predicate to calculate the row ids, (same as above)
Use row ids to read the columns needed for the remaining predicates, and use the pushed-down remaining predicates to reduce the number of row ids again.
Use row ids to read the remaining columns and pass them to the scanner.
This commit is contained in:
Xinyi Zou
2023-03-10 08:35:32 +08:00
committed by GitHub
parent 0334cde2b1
commit f9baf9c556
23 changed files with 571 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -229,6 +229,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String ENABLE_FUNCTION_PUSHDOWN = "enable_function_pushdown";
public static final String ENABLE_COMMON_EXPR_PUSHDOWN = "enable_common_expr_pushdown";
public static final String FRAGMENT_TRANSMISSION_COMPRESSION_CODEC = "fragment_transmission_compression_codec";
public static final String ENABLE_LOCAL_EXCHANGE = "enable_local_exchange";
@ -643,6 +645,9 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = ENABLE_FUNCTION_PUSHDOWN)
public boolean enableFunctionPushdown = true;
@VariableMgr.VarAttr(name = ENABLE_COMMON_EXPR_PUSHDOWN, fuzzy = true)
public boolean enableCommonExprPushdown = true;
@VariableMgr.VarAttr(name = ENABLE_LOCAL_EXCHANGE, fuzzy = true)
public boolean enableLocalExchange = true;
@ -765,6 +770,7 @@ public class SessionVariable implements Serializable, Writable {
public void initFuzzyModeVariables() {
Random random = new Random(System.currentTimeMillis());
this.parallelExecInstanceNum = random.nextInt(8) + 1;
this.enableCommonExprPushdown = random.nextBoolean();
this.enableLocalExchange = random.nextBoolean();
// This will cause be dead loop, disable it first
// this.disableJoinReorder = random.nextBoolean();
@ -1609,6 +1615,7 @@ public class SessionVariable implements Serializable, Writable {
}
tResult.setEnableFunctionPushdown(enableFunctionPushdown);
tResult.setEnableCommonExprPushdown(enableCommonExprPushdown);
tResult.setCheckOverflowForDecimal(checkOverflowForDecimal);
tResult.setFragmentTransmissionCompressionCodec(fragmentTransmissionCompressionCodec);
tResult.setEnableLocalExchange(enableLocalExchange);