database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](nereids) disable PROJECT_OTHER_JOIN_CONDITION rule if bitmap filter is enabled. #34189

Browse Source 
this pr is a quick solution, but not complete.
runtime filter on NestLoopJoin suffers this bug even without PROJECT_OTHER_JOIN_CONDITION rule.

for example, when enable Min_Max Runtime filter, the target Expression is n_regionkey, but it should be "n_regionkey - 28"
explain
select n_nationkey, nrkey
from (select n_regionkey -28 nrkey, n_nationkey from nation) T
join region on nrkey > r_regionkey;

we will refactor RuntimeFilterGenerator to completely solve this issue in following pr.
This commit is contained in:
minghong
2024-04-28 15:33:22 +08:00
committed by yiguolei
parent 74029f56d4
commit 8f40882701
3 changed files with 54 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ProjectOtherJoinConditionForNestedLoopJoin.java
View File

@ -26,6 +26,8 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TRuntimeFilterType;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@ -49,12 +51,14 @@ import java.util.Set;
public class ProjectOtherJoinConditionForNestedLoopJoin extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalJoin().when(
join -> join.getHashJoinConjuncts().isEmpty()
return logicalJoin()
.when(join -> join.getHashJoinConjuncts().isEmpty()
&& !join.isMarkJoin()
&& !join.getOtherJoinConjuncts().isEmpty()
).then(
join -> {
&& !join.getOtherJoinConjuncts().isEmpty())
.whenNot(join -> ConnectContext.get() != null
&& ConnectContext.get().getSessionVariable()
.allowedRuntimeFilterType(TRuntimeFilterType.BITMAP))
.then(join -> {
List<Expression> otherConjuncts = join.getOtherJoinConjuncts();
List<Expression> newOtherConjuncts = new ArrayList<>();
Set<Slot> leftSlots = join.child(0).getOutputSet();

4
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -2561,6 +2561,10 @@ public class SessionVariable implements Serializable, Writable {
return runtimeFilterType;
}
public boolean allowedRuntimeFilterType(TRuntimeFilterType type) {
return (runtimeFilterType & type.getValue()) != 0;
}
public boolean isRuntimeFilterTypeEnabled(TRuntimeFilterType type) {
return (runtimeFilterType & type.getValue()) == type.getValue();
}

87
regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy
View File

@ -16,84 +16,79 @@
// under the License.
suite("test_bitmap_filter_nereids") {
def tbl1 = "test_query_db.bigtable"
def tbl2 = "bitmap_table_nereids"
def tbl3 = "test_query_db.baseall"
multi_sql """
set runtime_filter_type = 16;
DROP TABLE IF EXISTS bitmap_table_nereids;
CREATE TABLE bitmap_table_nereids (
`k1` int(11) NULL,
`k2` bitmap BITMAP_UNION NULL,
`k3` bitmap BITMAP_UNION NULL
) ENGINE=OLAP
AGGREGATE KEY(`k1`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 2
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
);
insert into bitmap_table_nereids values
(1, bitmap_from_string('1, 3, 5, 7, 9, 11, 13, 99, 19910811, 20150402'),
bitmap_from_string('32767, 1985, 255, 789, 1991')),
(2, bitmap_from_string('10, 11, 12, 13, 14'), bitmap_empty());
sql "set runtime_filter_type = 16"
set enable_nereids_planner=true;
set enable_fallback_to_original_planner=false;
"""
qt_sql1 "select k1, k2 from test_query_db.bigtable where k1 in (select k2 from bitmap_table_nereids) order by k1;"
sql "DROP TABLE IF EXISTS ${tbl2}"
sql """
CREATE TABLE ${tbl2} (
`k1` int(11) NULL,
`k2` bitmap BITMAP_UNION NULL,
`k3` bitmap BITMAP_UNION NULL
) ENGINE=OLAP
AGGREGATE KEY(`k1`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 2
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
);
"""
sql """
insert into ${tbl2} values
(1, bitmap_from_string('1, 3, 5, 7, 9, 11, 13, 99, 19910811, 20150402'),
bitmap_from_string('32767, 1985, 255, 789, 1991')),
(2, bitmap_from_string('10, 11, 12, 13, 14'), bitmap_empty());"""
qt_sql2 "select k1, k2 from test_query_db.bigtable where k1 + 1 in (select k2 from bitmap_table_nereids) order by k1;"
sql "set enable_nereids_planner=true;"
sql "set enable_fallback_to_original_planner=false;"
qt_sql3 "select k1, k2 from test_query_db.bigtable where k1 not in (select k2 from bitmap_table_nereids where k1 = 1) order by k1;"
qt_sql1 "select k1, k2 from ${tbl1} where k1 in (select k2 from ${tbl2}) order by k1;"
qt_sql4 "select t1.k1, t1.k2 from test_query_db.bigtable t1 join test_query_db.baseall t3 on t1.k1 = t3.k1 where t1.k1 in (select k2 from bitmap_table_nereids where k1 = 1) order by t1.k1;"
qt_sql2 "select k1, k2 from ${tbl1} where k1 + 1 in (select k2 from ${tbl2}) order by k1;"
qt_sql5 "select k1, k2 from test_query_db.bigtable where k1 in (select k2 from bitmap_table_nereids) and k2 not in (select k3 from bitmap_table_nereids) order by k1;"
qt_sql3 "select k1, k2 from ${tbl1} where k1 not in (select k2 from ${tbl2} where k1 = 1) order by k1;"
qt_sql6 "select k2, count(k2) from test_query_db.bigtable where k1 in (select k2 from bitmap_table_nereids) group by k2 order by k2;"
qt_sql4 "select t1.k1, t1.k2 from ${tbl1} t1 join ${tbl3} t3 on t1.k1 = t3.k1 where t1.k1 in (select k2 from ${tbl2} where k1 = 1) order by t1.k1;"
qt_sql7 "select k1, k2 from (select 2 k1, 2 k2) t where k1 in (select k2 from bitmap_table_nereids) order by 1, 2;"
qt_sql5 "select k1, k2 from ${tbl1} where k1 in (select k2 from ${tbl2}) and k2 not in (select k3 from ${tbl2}) order by k1;"
qt_sql8 "select k1, k2 from (select 11 k1, 11 k2) t where k1 in (select k2 from bitmap_table_nereids) order by 1, 2;"
qt_sql6 "select k2, count(k2) from ${tbl1} where k1 in (select k2 from ${tbl2}) group by k2 order by k2;"
qt_sql9 "select k1, k2 from (select 2 k1, 11 k2) t where k1 not in (select k2 from bitmap_table_nereids) order by 1, 2;"
qt_sql7 "select k1, k2 from (select 2 k1, 2 k2) t where k1 in (select k2 from ${tbl2}) order by 1, 2;"
qt_sql10 "select k1, k2 from (select 1 k1, 11 k2) t where k1 not in (select k2 from bitmap_table_nereids) order by 1, 2;"
qt_sql8 "select k1, k2 from (select 11 k1, 11 k2) t where k1 in (select k2 from ${tbl2}) order by 1, 2;"
qt_sql9 "select k1, k2 from (select 2 k1, 11 k2) t where k1 not in (select k2 from ${tbl2}) order by 1, 2;"
qt_sql10 "select k1, k2 from (select 1 k1, 11 k2) t where k1 not in (select k2 from ${tbl2}) order by 1, 2;"
qt_sql11 "select k10 from ${tbl1} where cast(k10 as bigint) in (select bitmap_or(k2, to_bitmap(20120314)) from ${tbl2} b) order by 1;"
qt_sql11 "select k10 from test_query_db.bigtable where cast(k10 as bigint) in (select bitmap_or(k2, to_bitmap(20120314)) from bitmap_table_nereids b) order by 1;"
qt_sql12 """
with w1 as (select k1 from ${tbl1} where k1 in (select k2 from ${tbl2})), w2 as (select k2 from ${tbl1} where k2 in (select k3 from ${tbl2}))
with w1 as (select k1 from test_query_db.bigtable where k1 in (select k2 from bitmap_table_nereids)), w2 as (select k2 from test_query_db.bigtable where k2 in (select k3 from bitmap_table_nereids))
select * from (select * from w1 union select * from w2) tmp order by 1;
"""
qt_sql13 "select k1, k2 from ${tbl1} where k1 in (select to_bitmap(10)) order by 1, 2"
qt_sql13 "select k1, k2 from test_query_db.bigtable where k1 in (select to_bitmap(10)) order by 1, 2"
qt_sql14 "select k1, k2 from ${tbl1} where k1 in (select bitmap_from_string('1,10')) order by 1, 2"
qt_sql14 "select k1, k2 from test_query_db.bigtable where k1 in (select bitmap_from_string('1,10')) order by 1, 2"
test {
sql "select k1, count(*) from ${tbl1} b1 group by k1 having k1 in (select k2 from ${tbl2} b2) order by k1;"
sql "select k1, count(*) from test_query_db.bigtable b1 group by k1 having k1 in (select k2 from bitmap_table_nereids b2) order by k1;"
exception "Doris hll, bitmap, array, map, struct, jsonb, variant column must use with specific function, and don't support filter"
}
sql "set ignore_storage_data_distribution=false"
explain{
sql "select k1, k2 from ${tbl1} where k1 in (select k2 from ${tbl2}) order by k1;"
sql "select k1, k2 from test_query_db.bigtable where k1 in (select k2 from bitmap_table_nereids) order by k1;"
contains "RF000[bitmap]"
}
explain{
sql "select k1, k2 from ${tbl1} where k1 not in (select k2 from ${tbl2} where k1 = 1)"
sql "select k1, k2 from test_query_db.bigtable where k1 not in (select k2 from bitmap_table_nereids where k1 = 1)"
contains "RF000[bitmap]"
}
explain{
sql " select k1, k2 from (select 2 k1, 2 k2) t where k1 in (select k2 from ${tbl2})"
sql " select k1, k2 from (select 2 k1, 2 k2) t where k1 in (select k2 from bitmap_table_nereids)"
notContains "RF000[bitmap]"
}
}