database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](nereids) runtime filter prune skip filter with invisible column (#28010)

Browse Source 
if a conjunct only contains invisible column, this conjunct should not be used in runtime filter pruner
This commit is contained in:
minghong
2023-12-06 12:42:40 +08:00
committed by GitHub
parent 24fdb7ad4e
commit a0fee4c96e
2 changed files with 95 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.AbstractPlan;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows;
@ -103,10 +104,26 @@ public class RuntimeFilterPruner extends PlanPostProcessor {
return join;
}
private boolean isVisibleColumn(Slot slot) {
if (slot instanceof SlotReference) {
SlotReference slotReference = (SlotReference) slot;
if (slotReference.getColumn().isPresent()) {
return slotReference.getColumn().get().isVisible();
}
}
return true;
}
@Override
public PhysicalFilter visitPhysicalFilter(PhysicalFilter<? extends Plan> filter, CascadesContext context) {
filter.child().accept(this, context);
context.getRuntimeFilterContext().addEffectiveSrcNode(filter);
boolean visibleFilter = filter.getExpressions().stream()
.flatMap(expression -> expression.getInputSlots().stream())
.anyMatch(slot -> isVisibleColumn(slot));
if (visibleFilter) {
// skip filters like: __DORIS_DELETE_SIGN__ = 0
context.getRuntimeFilterContext().addEffectiveSrcNode(filter);
}
return filter;
}

77
regression-test/suites/nereids_rules_p0/runtime_filter/rf_prune.groovy Normal file
View File

@ -0,0 +1,77 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("rf_prune") {
String db = context.config.getDbNameByFile(context.file)
sql "use ${db}"
sql "SET enable_nereids_planner=true"
sql "SET enable_fallback_to_original_planner=false"
sql "set enable_runtime_filter_prune=true;"
sql "set disable_join_reorder=true"
sql "drop table if exists A"
sql """
CREATE TABLE A (
`id` INT NULL,
`msg` TEXT NULL
) ENGINE=OLAP
unique KEY(`id`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`id`) BUCKETS 10
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"min_load_replica_num" = "-1",
"is_being_synced" = "false",
"storage_format" = "V2",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false",
"group_commit_interval_ms" = "10000"
);
"""
sql "drop table if exists B"
sql """
CREATE TABLE B (
`id` INT NULL,
`msg` TEXT NULL
) ENGINE=OLAP
unique KEY(`id`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`id`) BUCKETS 10
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"min_load_replica_num" = "-1",
"is_being_synced" = "false",
"storage_format" = "V2",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false",
"group_commit_interval_ms" = "10000"
);
"""
explain{
sql " shape plan select * from A join B on A.id=B.id;"
notContains "build RFs"
}
explain{
sql " shape plan select * from A join B on A.id=B.id where B.msg='xyz';"
contains "PhysicalOlapScan[A] apply RFs: RF0"
}
}