From a0fee4c96ecd8ef229c2ba3dc407dc9873fe8b23 Mon Sep 17 00:00:00 2001
From: minghong <englefly@gmail.com>
Date: Wed, 6 Dec 2023 12:42:40 +0800
Subject: [PATCH] [fix](nereids) runtime filter prune skip filter with
 invisible column (#28010)

if a conjunct only contains invisible column, this conjunct should not be used in runtime filter pruner
---
 .../processor/post/RuntimeFilterPruner.java   | 19 ++++-
 .../runtime_filter/rf_prune.groovy            | 77 +++++++++++++++++++
 2 files changed, 95 insertions(+), 1 deletion(-)
 create mode 100644 regression-test/suites/nereids_rules_p0/runtime_filter/rf_prune.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
index 65c5e7ec1a..210ed4f6f3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.EqualTo;
 import org.apache.doris.nereids.trees.expressions.ExprId;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.plans.AbstractPlan;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows;
@@ -103,10 +104,26 @@ public class RuntimeFilterPruner extends PlanPostProcessor {
         return join;
     }
 
+    private boolean isVisibleColumn(Slot slot) {
+        if (slot instanceof SlotReference) {
+            SlotReference slotReference = (SlotReference) slot;
+            if (slotReference.getColumn().isPresent()) {
+                return slotReference.getColumn().get().isVisible();
+            }
+        }
+        return true;
+    }
+
     @Override
     public PhysicalFilter visitPhysicalFilter(PhysicalFilter<? extends Plan> filter, CascadesContext context) {
         filter.child().accept(this, context);
-        context.getRuntimeFilterContext().addEffectiveSrcNode(filter);
+        boolean visibleFilter = filter.getExpressions().stream()
+                .flatMap(expression -> expression.getInputSlots().stream())
+                .anyMatch(slot -> isVisibleColumn(slot));
+        if (visibleFilter) {
+            // skip filters like: __DORIS_DELETE_SIGN__ = 0
+            context.getRuntimeFilterContext().addEffectiveSrcNode(filter);
+        }
         return filter;
     }
 
diff --git a/regression-test/suites/nereids_rules_p0/runtime_filter/rf_prune.groovy b/regression-test/suites/nereids_rules_p0/runtime_filter/rf_prune.groovy
new file mode 100644
index 0000000000..a4cee458a1
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/runtime_filter/rf_prune.groovy
@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("rf_prune") {
+    String db = context.config.getDbNameByFile(context.file)
+    sql "use ${db}"
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+    sql "set enable_runtime_filter_prune=true;"
+    sql "set disable_join_reorder=true"
+    sql "drop table if exists A"
+    sql """
+        CREATE TABLE A (
+          `id` INT NULL,
+          `msg` TEXT NULL
+        ) ENGINE=OLAP
+        unique KEY(`id`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`id`) BUCKETS 10
+        PROPERTIES (
+        "replication_allocation" = "tag.location.default: 1",
+        "min_load_replica_num" = "-1",
+        "is_being_synced" = "false",
+        "storage_format" = "V2",
+        "light_schema_change" = "true",
+        "disable_auto_compaction" = "false",
+        "enable_single_replica_compaction" = "false",
+        "group_commit_interval_ms" = "10000"
+        ); 
+        """
+    sql "drop table if exists B"
+    sql """
+        CREATE TABLE B (
+          `id` INT NULL,
+          `msg` TEXT NULL
+        ) ENGINE=OLAP
+        unique KEY(`id`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`id`) BUCKETS 10
+        PROPERTIES (
+        "replication_allocation" = "tag.location.default: 1",
+        "min_load_replica_num" = "-1",
+        "is_being_synced" = "false",
+        "storage_format" = "V2",
+        "light_schema_change" = "true",
+        "disable_auto_compaction" = "false",
+        "enable_single_replica_compaction" = "false",
+        "group_commit_interval_ms" = "10000"
+        ); 
+        """
+
+    explain{
+      sql " shape plan select * from A join B on A.id=B.id;"
+      notContains "build RFs"
+    }
+
+    explain{
+      sql " shape plan select * from A join B on A.id=B.id where B.msg='xyz';"
+      contains "PhysicalOlapScan[A] apply RFs: RF0"
+    }
+
+
+}
\ No newline at end of file