From 228f78b80d8d4b6717b509473200724aefe2a34c Mon Sep 17 00:00:00 2001
From: Sun Chenyang <csun5285@gmail.com>
Date: Tue, 13 Aug 2024 10:57:53 +0800
Subject: [PATCH] [fix] (nereids) fix Match Expreesion in filter estimation
 (#39050) (#39215)

## Proposed changes

pick from master #39050
---
 .../doris/nereids/stats/FilterEstimation.java | 10 ++-
 .../inverted_index_p0/test_or_not_match.out   |  5 ++
 .../test_or_not_match.groovy                  | 69 +++++++++++++++++++
 3 files changed, 83 insertions(+), 1 deletion(-)
 create mode 100644 regression-test/data/inverted_index_p0/test_or_not_match.out
 create mode 100644 regression-test/suites/inverted_index_p0/test_or_not_match.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
index b8e08086d6..bda5330c8e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
@@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.expressions.IsNull;
 import org.apache.doris.nereids.trees.expressions.LessThan;
 import org.apache.doris.nereids.trees.expressions.LessThanEqual;
 import org.apache.doris.nereids.trees.expressions.Like;
+import org.apache.doris.nereids.trees.expressions.Match;
 import org.apache.doris.nereids.trees.expressions.Not;
 import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
 import org.apache.doris.nereids.trees.expressions.Or;
@@ -54,6 +55,8 @@ import org.apache.doris.statistics.StatisticsBuilder;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.HashMap;
 import java.util.List;
@@ -74,6 +77,7 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
 
     public static final double DEFAULT_LIKE_COMPARISON_SELECTIVITY = 0.2;
     public static final double DEFAULT_ISNULL_SELECTIVITY = 0.005;
+    private static final Logger LOG = LoggerFactory.getLogger(FilterEstimation.class);
     private Set<Slot> aggSlots;
 
     private boolean isOnBaseTable = false;
@@ -486,7 +490,8 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
                         child instanceof EqualPredicate
                                 || child instanceof InPredicate
                                 || child instanceof IsNull
-                                || child instanceof Like,
+                                || child instanceof Like
+                                || child instanceof Match,
                         "Not-predicate meet unexpected child: %s", child.toSql());
                 if (child instanceof Like) {
                     rowCount = context.statistics.getRowCount() - childStats.getRowCount();
@@ -509,6 +514,9 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
                             .setMinExpr(originColStats.minExpr)
                             .setMaxValue(originColStats.maxValue)
                             .setMaxExpr(originColStats.maxExpr);
+                } else if (child instanceof Match) {
+                    rowCount = context.statistics.getRowCount() - childStats.getRowCount();
+                    colBuilder.setNdv(Math.max(1.0, originColStats.ndv - childColStats.ndv));
                 }
                 if (not.child().getInputSlots().size() == 1 && !(child instanceof IsNull)) {
                     // only consider the single column numNull, otherwise, ignore
diff --git a/regression-test/data/inverted_index_p0/test_or_not_match.out b/regression-test/data/inverted_index_p0/test_or_not_match.out
new file mode 100644
index 0000000000..22dde7a8bf
--- /dev/null
+++ b/regression-test/data/inverted_index_p0/test_or_not_match.out
@@ -0,0 +1,5 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+GET / HTTP/1.0
+GET / HTTP/1.0
+
diff --git a/regression-test/suites/inverted_index_p0/test_or_not_match.groovy b/regression-test/suites/inverted_index_p0/test_or_not_match.groovy
new file mode 100644
index 0000000000..95af26480c
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/test_or_not_match.groovy
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_or_not_match", "p0") {
+    def tableName = "test_or_not_match"
+    sql "DROP TABLE IF EXISTS ${tableName}"
+    sql """
+      CREATE TABLE ${tableName} (
+      `@timestamp` int(11) NULL COMMENT "",
+      `clientip` varchar(20) NULL COMMENT "",
+      `request` text NULL COMMENT "",
+      `status` int(11) NULL COMMENT "",
+      `size` int(11) NULL COMMENT "",
+      INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true") COMMENT ''
+      ) ENGINE=OLAP
+      DUPLICATE KEY(`@timestamp`)
+      COMMENT "OLAP"
+      DISTRIBUTED BY RANDOM BUCKETS 1
+      PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+      );
+    """
+    // load the json data
+    streamLoad {
+        table "${tableName}"
+        
+        // set http request header params
+        set 'read_json_by_line', 'true'
+        set 'format', 'json'
+        file 'documents-1000.json' // import json file
+        time 10000 // limit inflight 10s
+
+        // if declared a check callback, the default check condition will ignore.
+        // So you must check all condition
+        check { result, exception, startTime, endTime ->
+            if (exception != null) {
+                throw exception
+            }
+            log.info("Stream load result: ${result}".toString())
+            def json = parseJson(result)
+            assertEquals("success", json.Status.toLowerCase())
+            assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+        }
+    }
+    for (int i = 0; i < 10; i++) {
+        sql "select request from ${tableName} where request like '1.0' or not request MATCH 'GETA';"
+    }
+
+    sql "set enable_nereids_planner = true"
+    sql "set enable_fallback_to_original_planner = false"
+
+    qt_sql "select request from ${tableName} where request like '1.0' or not request MATCH 'GETA' order by request limit 2;"
+}