From 85fe9d24968e951bf9e845d847376a8bb81eb97b Mon Sep 17 00:00:00 2001
From: Pxl <pxl290@qq.com>
Date: Tue, 3 Jan 2023 21:14:50 +0800
Subject: [PATCH] [Bug](filter) fix not in(null) return true (#15466)

fix not in(null) return true
---
 be/src/vec/exec/scan/vscan_node.cpp              |  3 +++
 be/src/vec/functions/in.h                        | 16 ++++------------
 .../data/query_p0/sql_functions/test_in_expr.out | 10 ++++++++++
 .../query_p0/sql_functions/test_in_expr.groovy   |  8 ++++++++
 4 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/be/src/vec/exec/scan/vscan_node.cpp b/be/src/vec/exec/scan/vscan_node.cpp
index b88a2d9546..62053cc36e 100644
--- a/be/src/vec/exec/scan/vscan_node.cpp
+++ b/be/src/vec/exec/scan/vscan_node.cpp
@@ -797,6 +797,9 @@ Status VScanNode::_normalize_not_in_and_not_eq_predicate(VExpr* expr, VExprConte
                         ->get_function_state(FunctionContext::FRAGMENT_LOCAL));
         HybridSetBase::IteratorBase* iter = state->hybrid_set->begin();
         auto fn_name = std::string("");
+        if (!is_fixed_range && state->null_in_set) {
+            _eos = true;
+        }
         while (iter->has_next()) {
             // column not in (nullptr) is always true
             if (nullptr == iter->get_value()) {
diff --git a/be/src/vec/functions/in.h b/be/src/vec/functions/in.h
index 79187abb2b..626060e399 100644
--- a/be/src/vec/functions/in.h
+++ b/be/src/vec/functions/in.h
@@ -23,6 +23,7 @@
 #include "exprs/create_predicate_function.h"
 #include "vec/columns/column_nullable.h"
 #include "vec/columns/columns_number.h"
+#include "vec/data_types/data_type.h"
 #include "vec/data_types/data_type_nullable.h"
 #include "vec/data_types/data_type_number.h"
 #include "vec/functions/function.h"
@@ -108,7 +109,7 @@ public:
         vec_res.resize(input_rows_count);
 
         ColumnUInt8::MutablePtr col_null_map_to;
-        col_null_map_to = ColumnUInt8::create(input_rows_count);
+        col_null_map_to = ColumnUInt8::create(input_rows_count, false);
         auto& vec_null_map_to = col_null_map_to->get_data();
 
         /// First argument may be a single column.
@@ -150,11 +151,12 @@ public:
                     }
                 } else {
                     for (size_t i = 0; i < input_rows_count; ++i) {
-                        vec_null_map_to[i] = null_bitmap[i] || (negative == vec_res[i]);
+                        vec_null_map_to[i] = null_bitmap[i] || negative == vec_res[i];
                     }
                 }
 
             } else { // non-nullable
+                DCHECK(!in_state->null_in_set);
 
                 auto search_hash_set = [&](auto* col_ptr) {
                     for (size_t i = 0; i < input_rows_count; ++i) {
@@ -175,16 +177,6 @@ public:
                 } else {
                     search_hash_set(materialized_column.get());
                 }
-
-                if (in_state->null_in_set) {
-                    for (size_t i = 0; i < input_rows_count; ++i) {
-                        vec_null_map_to[i] = negative == vec_res[i];
-                    }
-                } else {
-                    for (size_t i = 0; i < input_rows_count; ++i) {
-                        vec_null_map_to[i] = false;
-                    }
-                }
             }
         } else {
             std::vector<ColumnPtr> set_columns;
diff --git a/regression-test/data/query_p0/sql_functions/test_in_expr.out b/regression-test/data/query_p0/sql_functions/test_in_expr.out
index 5006d062dc..31d6bb5b1a 100644
--- a/regression-test/data/query_p0/sql_functions/test_in_expr.out
+++ b/regression-test/data/query_p0/sql_functions/test_in_expr.out
@@ -5,6 +5,16 @@
 -- !select --
 4
 
+-- !select --
+
+-- !select --
+103	4	d
+
+-- !select --
+103	4	d
+
+-- !select --
+
 -- !select --
 c
 
diff --git a/regression-test/suites/query_p0/sql_functions/test_in_expr.groovy b/regression-test/suites/query_p0/sql_functions/test_in_expr.groovy
index 74d366a1cb..46d5f95dd0 100644
--- a/regression-test/suites/query_p0/sql_functions/test_in_expr.groovy
+++ b/regression-test/suites/query_p0/sql_functions/test_in_expr.groovy
@@ -65,6 +65,14 @@ suite("test_in_expr", "query") {
     // 1.1.2 string + null_in_set
     qt_select "select t1.number from ${nullTableName} t1 left join ${nullTableName} t2 on t1.cid=t2.cid where t2.addr in ('d', null)"
 
+    qt_select "select * from ${nullTableName} where addr not in ('d', null)"
+
+    qt_select "select * from ${nullTableName} where not(addr not in ('d', null))"
+
+    qt_select "select * from ${nullTableName} where addr in ('d', null)"
+
+    qt_select "select * from ${nullTableName} where not(addr in ('d', null))"
+
     // 1.1.3 non-string
     qt_select "select t1.addr from ${nullTableName} t1 left join ${nullTableName} t2 on t1.cid=t2.cid where t2.number in (3)"