[branch-2.1][fix](jdbc scan) Remove the conjuncts.remove call in JdbcScan (#39407)

pick (#39180)

In #37565, due to the change in the calling order of finalize, the final
generated Plan will be missing the PREDICATES that have been pushed down
in Jdbc. Although this behavior is correct, before perfectly handling
the push down of various PREDICATES, we need to keep all conjuncts to
ensure that we can still filter data normally when the data returned by
Jdbc is a superset.

docker/thirdparties/docker-compose/mysql/init/03-create-table.sql

@@ -342,4 +342,13 @@ CREATE TABLE Doris.Doris (
 
 CREATE TABLE Doris.doris (
   id varchar(128)
-);
+);
+
+create table doris_test.compoundpredicate_test (
+pk int,
+col_int_undef_signed int,
+col_int_undef_signed2 int
+);
+
+create table doris_test.text_push (pk varchar(10));
+

docker/thirdparties/docker-compose/mysql/init/04-insert.sql

@@ -1164,3 +1164,7 @@ insert into doris_test.test_zd (id,d_z) VALUES (1,'0000-00-00'),(2,'2022-01-01')
 insert into Doris.DORIS values ('DORIS');
 insert into Doris.Doris values ('Doris');
 insert into Doris.doris values ('doris');
+
+insert into doris_test.compoundpredicate_test(pk,col_int_undef_signed,col_int_undef_signed2) values (0,null,23868),(1,68,-18),(2,19030,-125),(3,16539,null),(4,null,null),(5,null,-127),(6,14680,-26424),(7,-22270,12722),(8,null,null),(9,null,null),(10,null,7744),(11,null,-94),(12,16970,95),(13,null,7023),(14,null,1),(15,3679,-11),(16,null,-1079),(17,-22,null),(18,30995,null),(19,null,-79);
+
+insert into doris_test.text_push values('a'),('aa'),('aaa');

fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java

@@ -61,6 +61,7 @@ public class JdbcScanNode extends ExternalScanNode {
 
     private final List<String> columns = new ArrayList<String>();
     private final List<String> filters = new ArrayList<String>();
+    private final List<Expr> pushedDownConjuncts = new ArrayList<>();
     private String tableName;
     private TOdbcTableType jdbcType;
     private String graphQueryString = "";
@@ -128,7 +129,7 @@ public class JdbcScanNode extends ExternalScanNode {
         for (Expr individualConjunct : pushDownConjuncts) {
             String filter = conjunctExprToString(jdbcType, individualConjunct, tbl);
             filters.add(filter);
-            conjuncts.remove(individualConjunct);
+            pushedDownConjuncts.add(individualConjunct);
         }
     }
 
@@ -165,7 +166,7 @@ public class JdbcScanNode extends ExternalScanNode {
     }
 
     private boolean shouldPushDownLimit() {
-        return limit != -1 && conjuncts.isEmpty();
+        return limit != -1 && conjuncts.size() == pushedDownConjuncts.size();
     }
 
     private String getJdbcQueryStr() {

fe/fe-core/src/main/java/org/apache/doris/datasource/odbc/source/OdbcScanNode.java

@@ -60,6 +60,7 @@ public class OdbcScanNode extends ExternalScanNode {
 
     private final List<String> columns = new ArrayList<String>();
     private final List<String> filters = new ArrayList<String>();
+    private final List<Expr> pushedDownConjuncts = new ArrayList<>();
     private String tblName;
     private String connectString;
     private TOdbcTableType odbcType;
@@ -138,7 +139,7 @@ public class OdbcScanNode extends ExternalScanNode {
     // only all conjuncts be pushed down as filter, we can
     // push down limit operation to ODBC table
     private boolean shouldPushDownLimit() {
-        return limit != -1 && conjuncts.isEmpty();
+        return limit != -1 && conjuncts.size() == pushedDownConjuncts.size();
     }
 
     private String getOdbcQueryStr() {
@@ -208,7 +209,7 @@ public class OdbcScanNode extends ExternalScanNode {
             if (shouldPushDownConjunct(odbcType, p)) {
                 String filter = JdbcScanNode.conjunctExprToString(odbcType, p, tbl);
                 filters.add(filter);
-                conjuncts.remove(p);
+                pushedDownConjuncts.add(p);
             }
         }
     }

regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out

@@ -247,15 +247,6 @@ workload_policy
 张三6	11	124314567	123	321312	1999-02-13T00:00	中国	男	0
 张三7	11	123445167	123	321312	1998-02-13T00:00	中国	男	0
 
--- !test_filter_not_old_plan --
-张三1	11	12345678	123	321312	1999-02-13T00:00	中国	男	0
-张三2	11	12345671	123	321312	1999-02-13T00:00	中国	男	0
-张三3	11	12345673	123	321312	1999-02-13T00:00	中国	男	0
-张三4	11	123456711	123	321312	1999-02-13T00:00	中国	男	0
-张三5	11	1232134567	123	321312	1999-02-13T00:00	中国	男	0
-张三6	11	124314567	123	321312	1999-02-13T00:00	中国	男	0
-张三7	11	123445167	123	321312	1998-02-13T00:00	中国	男	0
-
 -- !test_insert1 --
 doris1	18
 
@@ -454,3 +445,22 @@ doris
 -- !sql --
 1
 
+-- !sql --
+10	\N	7744
+11	\N	-94
+12	16970	95
+13	\N	7023
+14	\N	1
+15	3679	-11
+16	\N	-1079
+17	-22	\N
+18	30995	\N
+19	\N	-79
+5	\N	-127
+6	14680	-26424
+7	-22270	12722
+8	\N	\N
+9	\N	\N
+
+-- !sql --
+

regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy

@@ -133,102 +133,89 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc
         qt_sql """select current_catalog()"""
         sql """switch ${catalog_name}"""
         qt_sql """select current_catalog()"""
-        def res_dbs_log = sql "show databases;"
-		for(int i = 0;i < res_dbs_log.size();i++) {
-			def tbs = sql "show tables from  `${res_dbs_log[i][0]}`"
-			log.info( "database = ${res_dbs_log[i][0]} => tables = "+tbs.toString())
-		}
-        try {
-        
-            sql """ use ${ex_db_name}"""
+        sql """ use ${ex_db_name}"""
 
-            order_qt_ex_tb0  """ select id, name from ${ex_tb0} order by id; """
-            sql  """ insert into internal.${internal_db_name}.${inDorisTable} select id, name from ${ex_tb0}; """
-            order_qt_in_tb  """ select id, name from internal.${internal_db_name}.${inDorisTable} order by id; """
+        order_qt_ex_tb0  """ select id, name from ${ex_tb0} order by id; """
+        sql  """ insert into internal.${internal_db_name}.${inDorisTable} select id, name from ${ex_tb0}; """
+        order_qt_in_tb  """ select id, name from internal.${internal_db_name}.${inDorisTable} order by id; """
 
-            order_qt_ex_tb1  """ select * from ${ex_tb1} order by id; """
-            order_qt_ex_tb2  """ select * from ${ex_tb2} order by id; """
-            order_qt_ex_tb3  """ select * from ${ex_tb3} order by game_code; """
-            order_qt_ex_tb4  """ select * from ${ex_tb4} order by products_id; """
-            order_qt_ex_tb5  """ select * from ${ex_tb5} order by id; """
-            order_qt_ex_tb6  """ select * from ${ex_tb6} order by id; """
-            order_qt_ex_tb7  """ select * from ${ex_tb7} order by id; """
-            order_qt_ex_tb8  """ select * from ${ex_tb8} order by uid; """
-            order_qt_ex_tb9  """ select * from ${ex_tb9} order by c_date; """
-            order_qt_ex_tb10  """ select * from ${ex_tb10} order by aa; """
-            order_qt_ex_tb11  """ select * from ${ex_tb11} order by aa; """
-            order_qt_ex_tb12  """ select * from ${ex_tb12} order by cc; """
-            order_qt_ex_tb13  """ select * from ${ex_tb13} order by name; """
-            order_qt_ex_tb14  """ select * from ${ex_tb14} order by tid; """
-            order_qt_ex_tb15  """ select * from ${ex_tb15} order by col1; """
-            order_qt_ex_tb16  """ select * from ${ex_tb16} order by id; """
-            order_qt_ex_tb17  """ select * from ${ex_tb17} order by id; """
-            order_qt_ex_tb18  """ select * from ${ex_tb18} order by num_tinyint; """
-            order_qt_ex_tb19  """ select * from ${ex_tb19} order by date_value; """
-            order_qt_ex_tb20  """ select * from ${ex_tb20} order by decimal_normal; """
-            order_qt_ex_tb21_1  """ select `key`, `id` from ${ex_tb21} where `key` = 2 order by id;"""
-            order_qt_ex_tb21_2  """ select `key`, `id` from ${ex_tb21} where `key` like 2 order by id;"""
-            order_qt_ex_tb21_3  """ select `key`, `id` from ${ex_tb21} where `key` in (1,2) order by id;"""
-            order_qt_ex_tb21_4  """ select `key`, `id` from ${ex_tb21} where abs(`key`) = 2 order by id;"""
-            order_qt_ex_tb21_5  """ select `key`, `id` from ${ex_tb21} where `key` between 1 and 2 order by id;"""
-            order_qt_ex_tb21_6  """ select `key`, `id` from ${ex_tb21} where `key` = case when id = 1 then 1 else 0 end order by id;"""
-            order_qt_ex_tb21_7  """ select (`key` +1) as k, `id` from ${ex_tb21} having abs(k) = 2 order by id;"""
-            order_qt_ex_tb21_8  """ select `key` as k, `id` from ${ex_tb21} having abs(k) = 2 order by id;"""
-            order_qt_information_schema """ show tables from information_schema; """
-            order_qt_dt """select * from ${dt}; """
-            order_qt_dt_null """select * from ${dt_null} order by 1; """
-            order_qt_test_dz """select * from ${test_zd} order by 1; """
-            order_qt_test_filter_not """select * from ${ex_tb13} where name not like '%张三0%' order by 1; """
-            explain {
-                sql("select `datetime` from all_types where to_date(`datetime`) = '2012-10-25';")
-                contains """ SELECT `datetime` FROM `doris_test`.`all_types` WHERE (date(`datetime`) = '2012-10-25')"""
+        order_qt_ex_tb1  """ select * from ${ex_tb1} order by id; """
+        order_qt_ex_tb2  """ select * from ${ex_tb2} order by id; """
+        order_qt_ex_tb3  """ select * from ${ex_tb3} order by game_code; """
+        order_qt_ex_tb4  """ select * from ${ex_tb4} order by products_id; """
+        order_qt_ex_tb5  """ select * from ${ex_tb5} order by id; """
+        order_qt_ex_tb6  """ select * from ${ex_tb6} order by id; """
+        order_qt_ex_tb7  """ select * from ${ex_tb7} order by id; """
+        order_qt_ex_tb8  """ select * from ${ex_tb8} order by uid; """
+        order_qt_ex_tb9  """ select * from ${ex_tb9} order by c_date; """
+        order_qt_ex_tb10  """ select * from ${ex_tb10} order by aa; """
+        order_qt_ex_tb11  """ select * from ${ex_tb11} order by aa; """
+        order_qt_ex_tb12  """ select * from ${ex_tb12} order by cc; """
+        order_qt_ex_tb13  """ select * from ${ex_tb13} order by name; """
+        order_qt_ex_tb14  """ select * from ${ex_tb14} order by tid; """
+        order_qt_ex_tb15  """ select * from ${ex_tb15} order by col1; """
+        order_qt_ex_tb16  """ select * from ${ex_tb16} order by id; """
+        order_qt_ex_tb17  """ select * from ${ex_tb17} order by id; """
+        order_qt_ex_tb18  """ select * from ${ex_tb18} order by num_tinyint; """
+        order_qt_ex_tb19  """ select * from ${ex_tb19} order by date_value; """
+        order_qt_ex_tb20  """ select * from ${ex_tb20} order by decimal_normal; """
+        order_qt_ex_tb21_1  """ select `key`, `id` from ${ex_tb21} where `key` = 2 order by id;"""
+        order_qt_ex_tb21_2  """ select `key`, `id` from ${ex_tb21} where `key` like 2 order by id;"""
+        order_qt_ex_tb21_3  """ select `key`, `id` from ${ex_tb21} where `key` in (1,2) order by id;"""
+        order_qt_ex_tb21_4  """ select `key`, `id` from ${ex_tb21} where abs(`key`) = 2 order by id;"""
+        order_qt_ex_tb21_5  """ select `key`, `id` from ${ex_tb21} where `key` between 1 and 2 order by id;"""
+        order_qt_ex_tb21_6  """ select `key`, `id` from ${ex_tb21} where `key` = case when id = 1 then 1 else 0 end order by id;"""
+        order_qt_ex_tb21_7  """ select (`key` +1) as k, `id` from ${ex_tb21} having abs(k) = 2 order by id;"""
+        order_qt_ex_tb21_8  """ select `key` as k, `id` from ${ex_tb21} having abs(k) = 2 order by id;"""
+        order_qt_information_schema """ show tables from information_schema; """
+        order_qt_dt """select * from ${dt}; """
+        order_qt_dt_null """select * from ${dt_null} order by 1; """
+        order_qt_test_dz """select * from ${test_zd} order by 1; """
+        order_qt_test_filter_not """select * from ${ex_tb13} where name not like '%张三0%' order by 1; """
+        explain {
+            sql("select `datetime` from all_types where to_date(`datetime`) = '2012-10-25';")
+            contains """ SELECT `datetime` FROM `doris_test`.`all_types` WHERE (date(`datetime`) = '2012-10-25')"""
+        }
+
+        explain {
+            sql("select /*+ SET_VAR(enable_ext_func_pred_pushdown = false) */ `datetime` from all_types where to_date(`datetime`) = '2012-10-25';")
+            contains """SELECT `datetime` FROM `doris_test`.`all_types`"""
+        }
+
+        // test insert
+        String uuid1 = UUID.randomUUID().toString();
+        connect(user=user, password="${pwd}", url=url) {
+            try {
+                sql """ insert into ${catalog_name}.${ex_db_name}.${test_insert} values ('${uuid1}', 'doris1', 18) """
+                fail()
+            } catch (Exception e) {
+                log.info(e.getMessage())
             }
+        }
 
-            explain {
-                sql("select /*+ SET_VAR(enable_ext_func_pred_pushdown = false) */ `datetime` from all_types where to_date(`datetime`) = '2012-10-25';")
-                contains """SELECT `datetime` FROM `doris_test`.`all_types`"""
+        sql """GRANT LOAD_PRIV ON ${catalog_name}.${ex_db_name}.${test_insert} TO ${user}"""
+
+        connect(user=user, password="${pwd}", url=url) {
+            try {
+                sql """ insert into ${catalog_name}.${ex_db_name}.${test_insert} values ('${uuid1}', 'doris1', 18) """
+            } catch (Exception e) {
+                fail();
             }
+        }
+        order_qt_test_insert1 """ select name, age from ${test_insert} where id = '${uuid1}' order by age """
 
-            // test insert
-            String uuid1 = UUID.randomUUID().toString();
-            connect(user=user, password="${pwd}", url=url) {
-                try {
-                    sql """ insert into ${catalog_name}.${ex_db_name}.${test_insert} values ('${uuid1}', 'doris1', 18) """
-                    fail()
-                } catch (Exception e) {
-                    log.info(e.getMessage())
-                }
-            }
+        String uuid2 = UUID.randomUUID().toString();
+        sql """ insert into ${test_insert} values ('${uuid2}', 'doris2', 19), ('${uuid2}', 'doris3', 20) """
+        order_qt_test_insert2 """ select name, age from ${test_insert} where id = '${uuid2}' order by age """
 
-            sql """GRANT LOAD_PRIV ON ${catalog_name}.${ex_db_name}.${test_insert} TO ${user}"""
+        sql """ insert into ${test_insert} select * from ${test_insert} where id = '${uuid2}' """
+        order_qt_test_insert3 """ select name, age from ${test_insert} where id = '${uuid2}' order by age """
 
-            connect(user=user, password="${pwd}", url=url) {
-                try {
-                    sql """ insert into ${catalog_name}.${ex_db_name}.${test_insert} values ('${uuid1}', 'doris1', 18) """
-                } catch (Exception e) {
-                    fail();
-                }
-            }
-            order_qt_test_insert1 """ select name, age from ${test_insert} where id = '${uuid1}' order by age """
+        String uuid3 = UUID.randomUUID().toString();
+        sql """ INSERT INTO ${test_insert2} VALUES
+                ('${uuid3}', true, 'abcHa1.12345', '1.123450xkalowadawd', '2022-10-01', 3.14159, 1, 2, 0, 100000, 1.2345678, 24.000, '07:09:51', '2022', '2022-11-27 07:09:51', '2022-11-27 07:09:51'); """
+        order_qt_test_insert4 """ select k1,k2,k3,k4,k5,k6,k7,k8,k9,k10,k11,k12,k13,k14,k15 from ${test_insert2} where id = '${uuid3}' """
 
-            String uuid2 = UUID.randomUUID().toString();
-            sql """ insert into ${test_insert} values ('${uuid2}', 'doris2', 19), ('${uuid2}', 'doris3', 20) """
-            order_qt_test_insert2 """ select name, age from ${test_insert} where id = '${uuid2}' order by age """
-
-            sql """ insert into ${test_insert} select * from ${test_insert} where id = '${uuid2}' """
-            order_qt_test_insert3 """ select name, age from ${test_insert} where id = '${uuid2}' order by age """
-
-            String uuid3 = UUID.randomUUID().toString();
-            sql """ INSERT INTO ${test_insert2} VALUES
-                    ('${uuid3}', true, 'abcHa1.12345', '1.123450xkalowadawd', '2022-10-01', 3.14159, 1, 2, 0, 100000, 1.2345678, 24.000, '07:09:51', '2022', '2022-11-27 07:09:51', '2022-11-27 07:09:51'); """
-            order_qt_test_insert4 """ select k1,k2,k3,k4,k5,k6,k7,k8,k9,k10,k11,k12,k13,k14,k15 from ${test_insert2} where id = '${uuid3}' """
-        } finally {
-			res_dbs_log = sql "show databases;"
-			for(int i = 0;i < res_dbs_log.size();i++) {
-				def tbs = sql "show tables from  `${res_dbs_log[i][0]}`"
-				log.info( "database = ${res_dbs_log[i][0]} => tables = "+tbs.toString())
-			}
-		}
         sql """ drop catalog if exists ${catalog_name} """
 
         // test only_specified_database argument
@@ -313,41 +300,29 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc
             "jdbc.driver_class" = "com.mysql.cj.jdbc.Driver");
         """
         sql """ switch ${catalog_name} """
-        
-        res_dbs_log = sql "show databases;"
-		for(int i = 0;i < res_dbs_log.size();i++) {
-			def tbs = sql "show tables from  `${res_dbs_log[i][0]}`"
-			log.info( "database = ${res_dbs_log[i][0]} => tables = "+tbs.toString())
-		}
-        try {
-            sql """ use ${ex_db_name} """
-            order_qt_ex_tb1  """ select * from ${ex_tb1} order by id; """
 
-            // test all types supported by MySQL
-            sql """use doris_test;"""
-            qt_mysql_all_types """select * from all_types order by tinyint_u;"""
+        sql """ use ${ex_db_name} """
+        order_qt_ex_tb1  """ select * from ${ex_tb1} order by id; """
 
-            // test insert into internal.db.table select * from all_types
-            sql """ insert into internal.${internal_db_name}.${test_insert_all_types} select * from all_types; """
-            order_qt_select_insert_all_types """ select * from internal.${internal_db_name}.${test_insert_all_types} order by tinyint_u; """
+        // test all types supported by MySQL
+        sql """use doris_test;"""
+        qt_mysql_all_types """select * from all_types order by tinyint_u;"""
 
-            // test CTAS
-            sql  """ drop table if exists internal.${internal_db_name}.${test_ctas} """
-            sql """ create table internal.${internal_db_name}.${test_ctas}
-                    PROPERTIES("replication_num" = "1")
-                    AS select * from all_types;
-                """
+        // test insert into internal.db.table select * from all_types
+        sql """ insert into internal.${internal_db_name}.${test_insert_all_types} select * from all_types; """
+        order_qt_select_insert_all_types """ select * from internal.${internal_db_name}.${test_insert_all_types} order by tinyint_u; """
 
-            order_qt_ctas """select * from internal.${internal_db_name}.${test_ctas} order by tinyint_u;"""
+        // test CTAS
+        sql  """ drop table if exists internal.${internal_db_name}.${test_ctas} """
+        sql """ create table internal.${internal_db_name}.${test_ctas}
+                PROPERTIES("replication_num" = "1")
+                AS select * from all_types;
+            """
+
+        order_qt_ctas """select * from internal.${internal_db_name}.${test_ctas} order by tinyint_u;"""
+
+        order_qt_ctas_desc """desc internal.${internal_db_name}.${test_ctas};"""
 
-            order_qt_ctas_desc """desc internal.${internal_db_name}.${test_ctas};"""
-        } finally {
-			res_dbs_log = sql "show databases;"
-			for(int i = 0;i < res_dbs_log.size();i++) {
-				def tbs = sql "show tables from  `${res_dbs_log[i][0]}`"
-				log.info( "database = ${res_dbs_log[i][0]} => tables = "+tbs.toString())
-			}
-		}
         sql """ drop catalog if exists ${catalog_name} """
 
         // test mysql view
@@ -624,6 +599,22 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc
 
         sql """drop catalog if exists mysql_rename2;"""
 
+        sql """drop catalog if exists mysql_conjuncts;"""
+
+        sql """create catalog if not exists mysql_conjuncts properties(
+            "type"="jdbc",
+            "user"="root",
+            "password"="123456",
+            "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}/doris_test?useSSL=false&zeroDateTimeBehavior=convertToNull",
+            "driver_url" = "${driver_url}",
+            "driver_class" = "com.mysql.cj.jdbc.Driver"
+        );"""
+
+        order_qt_sql """SELECT * FROM mysql_conjuncts.doris_test.compoundpredicate_test WHERE (pk > 4) OR ((pk < 6 OR pk > 7) AND col_int_undef_signed < 1);"""
+
+        order_qt_sql """select * from mysql_conjuncts.doris_test.text_push where pk <=7;"""
+
+        sql """drop catalog if exists mysql_conjuncts;"""
     }
 }