From cf677b327bc3d9fff0162da2332429bd95d18e49 Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Tue, 25 Jul 2023 22:45:22 +0800 Subject: [PATCH] [fix](jdbc catalog) Fixed mappings with type errors for bool and tinyint(1) (#22089) First of all, mysql does not have a boolean type, its boolean type is actually tinyint(1), in the previous logic, We force tinyint(1) to be a boolean by passing tinyInt1isBit=true, which causes an error if tinyint(1) is not a 0 or 1, Therefore, we need to match tinyint(1) according to tinyint instead of boolean, and this change will not affect the correctness of where k = 1 or where k = true queries --- be/src/vec/exec/vjdbc_connector.cpp | 3 ++- .../mysql/init/03-create-table.sql | 2 +- docs/en/docs/lakehouse/multi-catalog/jdbc.md | 2 +- .../docs/lakehouse/multi-catalog/jdbc.md | 2 +- .../org/apache/doris/jdbc/JdbcExecutor.java | 24 ++++++++++++++++--- .../apache/doris/catalog/JdbcResource.java | 2 +- .../test_mysql_jdbc_catalog.out | 24 +++++++++---------- .../test_mysql_jdbc_catalog_nereids.out | 18 +++++++------- 8 files changed, 48 insertions(+), 29 deletions(-) diff --git a/be/src/vec/exec/vjdbc_connector.cpp b/be/src/vec/exec/vjdbc_connector.cpp index 446c282ea0..cfdbcbc185 100644 --- a/be/src/vec/exec/vjdbc_connector.cpp +++ b/be/src/vec/exec/vjdbc_connector.cpp @@ -261,7 +261,8 @@ Status JdbcConnector::_check_type(SlotDescriptor* slot_desc, const std::string& type_str, slot_desc->type().debug_string(), slot_desc->col_name()); switch (slot_desc->type().type) { case TYPE_BOOLEAN: { - if (type_str != "java.lang.Boolean" && type_str != "java.lang.Byte") { + if (type_str != "java.lang.Boolean" && type_str != "java.lang.Byte" && + type_str != "java.lang.Integer") { return Status::InternalError(error_msg); } break; diff --git a/docker/thirdparties/docker-compose/mysql/init/03-create-table.sql b/docker/thirdparties/docker-compose/mysql/init/03-create-table.sql index f1c45b7b1e..0b013c827a 100644 --- a/docker/thirdparties/docker-compose/mysql/init/03-create-table.sql +++ b/docker/thirdparties/docker-compose/mysql/init/03-create-table.sql @@ -266,7 +266,7 @@ create table doris_test.all_types ( `int` int, `bigint` bigint, `date` date, - `timestamp` timestamp(4), + `timestamp` timestamp(4) null, `datetime` datetime, `float` float, `double` double, diff --git a/docs/en/docs/lakehouse/multi-catalog/jdbc.md b/docs/en/docs/lakehouse/multi-catalog/jdbc.md index 14377c718f..3f8c627ef9 100644 --- a/docs/en/docs/lakehouse/multi-catalog/jdbc.md +++ b/docs/en/docs/lakehouse/multi-catalog/jdbc.md @@ -167,7 +167,7 @@ CREATE CATALOG jdbc_mysql PROPERTIES ( | MYSQL Type | Doris Type | Comment | |-------------------------------------------|----------------|-------------------------------------------------------------------------------| -| BOOLEAN | BOOLEAN | | +| BOOLEAN | TINYINT | | | TINYINT | TINYINT | | | SMALLINT | SMALLINT | | | MEDIUMINT | INT | | diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md index e732cb5a60..377a495fbe 100644 --- a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md +++ b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md @@ -167,7 +167,7 @@ CREATE CATALOG jdbc_mysql PROPERTIES ( | MYSQL Type | Doris Type | Comment | |-------------------------------------------|----------------|-------------------------------------------------| -| BOOLEAN | BOOLEAN | | +| BOOLEAN | TINYINT | | | TINYINT | TINYINT | | | SMALLINT | SMALLINT | | | MEDIUMINT | INT | | diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java index 988e69ee07..56ad752a8a 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java @@ -469,9 +469,8 @@ public class JdbcExecutor { } } - public void copyBatchBooleanResult(Object columnObj, boolean isNullable, int numRows, long nullMapAddr, - long columnAddr) { - Object[] column = (Object[]) columnObj; + public void booleanPutToByte(Object[] column, boolean isNullable, int numRows, long nullMapAddr, long columnAddr, + int startRow) { if (isNullable) { for (int i = 0; i < numRows; i++) { if (column[i] == null) { @@ -487,6 +486,25 @@ public class JdbcExecutor { } } + public void copyBatchBooleanResult(Object columnObj, boolean isNullable, int numRows, long nullMapAddr, + long columnAddr) { + Object[] column = (Object[]) columnObj; + int firstNotNullIndex = 0; + if (isNullable) { + firstNotNullIndex = getFirstNotNullObject(column, numRows, nullMapAddr); + } + if (firstNotNullIndex == numRows) { + return; + } + if (column[firstNotNullIndex] instanceof Boolean) { + booleanPutToByte(column, isNullable, numRows, nullMapAddr, columnAddr, firstNotNullIndex); + } else if (column[firstNotNullIndex] instanceof Integer) { + integerPutToByte(column, isNullable, numRows, nullMapAddr, columnAddr, firstNotNullIndex); + } else if (column[firstNotNullIndex] instanceof Byte) { + bytePutToByte(column, isNullable, numRows, nullMapAddr, columnAddr, firstNotNullIndex); + } + } + private void bigDecimalPutToByte(Object[] column, boolean isNullable, int numRows, long nullMapAddr, long columnAddr, int startRowForNullable) { if (isNullable) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java index 777a23330c..459e5a6ba8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java @@ -303,7 +303,7 @@ public class JdbcResource extends Resource { // However when tinyInt1isBit=false, GetColumnClassName of MySQL returns java.lang.Boolean, // while that of Doris returns java.lang.Integer. In order to be compatible with both MySQL and Doris, // Jdbc params should set tinyInt1isBit=true&transformedBitIsBoolean=true - newJdbcUrl = checkAndSetJdbcBoolParam(newJdbcUrl, "tinyInt1isBit", "false", "true"); + newJdbcUrl = checkAndSetJdbcBoolParam(newJdbcUrl, "tinyInt1isBit", "true", "false"); newJdbcUrl = checkAndSetJdbcBoolParam(newJdbcUrl, "transformedBitIsBoolean", "false", "true"); // set useUnicode and characterEncoding to false and utf-8 newJdbcUrl = checkAndSetJdbcBoolParam(newJdbcUrl, "useUnicode", "false", "true"); diff --git a/regression-test/data/jdbc_catalog_p0/test_mysql_jdbc_catalog.out b/regression-test/data/jdbc_catalog_p0/test_mysql_jdbc_catalog.out index 7177d429e8..31d48434ca 100644 --- a/regression-test/data/jdbc_catalog_p0/test_mysql_jdbc_catalog.out +++ b/regression-test/data/jdbc_catalog_p0/test_mysql_jdbc_catalog.out @@ -81,14 +81,14 @@ b 1 c 1 -- !ex_tb13 -- -张三0 11 1234567 123 321312 1999-02-13T00:00 中国 男 false -张三1 11 12345678 123 321312 1999-02-13T00:00 中国 男 false -张三2 11 12345671 123 321312 1999-02-13T00:00 中国 男 false -张三3 11 12345673 123 321312 1999-02-13T00:00 中国 男 false -张三4 11 123456711 123 321312 1999-02-13T00:00 中国 男 false -张三5 11 1232134567 123 321312 1999-02-13T00:00 中国 男 false -张三6 11 124314567 123 321312 1999-02-13T00:00 中国 男 false -张三7 11 123445167 123 321312 1998-02-13T00:00 中国 男 false +张三0 11 1234567 123 321312 1999-02-13T00:00 中国 男 0 +张三1 11 12345678 123 321312 1999-02-13T00:00 中国 男 0 +张三2 11 12345671 123 321312 1999-02-13T00:00 中国 男 0 +张三3 11 12345673 123 321312 1999-02-13T00:00 中国 男 0 +张三4 11 123456711 123 321312 1999-02-13T00:00 中国 男 0 +张三5 11 1232134567 123 321312 1999-02-13T00:00 中国 男 0 +张三6 11 124314567 123 321312 1999-02-13T00:00 中国 男 0 +张三7 11 123445167 123 321312 1998-02-13T00:00 中国 男 0 -- !ex_tb14 -- 123 2022-11-02 2022-11-02 8011 oppo @@ -275,7 +275,7 @@ doris3 20 doris3 20 -- !test_insert4 -- -true abcHa1.12345 1.123450xkalowadawd 2022-10-01 3.14159 1 2 0 100000 1.2345678 24.000 07:09:51 2022 2022-11-27T07:09:51 2022-11-27T07:09:51 +1 abcHa1.12345 1.123450xkalowadawd 2022-10-01 3.14159 1 2 0 100000 1.2345678 24.000 07:09:51 2022 2022-11-27T07:09:51 2022-11-27T07:09:51 -- !specified_database_1 -- doris_test @@ -297,7 +297,7 @@ sys -- !mysql_all_types -- \N 302 \N 502 602 4.14159 \N 6.14159 \N -124 -302 2013 -402 -502 -602 \N 2012-10-26T02:08:39.345700 2013-10-26T08:09:18 -5.14145 \N -7.1400 row2 \N 09:11:09.567 text2 0xe86f6c6c6f20576f726c67 \N \N 0x2f \N 0x88656c6c9f Value3 -201 301 401 501 601 3.14159 4.1415926 5.14159 true -123 -301 2012 -401 -501 -601 2012-10-30 2012-10-25T12:05:36.345700 2012-10-25T08:08:08 -4.14145 -5.1400000001 -6.1400 row1 line1 09:09:09.567 text1 0x48656c6c6f20576f726c64 {"age":30,"city":"London","name":"Alice"} Option1,Option3 0x2a 0x48656c6c6f00000000000000 0x48656c6c6f Value2 -202 302 402 502 602 4.14159 5.1415926 6.14159 false -124 -302 2013 -402 -502 -602 2012-11-01 2012-10-26T02:08:39.345700 2013-10-26T08:09:18 -5.14145 -6.1400000001 -7.1400 row2 line2 09:11:09.567 text2 0xe86f6c6c6f20576f726c67 {"age":18,"city":"ChongQing","name":"Gaoxin"} Option1,Option2 0x2f 0x58676c6c6f00000000000000 0x88656c6c9f Value3 -203 303 403 503 603 7.14159 8.1415926 9.14159 false \N -402 2017 -602 -902 -1102 2012-11-02 2023-07-10T09:34:21.204900 2013-10-27T08:11:18 -5.14145 -6.1400000000001 -7.1400 row3 line3 09:11:09.567 text3 0xe86f6c6c6f20576f726c67 {"age":24,"city":"ChongQing","name":"ChenQi"} Option2 0x2f 0x58676c6c6f00000000000000 \N Value1 +201 301 401 501 601 3.14159 4.1415926 5.14159 1 -123 -301 2012 -401 -501 -601 2012-10-30 2012-10-25T12:05:36.345700 2012-10-25T08:08:08 -4.14145 -5.1400000001 -6.1400 row1 line1 09:09:09.567 text1 0x48656c6c6f20576f726c64 {"age":30,"city":"London","name":"Alice"} Option1,Option3 0x2a 0x48656c6c6f00000000000000 0x48656c6c6f Value2 +202 302 402 502 602 4.14159 5.1415926 6.14159 0 -124 -302 2013 -402 -502 -602 2012-11-01 2012-10-26T02:08:39.345700 2013-10-26T08:09:18 -5.14145 -6.1400000001 -7.1400 row2 line2 09:11:09.567 text2 0xe86f6c6c6f20576f726c67 {"age":18,"city":"ChongQing","name":"Gaoxin"} Option1,Option2 0x2f 0x58676c6c6f00000000000000 0x88656c6c9f Value3 +203 303 403 503 603 7.14159 8.1415926 9.14159 0 \N -402 2017 -602 -902 -1102 2012-11-02 \N 2013-10-27T08:11:18 -5.14145 -6.1400000000001 -7.1400 row3 line3 09:11:09.567 text3 0xe86f6c6c6f20576f726c67 {"age":24,"city":"ChongQing","name":"ChenQi"} Option2 0x2f 0x58676c6c6f00000000000000 \N Value1 diff --git a/regression-test/data/jdbc_catalog_p0/test_mysql_jdbc_catalog_nereids.out b/regression-test/data/jdbc_catalog_p0/test_mysql_jdbc_catalog_nereids.out index c1cfae034c..c25b068ade 100644 --- a/regression-test/data/jdbc_catalog_p0/test_mysql_jdbc_catalog_nereids.out +++ b/regression-test/data/jdbc_catalog_p0/test_mysql_jdbc_catalog_nereids.out @@ -98,14 +98,14 @@ b 1 c 1 -- !ex_tb13 -- -张三0 11 1234567 123 321312 1999-02-13T00:00 中国 男 false -张三1 11 12345678 123 321312 1999-02-13T00:00 中国 男 false -张三2 11 12345671 123 321312 1999-02-13T00:00 中国 男 false -张三3 11 12345673 123 321312 1999-02-13T00:00 中国 男 false -张三4 11 123456711 123 321312 1999-02-13T00:00 中国 男 false -张三5 11 1232134567 123 321312 1999-02-13T00:00 中国 男 false -张三6 11 124314567 123 321312 1999-02-13T00:00 中国 男 false -张三7 11 123445167 123 321312 1998-02-13T00:00 中国 男 false +张三0 11 1234567 123 321312 1999-02-13T00:00 中国 男 0 +张三1 11 12345678 123 321312 1999-02-13T00:00 中国 男 0 +张三2 11 12345671 123 321312 1999-02-13T00:00 中国 男 0 +张三3 11 12345673 123 321312 1999-02-13T00:00 中国 男 0 +张三4 11 123456711 123 321312 1999-02-13T00:00 中国 男 0 +张三5 11 1232134567 123 321312 1999-02-13T00:00 中国 男 0 +张三6 11 124314567 123 321312 1999-02-13T00:00 中国 男 0 +张三7 11 123445167 123 321312 1998-02-13T00:00 中国 男 0 -- !ex_tb14 -- 123 2022-11-02 2022-11-02 8011 oppo @@ -259,7 +259,7 @@ doris3 20 doris3 20 -- !test_insert4 -- -true abcHa1.12345 1.123450xkalowadawd 2022-10-01 3.14159 1 2 0 100000 1.2345678 24.000 07:09:51 2022 2022-11-27T07:09:51 2022-11-27T07:09:51 +1 abcHa1.12345 1.123450xkalowadawd 2022-10-01 3.14159 1 2 0 100000 1.2345678 24.000 07:09:51 2022 2022-11-27T07:09:51 2022-11-27T07:09:51 -- !ex_tb1 -- {"k1":"v1", "k2":"v2"}