From de5603da6bbf296243293e67a87e105344fb86f6 Mon Sep 17 00:00:00 2001 From: zhangguoqiang <18372634969@163.com> Date: Thu, 10 Aug 2023 15:23:30 +0800 Subject: [PATCH] [regresstion][external]fix jdbc cases fail external 0809 (#22761) fix jdbc cases fail external 0809 --- .../sqlserver/init/04-insert.sql | 1 + .../sqlserver/sqlserver.yaml.tpl | 27 ++++++++++++++++++- .../pipeline/p0/conf/regression-conf.groovy | 3 ++- .../hive/test_hive_schema_evolution.groovy | 3 ++- .../jdbc/test_clickhouse_jdbc_catalog.groovy | 8 +++--- .../jdbc/test_doris_jdbc_catalog.groovy | 8 +++--- .../jdbc/test_jdbc_query_mysql.groovy | 8 +++--- .../jdbc/test_mysql_jdbc_catalog.groovy | 7 ++--- .../test_mysql_jdbc_catalog_nereids.groovy | 6 ++--- .../jdbc/test_oracle_jdbc_catalog.groovy | 6 ++--- .../jdbc/test_pg_jdbc_catalog.groovy | 8 +++--- .../jdbc/test_sqlserver_jdbc_catalog.groovy | 6 ++--- .../nereids_p0/show/test_show_where.groovy | 10 ++++--- .../query_p0/show/test_show_where.groovy | 10 ++++--- 14 files changed, 74 insertions(+), 37 deletions(-) diff --git a/docker/thirdparties/docker-compose/sqlserver/init/04-insert.sql b/docker/thirdparties/docker-compose/sqlserver/init/04-insert.sql index 4a9b4d2298..d912c18fcc 100644 --- a/docker/thirdparties/docker-compose/sqlserver/init/04-insert.sql +++ b/docker/thirdparties/docker-compose/sqlserver/init/04-insert.sql @@ -14,6 +14,7 @@ -- KIND, either express or implied. See the License for the -- specific language governing permissions and limitations -- under the License. +use doris_test; Insert into dbo.student values (1, 'doris', 18), (2, 'alice', 19), (3, 'bob', 20); diff --git a/docker/thirdparties/docker-compose/sqlserver/sqlserver.yaml.tpl b/docker/thirdparties/docker-compose/sqlserver/sqlserver.yaml.tpl index 02f909f5d5..04b5eb976e 100644 --- a/docker/thirdparties/docker-compose/sqlserver/sqlserver.yaml.tpl +++ b/docker/thirdparties/docker-compose/sqlserver/sqlserver.yaml.tpl @@ -29,6 +29,31 @@ services: retries: 120 volumes: - ./init:/docker-entrypoint-initdb.d + command: + - /bin/bash + - -c + - | + # Launch MSSQL and send to background + /opt/mssql/bin/sqlservr & + # Wait for it to be available + echo "Waiting for MS SQL to be available ⏳" + /opt/mssql-tools/bin/sqlcmd -l 30 -S localhost -h-1 -V1 -U sa -P Doris123456 -Q "SET NOCOUNT ON SELECT \"YAY WE ARE UP\" , @@servername" + is_up=$$? + while [ $$is_up -ne 0 ] ; do + echo -e $$(date) + /opt/mssql-tools/bin/sqlcmd -l 30 -S localhost -h-1 -V1 -U sa -P Doris123456 -Q "SET NOCOUNT ON SELECT \"YAY WE ARE UP\" , @@servername" + is_up=$$? + sleep 5 + done + # Run every script in /scripts + # TODO set a flag so that this is only done once on creation, + # and not every time the container runs + for foo in /docker-entrypoint-initdb.d/*.sql + do /opt/mssql-tools/bin/sqlcmd -U sa -P Doris123456 -l 30 -e -i $$foo + done + # So that the container doesn't shut down, sleep this thread + sleep infinity + restart: always environment: # Accept the end user license Agreement @@ -44,4 +69,4 @@ services: condition: service_healthy networks: - doris--sqlserver_2022: \ No newline at end of file + doris--sqlserver_2022: diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy b/regression-test/pipeline/p0/conf/regression-conf.groovy index fc9c69fcf0..15a4e484a9 100644 --- a/regression-test/pipeline/p0/conf/regression-conf.groovy +++ b/regression-test/pipeline/p0/conf/regression-conf.groovy @@ -54,7 +54,8 @@ testDirectories = "" // this groups will not be executed excludeGroups = "" // this suites will not be executed -excludeSuites = "window_function,test_profile,test_broker_load,test_spark_load,test_analyze_stats_p1,test_refresh_mtmv,test_bitmap_filter,test_export_parquet,test_doris_jdbc_catalog" +excludeSuites = "test_profile,test_broker_load,test_spark_load,test_analyze_stats_p1,test_refresh_mtmv,test_bitmap_filter,test_export_parquet,test_doris_jdbc_catalog,test_transactional_hive" + // this directories will not be executed excludeDirectories = "workload_manager_p1" diff --git a/regression-test/suites/external_table_p0/hive/test_hive_schema_evolution.groovy b/regression-test/suites/external_table_p0/hive/test_hive_schema_evolution.groovy index 23dcfea991..d06e3759ee 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_schema_evolution.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_schema_evolution.groovy @@ -58,12 +58,13 @@ suite("test_hive_schema_evolution", "p0,external,hive,external_docker,external_d if (enabled != null && enabled.equalsIgnoreCase("true")) { try { String hms_port = context.config.otherConfigs.get("hms_port") - String catalog_name = "suites.external_table_p0.hive.test_hive_schema_evolution" + String catalog_name = "test_hive_schema_evolution" sql """drop catalog if exists ${catalog_name}""" sql """create catalog if not exists ${catalog_name} properties ( "type"="hms", 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' );""" + sql """switch ${catalog_name}""" sql """use `${catalog_name}`.`default`""" q_text() diff --git a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy index 909a57ffcf..7d52939efe 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy @@ -27,7 +27,7 @@ suite("test_clickhouse_jdbc_catalog", "p0,external,clickhouse,external_docker,ex String bucket = getS3BucketName() String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/clickhouse-jdbc-0.4.2-all.jar" - String inDorisTable = "doris_in_tb"; + String inDorisTable = "test_clickhouse_jdbc_doris_in_tb"; sql """create database if not exists ${internal_db_name}; """ @@ -41,10 +41,10 @@ suite("test_clickhouse_jdbc_catalog", "p0,external,clickhouse,external_docker,ex "driver_url" = "${driver_url}", "driver_class" = "com.clickhouse.jdbc.ClickHouseDriver" );""" - - sql """ drop table if exists ${inDorisTable} """ + sql """use ${internal_db_name}""" + sql """ drop table if exists ${internal_db_name}.${inDorisTable} """ sql """ - CREATE TABLE ${inDorisTable} ( + CREATE TABLE ${internal_db_name}.${inDorisTable} ( `id` INT NULL COMMENT "主键id", `name` string NULL COMMENT "名字", `age` INT NULL COMMENT "年龄" diff --git a/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog.groovy index 5d4aea50af..0eff0b18c1 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog.groovy @@ -31,7 +31,7 @@ suite("test_doris_jdbc_catalog", "p0,external,doris,external_docker,external_doc String catalog_name = "doris_jdbc_catalog"; String internal_db_name = "regression_test_jdbc_catalog_p0"; String doris_port = 9030; - String inDorisTable = "doris_in_tb"; + String inDorisTable = "test_doris_jdbc_doris_in_tb"; String hllTable = "bowen_hll_test" sql """create database if not exists ${internal_db_name}; """ @@ -47,10 +47,10 @@ suite("test_doris_jdbc_catalog", "p0,external,doris,external_docker,external_doc "driver_url" = "${driver_url}", "driver_class" = "com.mysql.jdbc.Driver" )""" - - sql """ drop table if exists ${inDorisTable} """ + sql """use ${internal_db_name}""" + sql """ drop table if exists ${internal_db_name}.${inDorisTable} """ sql """ - CREATE TABLE ${inDorisTable} ( + CREATE TABLE ${internal_db_name}.${inDorisTable} ( `id` INT NULL COMMENT "主键id", `name` string NULL COMMENT "名字" ) DISTRIBUTED BY HASH(id) BUCKETS 10 diff --git a/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_mysql.groovy b/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_mysql.groovy index 64831e839a..813824d6ab 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_mysql.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_mysql.groovy @@ -30,10 +30,10 @@ suite("test_jdbc_query_mysql", "p0,external,mysql,external_docker,external_docke String exMysqlTable = "doris_ex_tb"; String exMysqlTable1 = "doris_ex_tb1"; String exMysqlTable2 = "doris_ex_tb2"; - String exMysqlTypeTable = "doris_ex_type_tb"; - String inDorisTable = "doris_in_tb"; - String inDorisTable1 = "doris_in_tb1"; - String inDorisTable2 = "doris_in_tb2"; + String exMysqlTypeTable = "doris_ex_type_tb"; + String inDorisTable = "test_jdbc_mysql_doris_in_tb"; + String inDorisTable1 = "test_jdbc_mysql_doris_in_tb1"; + String inDorisTable2 = "test_jdbc_mysql_doris_in_tb2"; sql """drop resource if exists $jdbcResourceMysql57;""" sql """ diff --git a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy index cc16f91345..2997f2f785 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy @@ -28,7 +28,7 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc String internal_db_name = "regression_test_jdbc_catalog_p0"; String ex_db_name = "doris_test"; String mysql_port = context.config.otherConfigs.get("mysql_57_port"); - String inDorisTable = "doris_in_tb"; + String inDorisTable = "test_mysql_jdbc_doris_in_tb"; String ex_tb0 = "ex_tb0"; String ex_tb1 = "ex_tb1"; String ex_tb2 = "ex_tb2"; @@ -69,9 +69,10 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc "driver_class" = "com.mysql.cj.jdbc.Driver" );""" - sql """ drop table if exists ${inDorisTable} """ + sql """use ${internal_db_name}""" + sql """ drop table if exists ${internal_db_name}.${inDorisTable} """ sql """ - CREATE TABLE ${inDorisTable} ( + CREATE TABLE ${internal_db_name}.${inDorisTable} ( `id` INT NULL COMMENT "主键id", `name` string NULL COMMENT "名字" ) DISTRIBUTED BY HASH(id) BUCKETS 10 diff --git a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog_nereids.groovy b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog_nereids.groovy index 71191e6fa6..91d3075fe8 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog_nereids.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog_nereids.groovy @@ -67,10 +67,10 @@ suite("test_mysql_jdbc_catalog_nereids", "p0,external,mysql,external_docker,exte "driver_url" = "${driver_url}", "driver_class" = "com.mysql.cj.jdbc.Driver" );""" - - sql """ drop table if exists ${inDorisTable} """ + sql """use ${internal_db_name}""" + sql """ drop table if exists ${internal_db_name}.${inDorisTable} """ sql """ - CREATE TABLE ${inDorisTable} ( + CREATE TABLE ${internal_db_name}.${inDorisTable} ( `id` INT NULL COMMENT "主键id", `name` string NULL COMMENT "名字" ) DISTRIBUTED BY HASH(id) BUCKETS 10 diff --git a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy index a8f371db94..ff6e88118a 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy @@ -44,10 +44,10 @@ suite("test_oracle_jdbc_catalog", "p0,external,oracle,external_docker,external_d "driver_url" = "${driver_url}", "driver_class" = "oracle.jdbc.driver.OracleDriver" );""" - - sql """ drop table if exists ${inDorisTable} """ + sql """use ${internal_db_name}""" + sql """ drop table if exists ${internal_db_name}.${inDorisTable} """ sql """ - CREATE TABLE ${inDorisTable} ( + CREATE TABLE ${internal_db_name}.${inDorisTable} ( `id` INT NULL COMMENT "主键id", `name` string NULL COMMENT "名字", `age` INT NULL COMMENT "年龄" diff --git a/regression-test/suites/external_table_p0/jdbc/test_pg_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_pg_jdbc_catalog.groovy index 730f5b34c1..671fab1aa5 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_pg_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_pg_jdbc_catalog.groovy @@ -27,7 +27,7 @@ suite("test_pg_jdbc_catalog", "p0,external,pg,external_docker,external_docker_pg String ex_schema_name = "doris_test"; String ex_schema_name2 = "catalog_pg_test"; String pg_port = context.config.otherConfigs.get("pg_14_port"); - String inDorisTable = "doris_in_tb"; + String inDorisTable = "test_pg_jdbc_doris_in_tb"; String test_insert = "test_insert"; sql """create database if not exists ${internal_db_name}; """ @@ -42,10 +42,10 @@ suite("test_pg_jdbc_catalog", "p0,external,pg,external_docker,external_docker_pg "driver_url" = "${driver_url}", "driver_class" = "org.postgresql.Driver" );""" - - sql """ drop table if exists ${inDorisTable} """ + sql """use ${internal_db_name}""" + sql """ drop table if exists ${internal_db_name}.${inDorisTable} """ sql """ - CREATE TABLE ${inDorisTable} ( + CREATE TABLE ${internal_db_name}.${inDorisTable} ( `id` INT NULL COMMENT "主键id", `name` string NULL COMMENT "名字" ) DISTRIBUTED BY HASH(id) BUCKETS 10 diff --git a/regression-test/suites/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.groovy index 4fab5316a9..af0cf21aaf 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.groovy @@ -24,10 +24,10 @@ suite("test_sqlserver_jdbc_catalog", "p0,external,sqlserver,external_docker,exte if (enabled != null && enabled.equalsIgnoreCase("true")) { String catalog_name = "sqlserver_catalog"; String internal_db_name = "regression_test_jdbc_catalog_p0"; - String ex_db_name = "dbo"; + String ex_db_name = "doris_test"; String sqlserver_port = context.config.otherConfigs.get("sqlserver_2022_port"); - String inDorisTable = "doris_in_tb"; + String inDorisTable = "test_sqlserver_doris_in_tb"; sql """ drop catalog if exists ${catalog_name} """ @@ -39,7 +39,7 @@ suite("test_sqlserver_jdbc_catalog", "p0,external,sqlserver,external_docker,exte "driver_url" = "${driver_url}", "driver_class" = "com.microsoft.sqlserver.jdbc.SQLServerDriver" );""" - + sql """use ${internal_db_name}""" sql """ drop table if exists ${inDorisTable} """ sql """ CREATE TABLE ${inDorisTable} ( diff --git a/regression-test/suites/nereids_p0/show/test_show_where.groovy b/regression-test/suites/nereids_p0/show/test_show_where.groovy index c65c23c68f..5f555a4e72 100644 --- a/regression-test/suites/nereids_p0/show/test_show_where.groovy +++ b/regression-test/suites/nereids_p0/show/test_show_where.groovy @@ -16,13 +16,17 @@ // under the License. -suite("test_show_where", "query,external,mysql") { +suite("test_show_where", "query,external,mysql,external_docker,external_docker_mysql") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" String ex_db_name = "doris_test"; String ex_tb0 = "ex_tb0"; String ex_tb1 = "ex_tb1"; String catalog_name = "test_show_where_mysql_jdbc_catalog"; + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-java-8.0.25.jar" try { sql """ drop database if exists ${ex_db_name} """ sql """ create database ${ex_db_name} """ @@ -62,8 +66,8 @@ suite("test_show_where", "query,external,mysql") { "type"="jdbc", "jdbc.user"="root", "jdbc.password"="123456", - "jdbc.jdbc_url" = "jdbc:mysql://127.0.0.1:${mysql_port}/doris_test?useSSL=false", - "jdbc.driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar", + "jdbc.jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}/doris_test?useSSL=false", + "jdbc.driver_url" = "${driver_url}", "jdbc.driver_class" = "com.mysql.cj.jdbc.Driver"); """ diff --git a/regression-test/suites/query_p0/show/test_show_where.groovy b/regression-test/suites/query_p0/show/test_show_where.groovy index 665ddab0d5..9fac7148fe 100644 --- a/regression-test/suites/query_p0/show/test_show_where.groovy +++ b/regression-test/suites/query_p0/show/test_show_where.groovy @@ -16,11 +16,15 @@ // under the License. -suite("test_show_where", "query,external,mysql") { +suite("test_show_where", "query,external,mysql,external_docker,external_docker_mysql") { String ex_db_name = "doris_test"; String ex_tb0 = "ex_tb0"; String ex_tb1 = "ex_tb1"; String catalog_name = "test_show_where_mysql_jdbc_catalog"; + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-java-8.0.25.jar" try { sql """ drop database if exists ${ex_db_name} """ sql """ create database ${ex_db_name} """ @@ -60,8 +64,8 @@ suite("test_show_where", "query,external,mysql") { "type"="jdbc", "jdbc.user"="root", "jdbc.password"="123456", - "jdbc.jdbc_url" = "jdbc:mysql://127.0.0.1:${mysql_port}/doris_test?useSSL=false", - "jdbc.driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar", + "jdbc.jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}/doris_test?useSSL=false", + "jdbc.driver_url" = "${driver_url}", "jdbc.driver_class" = "com.mysql.cj.jdbc.Driver"); """