[branch-2.1][test](jdbc catalog) add oceanbase ce jdbc catalog test (#40978)

pick #34972)
2024-09-19 22:11:24 +08:00
parent e7d41eb143
commit 5f583fa329
11 changed files with 259 additions and 3 deletions
--- a/docker/thirdparties/docker-compose/oceanbase/init/01-drop-db.sql
+++ b/docker/thirdparties/docker-compose/oceanbase/init/01-drop-db.sql
@ -0,0 +1,18 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+drop database if exists doris_test;
--- a/docker/thirdparties/docker-compose/oceanbase/init/02-create-db.sql
+++ b/docker/thirdparties/docker-compose/oceanbase/init/02-create-db.sql
@ -0,0 +1,18 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+create database doris_test;
--- a/docker/thirdparties/docker-compose/oceanbase/init/03-create-table.sql
+++ b/docker/thirdparties/docker-compose/oceanbase/init/03-create-table.sql
@ -0,0 +1,51 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+create table doris_test.all_types (
+  `tinyint_u` tinyint unsigned,
+  `smallint_u` smallint unsigned,
+  `mediumint_u` mediumint unsigned,
+  `int_u` int unsigned,
+  `bigint_u` bigint unsigned,
+  `decimal_u` decimal(18, 5) unsigned,
+  `double_u` double unsigned,
+  `float_u` float unsigned,
+  `boolean` boolean,
+  `tinyint` tinyint,
+  `smallint` smallint,
+  `year` year,
+  `mediumint` mediumint,
+  `int` int,
+  `bigint` bigint,
+  `date` date,
+  `timestamp` timestamp(4) null,
+  `datetime` datetime,
+  `float` float,
+  `double` double,
+  `decimal` decimal(12, 4),
+  `char` char(5),
+  `varchar` varchar(10),
+  `time` time(4),
+  `text` text,
+  `blob` blob,
+  `json` json,
+  `set` set('Option1', 'Option2', 'Option3'),
+  `bit` bit(6),
+  `binary` binary(12),
+  `varbinary` varbinary(12),
+  `enum` enum('Value1', 'Value2', 'Value3')
+) engine=innodb charset=utf8;
--- a/docker/thirdparties/docker-compose/oceanbase/init/04-insert.sql
+++ b/docker/thirdparties/docker-compose/oceanbase/init/04-insert.sql
@ -0,0 +1,30 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+INSERT INTO doris_test.all_types VALUES
+(201, 301, 401, 501, 601, 3.14159, 4.1415926, 5.141592, true, -123, -301, 2012, -401, -501, -601, '2012-10-30', '2012-10-25 12:05:36.3456712', '2012-10-25 08:08:08.3456712',
+ -4.14145001, -5.1400000001, -6.140000001, 'row1', 'line1', '09:09:09.56782346', 'text1', X'48656C6C6F20576F726C64', '{"name": "Alice", "age": 30, "city": "London"}',
+ 'Option1,Option3', b'101010', X'48656C6C6F', X'48656C6C6F', 'Value2'),
+(202, 302, 402, 502, 602, 4.14159, 5.1415926, 6.141592, false, -124, -302, 2013, -402, -502, -602, '2012-11-01', '2012-10-26 02:08:39.3456712', '2013-10-26 08:09:18.3456712',
+ -5.14145001, -6.1400000001, -7.140000001, 'row2', 'line2', '09:11:09.56782346', 'text2', X'E86F6C6C6F20576F726C67', '{"name": "Gaoxin", "age": 18, "city": "ChongQing"}',
+ 'Option1,Option2', b'101111', X'58676C6C6F', X'88656C6C9F', 'Value3'),
+(null, 302, null, 502, 602, 4.14159, null, 6.141592, null, -124, -302, 2013, -402, -502, -602, null, '2012-10-26 02:08:39.3456712', '2013-10-26 08:09:18.3456712',
+ -5.14145001, null, -7.140000001, 'row2', null, '09:11:09.56782346', 'text2', X'E86F6C6C6F20576F726C67', null,
+ null, b'101111', null, X'88656C6C9F', 'Value3'),
+(203, 303, 403, 503, 603, 7.14159, 8.1415926, 9.141592, false, null, -402, 2017, -602, -902, -1102, '2012-11-02', null, '2013-10-27 08:11:18.3456712',
+ -5.14145000001, -6.1400000000001, -7.140000000001, 'row3', 'line3', '09:11:09.56782346', 'text3', X'E86F6C6C6F20576F726C67', '{"name": "ChenQi", "age": 24, "city": "ChongQing"}',
+ 'Option2', b'101111', X'58676C6C6F', null, 'Value1');
--- a/docker/thirdparties/docker-compose/oceanbase/oceanbase.env
+++ b/docker/thirdparties/docker-compose/oceanbase/oceanbase.env
@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+DOCKER_OCEANBASE_EXTERNAL_PORT=2881
--- a/docker/thirdparties/docker-compose/oceanbase/oceanbase.yaml.tpl
+++ b/docker/thirdparties/docker-compose/oceanbase/oceanbase.yaml.tpl
@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "2.1"
+
+services:
+  doris--oceanbase:
+    image: oceanbase/oceanbase-ce:4.2.1
+    restart: always
+    environment:
+      MODE: slim
+      OB_MEMORY_LIMIT: 5G
+      TZ: Asia/Shanghai
+    ports:
+      - ${DOCKER_OCEANBASE_EXTERNAL_PORT}:2881
+    healthcheck:
+      test: ["CMD-SHELL", "obclient -h127.1 -uroot@sys -P2881  -e 'SELECT 1'"]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+    volumes:
+      - ./init:/root/boot/init.d
+    networks:
+      - doris--oceanbase
+  doris--oceanbase-hello-world:
+    image: hello-world
+    depends_on:
+      doris--oceanbase:
+        condition: service_healthy
+    networks:
+      - doris--oceanbase
+networks:
+  doris--oceanbase:
+    ipam:
+      driver: default
+      config:
+        - subnet: 168.32.0.0/24
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@ -37,7 +37,7 @@ Usage: $0 <options>
     --stop             stop the specified components

  All valid components:
-    mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,hudi,trino,kafka,mariadb,db2,kerberos
+    mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,hudi,trino,kafka,mariadb,db2,kerberos,oceanbase
  "
    exit 1
 }
@ -60,7 +60,7 @@ STOP=0

 if [[ "$#" == 1 ]]; then
    # default
-    COMPONENTS="mysql,es,hive2,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2,kerberos"
+    COMPONENTS="mysql,es,hive2,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2,oceanbase,kerberos"
 else
    while true; do
        case "$1" in
@ -92,7 +92,7 @@ else
    done
    if [[ "${COMPONENTS}"x == ""x ]]; then
        if [[ "${STOP}" -eq 1 ]]; then
-            COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive2,hive3,iceberg,hudi,trino,kafka,mariadb,db2,kerberos,lakesoul"
+            COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive2,hive3,iceberg,hudi,trino,kafka,mariadb,db2,oceanbase,kerberos,lakesoul"
        fi
    fi
 fi
@ -136,6 +136,7 @@ RUN_SPARK=0
 RUN_MARIADB=0
 RUN_DB2=0
 RUN_KERBEROS=0
+RUN_OCENABASE=0

 for element in "${COMPONENTS_ARR[@]}"; do
    if [[ "${element}"x == "mysql"x ]]; then
@ -170,6 +171,8 @@ for element in "${COMPONENTS_ARR[@]}"; do
        RUN_DB2=1
    elif [[ "${element}"x == "kerberos"x ]]; then
        RUN_KERBEROS=1
+    elif [[ "${element}"x == "oceanbase"x ]];then
+        RUN_OCEANBASE=1
    else
        echo "Invalid component: ${element}"
        usage
@ -249,6 +252,18 @@ if [[ "${RUN_DB2}" -eq 1 ]]; then
    fi
 fi

+if [[ "${RUN_OCEANBASE}" -eq 1 ]]; then
+    # oceanbase
+    cp "${ROOT}"/docker-compose/oceanbase/oceanbase.yaml.tpl "${ROOT}"/docker-compose/oceanbase/oceanbase.yaml
+    sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/oceanbase/oceanbase.yaml
+    sudo docker compose -f "${ROOT}"/docker-compose/oceanbase/oceanbase.yaml --env-file "${ROOT}"/docker-compose/oceanbase/oceanbase.env down
+    if [[ "${STOP}" -ne 1 ]]; then
+        sudo rm "${ROOT}"/docker-compose/oceanbase/data/* -rf
+        sudo mkdir -p "${ROOT}"/docker-compose/oceanbase/data/
+        sudo docker compose -f "${ROOT}"/docker-compose/oceanbase/oceanbase.yaml --env-file "${ROOT}"/docker-compose/oceanbase/oceanbase.env up -d
+    fi
+fi
+
 if [[ "${RUN_SQLSERVER}" -eq 1 ]]; then
    # sqlserver
    cp "${ROOT}"/docker-compose/sqlserver/sqlserver.yaml.tpl "${ROOT}"/docker-compose/sqlserver/sqlserver.yaml
--- a/regression-test/conf/regression-conf.groovy
+++ b/regression-test/conf/regression-conf.groovy
@ -125,6 +125,7 @@ clickhouse_22_port=8123
 doris_port=9030
 mariadb_10_port=3326
 db2_11_port=50000
+oceanbase_port=2881

 // hive catalog test config
 // To enable hive/paimon test, you need first start hive container.
--- a/regression-test/data/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.out
+++ b/regression-test/data/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.out
@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !query --
+\N	302	\N	502	602	4.14159	\N	6.141592	\N	-124	-302	2013	-402	-502	-602	\N	2012-10-26T02:08:39.345700	2013-10-26T08:09:18	-5.14145	\N	-7.1400	row2	\N	09:11:09.567	text2	0xE86F6C6C6F20576F726C67	\N	\N	0x2F	\N	0x88656C6C9F	Value3
+201	301	401	501	601	3.14159	4.1415926	5.141592	1	-123	-301	2012	-401	-501	-601	2012-10-30	2012-10-25T12:05:36.345700	2012-10-25T08:08:08	-4.14145	-5.1400000001	-6.1400	row1	line1	09:09:09.567	text1	0x48656C6C6F20576F726C64	{"age": 30, "city": "London", "name": "Alice"}	Option1,Option3	0x2A	0x48656C6C6F00000000000000	0x48656C6C6F	Value2
+202	302	402	502	602	4.14159	5.1415926	6.141592	0	-124	-302	2013	-402	-502	-602	2012-11-01	2012-10-26T02:08:39.345700	2013-10-26T08:09:18	-5.14145	-6.1400000001	-7.1400	row2	line2	09:11:09.567	text2	0xE86F6C6C6F20576F726C67	{"age": 18, "city": "ChongQing", "name": "Gaoxin"}	Option1,Option2	0x2F	0x58676C6C6F00000000000000	0x88656C6C9F	Value3
+203	303	403	503	603	7.14159	8.1415926	9.141592	0	\N	-402	2017	-602	-902	-1102	2012-11-02	\N	2013-10-27T08:11:18	-5.14145	-6.1400000000001	-7.1400	row3	line3	09:11:09.567	text3	0xE86F6C6C6F20576F726C67	{"age": 24, "city": "ChongQing", "name": "ChenQi"}	Option2	0x2F	0x58676C6C6F00000000000000	\N	Value1
+
--- a/regression-test/pipeline/external/conf/regression-conf.groovy
+++ b/regression-test/pipeline/external/conf/regression-conf.groovy
@ -90,6 +90,7 @@ sqlserver_2022_port=1433
 clickhouse_22_port=8123
 mariadb_10_port=3326
 db2_11_port=50000
+oceanbase_port=2881

 // hive catalog test config
 // To enable jdbc test, you need first start hive container.
--- a/regression-test/suites/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.groovy
+++ b/regression-test/suites/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.groovy
@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_oceanbase_jdbc_catalog", "p0,external,oceanbase,external_docker,external_docker_oceanbase") {
+    String enabled = context.config.otherConfigs.get("enableJdbcTest");
+    String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+    String s3_endpoint = getS3Endpoint()
+    String bucket = getS3BucketName()
+    String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/oceanbase-client-2.4.8.jar"
+    if (enabled != null && enabled.equalsIgnoreCase("true")) {
+        String catalog_name = "oceanbase_catalog";
+        String ex_db_name = "doris_test";
+        String oceanbase_port = context.config.otherConfigs.get("oceanbase_port");
+
+
+        sql """ drop catalog if exists ${catalog_name} """
+
+        sql """ create catalog if not exists ${catalog_name} properties(
+                    "type"="jdbc",
+                    "user"="root@test",
+                    "password"="",
+                    "jdbc_url" = "jdbc:oceanbase://${externalEnvIp}:${oceanbase_port}/doris_test",
+                    "driver_url" = "${driver_url}",
+                    "driver_class" = "com.oceanbase.jdbc.Driver"
+        );"""
+
+        order_qt_query """ select * from ${catalog_name}.doris_test.all_types order by 1; """
+
+        sql """ drop catalog if exists ${catalog_name} """
+    }
+}