[pick](branch-2.1) pick #44489 #44894 (#45024)

### What problem does this PR solve?

pick #44489 #44894
This commit is contained in:
Xinyi Zou
2024-12-05 17:04:15 +08:00
committed by GitHub
parent a8f36a4813
commit fa633d751d
49 changed files with 106 additions and 50 deletions

View File

@ -182,11 +182,11 @@ public class FlightSqlConnectProcessor extends ConnectProcessor implements AutoC
public void close() throws Exception {
ctx.setCommand(MysqlCommand.COM_SLEEP);
ctx.clear();
// TODO support query profile
for (StmtExecutor asynExecutor : returnResultFromRemoteExecutor) {
asynExecutor.finalizeQuery();
}
returnResultFromRemoteExecutor.clear();
executor.finalizeQuery();
ConnectContext.remove();
}
}

View File

@ -0,0 +1,8 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !arrow_flight_sql --
777 4
-- !arrow_flight_sql_datetime --
333 plsql333 2024-07-21 12:00:00.123456 2024-07-21 12:00:00.0
222 plsql222 2024-07-20 12:00:00.123456 2024-07-20 12:00:00.0
111 plsql111 2024-07-19 12:00:00.123456 2024-07-19 12:00:00.0

View File

@ -74,6 +74,7 @@ under the License.
<groovy.version>4.0.19</groovy.version>
<antlr.version>4.9.3</antlr.version>
<hadoop.version>2.8.0</hadoop.version>
<!-- Arrow 18 only supports jdk17 -->
<arrow.version>17.0.0</arrow.version>
</properties>
<build>

View File

@ -706,19 +706,23 @@ class Config {
return DriverManager.getConnection(dbUrl, jdbcUser, jdbcPassword)
}
Connection getConnectionByArrowFlightSql(String dbName) {
Connection getConnectionByArrowFlightSqlDbName(String dbName) {
Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver")
String arrowFlightSqlHost = otherConfigs.get("extArrowFlightSqlHost")
String arrowFlightSqlPort = otherConfigs.get("extArrowFlightSqlPort")
String arrowFlightSqlUrl = "jdbc:arrow-flight-sql://${arrowFlightSqlHost}:${arrowFlightSqlPort}" +
"/?useServerPrepStmts=false&useSSL=false&useEncryption=false"
// TODO jdbc:arrow-flight-sql not support connect db
String dbUrl = buildUrlWithDbImpl(arrowFlightSqlUrl, dbName)
// Arrow 17.0.0-rc03 support jdbc:arrow-flight-sql connect db
// https://github.com/apache/arrow/issues/41947
if (dbName?.trim()) {
arrowFlightSqlUrl = "jdbc:arrow-flight-sql://${arrowFlightSqlHost}:${arrowFlightSqlPort}" +
"/catalog=" + dbName + "?useServerPrepStmts=false&useSSL=false&useEncryption=false"
}
tryCreateDbIfNotExist(dbName)
log.info("connect to ${dbUrl}".toString())
log.info("connect to ${arrowFlightSqlUrl}".toString())
String arrowFlightSqlJdbcUser = otherConfigs.get("extArrowFlightSqlUser")
String arrowFlightSqlJdbcPassword = otherConfigs.get("extArrowFlightSqlPassword")
return DriverManager.getConnection(dbUrl, arrowFlightSqlJdbcUser, arrowFlightSqlJdbcPassword)
return DriverManager.getConnection(arrowFlightSqlUrl, arrowFlightSqlJdbcUser, arrowFlightSqlJdbcPassword)
}
Connection getDownstreamConnection() {

View File

@ -156,7 +156,7 @@ class SuiteContext implements Closeable {
def threadConnInfo = threadArrowFlightSqlConn.get()
if (threadConnInfo == null) {
threadConnInfo = new ConnectionInfo()
threadConnInfo.conn = config.getConnectionByArrowFlightSql(dbName)
threadConnInfo.conn = config.getConnectionByArrowFlightSqlDbName(dbName)
threadConnInfo.username = config.jdbcUser
threadConnInfo.password = config.jdbcPassword
threadArrowFlightSqlConn.set(threadConnInfo)

View File

@ -0,0 +1,43 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_select", "arrow_flight_sql") {
def tableName = "test_select"
sql "DROP TABLE IF EXISTS ${tableName}"
sql """
create table ${tableName} (id int, name varchar(20)) DUPLICATE key(`id`) distributed by hash (`id`) buckets 4
properties ("replication_num"="1");
"""
sql """INSERT INTO ${tableName} VALUES(111, "plsql111")"""
sql """INSERT INTO ${tableName} VALUES(222, "plsql222")"""
sql """INSERT INTO ${tableName} VALUES(333, "plsql333")"""
sql """INSERT INTO ${tableName} VALUES(111, "plsql333")"""
qt_arrow_flight_sql "select sum(id) as a, count(1) as b from ${tableName}"
tableName = "test_select_datetime"
sql "DROP TABLE IF EXISTS ${tableName}"
sql """
create table ${tableName} (id int, name varchar(20), f_datetime_p datetime(6), f_datetime datetime) DUPLICATE key(`id`) distributed by hash (`id`) buckets 4
properties ("replication_num"="1");
"""
sql """INSERT INTO ${tableName} VALUES(111, "plsql111","2024-07-19 12:00:00.123456","2024-07-19 12:00:00")"""
sql """INSERT INTO ${tableName} VALUES(222, "plsql222","2024-07-20 12:00:00.123456","2024-07-20 12:00:00")"""
sql """INSERT INTO ${tableName} VALUES(333, "plsql333","2024-07-21 12:00:00.123456","2024-07-21 12:00:00")"""
qt_arrow_flight_sql_datetime "select * from ${tableName} order by id desc"
}

View File

@ -17,7 +17,7 @@
* under the License.
*/
suite("aggregate_count1", "query,arrow_flight_sql") {
suite("aggregate_count1", "query") {
sql """ DROP TABLE IF EXISTS aggregate_count1 """
sql """create table if not exists aggregate_count1 (
name varchar(128),

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("select_distinct", "arrow_flight_sql") {
suite("select_distinct") {
sql """DROP TABLE IF EXISTS decimal_a;"""
sql """DROP TABLE IF EXISTS decimal_b;"""
sql """DROP TABLE IF EXISTS decimal_c;"""

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("join_with_column_casesensetive", "arrow_flight_sql") {
suite("join_with_column_casesensetive") {
def tables=["ad_order_data_v1","ad_order_data"]
for (String table in tables) {

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite('test_cast', "arrow_flight_sql") {
suite('test_cast') {
def date = "date '2020-01-01'"
def datev2 = "datev2 '2020-01-01'"
def datetime = "timestamp '2020-01-01 12:34:45'"

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_query_except", "arrow_flight_sql") {
suite("test_query_except") {
// test query except, depend on query_test_data_load.groovy
sql "use test_query_db"
qt_select_except1 """

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_group_concat", "query,p0,arrow_flight_sql") {
suite("test_group_concat", "query,p0") {
qt_select """
SELECT group_concat(k6) FROM test_query_db.test where k6='false'
"""

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_grouping_sets1", "arrow_flight_sql") {
suite("test_grouping_sets1") {
qt_select """
select
col1

View File

@ -19,7 +19,7 @@
// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/aggregate
// and modified by Doris.
suite("having", "query,p0,arrow_flight_sql") {
suite("having", "query,p0") {
sql """DROP TABLE IF EXISTS supplier"""
sql """CREATE TABLE `supplier` (
`s_suppkey` int(11) NOT NULL,

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_intersect", "arrow_flight_sql") {
suite("test_intersect") {
qt_select """
SELECT * FROM (SELECT k1 FROM test_query_db.baseall
INTERSECT SELECT k1 FROM test_query_db.test) a ORDER BY k1

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_join2", "query,p0,arrow_flight_sql") {
suite("test_join2", "query,p0") {
def DBname = "regression_test_join2"
def TBname1 = "J1_TBL"
def TBname2 = "J2_TBL"

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_left_join1", "query,p0,arrow_flight_sql") {
suite("test_left_join1", "query,p0") {
def tableName = "test_left_join1"
sql """drop table if exists ${tableName}"""

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_nestedloop_outer_join", "query_p0,arrow_flight_sql") {
suite("test_nestedloop_outer_join", "query_p0") {
def tbl1 = "test_nestedloop_outer_join1"
def tbl2 = "test_nestedloop_outer_join2"

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_partitioned_hash_join", "query,p0,arrow_flight_sql") {
suite("test_partitioned_hash_join", "query,p0") {
sql "drop table if exists test_partitioned_hash_join_l"
sql "drop table if exists test_partitioned_hash_join_r"
sql """ create table test_partitioned_hash_join_l (

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("lateral_view", "arrow_flight_sql") {
suite("lateral_view") {
sql """ DROP TABLE IF EXISTS `test_explode_bitmap` """
sql """
CREATE TABLE `test_explode_bitmap` (

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_offset_in_subquery_with_join", "query,arrow_flight_sql") {
suite("test_offset_in_subquery_with_join", "query") {
// define a sql table
def testTable = "test_offset_in_subquery_with_join"

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("literal_view_test", "arrow_flight_sql") {
suite("literal_view_test") {
sql """DROP TABLE IF EXISTS table1"""

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_set_operators", "query,p0,arrow_flight_sql") {
suite("test_set_operators", "query,p0") {
sql """
DROP TABLE IF EXISTS t1;

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_sort_operator", "query,p0,arrow_flight_sql") {
suite("test_sort_operator", "query,p0") {
sql """
DROP TABLE IF EXISTS dim_org_ful;

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite('test_default_limit', "arrow_flight_sql") {
suite('test_default_limit') {
sql 'drop table if exists baseall'
sql 'drop table if exists bigtable'

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_show_create_table", "query,arrow_flight_sql") {
suite("test_show_create_table", "query") {
String tb_name = "tb_show_create_table";
try {
sql """drop table if exists ${tb_name} """

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_aggregate_all_functions", "arrow_flight_sql") {
suite("test_aggregate_all_functions") {
sql "set batch_size = 4096"

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_case_function_null", "query,p0,arrow_flight_sql") {
suite("test_case_function_null", "query,p0") {
sql """ drop table if exists case_null0 """
sql """ create table case_null0 (
`c0` decimalv3(17, 1) NULL,

View File

@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_hash_function", "arrow_flight_sql") {
suite("test_hash_function") {
sql "set batch_size = 4096;"
sql "set enable_profile = true;"

View File

@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_ip_functions", "arrow_flight_sql") {
suite("test_ip_functions") {
sql "set batch_size = 4096;"
qt_sql "SELECT ipv4_num_to_string(-1);"

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_query_json_insert", "query,arrow_flight_sql") {
suite("test_query_json_insert", "query") {
qt_sql "select json_insert('{\"a\": 1, \"b\": [2, 3]}', '\$', null);"
qt_sql "select json_insert('{\"k\": [1, 2]}', '\$.k[0]', null, '\$.[1]', null);"
def tableName = "test_query_json_insert"

View File

@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_json_function", "arrow_flight_sql") {
suite("test_json_function") {
sql "set batch_size = 4096;"
qt_sql "SELECT get_json_double('{\"k1\":1.3, \"k2\":\"2\"}', \"\$.k1\");"

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_conv", "arrow_flight_sql") {
suite("test_conv") {
qt_select "SELECT CONV(15,10,2)"
sql """ drop table if exists test_conv; """

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_multi_string_search", "arrow_flight_sql") {
suite("test_multi_string_search") {
def table_name = "test_multi_string_search_strings"
sql """ DROP TABLE IF EXISTS ${table_name} """

View File

@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_gis_function", "arrow_flight_sql") {
suite("test_gis_function") {
sql "set batch_size = 4096;"
qt_sql "SELECT ST_AsText(ST_Point(24.7, 56.7));"

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_string_function", "arrow_flight_sql") {
suite("test_string_function") {
sql "set batch_size = 4096;"
qt_sql "select elt(0, \"hello\", \"doris\");"

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("explode_split", "arrow_flight_sql") {
suite("explode_split") {
def tableName = "test_lv_str"
sql """ DROP TABLE IF EXISTS ${tableName} """

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite('test_alias_function', "arrow_flight_sql") {
suite('test_alias_function') {
sql '''
CREATE ALIAS FUNCTION IF NOT EXISTS f1(DATETIMEV2(3), INT)
with PARAMETER (datetime1, int1) as date_trunc(days_sub(datetime1, int1), 'day')'''

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_predicate", "arrow_flight_sql") {
suite("test_predicate") {
sql """drop table if exists t1;"""
sql """
create table t1 (

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_width_bucket_function", "arrow_flight_sql") {
suite("test_width_bucket_function") {
qt_sql "select width_bucket(1, 2, 3, 2)"
qt_sql "select width_bucket(null, 2, 3, 2)"
qt_sql "select width_bucket(6, 2, 6, 4)"

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_subquery2", "arrow_flight_sql") {
suite("test_subquery2") {
sql """DROP TABLE IF EXISTS subquerytest2"""

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_data_type_marks", "arrow_flight_sql") {
suite("test_data_type_marks") {
def tbName = "org"
sql "DROP TABLE IF EXISTS ${tbName}"
sql """

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("dict_with_null", "query,arrow_flight_sql") {
suite("dict_with_null", "query") {
def tableName = "test_dict_with_null"
sql "DROP TABLE IF EXISTS ${tableName}"
sql """

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("orderby_nullliteral", "query,arrow_flight_sql") {
suite("orderby_nullliteral", "query") {
def tableName = "test_orderby_nullliteral"
sql "DROP TABLE IF EXISTS ${tableName}"

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_select_constant", "arrow_flight_sql") {
suite("test_select_constant") {
qt_select1 'select 100, "test", date("2021-01-02");'
qt_select_geo1 'SELECT ST_AsText(ST_Point(123.12345678901234567890,89.1234567890));'
}

View File

@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_select_with_predicate_like", "arrow_flight_sql") {
suite("test_select_with_predicate_like") {
def tables=["test_basic_agg"]
for (String table in tables) {

View File

@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_select_with_predicate_prune", "arrow_flight_sql") {
suite("test_select_with_predicate_prune") {
sql """
drop table if exists `test_select_with_predicate_prune`;
"""

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_largeint", "arrow_flight_sql") {
suite("test_largeint") {
def tbName = "test_largeint"
sql "DROP TABLE IF EXISTS ${tbName}"
sql """

View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
suite("test_with_and_two_phase_agg", "arrow_flight_sql") {
suite("test_with_and_two_phase_agg") {
def tableName = "test_with_and_two_phase_agg_table"
sql """ DROP TABLE IF EXISTS ${tableName} """
sql """