From 428d2e8a7345c3f1933631b61a39ca88c5db8ea2 Mon Sep 17 00:00:00 2001 From: yujun Date: Mon, 21 Jul 2025 12:10:31 +0800 Subject: [PATCH] branch-2.1: [fix](regress) new thread should connect to cluster.jdbcUrl in docker regression suite #53234 (#53316) cherry pick from #53234 --- .../org/apache/doris/regression/Config.groovy | 7 --- .../regression/action/WaitForAction.groovy | 8 +++- .../doris/regression/suite/Suite.groovy | 45 ++++++++++++++----- .../regression/suite/SuiteCluster.groovy | 2 + .../regression/suite/SuiteContext.groovy | 27 ++++++++++- .../suites/demo_p0/docker_action.groovy | 7 +++ ...l_multi_segments_re_calc_in_publish.groovy | 2 +- .../suites/trash_p0/clean_trash.groovy | 2 +- 8 files changed, 78 insertions(+), 22 deletions(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy index acd7076ab1..d45f187304 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy @@ -706,13 +706,6 @@ class Config { return DriverManager.getConnection(jdbcUrl, 'root', '') } - Connection getConnectionByDbName(String dbName) { - String dbUrl = buildUrlWithDb(jdbcUrl, dbName) - tryCreateDbIfNotExist(dbName) - log.info("connect to ${dbUrl}".toString()) - return DriverManager.getConnection(dbUrl, jdbcUser, jdbcPassword) - } - Connection getConnectionByArrowFlightSqlDbName(String dbName) { Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver") String arrowFlightSqlHost = otherConfigs.get("extArrowFlightSqlHost") diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/WaitForAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/WaitForAction.groovy index fb62081896..ee92016e27 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/WaitForAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/WaitForAction.groovy @@ -62,8 +62,12 @@ class WaitForAction implements SuiteAction{ if (forRollUp) { num = 8 } - Awaitility.await().atMost(time, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).and() - .pollInterval(100, TimeUnit.MILLISECONDS).await().until(() -> { + Awaitility + .with().pollInSameThread() + .await() + .atMost(time, TimeUnit.SECONDS) + .with().pollDelay(100, TimeUnit.MILLISECONDS).and() + .pollInterval(100, TimeUnit.MILLISECONDS).await().until({ log.info("sql is :\n${sql}") def (result, meta) = JdbcUtils.executeToList(context.getConnection(), sql) String res = result.get(0).get(num) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 6496360e9b..897addd6b4 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -262,6 +262,25 @@ class Suite implements GroovyInterceptable { return context.connect(user, password, url, actionSupplier) } + // delete 'dockerAwaitUntil', should call 'Awaitility.await()...' directly or use 'awaitUntil(..., f)' + // public void dockerAwaitUntil(int atMostSeconds, int intervalSecond = 1, Closure actionSupplier) { + // def connInfo = context.threadLocalConn.get() + // Awaitility.await().atMost(atMostSeconds, SECONDS).pollInterval(intervalSecond, SECONDS).until( + // { + // connect(connInfo.username, connInfo.password, connInfo.conn.getMetaData().getURL(), actionSupplier) + // } + // ) + // } + public void awaitUntil(int atMostSeconds, double intervalSecond = 1, Closure actionSupplier) { + Awaitility + .with().pollInSameThread() + .await() + .atMost(atMostSeconds, TimeUnit.SECONDS) + .pollInterval((int) (1000 * intervalSecond), TimeUnit.MILLISECONDS) + .until(actionSupplier) + } + + // more explaination can see example file: demo_p0/docker_action.groovy public void docker(ClusterOptions options = new ClusterOptions(), Closure actionSupplier) throws Exception { if (context.config.excludeDockerTest) { return @@ -283,6 +302,8 @@ class Suite implements GroovyInterceptable { } } + logger.info("=== start run suite {} in not_cloud mode. ===", name) + def originConnection = context.threadLocalConn.get() try { cluster.destroy(true) cluster.init(options, dockerIsCloud) @@ -308,18 +329,18 @@ class Suite implements GroovyInterceptable { // wait be report Thread.sleep(5000) - def url = String.format( - "jdbc:mysql://%s:%s/?useLocalSessionState=false&allowLoadLocalInfile=false", + def jdbcUrl = String.format( + "jdbc:mysql://%s:%s/?useLocalSessionState=true&allowLoadLocalInfile=false", fe.host, fe.queryPort) - def conn = DriverManager.getConnection(url, user, password) - def sql = "CREATE DATABASE IF NOT EXISTS " + context.dbName - logger.info("try create database if not exists {}", context.dbName) - JdbcUtils.executeToList(conn, sql) - url = Config.buildUrlWithDb(url, context.dbName) - - logger.info("connect to docker cluster: suite={}, url={}", name, url) - connect(user, password, url, actionSupplier) + cluster.jdbcUrl = jdbcUrl + context.threadLocalConn.remove() + actionSupplier.call() } finally { + if (originConnection == null) { + context.threadLocalConn.remove() + } else { + context.threadLocalConn.set(originConnection) + } if (!context.config.dockerEndNoKill) { cluster.destroy(context.config.dockerEndDeleteFiles) } @@ -603,6 +624,10 @@ class Suite implements GroovyInterceptable { return context.dbName } + String getCurDbConnectUrl() { + return Config.buildUrlWithDb(context.getJdbcUrl(), getCurDbName()) + } + long getDbId() { def dbInfo = sql "show proc '/dbs'" for(List row : dbInfo) { diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy index 856b0e7695..4fd8107541 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy @@ -266,6 +266,8 @@ class SuiteCluster { static final Logger logger = LoggerFactory.getLogger(this.class) + // dockerImpl() will set jdbcUrl + String jdbcUrl = "" final String name final Config config private boolean running diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy index 96d8607c82..72e6bf70d0 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy @@ -145,7 +145,7 @@ class SuiteContext implements Closeable { def threadConnInfo = threadLocalConn.get() if (threadConnInfo == null) { threadConnInfo = new ConnectionInfo() - threadConnInfo.conn = config.getConnectionByDbName(dbName) + threadConnInfo.conn = getConnectionByDbName(dbName) threadConnInfo.username = config.jdbcUser threadConnInfo.password = config.jdbcPassword threadLocalConn.set(threadConnInfo) @@ -153,6 +153,31 @@ class SuiteContext implements Closeable { return threadConnInfo.conn } + Connection getConnectionByDbName(String dbName) { + def jdbcUrl = getJdbcUrl() + def jdbcConn = DriverManager.getConnection(jdbcUrl, config.jdbcUser, config.jdbcPassword) + try { + String sql = "CREATE DATABASE IF NOT EXISTS ${dbName}" + log.info("Try to create db, sql: ${sql}".toString()) + if (!config.dryRun) { + jdbcConn.withCloseable { conn -> JdbcUtils.executeToList(conn, sql) } + } + } catch (Throwable t) { + throw new IllegalStateException("Create database failed, jdbcUrl: ${jdbcUrl}", t) + } + def dbUrl = Config.buildUrlWithDb(jdbcUrl, dbName) + log.info("connect to ${dbUrl}".toString()) + return DriverManager.getConnection(dbUrl, config.jdbcUser, config.jdbcPassword) + } + + String getJdbcUrl() { + if (cluster.isRunning()) { + return cluster.jdbcUrl + } else { + return config.jdbcUrl + } + } + // like getConnection, but connect to FE master Connection getMasterConnection() { def threadConnInfo = threadLocalMasterConn.get() diff --git a/regression-test/suites/demo_p0/docker_action.groovy b/regression-test/suites/demo_p0/docker_action.groovy index 6f293c4748..8d69a4b2f1 100644 --- a/regression-test/suites/demo_p0/docker_action.groovy +++ b/regression-test/suites/demo_p0/docker_action.groovy @@ -17,6 +17,13 @@ import org.apache.doris.regression.suite.ClusterOptions +// Run docker suite steps: +// 1. Read 'docker/runtime/doris-compose/Readme.md', make sure you can setup a doris docker cluster; +// 2. update regression-conf-custom.groovy with config: +// image = "xxxx" // your doris docker image +// excludeDockerTest = false // do run docker suite, default is true +// dockerEndDeleteFiles = false // after run docker suite, whether delete contains's log and data in directory '/tmp/doris/' + suite('docker_action', 'docker') { // run a new docker docker { diff --git a/regression-test/suites/fault_injection_p0/test_local_multi_segments_re_calc_in_publish.groovy b/regression-test/suites/fault_injection_p0/test_local_multi_segments_re_calc_in_publish.groovy index ff1d9531fb..c5fe06b19c 100644 --- a/regression-test/suites/fault_injection_p0/test_local_multi_segments_re_calc_in_publish.groovy +++ b/regression-test/suites/fault_injection_p0/test_local_multi_segments_re_calc_in_publish.groovy @@ -155,7 +155,7 @@ suite("test_local_multi_segments_re_calc_in_publish", "docker") { Thread.sleep(1000) do_streamload_2pc_commit(txnId) - dockerAwaitUntil(30) { + awaitUntil(30) { def result = sql_return_maparray "show transaction from ${dbName} where id = ${txnId}" result[0].TransactionStatus as String == "VISIBLE" } diff --git a/regression-test/suites/trash_p0/clean_trash.groovy b/regression-test/suites/trash_p0/clean_trash.groovy index 525e8054ce..4c03f43b80 100644 --- a/regression-test/suites/trash_p0/clean_trash.groovy +++ b/regression-test/suites/trash_p0/clean_trash.groovy @@ -40,7 +40,7 @@ suite("test_clean_trash", "docker") { def checkFunc = { boolean trashZero -> def succ = false - dockerAwaitUntil(300) { + awaitUntil(300) { def bes = sql_return_maparray """show backends""" succ = bes.every { if (trashZero) {