From ea8d4f2d0bb53bc6ba4808a0e9fcc1be935da276 Mon Sep 17 00:00:00 2001 From: XuJianxu Date: Wed, 20 Mar 2024 10:04:19 +0800 Subject: [PATCH] [fix][regression]update ccr test project (#32445) --- regression-test/conf/regression-conf.groovy | 5 ----- .../org/apache/doris/regression/Config.groovy | 14 ++++++++++++++ .../apache/doris/regression/suite/Suite.groovy | 14 +++++++++++--- .../doris/regression/suite/SuiteContext.groovy | 17 ++++++++++++----- .../doris/regression/suite/SyncerContext.groovy | 6 +++--- .../pipeline/p0/conf/regression-conf.groovy | 5 +++++ .../pipeline/p1/conf/regression-conf.groovy | 5 +++++ 7 files changed, 50 insertions(+), 16 deletions(-) diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index e825623374..c6764a2ce8 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -29,11 +29,6 @@ targetJdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoa jdbcUser = "root" jdbcPassword = "" -ccrDownstreamUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true" -ccrDownstreamUser = "root" -ccrDownstreamPassword = "" -ccrDownstreamFeThriftAddress = "127.0.0.1:9020" - feSourceThriftAddress = "127.0.0.1:9020" feTargetThriftAddress = "127.0.0.1:9020" syncerAddress = "127.0.0.1:9190" diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy index b1c9f5822e..615f12b788 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy @@ -41,6 +41,10 @@ class Config { public String jdbcPassword public String defaultDb + public String ccrDownstreamUrl + public String ccrDownstreamUser + public String ccrDownstreamPassword + public String feSourceThriftAddress public String feTargetThriftAddress public String feSyncerUser @@ -315,6 +319,9 @@ class Config { configToString(obj.sslCertificatePath) ) + config.ccrDownstreamUrl = configToString(obj.ccrDownstreamUrl) + config.ccrDownstreamUser = configToString(obj.ccrDownstreamUser) + config.ccrDownstreamPassword = configToString(obj.ccrDownstreamPassword) config.image = configToString(obj.image) config.dockerCoverageOutputDir = configToString(obj.dockerCoverageOutputDir) config.dockerEndDeleteFiles = configToBoolean(obj.dockerEndDeleteFiles) @@ -560,6 +567,13 @@ class Config { return DriverManager.getConnection(dbUrl, arrowFlightSqlJdbcUser, arrowFlightSqlJdbcPassword) } + Connection getDownstreamConnectionByDbName(String dbName) { + String dbUrl = buildUrlWithDb(ccrDownstreamUrl, dbName) + tryCreateDbIfNotExist(dbName) + log.info("connect to ${dbUrl}".toString()) + return DriverManager.getConnection(dbUrl, ccrDownstreamUser, ccrDownstreamPassword) + } + String getDbNameByFile(File suiteFile) { String dir = new File(suitePath).relativePath(suiteFile.parentFile) // We put sql files under sql dir, so dbs and tables used by cases diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 453809b6c0..4c6f7b72e4 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -336,8 +336,11 @@ class Suite implements GroovyInterceptable { } } - def sql_return_maparray_impl(Connection conn, String sqlStr) { + def sql_return_maparray_impl(String sqlStr, Connection conn = null) { logger.info("Execute sql: ${sqlStr}".toString()) + if (conn == null) { + conn = context.getConnection() + } def (result, meta) = JdbcUtils.executeToList(conn, sqlStr) // get all column names as list @@ -359,11 +362,11 @@ class Suite implements GroovyInterceptable { } def jdbc_sql_return_maparray(String sqlStr) { - return sql_return_maparray_impl(context.getConnection(), sqlStr) + return sql_return_maparray_impl(sqlStr, context.getConnection()) } def arrow_flight_sql_return_maparray(String sqlStr) { - return sql_return_maparray_impl(context.getArrowFlightSqlConnection(), (String) ("USE ${context.dbName};" + sqlStr)) + return sql_return_maparray_impl((String) ("USE ${context.dbName};" + sqlStr), context.getArrowFlightSqlConnection()) } def sql_return_maparray(String sqlStr) { @@ -706,6 +709,11 @@ class Suite implements GroovyInterceptable { return lines; } + + Connection getTargetConnection() { + return context.getTargetConnection(this) + } + boolean deleteFile(String filePath) { def file = new File(filePath) file.delete() diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy index d61f586b41..512dcca68d 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy @@ -195,9 +195,14 @@ class SuiteContext implements Closeable { return subJdbc.substring(0, subJdbc.indexOf("/")) } - private Map getSpec() { + private String getDownstreamJdbcNetInfo() { + String subJdbc = config.ccrDownstreamUrl.substring(config.ccrDownstreamUrl.indexOf("://") + 3) + return subJdbc.substring(0, subJdbc.indexOf("/")) + } + + private Map getSpec(String[] jdbc) { Map spec = Maps.newHashMap() - String[] jdbc = getJdbcNetInfo().split(":") + spec.put("host", jdbc[0]) spec.put("port", jdbc[1]) spec.put("user", config.feSyncerUser) @@ -208,7 +213,8 @@ class SuiteContext implements Closeable { } Map getSrcSpec() { - Map spec = getSpec() + String[] jdbc = getJdbcNetInfo().split(":") + Map spec = getSpec(jdbc) spec.put("thrift_port", config.feSourceThriftNetworkAddress.port.toString()) spec.put("database", dbName) @@ -216,7 +222,8 @@ class SuiteContext implements Closeable { } Map getDestSpec() { - Map spec = getSpec() + String[] jdbc = getDownstreamJdbcNetInfo().split(":") + Map spec = getSpec(jdbc) spec.put("thrift_port", config.feTargetThriftNetworkAddress.port.toString()) spec.put("database", "TEST_" + dbName) @@ -256,7 +263,7 @@ class SuiteContext implements Closeable { Connection getTargetConnection(Suite suite) { def context = getSyncer(suite).context if (context.targetConnection == null) { - context.targetConnection = config.getConnectionByDbName("TEST_" + dbName) + context.targetConnection = config.getDownstreamConnectionByDbName("TEST_" + dbName) } return context.targetConnection } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SyncerContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SyncerContext.groovy index 388904ec2d..93108e67fc 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SyncerContext.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SyncerContext.groovy @@ -147,8 +147,8 @@ class SyncerContext { return info } - FrontendClientImpl getMasterFrontClient() { - def result = suite.sql_return_maparray "select Host, RpcPort, IsMaster from frontends();" + FrontendClientImpl getMasterFrontClient(Connection conn) { + def result = suite.sql_return_maparray_impl("select Host, RpcPort, IsMaster from frontends();", conn) logger.info("get master fe: ${result}") def masterHost = "" @@ -179,7 +179,7 @@ class SyncerContext { FrontendClientImpl getTargetFrontClient() { if (targetFrontendClient == null) { - targetFrontendClient = getMasterFrontClient() + targetFrontendClient = getMasterFrontClient(suite.getTargetConnection()) } return targetFrontendClient } diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy b/regression-test/pipeline/p0/conf/regression-conf.groovy index b99e21c4e7..36035db9d4 100644 --- a/regression-test/pipeline/p0/conf/regression-conf.groovy +++ b/regression-test/pipeline/p0/conf/regression-conf.groovy @@ -25,6 +25,11 @@ targetJdbcUrl = "jdbc:mysql://172.19.0.2:9131/?useLocalSessionState=true&allowLo jdbcUser = "root" jdbcPassword = "" +ccrDownstreamUrl = "jdbc:mysql://172.19.0.2:9131/?useLocalSessionState=true&allowLoadLocalInfile=true" +ccrDownstreamUser = "root" +ccrDownstreamPassword = "" +ccrDownstreamFeThriftAddress = "127.0.0.1:9020" + feSourceThriftAddress = "127.0.0.1:9020" feTargetThriftAddress = "127.0.0.1:9020" feSyncerUser = "root" diff --git a/regression-test/pipeline/p1/conf/regression-conf.groovy b/regression-test/pipeline/p1/conf/regression-conf.groovy index b7eec21a41..517fd4b4dc 100644 --- a/regression-test/pipeline/p1/conf/regression-conf.groovy +++ b/regression-test/pipeline/p1/conf/regression-conf.groovy @@ -25,6 +25,11 @@ targetJdbcUrl = "jdbc:mysql://172.19.0.2:9132/?useLocalSessionState=true&allowLo jdbcUser = "root" jdbcPassword = "" +ccrDownstreamUrl = "jdbc:mysql://172.19.0.2:9132/?useLocalSessionState=true&allowLoadLocalInfile=true" +ccrDownstreamUser = "root" +ccrDownstreamPassword = "" +ccrDownstreamFeThriftAddress = "127.0.0.1:9020" + feSourceThriftAddress = "127.0.0.1:9020" feTargetThriftAddress = "127.0.0.1:9020" feSyncerUser = "root"