From 26ed4b69b1b4f381e94fbe1250fe91786e47c7f9 Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Mon, 18 Mar 2024 15:22:36 +0800 Subject: [PATCH] [opt](jdbc catalog) filter jdbc datasource internal database (#32294) --- .../doris/datasource/jdbc/client/JdbcClient.java | 12 ++++++++++++ .../datasource/jdbc/client/JdbcMySQLClient.java | 9 +++++++++ .../datasource/jdbc/client/JdbcOracleClient.java | 16 ++++++++++++++++ .../jdbc/test_mysql_jdbc_catalog.out | 5 +---- .../jdbc/test_mysql_jdbc_driver5_catalog.out | 4 ---- .../jdbc/test_sqlserver_jdbc_catalog.out | 1 - 6 files changed, 38 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java index a211c0d0e3..7e44c89b37 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java @@ -26,6 +26,7 @@ import org.apache.doris.common.util.Util; import org.apache.doris.datasource.jdbc.JdbcIdentifierMapping; import com.alibaba.druid.pool.DruidDataSource; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import lombok.Data; import lombok.Getter; @@ -44,6 +45,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.Consumer; @Getter @@ -400,6 +402,7 @@ public abstract class JdbcClient { } protected List filterDatabaseNames(List remoteDbNames) { + Set filterInternalDatabases = getFilterInternalDatabases(); List filteredDatabaseNames = Lists.newArrayList(); for (String databaseName : remoteDbNames) { if (isOnlySpecifiedDatabase) { @@ -410,11 +413,20 @@ public abstract class JdbcClient { continue; } } + if (filterInternalDatabases.contains(databaseName.toLowerCase())) { + continue; + } filteredDatabaseNames.add(databaseName); } return jdbcLowerCaseMetaMatching.setDatabaseNameMapping(filteredDatabaseNames); } + protected Set getFilterInternalDatabases() { + return ImmutableSet.builder() + .add("information_schema") + .build(); + } + protected List filterTableNames(String remoteDbName, List remoteTableNames) { return jdbcLowerCaseMetaMatching.setTableNameMapping(remoteDbName, remoteTableNames); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java index 0859f49a9d..0eb6c4c8bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.util.Util; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -33,6 +34,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; public class JdbcMySQLClient extends JdbcClient { @@ -160,6 +162,13 @@ public class JdbcMySQLClient extends JdbcClient { return tableSchema; } + protected Set getFilterInternalDatabases() { + return ImmutableSet.builder() + .add("mysql") + .add("sys") + .build(); + } + @Override protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { // For Doris type diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java index 4afec31c9d..8406df7c33 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.util.Util; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import java.sql.Connection; @@ -28,6 +29,7 @@ import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; +import java.util.Set; public class JdbcOracleClient extends JdbcClient { @@ -139,6 +141,20 @@ public class JdbcOracleClient extends JdbcClient { return !modifiedTableName.equals(actualTableName); } + @Override + protected Set getFilterInternalDatabases() { + return ImmutableSet.builder() + .add("ctxsys") + .add("flows_files") + .add("mdsys") + .add("outln") + .add("sys") + .add("system") + .add("xdb") + .add("xs$null") + .build(); + } + @Override protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { String oracleType = fieldSchema.getDataTypeName(); diff --git a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out index e2e853d7cf..95b2ac788f 100644 --- a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out @@ -284,10 +284,8 @@ Doris doris information_schema init_db -mysql performance_schema show_test_do_not_modify -sys -- !specified_database_4 -- information_schema @@ -428,10 +426,8 @@ doris_3 doris_test information_schema init_db -mysql performance_schema show_test_do_not_modify -sys -- !sql -- doris_1 @@ -446,3 +442,4 @@ Doris -- !sql -- doris + diff --git a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_driver5_catalog.out b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_driver5_catalog.out index 2665f803a6..9dc2c6d02c 100644 --- a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_driver5_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_driver5_catalog.out @@ -284,10 +284,8 @@ Doris doris information_schema init_db -mysql performance_schema show_test_do_not_modify -sys -- !specified_database_4 -- information_schema @@ -428,10 +426,8 @@ doris_3 doris_test information_schema init_db -mysql performance_schema show_test_do_not_modify -sys -- !sql -- doris_1 diff --git a/regression-test/data/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.out index 3b9c571e59..b7c6c7b793 100644 --- a/regression-test/data/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.out @@ -115,7 +115,6 @@ timestamp_col TEXT Yes true \N 1 -- !sql -- -INFORMATION_SCHEMA db_accessadmin db_backupoperator db_datareader