diff --git a/be/src/common/config.h b/be/src/common/config.h index 0f3a04ccd5..debcdb67d6 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -24,6 +24,9 @@ namespace config { // Dir of custom config file CONF_String(custom_config_dir, "${DORIS_HOME}/conf"); +// Dir of jdbc drivers +CONF_String(jdbc_drivers_dir, "${DORIS_HOME}/jdbc_drivers"); + // cluster id CONF_Int32(cluster_id, "-1"); // port on which BackendService is exported diff --git a/be/src/runtime/user_function_cache.cpp b/be/src/runtime/user_function_cache.cpp index d90e8eddac..ff2f575bc2 100644 --- a/be/src/runtime/user_function_cache.cpp +++ b/be/src/runtime/user_function_cache.cpp @@ -327,9 +327,11 @@ Status UserFunctionCache::_download_lib(const std::string& url, UserFunctionCach return Status::InternalError("fail to open file"); } + std::string real_url = _get_real_url(url); + Md5Digest digest; HttpClient client; - RETURN_IF_ERROR(client.init(url)); + RETURN_IF_ERROR(client.init(real_url)); Status status; auto download_cb = [&status, &tmp_file, &fp, &digest](const void* data, size_t length) { digest.update(data, length); @@ -367,6 +369,13 @@ Status UserFunctionCache::_download_lib(const std::string& url, UserFunctionCach return Status::OK(); } +std::string UserFunctionCache::_get_real_url(const std::string& url) { + if (url.find(":/") == std::string::npos) { + return "file://" + config::jdbc_drivers_dir + "/" + url; + } + return url; +} + // entry's lock must be held Status UserFunctionCache::_load_cache_entry_internal(UserFunctionCacheEntry* entry) { RETURN_IF_ERROR(dynamic_open(entry->lib_file.c_str(), &entry->lib_handle)); diff --git a/be/src/runtime/user_function_cache.h b/be/src/runtime/user_function_cache.h index c9ea2ca189..59f0a1deb9 100644 --- a/be/src/runtime/user_function_cache.h +++ b/be/src/runtime/user_function_cache.h @@ -83,6 +83,8 @@ private: std::string _make_lib_file(int64_t function_id, const std::string& checksum, LibType type); void _destroy_cache_entry(UserFunctionCacheEntry* entry); + std::string _get_real_url(const std::string& url); + private: std::string _lib_dir; void* _current_process_handle = nullptr; diff --git a/docs/en/docs/admin-manual/config/be-config.md b/docs/en/docs/admin-manual/config/be-config.md index 3940612dd1..4d109b851d 100644 --- a/docs/en/docs/admin-manual/config/be-config.md +++ b/docs/en/docs/admin-manual/config/be-config.md @@ -1363,3 +1363,7 @@ Indicates how many tablets failed to load in the data directory. At the same tim * Description: the increased frequency of priority for remaining tasks in BlockingPriorityQueue * Default value: 512 +#### `jdbc_drivers_dir + +* Description: Default dirs to put jdbc drivers. +* Default value: `${DORIS_HOME}/jdbc_drivers` diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md index 8958f7ae8e..75bea80386 100644 --- a/docs/en/docs/admin-manual/config/fe-config.md +++ b/docs/en/docs/admin-manual/config/fe-config.md @@ -2582,3 +2582,15 @@ IsMutable：true MasterOnly：false Whether to push the filter conditions with functions down to MYSQL, when exectue query of ODBC、JDBC external tables + +#### `jdbc_drivers_dir` + +Default: `${DORIS_HOME}/jdbc_drivers`; + +IsMutable：false + +MasterOnly：false + +The default dir to put jdbc drivers. + + diff --git a/docs/en/docs/ecosystem/external-table/jdbc-of-doris.md b/docs/en/docs/ecosystem/external-table/jdbc-of-doris.md index 0f02e66168..eaee590a48 100644 --- a/docs/en/docs/ecosystem/external-table/jdbc-of-doris.md +++ b/docs/en/docs/ecosystem/external-table/jdbc-of-doris.md @@ -23,10 +23,11 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> - # JDBC External Table Of Doris + + JDBC External Table Of Doris provides Doris to access external tables through the standard interface (JDBC) of database access. External tables save the tedious data import work, allowing Doris to have the ability to access various databases, and with the help of Doris's capabilities to solve data analysis problems with external tables: 1. Support various data sources to access Doris @@ -34,6 +35,7 @@ JDBC External Table Of Doris provides Doris to access external tables through th This document mainly introduces how to use this function. + ## Instructions @@ -85,6 +87,12 @@ Parameter Description： > >If you use the local path method, the jar package that the database driver depends on, the FE and BE nodes must be placed here + + +> After 1.2.1, you can put the driver in the `jdbc_drivers` directory of FE/BE, and directly specify the file name, such as: `"driver_url" = "mysql-connector-java-5.1.47.jar "`. The system will automatically look for files in the `jdbc_drivers` directory. + + + ### Query usage ``` @@ -288,4 +296,4 @@ ALTER TABLE table_name CHARSET=utf8mb4; SET NAMES utf8mb4 ``` - + diff --git a/docs/zh-CN/docs/admin-manual/config/be-config.md b/docs/zh-CN/docs/admin-manual/config/be-config.md index c1d45e4292..bed8d275bb 100644 --- a/docs/zh-CN/docs/admin-manual/config/be-config.md +++ b/docs/zh-CN/docs/admin-manual/config/be-config.md @@ -1380,3 +1380,8 @@ load tablets from header failed, failed tablets size: xxx, path=xxx * 描述: BlockingPriorityQueue中剩余任务的优先级频率增加 * 默认值:512 + +#### `jdbc_drivers_dir + +* 描述: 存放 jdbc driver 的默认目录。 +* 默认值: `${DORIS_HOME}/jdbc_drivers` diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md index 0ba2783533..8dbb37b621 100644 --- a/docs/zh-CN/docs/admin-manual/config/fe-config.md +++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md @@ -2582,3 +2582,15 @@ SmallFileMgr 中存储的最大文件数 是否为 Master FE 节点独有的配置项：false 在ODBC、JDBC的MYSQL外部表查询时，是否将带函数的过滤条件下推到MYSQL中执行 + +#### `jdbc_drivers_dir` + +默认值：`${DORIS_HOME}/jdbc_drivers`; + +是否可以动态配置：false + +是否为 Master FE 节点独有的配置项：false + +用于存放默认的 jdbc drivers + + diff --git a/docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md b/docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md index 6104f863ae..1b299792c3 100644 --- a/docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md +++ b/docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md @@ -67,6 +67,7 @@ PROPERTIES ( "table_type"="mysql" ); ``` + 参数说明： | 参数 | 说明| @@ -81,9 +82,15 @@ PROPERTIES ( | **table** | 在Doris中建立外表时，与外部数据库相映射的表名。| | **table_type** | 在Doris中建立外表时，该表来自那个数据库。例如mysql,postgresql,sqlserver,oracle| ->**注意：** +> **注意：** > ->如果你是本地路径方式，这里数据库驱动依赖的jar包，FE、BE节点都要放置 +> 如果你是本地路径方式，这里数据库驱动依赖的jar包，FE、BE节点都要放置 + + + +> 在1.2.1及之后的版本中，可以将 driver 放到 FE/BE 的 `jdbc_drivers` 目录下，并直接指定文件名，如：`"driver_url" = "mysql-connector-java-5.1.47.jar"`。系统会自动在 `jdbc_drivers` 目录寻找文件。 + + ### 查询用法 diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java index 7c79ce19b4..fe2553f339 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java @@ -19,6 +19,7 @@ package org.apache.doris.catalog; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.proc.BaseProcResult; @@ -29,9 +30,13 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; import org.apache.commons.codec.binary.Hex; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.io.IOException; import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Map; @@ -53,8 +58,10 @@ import java.util.Map; * DROP RESOURCE "jdbc_mysql"; */ public class JdbcResource extends Resource { + private static final Logger LOG = LogManager.getLogger(JdbcResource.class); + public static final String JDBC_PROPERTIES_PREFIX = "jdbc."; - public static final String URL = "jdbc_url"; + public static final String JDBC_URL = "jdbc_url"; public static final String USER = "user"; public static final String PASSWORD = "password"; public static final String DRIVER_CLASS = "driver_class"; @@ -97,7 +104,7 @@ public class JdbcResource extends Resource { // modify properties replaceIfEffectiveValue(this.configs, DRIVER_URL, properties.get(DRIVER_URL)); replaceIfEffectiveValue(this.configs, DRIVER_CLASS, properties.get(DRIVER_CLASS)); - replaceIfEffectiveValue(this.configs, URL, properties.get(URL)); + replaceIfEffectiveValue(this.configs, JDBC_URL, properties.get(JDBC_URL)); replaceIfEffectiveValue(this.configs, USER, properties.get(USER)); replaceIfEffectiveValue(this.configs, PASSWORD, properties.get(PASSWORD)); replaceIfEffectiveValue(this.configs, TYPE, properties.get(TYPE)); @@ -110,7 +117,7 @@ public class JdbcResource extends Resource { // check properties copiedProperties.remove(DRIVER_URL); copiedProperties.remove(DRIVER_CLASS); - copiedProperties.remove(URL); + copiedProperties.remove(JDBC_URL); copiedProperties.remove(USER); copiedProperties.remove(PASSWORD); copiedProperties.remove(TYPE); @@ -123,19 +130,19 @@ public class JdbcResource extends Resource { protected void setProperties(Map properties) throws DdlException { Preconditions.checkState(properties != null); for (String key : properties.keySet()) { - if (!DRIVER_URL.equals(key) && !URL.equals(key) && !USER.equals(key) && !PASSWORD.equals(key) + if (!DRIVER_URL.equals(key) && !JDBC_URL.equals(key) && !USER.equals(key) && !PASSWORD.equals(key) && !TYPE.equals(key) && !DRIVER_CLASS.equals(key)) { throw new DdlException("JDBC resource Property of " + key + " is unknown"); } } configs = properties; - computeObjectChecksum(); checkProperties(DRIVER_URL); checkProperties(DRIVER_CLASS); - checkProperties(URL); + checkProperties(JDBC_URL); checkProperties(USER); checkProperties(PASSWORD); checkProperties(TYPE); + computeObjectChecksum(); } @Override @@ -168,10 +175,10 @@ public class JdbcResource extends Resource { // skip checking checksum when running ut return; } - + String realDriverPath = getRealDriverPath(); InputStream inputStream = null; try { - inputStream = Util.getInputStreamFromUrl(getProperty(DRIVER_URL), null, HTTP_TIMEOUT_MS, HTTP_TIMEOUT_MS); + inputStream = Util.getInputStreamFromUrl(realDriverPath, null, HTTP_TIMEOUT_MS, HTTP_TIMEOUT_MS); MessageDigest digest = MessageDigest.getInstance("MD5"); byte[] buf = new byte[4096]; int bytesRead = 0; @@ -185,11 +192,26 @@ public class JdbcResource extends Resource { String checkSum = Hex.encodeHexString(digest.digest()); configs.put(CHECK_SUM, checkSum); } catch (IOException e) { - throw new DdlException( - "compute driver checksum from url: " + getProperty(DRIVER_URL) + " meet an IOException."); + throw new DdlException("compute driver checksum from url: " + getProperty(DRIVER_URL) + + " meet an IOException: " + e.getMessage()); } catch (NoSuchAlgorithmException e) { - throw new DdlException( - "compute driver checksum from url: " + getProperty(DRIVER_URL) + " could not find algorithm."); + throw new DdlException("compute driver checksum from url: " + getProperty(DRIVER_URL) + + " could not find algorithm: " + e.getMessage()); + } + } + + private String getRealDriverPath() { + String path = getProperty(DRIVER_URL); + try { + URI uri = new URI(path); + String schema = uri.getScheme(); + if (schema == null && !path.startsWith("/")) { + return "file://" + Config.jdbc_drivers_dir + "/" + path; + } + return path; + } catch (URISyntaxException e) { + LOG.warn("invalid jdbc driver url: " + path); + return path; } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java index c2851bb03a..21c7112f88 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java @@ -104,37 +104,47 @@ public class JdbcTable extends Table { } public String getJdbcUrl() { - return jdbcUrl; + return getFromJdbcResourceOrDefault(JdbcResource.JDBC_URL, jdbcUrl); } public String getJdbcUser() { - return jdbcUser; + return getFromJdbcResourceOrDefault(JdbcResource.USER, jdbcUser); } public String getJdbcPasswd() { - return jdbcPasswd; + return getFromJdbcResourceOrDefault(JdbcResource.PASSWORD, jdbcPasswd); } public String getDriverClass() { - return driverClass; + return getFromJdbcResourceOrDefault(JdbcResource.DRIVER_CLASS, driverClass); } public String getDriverUrl() { - return driverUrl; + return getFromJdbcResourceOrDefault(JdbcResource.DRIVER_URL, driverUrl); + } + + private String getFromJdbcResourceOrDefault(String key, String defaultVal) { + if (Strings.isNullOrEmpty(resourceName)) { + return defaultVal; + } + Resource resource = Env.getCurrentEnv().getResourceMgr().getResource(resourceName); + if (resource instanceof JdbcResource) { + return ((JdbcResource) resource).getProperty(key); + } + return defaultVal; } @Override public TTableDescriptor toThrift() { TJdbcTable tJdbcTable = new TJdbcTable(); - tJdbcTable.setJdbcUrl(jdbcUrl); - tJdbcTable.setJdbcUser(jdbcUser); - tJdbcTable.setJdbcPassword(jdbcPasswd); + tJdbcTable.setJdbcUrl(getJdbcUrl()); + tJdbcTable.setJdbcUser(getJdbcUser()); + tJdbcTable.setJdbcPassword(getJdbcPasswd()); tJdbcTable.setJdbcTableName(externalTableName); - tJdbcTable.setJdbcDriverClass(driverClass); - tJdbcTable.setJdbcDriverUrl(driverUrl); + tJdbcTable.setJdbcDriverClass(getDriverClass()); + tJdbcTable.setJdbcDriverUrl(getDriverUrl()); tJdbcTable.setJdbcResourceName(resourceName); tJdbcTable.setJdbcDriverChecksum(checkSum); - TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.JDBC_TABLE, fullSchema.size(), 0, getName(), ""); tTableDescriptor.setJdbcTable(tJdbcTable); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index d6e043509d..d1745d0f89 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -118,7 +118,7 @@ public class Config extends ConfigBase { /** * plugin_dir: - * plugin install directory + * plugin install directory */ @ConfField public static String plugin_dir = System.getenv("DORIS_HOME") + "/plugins"; @@ -126,6 +126,14 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static boolean plugin_enable = true; + /** + * The default path to save jdbc drivers. + * You can put all jdbc drivers in this path, and when creating jdbc resource with only jdbc driver file name, + * Doris will find jars from this path. + */ + @ConfField + public static String jdbc_drivers_dir = System.getenv("DORIS_HOME") + "/jdbc_drivers"; + /** * The default parallelism of the load execution plan * on a single node when the broker load is submitted diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java index 488e73bd11..9b9826ab2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java @@ -79,9 +79,9 @@ public class JdbcExternalCatalog extends ExternalCatalog { } jdbcUser = properties.getOrDefault(JdbcResource.USER, ""); jdbcPasswd = properties.getOrDefault(JdbcResource.PASSWORD, ""); - jdbcUrl = properties.getOrDefault(JdbcResource.URL, ""); + jdbcUrl = properties.getOrDefault(JdbcResource.JDBC_URL, ""); handleJdbcUrl(); - properties.put(JdbcResource.URL, jdbcUrl); + properties.put(JdbcResource.JDBC_URL, jdbcUrl); driverUrl = properties.getOrDefault(JdbcResource.DRIVER_URL, ""); driverClass = properties.getOrDefault(JdbcResource.DRIVER_CLASS, ""); return properties; diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/FsBrokerTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/FsBrokerTest.java index 9274fef823..32a45cbfee 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/FsBrokerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/FsBrokerTest.java @@ -32,6 +32,7 @@ import java.io.DataOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.net.URI; public class FsBrokerTest { @@ -105,4 +106,22 @@ public class FsBrokerTest { Assert.assertEquals(-1, readBroker.lastUpdateTime); dis.close(); } + + @Test + public void test() throws Exception { + URI url = new URI("/tmp/LetsVPNHelper.sent.socket"); + System.out.println(url.getScheme()); + url = new URI("file:/tmp/LetsVPNHelper.sent.socket"); + System.out.println(url.getScheme()); + url = new URI("file:///tmp/LetsVPNHelper.sent.socket"); + System.out.println(url.getScheme()); + url = new URI("asczcsad:/tmp/LetsVPNHelper.sent.socket"); + System.out.println(url.getScheme()); + url = new URI("http:/tmp/LetsVPNHelper.sent.socket"); + System.out.println(url.getScheme()); + url = new URI("http:///tmp/LetsVPNHelper.sent.socket"); + System.out.println(url.getScheme()); + url = new URI("LetsVPNHelper.sent.socket"); + System.out.println(url.getScheme()); + } } diff --git a/regression-test/data/jdbc_p0/test_jdbc_query_pg.out b/regression-test/data/jdbc_p0/test_jdbc_query_pg.out index fb4a86e30f..45528aea12 100644 --- a/regression-test/data/jdbc_p0/test_jdbc_query_pg.out +++ b/regression-test/data/jdbc_p0/test_jdbc_query_pg.out @@ -1076,7 +1076,7 @@ true abc 0 -- !sql5 -- -1026 +0 -- !sql6 -- 1026 diff --git a/regression-test/suites/jdbc_p0/test_jdbc_query_mysql.groovy b/regression-test/suites/jdbc_p0/test_jdbc_query_mysql.groovy index a1d0b0e41e..d2fc9809c2 100644 --- a/regression-test/suites/jdbc_p0/test_jdbc_query_mysql.groovy +++ b/regression-test/suites/jdbc_p0/test_jdbc_query_mysql.groovy @@ -934,6 +934,15 @@ suite("test_jdbc_query_mysql", "p0") { order_qt_sql111 """ SELECT rank() OVER () FROM (SELECT k8 FROM $jdbcMysql57Table1 LIMIT 10) as t LIMIT 3 """ order_qt_sql112 """ SELECT k7, count(DISTINCT k8) FROM $jdbcMysql57Table1 WHERE k8 > 110 GROUP BY GROUPING SETS ((), (k7)) """ + + // test alter resource + sql """alter resource $jdbcResourceMysql57 properties("password" = "1234567")""" + test { + sql """select count(*) from $jdbcMysql57Table1""" + exception "Access denied for user" + } + sql """alter resource $jdbcResourceMysql57 properties("password" = "123456")""" + // test for type check sql """ drop table if exists ${exMysqlTypeTable} """ sql """