database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](jdbc) fix jdbc driver bug and external datasource p2 test case issue (#16033)

Browse Source 
Fix bug that when create jdbc resource with only jdbc driver file name, it will failed to do checksum
This is because we forgot the pass the full driver url to JdbcClient.

Add ResultSet.FETCH_FORWARD and set AutoCommit to false to jdbc connection, so to avoid OOM when fetching large amount of data

set useCursorFetch in jdbc url for both MySQL and PostgreSQL.

Fix some p2 external datasource bug
This commit is contained in:
Mingyu Chen
2023-01-18 17:48:06 +08:00
committed by GitHub
parent bac2adfc74
commit 4035bd83c3
10 changed files with 44 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
View File

@ -189,10 +189,10 @@ public class JdbcResource extends Resource {
// skip checking checksum when running ut
return "";
}
String fullDriverPath = getRealDriverPath(driverPath);
String fullDriverUrl = getFullDriverUrl(driverPath);
InputStream inputStream = null;
try {
inputStream = Util.getInputStreamFromUrl(fullDriverPath, null, HTTP_TIMEOUT_MS, HTTP_TIMEOUT_MS);
inputStream = Util.getInputStreamFromUrl(fullDriverUrl, null, HTTP_TIMEOUT_MS, HTTP_TIMEOUT_MS);
MessageDigest digest = MessageDigest.getInstance("MD5");
byte[] buf = new byte[4096];
int bytesRead = 0;
@ -213,7 +213,7 @@ public class JdbcResource extends Resource {
}
}
private static String getRealDriverPath(String driverUrl) {
public static String getFullDriverUrl(String driverUrl) {
try {
URI uri = new URI(driverUrl);
String schema = uri.getScheme();
@ -254,6 +254,8 @@ public class JdbcResource extends Resource {
// it will convert to Doris tinyint, not bit.
newJdbcUrl = checkJdbcUrlParam(newJdbcUrl, "yearIsDateType", "true", "false");
newJdbcUrl = checkJdbcUrlParam(newJdbcUrl, "tinyInt1isBit", "true", "false");
}
if (dbType.equals(MYSQL) || dbType.equals(POSTGRESQL)) {
newJdbcUrl = checkJdbcUrlParam(newJdbcUrl, "useCursorFetch", "false", "true");
}
return newJdbcUrl;

13
fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java vendored
View File

@ -50,29 +50,20 @@ public class JdbcClient {
private String dbType;
private String jdbcUser;
private String jdbcPasswd;
private String jdbcUrl;
private String driverUrl;
private String driverClass;
private URLClassLoader classLoader = null;
private HikariDataSource dataSource = null;
public JdbcClient(String user, String password, String jdbcUrl, String driverUrl, String driverClass) {
this.jdbcUser = user;
this.jdbcPasswd = password;
this.jdbcUrl = jdbcUrl;
this.dbType = parseDbType(jdbcUrl);
this.driverUrl = driverUrl;
this.driverClass = driverClass;
ClassLoader oldClassLoader = Thread.currentThread().getContextClassLoader();
try {
// TODO(ftw): The problem here is that the jar package is handled by FE
// and URLClassLoader may load the jar package directly into memory
URL[] urls = {new URL(driverUrl)};
URL[] urls = {new URL(JdbcResource.getFullDriverUrl(driverUrl))};
// set parent ClassLoader to null, we can achieve class loading isolation.
classLoader = URLClassLoader.newInstance(urls, null);
Thread.currentThread().setContextClassLoader(classLoader);
@ -80,7 +71,7 @@ public class JdbcClient {
config.setDriverClassName(driverClass);
config.setJdbcUrl(jdbcUrl);
config.setUsername(jdbcUser);
config.setPassword(jdbcPasswd);
config.setPassword(password);
config.setMaximumPoolSize(1);
dataSource = new HikariDataSource(config);
} catch (MalformedURLException e) {