database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](information_schema) catch and skip exception when getting schema from FE catalog (#16647)

Browse Source 
When querying information_schema database, BE will call FE RPC
to get schema info such as db name list, table name list, etc.
But some external catalog when failed to get these info because of wrong connection info.
We should catch these kind of exception and skip it, so that it can continue to
get schema info of other catalogs.
Otherwise, the whole query on information_schema will fail, even if user just want to get
info of internal catalog.

And set jdbc connection timeout to 5s, to avoid thrift rpc timeout from BE to FE(default is 30s)
This commit is contained in:
Mingyu Chen
2023-02-21 08:43:09 +08:00
committed by GitHub
parent c618e69f59
commit 57519fcf50
2 changed files with 67 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java vendored
View File

@ -77,6 +77,12 @@ public class JdbcClient {
config.setUsername(jdbcUser);
config.setPassword(password);
config.setMaximumPoolSize(1);
// set connection timeout to 5s.
// The default is 30s, which is too long.
// Because when querying information_schema db, BE will call thrift rpc(default timeout is 30s)
// to FE to get schema info, and may create connection here, if we set it too long and the url is invalid,
// it may cause the thrift rpc timeout.
config.setConnectionTimeout(5000);
dataSource = new HikariDataSource(config);
} catch (MalformedURLException e) {
throw new JdbcClientException("MalformedURLException to load class about " + driverUrl, e);

112
fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
View File

@ -272,7 +272,15 @@ public class FrontendServiceImpl implements FrontendService.Iface {
.getCatalogOrException(params.catalog, catalog -> new TException("Unknown catalog " + catalog)));
}
for (CatalogIf catalog : catalogIfs) {
List<String> dbNames = catalog.getDbNamesOrEmpty();
List<String> dbNames;
try {
dbNames = catalog.getDbNamesOrEmpty();
} catch (Exception e) {
LOG.warn("failed to get database names for catalog {}", catalog.getName(), e);
// Some external catalog may fail to get databases due to wrong connection info.
// So continue here to get databases of other catalogs.
continue;
}
LOG.debug("get db names: {}, in catalog: {}", dbNames, catalog.getName());
UserIdentity currentUser = null;
@ -491,18 +499,22 @@ public class FrontendServiceImpl implements FrontendService.Iface {
.getDbNullable(params.db);
if (db != null) {
Set<String> tableNames = db.getTableNamesOrEmptyWithLock();
for (String tableName : tableNames) {
LOG.debug("get table: {}, wait to check", tableName);
if (!Env.getCurrentEnv().getAccessManager()
.checkTblPriv(currentUser, params.db, tableName, PrivPredicate.SHOW)) {
continue;
Set<String> tableNames;
try {
tableNames = db.getTableNamesOrEmptyWithLock();
for (String tableName : tableNames) {
LOG.debug("get table: {}, wait to check", tableName);
if (!Env.getCurrentEnv().getAccessManager()
.checkTblPriv(currentUser, params.db, tableName, PrivPredicate.SHOW)) {
continue;
}
if (matcher != null && !matcher.match(tableName)) {
continue;
}
tablesResult.add(tableName);
}
if (matcher != null && !matcher.match(tableName)) {
continue;
}
tablesResult.add(tableName);
} catch (Exception e) {
LOG.warn("failed to get table names for db {} in catalog {}", params.db, catalogName, e);
}
}
return result;
@ -540,51 +552,50 @@ public class FrontendServiceImpl implements FrontendService.Iface {
if (catalog != null) {
DatabaseIf db = catalog.getDbNullable(params.db);
if (db != null) {
List<TableIf> tables = Lists.newArrayList();
if (!params.isSetType() || params.getType() == null || params.getType().isEmpty()) {
tables = db.getTablesOrEmpty();
} else {
switch (params.getType()) {
case "VIEW":
tables = db.getViewsOrEmpty();
break;
default:
tables = db.getTablesOrEmpty();
try {
List<TableIf> tables;
if (!params.isSetType() || params.getType() == null || params.getType().isEmpty()) {
tables = db.getTablesOrEmpty();
} else {
switch (params.getType()) {
case "VIEW":
tables = db.getViewsOrEmpty();
break;
default:
tables = db.getTablesOrEmpty();
}
}
}
for (TableIf table : tables) {
if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(currentUser, params.db,
table.getName(), PrivPredicate.SHOW)) {
continue;
}
table.readLock();
try {
for (TableIf table : tables) {
if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(currentUser, params.db,
table.getName(), PrivPredicate.SHOW)) {
continue;
}
if (matcher != null && !matcher.match(table.getName())) {
continue;
table.readLock();
try {
if (matcher != null && !matcher.match(table.getName())) {
continue;
}
long lastCheckTime = table.getLastCheckTime() <= 0 ? 0 : table.getLastCheckTime();
TTableStatus status = new TTableStatus();
status.setName(table.getName());
status.setType(table.getMysqlType());
status.setEngine(table.getEngine());
status.setComment(table.getComment());
status.setCreateTime(table.getCreateTime());
status.setLastCheckTime(lastCheckTime / 1000);
status.setUpdateTime(table.getUpdateTime() / 1000);
status.setCheckTime(lastCheckTime / 1000);
status.setCollation("utf-8");
status.setRows(table.getRowCount());
status.setDataLength(table.getDataLength());
status.setAvgRowLength(table.getAvgRowLength());
tablesResult.add(status);
} finally {
table.readUnlock();
}
long lastCheckTime = table.getLastCheckTime() <= 0 ? 0 : table.getLastCheckTime();
TTableStatus status = new TTableStatus();
status.setName(table.getName());
status.setType(table.getMysqlType());
status.setEngine(table.getEngine());
status.setComment(table.getComment());
status.setCreateTime(table.getCreateTime());
status.setLastCheckTime(lastCheckTime / 1000);
status.setUpdateTime(table.getUpdateTime() / 1000);
status.setCheckTime(lastCheckTime / 1000);
status.setCollation("utf-8");
status.setRows(table.getRowCount());
status.setDataLength(table.getDataLength());
status.setAvgRowLength(table.getAvgRowLength());
tablesResult.add(status);
} finally {
table.readUnlock();
}
} catch (Exception e) {
LOG.warn("failed to get tables for db {} in catalog {}", db.getFullName(), catalogName, e);
}
}
}
@ -646,7 +657,6 @@ public class FrontendServiceImpl implements FrontendService.Iface {
public TFeResult updateExportTaskStatus(TUpdateExportTaskStatusRequest request) throws TException {
TStatus status = new TStatus(TStatusCode.OK);
TFeResult result = new TFeResult(FrontendServiceVersion.V1, status);
return result;
}