database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](info_db) do not fetch external catalog's info in information_schema (#25844)

Browse Source 
There is FE config `infodb_support_ext_catalog`, the default is false.
Which means that the tables in `information_schema` database will not return info of external catalog.
Because if there are too many external catalogs in Doris with lots of db/tbl (like running p0 regression tests),
querying infomation_schema db will take a long time and may causing rpc timeout.

And there is an unresolved issue that if thrift rpc timeout, the BE may be crashed in ASAN mode.
So to avoid this issue(not fix yet), this PR mainly changes:

if `infodb_support_ext_catalog` is false,
1. query info of external catalog in information_schema db is not allowed, such as

	show database like "external_catalog";
	show tables like "xxx"

2. select * from information_schema.tbl will not contains external catalogs' info

3. For external p0 regression test pipeline, set `infodb_support_ext_catalog` to true to run the tests related to external catalog
This commit is contained in:
Mingyu Chen
2023-10-25 21:34:36 +08:00
committed by GitHub
parent 018abdb679
commit 04a70ecaf0
2 changed files with 37 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
View File

@ -378,12 +378,23 @@ public class FrontendServiceImpl implements FrontendService.Iface {
Env env = Env.getCurrentEnv();
List<CatalogIf> catalogIfs = Lists.newArrayList();
if (Strings.isNullOrEmpty(params.catalog)) {
catalogIfs = env.getCatalogMgr().listCatalogs();
// If infodb_support_ext_catalog is true, we will list all catalogs or the specified catalog.
// Otherwise, we will only list internal catalog, or if the specified catalog is internal catalog.
if (Config.infodb_support_ext_catalog) {
if (Strings.isNullOrEmpty(params.catalog)) {
catalogIfs = env.getCatalogMgr().listCatalogs();
} else {
catalogIfs.add(env.getCatalogMgr()
.getCatalogOrException(params.catalog,
catalog -> new TException("Unknown catalog " + catalog)));
}
} else {
catalogIfs.add(env.getCatalogMgr()
.getCatalogOrException(params.catalog, catalog -> new TException("Unknown catalog " + catalog)));
if (Strings.isNullOrEmpty(params.catalog)
|| params.catalog.equalsIgnoreCase(InternalCatalog.INTERNAL_CATALOG_NAME)) {
catalogIfs.add(env.getInternalCatalog());
}
}
for (CatalogIf catalog : catalogIfs) {
Collection<DatabaseIf> dbs = new HashSet<DatabaseIf>();
try {
@ -621,6 +632,11 @@ public class FrontendServiceImpl implements FrontendService.Iface {
}
String catalogName = Strings.isNullOrEmpty(params.catalog) ? InternalCatalog.INTERNAL_CATALOG_NAME
: params.catalog;
if (!Config.infodb_support_ext_catalog
&& !catalogName.equalsIgnoreCase(InternalCatalog.INTERNAL_CATALOG_NAME)) {
throw new TException("Not support getting external catalog info when "
+ "infodb_support_ext_catalog is false");
}
DatabaseIf<TableIf> db = Env.getCurrentEnv().getCatalogMgr()
.getCatalogOrException(catalogName, catalog -> new TException("Unknown catalog " + catalog))
@ -676,6 +692,12 @@ public class FrontendServiceImpl implements FrontendService.Iface {
if (params.isSetCatalog()) {
catalogName = params.catalog;
}
if (!Config.infodb_support_ext_catalog
&& !catalogName.equalsIgnoreCase(InternalCatalog.INTERNAL_CATALOG_NAME)) {
throw new TException("Not support getting external catalog info when "
+ "infodb_support_ext_catalog is false");
}
CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName);
if (catalog != null) {
DatabaseIf db = catalog.getDbNullable(params.db);
@ -731,7 +753,6 @@ public class FrontendServiceImpl implements FrontendService.Iface {
}
public TListTableMetadataNameIdsResult listTableMetadataNameIds(TGetTablesParams params) throws TException {
LOG.debug("get list simple table request: {}", params);
TListTableMetadataNameIdsResult result = new TListTableMetadataNameIdsResult();
@ -890,6 +911,11 @@ public class FrontendServiceImpl implements FrontendService.Iface {
String catalogName = Strings.isNullOrEmpty(params.catalog) ? InternalCatalog.INTERNAL_CATALOG_NAME
: params.catalog;
if (!Config.infodb_support_ext_catalog
&& !catalogName.equalsIgnoreCase(InternalCatalog.INTERNAL_CATALOG_NAME)) {
throw new TException("Not support getting external catalog info when "
+ "infodb_support_ext_catalog is false");
}
DatabaseIf<TableIf> db = Env.getCurrentEnv().getCatalogMgr()
.getCatalogOrException(catalogName, catalog -> new TException("Unknown catalog " + catalog))
.getDbNullable(params.db);
@ -960,6 +986,11 @@ public class FrontendServiceImpl implements FrontendService.Iface {
String catalogName = Strings.isNullOrEmpty(params.catalog) ? InternalCatalog.INTERNAL_CATALOG_NAME
: params.catalog;
if (!Config.infodb_support_ext_catalog
&& !catalogName.equalsIgnoreCase(InternalCatalog.INTERNAL_CATALOG_NAME)) {
throw new TException("Not support getting external catalog info when "
+ "infodb_support_ext_catalog is false");
}
DatabaseIf<TableIf> db = Env.getCurrentEnv().getCatalogMgr()
.getCatalogOrException(catalogName, catalog -> new TException("Unknown catalog " + catalog))
.getDbNullable(params.db);

1
regression-test/pipeline/external/conf/fe.conf vendored
View File

@ -87,3 +87,4 @@ dynamic_partition_check_interval_seconds=3
enable_feature_binlog=true
auth_token = 5ff161c3-2c08-4079-b108-26c8850b6598
infodb_support_ext_catalog=true