database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](multi-catalog) fix image loading failture when create catalog with resource (#15692)

Browse Source 
Bug fix
fix image loading failture when create catalog with resource
When creating jdbc catalog with resource, the metadata image will failed to be loaded.
Because when loading jdbc catalog image, it will try to get resource from ResourceMgr,
but ResourceMgr has not been loaded, so NPE will be thrown.

This PR fix this bug, and refactor some logic about catalog and resource.

When loading jdbc catalog image, it will not get resource from ResourceMgr.
And now user can create catalog with resource and properties, like:

create catalog jdbc_catalog with resource jdbc_resource
properites("user" = "user1");
The properties in "properties" clause will overwrite the properties in "jdbc_resource".

force adding tinyInt1isBit=false to jdbc url
The default value of tinyInt1isBit is true, and it will cause tinyint in mysql to be bit type.
force adding tinyInt1isBit=false to jdbc url so that the tinyint in mysql will be tinyint in Doris.

Avoid calculate checksum of jdbc driver jar multiple times
Refactor
Refactor the notification logic when updating properties in resource.
When updating properties in resource, it will notify the corresponding catalog to update its own properties.
This PR change this logic. After updating properties in resource, it will only uninitialize the catalog's internal
objects such "jdbc client" or "hms client". And this objects will be re-initialized lazily.

And all properties will be got from Resource at runtime, so that it will always get the latest properties

Regression test cases
Because we add tinyInt1isBit=false to jdbc url, some of cases need to be changed.
This commit is contained in:
Mingyu Chen
2023-01-09 09:56:26 +08:00
committed by GitHub
parent 1514b5ab5c
commit 211cc66d02
29 changed files with 2608 additions and 2457 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/test/java/org/apache/doris/analysis/CreateCatalogStmtTest.java
View File

@ -53,7 +53,7 @@ public class CreateCatalogStmtTest {
Map<String, String> props = Maps.newHashMap();
props.put("type", "hms");
props.put("hive.metastore.uris", "thrift://localhost:9083");
CreateCatalogStmt stmt = new CreateCatalogStmt(false, "testCatalog", props);
CreateCatalogStmt stmt = new CreateCatalogStmt(false, "testCatalog", null, props);
stmt.analyze(analyzer);
Assert.assertEquals("testCatalog", stmt.getCatalogName());
Assert.assertNotNull(stmt.getProperties());
@ -65,7 +65,7 @@ public class CreateCatalogStmtTest {
Map<String, String> props = Maps.newHashMap();
props.put("type", "hms");
props.put("hive.metastore.uris", "thrift://localhost:9083");
CreateCatalogStmt stmt = new CreateCatalogStmt(false, "", props);
CreateCatalogStmt stmt = new CreateCatalogStmt(false, "", null, props);
stmt.analyze(analyzer);
Assert.fail("no exception");
}
@ -75,7 +75,7 @@ public class CreateCatalogStmtTest {
Map<String, String> props = Maps.newHashMap();
props.put("type", "hms");
props.put("hive.metastore.uris", "thrift://localhost:9083");
CreateCatalogStmt stmt = new CreateCatalogStmt(false, InternalCatalog.INTERNAL_CATALOG_NAME, props);
CreateCatalogStmt stmt = new CreateCatalogStmt(false, InternalCatalog.INTERNAL_CATALOG_NAME, null, props);
stmt.analyze(analyzer);
Assert.fail("no exception");
}
@ -84,7 +84,7 @@ public class CreateCatalogStmtTest {
public void testPropsTypeException() throws UserException {
Map<String, String> props = Maps.newHashMap();
props.put("hive.metastore.uris", "thrift://localhost:9083");
CreateCatalogStmt stmt = new CreateCatalogStmt(false, InternalCatalog.INTERNAL_CATALOG_NAME, props);
CreateCatalogStmt stmt = new CreateCatalogStmt(false, InternalCatalog.INTERNAL_CATALOG_NAME, null, props);
stmt.analyze(analyzer);
Assert.fail("no exception");
}

14
fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java
View File

@ -33,6 +33,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.EsResource;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ResourceMgr;
import org.apache.doris.catalog.external.EsExternalDatabase;
import org.apache.doris.catalog.external.EsExternalTable;
import org.apache.doris.catalog.external.HMSExternalDatabase;
@ -40,7 +41,6 @@ import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.UserException;
import org.apache.doris.mysql.privilege.PaloAuth;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSet;
@ -68,11 +68,13 @@ public class CatalogMgrTest extends TestWithFeService {
private static UserIdentity user1;
private static UserIdentity user2;
private CatalogMgr mgr;
private ResourceMgr resourceMgr;
@Override
protected void runBeforeAll() throws Exception {
FeConstants.runningUnitTest = true;
mgr = Env.getCurrentEnv().getCatalogMgr();
resourceMgr = Env.getCurrentEnv().getResourceMgr();
ConnectContext rootCtx = createDefaultCtx();
env = Env.getCurrentEnv();
@ -188,13 +190,9 @@ public class CatalogMgrTest extends TestWithFeService {
// Can't alter catalog with resource directly
String alterCltWithResource = "ALTER CATALOG hive SET PROPERTIES"
+ " ('hive.metastore.uris' = 'thrift://192.168.0.2:9084');";
try {
mgr.alterCatalogProps((AlterCatalogPropertyStmt) parseAndAnalyzeStmt(alterCltWithResource));
Assert.fail("Can't alter catalog with resource directly");
} catch (UserException e) {
Assert.assertEquals(e.getMessage(),
"errCode = 2, detailMessage = Catalog hive has hms_resource resource, please change the resource properties directly.");
}
mgr.alterCatalogProps((AlterCatalogPropertyStmt) parseAndAnalyzeStmt(alterCltWithResource));
Assertions.assertEquals("thrift://192.168.0.2:9084",
mgr.getCatalog("hive").getProperties().get("hive.metastore.uris"));
showCatalogSql = "SHOW CATALOGS LIKE 'hms%'";
showStmt = (ShowCatalogStmt) parseAndAnalyzeStmt(showCatalogSql);