database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](multi-catalog) add catalog comment and create time info (#18778)

Browse Source 
add catalog comment and create time info
```
create catalog hms_ctl
comment 'your comment' 
properties (
'type'='hms',
'hive.metastore.uris' = 'thrift://xx:1234' );
```
Create Time will generate when the catalog is created.

use show catalogs and show create catalog to get these info.
This commit is contained in:
slothever
2023-04-19 15:08:42 +08:00
committed by GitHub
parent 1a25f110ec
commit 93b35bbfbf
14 changed files with 121 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/cup/sql_parser.cup
View File

@ -1789,13 +1789,13 @@ create_stmt ::=
RESULT = new CreateDbStmt(ifNotExists, db, null);
:}
/* Catalog */
| KW_CREATE KW_CATALOG opt_if_not_exists:ifNotExists ident:catalogName opt_properties:properties
| KW_CREATE KW_CATALOG opt_if_not_exists:ifNotExists ident:catalogName opt_comment:comment opt_properties:properties
{:
RESULT = new CreateCatalogStmt(ifNotExists, catalogName, null, properties);
RESULT = new CreateCatalogStmt(ifNotExists, catalogName, null, properties, comment);
:}
| KW_CREATE KW_CATALOG opt_if_not_exists:ifNotExists ident:catalogName KW_WITH KW_RESOURCE ident:resourceName opt_properties:properties
| KW_CREATE KW_CATALOG opt_if_not_exists:ifNotExists ident:catalogName KW_WITH KW_RESOURCE ident:resourceName opt_comment:comment opt_properties:properties
{:
RESULT = new CreateCatalogStmt(ifNotExists, catalogName, resourceName, properties);
RESULT = new CreateCatalogStmt(ifNotExists, catalogName, resourceName, properties, comment);
:}
/* cluster */
/* KW_CREATE KW_CLUSTER ident:name opt_properties:properties KW_IDENTIFIED KW_BY STRING_LITERAL:password

6
fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogPropertyStmt.java
View File

@ -36,17 +36,23 @@ import java.util.Map;
*/
public class AlterCatalogPropertyStmt extends DdlStmt {
private final String catalogName;
private final String comment;
private final Map<String, String> newProperties;
public AlterCatalogPropertyStmt(String catalogName, Map<String, String> newProperties) {
this.catalogName = catalogName;
this.newProperties = newProperties;
this.comment = newProperties.getOrDefault("comment", "");
}
public String getCatalogName() {
return catalogName;
}
public String getComment() {
return comment;
}
public Map<String, String> getNewProperties() {
return newProperties;
}

19
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateCatalogStmt.java
View File

@ -32,24 +32,30 @@ import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Strings;
import com.google.common.collect.Maps;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.Map;
/**
* Statement for create a new catalog.
*/
public class CreateCatalogStmt extends DdlStmt {
public static final String CREATE_TIME_PROP = "create_time";
private final boolean ifNotExists;
private final String catalogName;
private final String resource;
private final String comment;
private final Map<String, String> properties;
/**
* Statement for create a new catalog.
*/
public CreateCatalogStmt(boolean ifNotExists, String catalogName, String resource, Map<String, String> properties) {
public CreateCatalogStmt(boolean ifNotExists, String catalogName, String resource, Map<String, String> properties,
String comment) {
this.ifNotExists = ifNotExists;
this.catalogName = catalogName;
this.resource = resource == null ? "" : resource;
this.resource = Strings.nullToEmpty(resource);
this.comment = Strings.nullToEmpty(comment);
this.properties = properties == null ? Maps.newHashMap() : properties;
}
@ -61,6 +67,10 @@ public class CreateCatalogStmt extends DdlStmt {
return resource;
}
public String getComment() {
return comment;
}
public Map<String, String> getProperties() {
return properties;
}
@ -82,6 +92,8 @@ public class CreateCatalogStmt extends DdlStmt {
ErrorReport.reportAnalysisException(ErrorCode.ERR_CATALOG_ACCESS_DENIED,
analyzer.getQualifiedUser(), catalogName);
}
String currentDateTime = LocalDateTime.now(ZoneId.systemDefault()).toString().replace("T", " ");
properties.put(CREATE_TIME_PROP, currentDateTime);
PropertyAnalyzer.checkCatalogProperties(properties, false);
}
@ -97,6 +109,9 @@ public class CreateCatalogStmt extends DdlStmt {
if (!Strings.isNullOrEmpty(resource)) {
stringBuilder.append(" WITH RESOURCE `").append(resource).append("`");
}
if (!Strings.isNullOrEmpty(comment)) {
stringBuilder.append("\nCOMMENT \"").append(comment).append("\"");
}
if (properties.size() > 0) {
stringBuilder.append("\nPROPERTIES (\n");
stringBuilder.append(new PrintableMap<>(properties, "=", true, true, false));

2
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java
View File

@ -31,6 +31,8 @@ public class ShowCatalogStmt extends ShowStmt {
.addColumn(new Column("CatalogName", ScalarType.createVarchar(64)))
.addColumn(new Column("Type", ScalarType.createStringType()))
.addColumn(new Column("IsCurrent", ScalarType.createStringType()))
.addColumn(new Column("CreateTime", ScalarType.createStringType()))
.addColumn(new Column("Comment", ScalarType.createStringType()))
.build();
private static final ShowResultSetMetaData META_DATA_SPECIFIC =

15
fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java
View File

@ -48,12 +48,17 @@ public class CatalogFactory {
log.setCatalogId(catalogId);
log.setCatalogName(((CreateCatalogStmt) stmt).getCatalogName());
log.setResource(((CreateCatalogStmt) stmt).getResource());
log.setComment(((CreateCatalogStmt) stmt).getComment());
log.setProps(((CreateCatalogStmt) stmt).getProperties());
} else if (stmt instanceof DropCatalogStmt) {
log.setCatalogId(catalogId);
} else if (stmt instanceof AlterCatalogPropertyStmt) {
log.setCatalogId(catalogId);
log.setNewProps(((AlterCatalogPropertyStmt) stmt).getNewProperties());
String newComment = ((AlterCatalogPropertyStmt) stmt).getComment();
if (!Strings.isNullOrEmpty(newComment)) {
log.setComment(newComment);
}
} else if (stmt instanceof AlterCatalogNameStmt) {
log.setCatalogId(catalogId);
log.setNewCatalogName(((AlterCatalogNameStmt) stmt).getNewCatalogName());
@ -70,11 +75,12 @@ public class CatalogFactory {
* create the catalog instance from catalog log.
*/
public static CatalogIf constructorFromLog(CatalogLog log) throws DdlException {
return constructorCatalog(log.getCatalogId(), log.getCatalogName(), log.getResource(), log.getProps());
return constructorCatalog(log.getCatalogId(), log.getCatalogName(), log.getResource(),
log.getComment(), log.getProps());
}
private static CatalogIf constructorCatalog(
long catalogId, String name, String resource, Map<String, String> props) throws DdlException {
private static CatalogIf constructorCatalog(long catalogId, String name, String resource, String comment,
Map<String, String> props) throws DdlException {
// get catalog type from resource or properties
String catalogType;
if (!Strings.isNullOrEmpty(resource)) {
@ -116,6 +122,9 @@ public class CatalogFactory {
default:
throw new DdlException("Unknown catalog type: " + catalogType);
}
if (catalog instanceof ExternalCatalog && !Strings.isNullOrEmpty(comment)) {
((ExternalCatalog) catalog).setComment(comment);
}
return catalog;
}
}

2
fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
View File

@ -143,4 +143,6 @@ public interface CatalogIf<T extends DatabaseIf> {
default void onClose() {
Env.getCurrentEnv().getRefreshManager().removeFromRefreshMap(getId());
}
String getComment();
}

2
fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogLog.java
View File

@ -59,6 +59,8 @@ public class CatalogLog implements Writable {
@SerializedName(value = "resource")
private String resource;
@SerializedName(value = "comment")
private String comment;
@Override
public void write(DataOutput out) throws IOException {

9
fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
View File

@ -373,6 +373,10 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
} else {
row.add("");
}
Map<String, String> props = catalog.getProperties();
String createTime = props.getOrDefault(CreateCatalogStmt.CREATE_TIME_PROP, "UNRECORDED");
row.add(createTime);
row.add(catalog.getComment());
rows.add(row);
}
@ -421,8 +425,11 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
throw new AnalysisException("No catalog found with name " + showStmt.getCatalog());
}
StringBuilder sb = new StringBuilder();
sb.append("CREATE CATALOG `").append(ClusterNamespace.getNameFromFullName(showStmt.getCatalog()))
sb.append("\nCREATE CATALOG `").append(ClusterNamespace.getNameFromFullName(showStmt.getCatalog()))
.append("`");
if (!com.google.common.base.Strings.isNullOrEmpty(catalog.getComment())) {
sb.append("\nCOMMENT \"").append(catalog.getComment()).append("\"\n");
}
if (catalog.getProperties().size() > 0) {
sb.append(" PROPERTIES (\n");
sb.append(new PrintableMap<>(catalog.getProperties(), "=", true, true, true, true));

16
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
View File

@ -82,6 +82,7 @@ public abstract class ExternalCatalog implements CatalogIf<ExternalDatabase>, Wr
protected boolean invalidCacheInInit = true;
private ExternalSchemaCache schemaCache;
private String comment;
public ExternalCatalog(long catalogId, String name) {
this.id = catalogId;
@ -251,6 +252,15 @@ public abstract class ExternalCatalog implements CatalogIf<ExternalDatabase>, Wr
return type;
}
@Override
public String getComment() {
return comment;
}
public void setComment(String comment) {
this.comment = comment;
}
@Override
public List<String> getDbNames() {
return listDatabaseNames(null);
@ -321,10 +331,16 @@ public abstract class ExternalCatalog implements CatalogIf<ExternalDatabase>, Wr
@Override
public void modifyCatalogProps(Map<String, String> props) {
modifyComment(props);
catalogProperty.modifyCatalogProps(props);
notifyPropertiesUpdated(props);
}
private void modifyComment(Map<String, String> props) {
setComment(props.getOrDefault("comment", comment));
props.remove("comment");
}
@Override
public void write(DataOutput out) throws IOException {
Text.writeString(out, GsonUtils.GSON.toJson(this));

5
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -236,6 +236,11 @@ public class InternalCatalog implements CatalogIf<Database> {
return "internal";
}
@Override
public String getComment() {
return "Doris internal catalog";
}
@Override
public String getName() {
return INTERNAL_CATALOG_NAME;

12
fe/fe-core/src/test/java/org/apache/doris/analysis/CreateCatalogStmtTest.java
View File

@ -53,11 +53,13 @@ public class CreateCatalogStmtTest {
Map<String, String> props = Maps.newHashMap();
props.put("type", "hms");
props.put("hive.metastore.uris", "thrift://localhost:9083");
CreateCatalogStmt stmt = new CreateCatalogStmt(false, "testCatalog", null, props);
CreateCatalogStmt stmt = new CreateCatalogStmt(false, "testCatalog", null, props, "comment");
stmt.analyze(analyzer);
Assert.assertEquals("testCatalog", stmt.getCatalogName());
Assert.assertNotNull(stmt.getComment());
Assert.assertNotNull(stmt.getProperties());
Assert.assertEquals(2, stmt.getProperties().size());
Assert.assertTrue(stmt.getProperties().containsKey("create_time"));
Assert.assertEquals(3, stmt.getProperties().size());
}
@Test(expected = AnalysisException.class)
@ -65,7 +67,7 @@ public class CreateCatalogStmtTest {
Map<String, String> props = Maps.newHashMap();
props.put("type", "hms");
props.put("hive.metastore.uris", "thrift://localhost:9083");
CreateCatalogStmt stmt = new CreateCatalogStmt(false, "", null, props);
CreateCatalogStmt stmt = new CreateCatalogStmt(false, "", null, props, "comment");
stmt.analyze(analyzer);
Assert.fail("no exception");
}
@ -75,7 +77,7 @@ public class CreateCatalogStmtTest {
Map<String, String> props = Maps.newHashMap();
props.put("type", "hms");
props.put("hive.metastore.uris", "thrift://localhost:9083");
CreateCatalogStmt stmt = new CreateCatalogStmt(false, InternalCatalog.INTERNAL_CATALOG_NAME, null, props);
CreateCatalogStmt stmt = new CreateCatalogStmt(false, InternalCatalog.INTERNAL_CATALOG_NAME, null, props, "comment");
stmt.analyze(analyzer);
Assert.fail("no exception");
}
@ -84,7 +86,7 @@ public class CreateCatalogStmtTest {
public void testPropsTypeException() throws UserException {
Map<String, String> props = Maps.newHashMap();
props.put("hive.metastore.uris", "thrift://localhost:9083");
CreateCatalogStmt stmt = new CreateCatalogStmt(false, InternalCatalog.INTERNAL_CATALOG_NAME, null, props);
CreateCatalogStmt stmt = new CreateCatalogStmt(false, InternalCatalog.INTERNAL_CATALOG_NAME, null, props, "comment");
stmt.analyze(analyzer);
Assert.fail("no exception");
}

4
fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java
View File

@ -30,14 +30,14 @@ public class ShowCatalogStmtTest {
ShowCatalogStmt stmt = new ShowCatalogStmt();
stmt.analyze(analyzer);
Assert.assertNull(stmt.getCatalogName());
Assert.assertEquals(4, stmt.getMetaData().getColumnCount());
Assert.assertEquals(6, stmt.getMetaData().getColumnCount());
Assert.assertEquals("SHOW CATALOGS", stmt.toSql());
stmt = new ShowCatalogStmt(null, "%hive%");
stmt.analyze(analyzer);
Assert.assertNull(stmt.getCatalogName());
Assert.assertNotNull(stmt.getPattern());
Assert.assertEquals(4, stmt.getMetaData().getColumnCount());
Assert.assertEquals(6, stmt.getMetaData().getColumnCount());
Assert.assertEquals("SHOW CATALOGS LIKE '%hive%'", stmt.toSql());
stmt = new ShowCatalogStmt("testCatalog", null);

30
fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java
View File

@ -193,6 +193,7 @@ public class CatalogMgrTest extends TestWithFeService {
@Test
public void testNormalCase() throws Exception {
String createCatalogSql = "CREATE CATALOG hms_catalog "
+ "comment 'hms comment'"
+ "properties( \"type\" = \"hms\", \"hive.metastore.uris\"=\"thrift://localhost:9083\" )";
CreateCatalogStmt createStmt = (CreateCatalogStmt) parseAndAnalyzeStmt(createCatalogSql);
mgr.createCatalog(createStmt);
@ -240,7 +241,8 @@ public class CatalogMgrTest extends TestWithFeService {
mgr.alterCatalogProps(alterPropStmt);
CatalogIf catalog = env.getCatalogMgr().getCatalog(MY_CATALOG);
Assert.assertEquals(2, catalog.getProperties().size());
// type, hive.metastore.uris and create_time
Assert.assertEquals(3, catalog.getProperties().size());
Assert.assertEquals("thrift://172.16.5.9:9083", catalog.getProperties().get("hive.metastore.uris"));
// test add property
@ -250,14 +252,14 @@ public class CatalogMgrTest extends TestWithFeService {
AlterCatalogPropertyStmt alterStmt = new AlterCatalogPropertyStmt(MY_CATALOG, alterProps2);
mgr.alterCatalogProps(alterStmt);
catalog = env.getCatalogMgr().getCatalog(MY_CATALOG);
Assert.assertEquals(4, catalog.getProperties().size());
Assert.assertEquals(5, catalog.getProperties().size());
Assert.assertEquals("service1", catalog.getProperties().get("dfs.nameservices"));
String showDetailCatalog = "SHOW CATALOG my_catalog";
ShowCatalogStmt showDetailStmt = (ShowCatalogStmt) parseAndAnalyzeStmt(showDetailCatalog);
showResultSet = mgr.showCatalogs(showDetailStmt);
Assert.assertEquals(4, showResultSet.getResultRows().size());
Assert.assertEquals(5, showResultSet.getResultRows().size());
for (List<String> row : showResultSet.getResultRows()) {
Assertions.assertEquals(2, row.size());
if (row.get(0).equalsIgnoreCase("type")) {
@ -274,7 +276,7 @@ public class CatalogMgrTest extends TestWithFeService {
Assert.assertEquals(1, showResultSet.getResultRows().size());
List<String> result = showResultSet.getResultRows().get(0);
Assertions.assertEquals("my_catalog", result.get(0));
Assertions.assertTrue(result.get(1).startsWith("CREATE CATALOG `my_catalog` PROPERTIES ("));
Assertions.assertTrue(result.get(1).startsWith("\nCREATE CATALOG `my_catalog`\nCOMMENT \"hms comment\"\n PROPERTIES ("));
testCatalogMgrPersist();
@ -326,7 +328,7 @@ public class CatalogMgrTest extends TestWithFeService {
CatalogIf hms = mgr2.getCatalog(MY_CATALOG);
properties = hms.getProperties();
Assert.assertEquals(4, properties.size());
Assert.assertEquals(5, properties.size());
Assert.assertEquals("hms", properties.get("type"));
Assert.assertEquals("thrift://172.16.5.9:9083", properties.get("hive.metastore.uris"));
@ -659,7 +661,9 @@ public class CatalogMgrTest extends TestWithFeService {
public void testAlterFileCache() throws Exception {
String catalogName = "good_hive_3";
String createCatalogSql = "CREATE CATALOG " + catalogName + " PROPERTIES (\n"
String createCatalogSql = "CREATE CATALOG " + catalogName
+ " COMMENT 'create comment'\n"
+ " PROPERTIES (\n"
+ " 'type'='hms',\n"
+ " 'hive.metastore.uris' = 'thrift://172.21.0.1:7004',\n"
+ " 'hadoop.username' = 'hive',\n"
@ -697,4 +701,18 @@ public class CatalogMgrTest extends TestWithFeService {
}
@Test
public void testCatalogWithComment() throws Exception {
ConnectContext rootCtx = createDefaultCtx();
CreateCatalogStmt catalogWithComment = (CreateCatalogStmt) parseAndAnalyzeStmt(
"create catalog hive_c comment 'create' properties('type' = 'hms', 'hive.metastore.uris' = 'thrift://192.168.0.1:9083');",
rootCtx);
env.getCatalogMgr().createCatalog(catalogWithComment);
Assertions.assertNotNull(env.getCatalogMgr().getCatalog("hive_c").getComment());
String alterComment = "ALTER CATALOG hive_c SET PROPERTIES"
+ " (\"comment\" = \"alter comment\");";
mgr.alterCatalogProps((AlterCatalogPropertyStmt) parseAndAnalyzeStmt(alterComment));
Assertions.assertEquals(env.getCatalogMgr().getCatalog("hive_c").getComment(), "alter comment");
}
}

28
fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
View File

@ -237,10 +237,10 @@ public class PropertyConverterTest extends TestWithFeService {
CreateCatalogStmt analyzedStmt = createStmt(queryOld);
HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_s3_old");
Map<String, String> properties = catalog.getCatalogProperty().getProperties();
Assertions.assertEquals(properties.size(), 6);
Assertions.assertEquals(properties.size(), 7);
Map<String, String> hdProps = catalog.getCatalogProperty().getHadoopProperties();
Assertions.assertEquals(hdProps.size(), 19);
Assertions.assertEquals(hdProps.size(), 20);
}
@Test
@ -255,10 +255,10 @@ public class PropertyConverterTest extends TestWithFeService {
CreateCatalogStmt analyzedStmt = createStmt(query);
HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_s3");
Map<String, String> properties = catalog.getCatalogProperty().getProperties();
Assertions.assertEquals(properties.size(), 9);
Assertions.assertEquals(properties.size(), 10);
Map<String, String> hdProps = catalog.getCatalogProperty().getHadoopProperties();
Assertions.assertEquals(hdProps.size(), 18);
Assertions.assertEquals(hdProps.size(), 19);
}
@Test
@ -275,10 +275,10 @@ public class PropertyConverterTest extends TestWithFeService {
CreateCatalogStmt analyzedStmt = createStmt(queryOld);
HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_glue_old");
Map<String, String> properties = catalog.getCatalogProperty().getProperties();
Assertions.assertEquals(properties.size(), 18);
Assertions.assertEquals(properties.size(), 19);
Map<String, String> hdProps = catalog.getCatalogProperty().getHadoopProperties();
Assertions.assertEquals(hdProps.size(), 27);
Assertions.assertEquals(hdProps.size(), 28);
String query = "create catalog hms_glue properties (\n"
+ " 'type'='hms',\n"
@ -291,10 +291,10 @@ public class PropertyConverterTest extends TestWithFeService {
CreateCatalogStmt analyzedStmtNew = createStmt(query);
HMSExternalCatalog catalogNew = createAndGetCatalog(analyzedStmtNew, "hms_glue");
Map<String, String> propertiesNew = catalogNew.getCatalogProperty().getProperties();
Assertions.assertEquals(propertiesNew.size(), 18);
Assertions.assertEquals(propertiesNew.size(), 19);
Map<String, String> hdPropsNew = catalogNew.getCatalogProperty().getHadoopProperties();
Assertions.assertEquals(hdPropsNew.size(), 27);
Assertions.assertEquals(hdPropsNew.size(), 28);
}
@Test
@ -309,10 +309,10 @@ public class PropertyConverterTest extends TestWithFeService {
CreateCatalogStmt analyzedStmt = createStmt(query);
HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_obs");
Map<String, String> properties = catalog.getCatalogProperty().getProperties();
Assertions.assertEquals(properties.size(), 9);
Assertions.assertEquals(properties.size(), 10);
Map<String, String> hdProps = catalog.getCatalogProperty().getHadoopProperties();
Assertions.assertEquals(hdProps.size(), 13);
Assertions.assertEquals(hdProps.size(), 14);
}
@Test
@ -328,10 +328,10 @@ public class PropertyConverterTest extends TestWithFeService {
CreateCatalogStmt analyzedStmt = createStmt(query);
HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_cos");
Map<String, String> properties = catalog.getCatalogProperty().getProperties();
Assertions.assertEquals(properties.size(), 10);
Assertions.assertEquals(properties.size(), 11);
Map<String, String> hdProps = catalog.getCatalogProperty().getHadoopProperties();
Assertions.assertEquals(hdProps.size(), 22);
Assertions.assertEquals(hdProps.size(), 23);
}
@Test
@ -346,10 +346,10 @@ public class PropertyConverterTest extends TestWithFeService {
CreateCatalogStmt analyzedStmt = createStmt(query);
HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_oss");
Map<String, String> properties = catalog.getCatalogProperty().getProperties();
Assertions.assertEquals(properties.size(), 9);
Assertions.assertEquals(properties.size(), 10);
Map<String, String> hdProps = catalog.getCatalogProperty().getHadoopProperties();
Assertions.assertEquals(hdProps.size(), 21);
Assertions.assertEquals(hdProps.size(), 22);
}
private static HMSExternalCatalog createAndGetCatalog(CreateCatalogStmt analyzedStmt, String name)