From f40ae7c654e5c5e49a916fc628846413f7b5577f Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Tue, 5 Jul 2022 13:40:24 +0800 Subject: [PATCH] [feature-wip](multi-catalog) support "show proc 'catalogs/'" (#10596) --- .../apache/doris/analysis/DescribeStmt.java | 64 ++---- .../org/apache/doris/catalog/Database.java | 95 +-------- .../org/apache/doris/catalog/DatabaseIf.java | 89 ++++++-- .../catalog/external/ExternalDatabase.java | 72 ------- .../catalog/external/HMSExternalDatabase.java | 167 +++------------ .../catalog/external/HMSExternalTable.java | 89 ++++---- .../doris/common/proc/CatalogsProcDir.java | 110 ++++++++++ .../apache/doris/common/proc/DbsProcDir.java | 23 ++- .../doris/common/proc/IndexInfoProcDir.java | 2 +- .../apache/doris/common/proc/ProcService.java | 3 +- .../apache/doris/datasource/DataSourceIf.java | 76 +++++-- .../doris/datasource/DataSourceMgr.java | 8 +- .../doris/datasource/ExternalDataSource.java | 55 ----- .../datasource/HMSExternalDataSource.java | 91 +-------- .../doris/datasource/InternalDataSource.java | 68 +------ .../org/apache/doris/qe/ShowExecutor.java | 1 - .../org/apache/doris/alter/AlterTest.java | 192 +++++++++--------- .../ShowCreateMaterializedViewStmtTest.java | 97 --------- .../doris/common/proc/DbsProcDirTest.java | 16 +- .../routineload/RoutineLoadManagerTest.java | 41 +++- 20 files changed, 507 insertions(+), 852 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateMaterializedViewStmtTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java index 48b4e4d664..015f238a31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java @@ -37,12 +37,12 @@ import org.apache.doris.common.proc.ProcResult; import org.apache.doris.common.proc.ProcService; import org.apache.doris.common.proc.TableProcDir; import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.DataSourceIf; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSetMetaData; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; @@ -50,10 +50,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedList; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; public class DescribeStmt extends ShowStmt { private static final ShowResultSetMetaData DESC_OLAP_TABLE_ALL_META_DATA = @@ -79,17 +77,6 @@ public class DescribeStmt extends ShowStmt { .addColumn(new Column("Table", ScalarType.createVarchar(30))) .build(); - // The same columns in IndexSchemaProcNode.TITLE_NAMES - private static final ShowResultSetMetaData HMS_EXTERNAL_TABLE_META_DATA = - ShowResultSetMetaData.builder() - .addColumn(new Column("Field", ScalarType.createVarchar(20))) - .addColumn(new Column("Type", ScalarType.createVarchar(20))) - .addColumn(new Column("Null", ScalarType.createVarchar(20))) - .addColumn(new Column("Key", ScalarType.createVarchar(20))) - .addColumn(new Column("Default", ScalarType.createVarchar(20))) - .addColumn(new Column("Extra", ScalarType.createVarchar(20))) - .build(); - // empty col num equals to DESC_OLAP_TABLE_ALL_META_DATA.size() private static final List EMPTY_ROW = initEmptyRow(); @@ -114,47 +101,35 @@ public class DescribeStmt extends ShowStmt { } @Override - public void analyze(Analyzer analyzer) throws AnalysisException, UserException { + public void analyze(Analyzer analyzer) throws UserException { dbTableName.analyze(analyzer); - if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv( - ConnectContext.get(), dbTableName, PrivPredicate.SHOW)) { + if (!Catalog.getCurrentCatalog().getAuth() + .checkTblPriv(ConnectContext.get(), dbTableName, PrivPredicate.SHOW)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "DESCRIBE", - ConnectContext.get().getQualifiedUser(), - ConnectContext.get().getRemoteIP(), - dbTableName.toString()); + ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(), + dbTableName.toString()); } - DatabaseIf db = Catalog.getCurrentCatalog().getDataSourceMgr() - .getCatalogOrAnalysisException(dbTableName.getCtl()).getDbOrAnalysisException(dbTableName.getDb()); + DataSourceIf ds = Catalog.getCurrentCatalog().getDataSourceMgr().getCatalog(dbTableName.getCtl()); + DatabaseIf db = ds.getDbOrAnalysisException(dbTableName.getDb()); TableIf table = db.getTableOrAnalysisException(dbTableName.getTbl()); table.readLock(); try { if (!isAllTables) { - if (table.getType() == TableType.HMS_EXTERNAL_TABLE) { - hmsSchema = table.getFullSchema().stream().map(col -> Arrays.asList( - col.getName(), - col.getType().toSql().toUpperCase(Locale.ROOT), - Boolean.toString(col.isAllowNull()), - Boolean.toString(col.isKey()), - Strings.nullToEmpty(col.getDefaultValue()), - "" /* no extra field */)) - .collect(Collectors.toList()); + // show base table schema only + String procString = "/catalogs/" + ds.getId() + "/" + db.getId() + "/" + table.getId() + "/" + + TableProcDir.INDEX_SCHEMA + "/"; + if (table.getType() == TableType.OLAP) { + procString += ((OlapTable) table).getBaseIndexId(); } else { - // show base table schema only - String procString = "/dbs/" + db.getId() + "/" + table.getId() + "/" + TableProcDir.INDEX_SCHEMA - + "/"; - if (table.getType() == TableType.OLAP) { - procString += ((OlapTable) table).getBaseIndexId(); - } else { - procString += table.getId(); - } + procString += table.getId(); + } - node = ProcService.getInstance().open(procString); - if (node == null) { - throw new AnalysisException("Describe table[" + dbTableName.getTbl() + "] failed"); - } + node = ProcService.getInstance().open(procString); + if (node == null) { + throw new AnalysisException("Describe table[" + dbTableName.getTbl() + "] failed"); } } else { Util.prohibitExternalCatalog(dbTableName.getCtl(), this.getClass().getSimpleName() + " ALL"); @@ -272,9 +247,6 @@ public class DescribeStmt extends ShowStmt { @Override public ShowResultSetMetaData getMetaData() { if (!isAllTables) { - if (hmsSchema != null) { - return HMS_EXTERNAL_TABLE_META_DATA; - } ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); ProcResult result = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java index 48a2f61194..50e4faa619 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java @@ -22,7 +22,6 @@ import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; -import org.apache.doris.common.ErrorCode; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.Pair; @@ -50,13 +49,11 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; -import javax.annotation.Nullable; /** * Internal representation of db-related metadata. Owned by Catalog instance. @@ -457,7 +454,7 @@ public class Database extends MetaObject implements Writable, DatabaseIf /** * This is a thread-safe method when nameToTable is a concurrent hash map */ - @Nullable + @Override public Table getTableNullable(String tableName) { if (Catalog.isStoredTableNamesLowerCase()) { tableName = tableName.toLowerCase(); @@ -474,99 +471,11 @@ public class Database extends MetaObject implements Writable, DatabaseIf
/** * This is a thread-safe method when idToTable is a concurrent hash map */ - @Nullable + @Override public Table getTableNullable(long tableId) { return idToTable.get(tableId); } - public Optional
getTable(String tableName) { - return Optional.ofNullable(getTableNullable(tableName)); - } - - public Optional
getTable(long tableId) { - return Optional.ofNullable(getTableNullable(tableId)); - } - - public Table getTableOrException( - String tableName, java.util.function.Function e) throws E { - Table table = getTableNullable(tableName); - if (table == null) { - throw e.apply(tableName); - } - return table; - } - - public Table getTableOrException( - long tableId, java.util.function.Function e) throws E { - Table table = getTableNullable(tableId); - if (table == null) { - throw e.apply(tableId); - } - return table; - } - - public Table getTableOrMetaException(String tableName) throws MetaNotFoundException { - return getTableOrException(tableName, t -> new MetaNotFoundException("unknown table, tableName=" + t)); - } - - public Table getTableOrMetaException(long tableId) throws MetaNotFoundException { - return getTableOrException(tableId, t -> new MetaNotFoundException("unknown table, tableId=" + t)); - } - - @SuppressWarnings("unchecked") - public Table getTableOrMetaException(String tableName, TableType tableType) throws MetaNotFoundException { - Table table = getTableOrMetaException(tableName); - if (table.getType() != tableType) { - throw new MetaNotFoundException("table type is not " - + tableType + ", tableName=" + tableName + ", type=" + table.getType()); - } - return table; - } - - @SuppressWarnings("unchecked") - public Table getTableOrMetaException(long tableId, TableType tableType) throws MetaNotFoundException { - Table table = getTableOrMetaException(tableId); - if (table.getType() != tableType) { - throw new MetaNotFoundException("table type is not " + tableType - + ", tableId=" + tableId + ", type=" + table.getType()); - } - return table; - } - - public Table getTableOrDdlException(String tableName) throws DdlException { - return getTableOrException(tableName, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t))); - } - - public OlapTable getOlapTableOrDdlException(String tableName) throws DdlException { - Table table = getTableOrDdlException(tableName); - if (!(table instanceof OlapTable)) { - throw new DdlException(ErrorCode.ERR_NOT_OLAP_TABLE.formatErrorMsg(tableName)); - } - return (OlapTable) table; - } - - public Table getTableOrDdlException(long tableId) throws DdlException { - return getTableOrException(tableId, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t))); - } - - public Table getTableOrAnalysisException(String tableName) throws AnalysisException { - return getTableOrException(tableName, - t -> new AnalysisException(ErrorCode.ERR_UNKNOWN_TABLE.formatErrorMsg(t, fullQualifiedName))); - } - - public OlapTable getOlapTableOrAnalysisException(String tableName) throws AnalysisException { - Table table = getTableOrAnalysisException(tableName); - if (!(table instanceof OlapTable)) { - throw new AnalysisException(ErrorCode.ERR_NOT_OLAP_TABLE.formatErrorMsg(tableName)); - } - return (OlapTable) table; - } - - public Table getTableOrAnalysisException(long tableId) throws AnalysisException { - return getTableOrException(tableId, - t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t))); - } - public int getMaxReplicationNum() { int ret = 0; readLock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java index caeb3862b5..9fad070fe9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java @@ -19,12 +19,14 @@ package org.apache.doris.catalog; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; +import org.apache.doris.common.ErrorCode; import org.apache.doris.common.MetaNotFoundException; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Function; /** * Database interface. @@ -76,31 +78,88 @@ public interface DatabaseIf { T getTableNullable(long tableId); - Optional getTable(String tableName); + default Optional getTable(String tableName) { + return Optional.ofNullable(getTableNullable(tableName)); + } - Optional getTable(long tableId); + default Optional getTable(long tableId) { + return Optional.ofNullable(getTableNullable(tableId)); + } - T getTableOrException(String tableName, java.util.function.Function e) throws E; + default T getTableOrException(String tableName, java.util.function.Function e) + throws E { + T table = getTableNullable(tableName); + if (table == null) { + throw e.apply(tableName); + } + return table; + } - T getTableOrException(long tableId, java.util.function.Function e) throws E; + default T getTableOrException(long tableId, Function e) throws E { + T table = getTableNullable(tableId); + if (table == null) { + throw e.apply(tableId); + } + return table; + } - T getTableOrMetaException(String tableName) throws MetaNotFoundException; + default T getTableOrMetaException(String tableName) throws MetaNotFoundException { + return getTableOrException(tableName, t -> new MetaNotFoundException("unknown table, tableName=" + t)); + } - T getTableOrMetaException(long tableId) throws MetaNotFoundException; + default T getTableOrMetaException(long tableId) throws MetaNotFoundException { + return getTableOrException(tableId, t -> new MetaNotFoundException("unknown table, tableId=" + t)); + } - @SuppressWarnings("unchecked") - T getTableOrMetaException(String tableName, Table.TableType tableType) throws MetaNotFoundException; + default T getTableOrMetaException(String tableName, TableIf.TableType tableType) throws MetaNotFoundException { + T table = getTableOrMetaException(tableName); + if (table.getType() != tableType) { + throw new MetaNotFoundException( + "table type is not " + tableType + ", tableName=" + tableName + ", type=" + table.getType()); + } + return table; + } - @SuppressWarnings("unchecked") - T getTableOrMetaException(long tableId, Table.TableType tableType) throws MetaNotFoundException; + default T getTableOrMetaException(long tableId, TableIf.TableType tableType) throws MetaNotFoundException { + T table = getTableOrMetaException(tableId); + if (table.getType() != tableType) { + throw new MetaNotFoundException( + "table type is not " + tableType + ", tableId=" + tableId + ", type=" + table.getType()); + } + return table; + } - T getTableOrDdlException(String tableName) throws DdlException; + default T getTableOrDdlException(String tableName) throws DdlException { + return getTableOrException(tableName, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t))); + } - T getTableOrDdlException(long tableId) throws DdlException; + default T getTableOrDdlException(long tableId) throws DdlException { + return getTableOrException(tableId, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t))); + } - T getTableOrAnalysisException(String tableName) throws AnalysisException; + default T getTableOrAnalysisException(String tableName) throws AnalysisException { + return getTableOrException(tableName, + t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t))); + } - T getTableOrAnalysisException(long tableId) throws AnalysisException; + default T getTableOrAnalysisException(long tableId) throws AnalysisException { + return getTableOrException(tableId, + t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t))); + } - OlapTable getOlapTableOrAnalysisException(String tableName) throws AnalysisException; + default OlapTable getOlapTableOrDdlException(String tableName) throws DdlException { + T table = getTableOrDdlException(tableName); + if (!(table instanceof OlapTable)) { + throw new DdlException(ErrorCode.ERR_NOT_OLAP_TABLE.formatErrorMsg(tableName)); + } + return (OlapTable) table; + } + + default OlapTable getOlapTableOrAnalysisException(String tableName) throws AnalysisException { + T table = getTableOrAnalysisException(tableName); + if (!(table instanceof OlapTable)) { + throw new AnalysisException(ErrorCode.ERR_NOT_OLAP_TABLE.formatErrorMsg(tableName)); + } + return (OlapTable) table; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java index dd3d78ff9d..e844da2318 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java @@ -19,9 +19,6 @@ package org.apache.doris.catalog.external; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.DatabaseProperty; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.datasource.ExternalDataSource; @@ -32,11 +29,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Function; /** * Base class of external database. @@ -177,71 +172,4 @@ public class ExternalDatabase implements DatabaseIf public T getTableNullable(long tableId) { throw new NotImplementedException(); } - - @Override - public Optional getTable(String tableName) { - throw new NotImplementedException(); - } - - @Override - public Optional getTable(long tableId) { - throw new NotImplementedException(); - } - - @Override - public T getTableOrException(String tableName, Function e) throws E { - throw new NotImplementedException(); - } - - @Override - public T getTableOrException(long tableId, Function e) throws E { - throw new NotImplementedException(); - } - - @Override - public T getTableOrMetaException(String tableName) throws MetaNotFoundException { - throw new NotImplementedException(); - } - - @Override - public T getTableOrMetaException(long tableId) throws MetaNotFoundException { - throw new NotImplementedException(); - } - - @Override - public T getTableOrMetaException(String tableName, Table.TableType tableType) - throws MetaNotFoundException { - throw new NotImplementedException(); - } - - @Override - public T getTableOrMetaException(long tableId, Table.TableType tableType) - throws MetaNotFoundException { - throw new NotImplementedException(); - } - - @Override - public T getTableOrDdlException(String tableName) throws DdlException { - throw new NotImplementedException(); - } - - @Override - public T getTableOrDdlException(long tableId) throws DdlException { - throw new NotImplementedException(); - } - - @Override - public T getTableOrAnalysisException(String tableName) throws AnalysisException { - throw new NotImplementedException(); - } - - @Override - public T getTableOrAnalysisException(long tableId) throws AnalysisException { - throw new NotImplementedException(); - } - - @Override - public OlapTable getOlapTableOrAnalysisException(String tableName) throws AnalysisException { - throw new NotImplementedException(); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java index 8a142e78bd..abe9fad243 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java @@ -17,40 +17,33 @@ package org.apache.doris.catalog.external; +import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.TableIf; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.ErrorCode; -import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.datasource.ExternalDataSource; import org.apache.doris.datasource.HMSExternalDataSource; +import com.clearspring.analytics.util.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.ArrayList; import java.util.Comparator; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Function; import java.util.stream.Collectors; /** * Hive metastore external database. */ public class HMSExternalDatabase extends ExternalDatabase { - private static final Logger LOG = LogManager.getLogger(HMSExternalDatabase.class); // Cache of table name to table id. - private ConcurrentHashMap tableNameToId = new ConcurrentHashMap<>(); - private AtomicLong nextId = new AtomicLong(0); - private final String hiveMetastoreUri; + private Map tableNameToId = Maps.newConcurrentMap(); + private Map idToTbl = Maps.newHashMap(); + private boolean initialized = false; /** * Create HMS external database. @@ -58,37 +51,33 @@ public class HMSExternalDatabase extends ExternalDatabase { * @param extDataSource External data source this database belongs to. * @param id database id. * @param name database name. - * @param uri Hive metastore uri. */ - public HMSExternalDatabase(ExternalDataSource extDataSource, long id, String name, String uri) { + public HMSExternalDatabase(ExternalDataSource extDataSource, long id, String name) { super(extDataSource, id, name); - this.hiveMetastoreUri = uri; - init(); + } + + private synchronized void makeSureInitialized() { + if (!initialized) { + init(); + initialized = true; + } } private void init() { List tableNames = extDataSource.listTableNames(null, name); if (tableNames != null) { for (String tableName : tableNames) { - tableNameToId.put(tableName, nextId.incrementAndGet()); + long tblId = Catalog.getCurrentCatalog().getNextId(); + tableNameToId.put(tableName, tblId); + idToTbl.put(tblId, new HMSExternalTable(tblId, tableName, name, (HMSExternalDataSource) extDataSource)); } } } @Override public List getTables() { - List tables = new ArrayList<>(); - List tableNames = extDataSource.listTableNames(null, name); - for (String tableName : tableNames) { - tableNameToId.putIfAbsent(tableName, nextId.incrementAndGet()); - try { - tables.add(new HMSExternalTable(tableNameToId.get(tableName), tableName, name, - (HMSExternalDataSource) extDataSource)); - } catch (MetaNotFoundException e) { - LOG.warn("Table {} in db {} not found in Hive metastore.", tableName, name, e); - } - } - return tables; + makeSureInitialized(); + return Lists.newArrayList(idToTbl.values()); } @Override @@ -99,122 +88,22 @@ public class HMSExternalDatabase extends ExternalDatabase { @Override public Set getTableNamesWithLock() { - // Doesn't need to lock because everytime we call the hive metastore api to get table names. - return new HashSet<>(extDataSource.listTableNames(null, name)); + makeSureInitialized(); + return Sets.newHashSet(tableNameToId.keySet()); } @Override public HMSExternalTable getTableNullable(String tableName) { - if (extDataSource.tableExist(null, name, tableName)) { - tableNameToId.putIfAbsent(tableName, nextId.incrementAndGet()); - try { - return new HMSExternalTable(tableNameToId.get(tableName), tableName, name, - (HMSExternalDataSource) extDataSource); - } catch (MetaNotFoundException e) { - LOG.warn("Table {} in db {} not found in Hive metastore.", tableName, name, e); - } + makeSureInitialized(); + if (!tableNameToId.containsKey(tableName)) { + return null; } - return null; + return idToTbl.get(tableNameToId.get(tableName)); } @Override public HMSExternalTable getTableNullable(long tableId) { - for (Map.Entry entry : tableNameToId.entrySet()) { - if (entry.getValue() == tableId) { - if (extDataSource.tableExist(null, name, entry.getKey())) { - try { - return new HMSExternalTable(tableId, entry.getKey(), name, - (HMSExternalDataSource) extDataSource); - } catch (MetaNotFoundException e) { - LOG.warn("Table {} in db {} not found in Hive metastore.", entry.getKey(), name, e); - } - } - return null; - } - } - return null; - } - - @Override - public Optional getTable(String tableName) { - return Optional.ofNullable(getTableNullable(tableName)); - } - - @Override - public Optional getTable(long tableId) { - return Optional.ofNullable(getTableNullable(tableId)); - } - - @Override - public HMSExternalTable getTableOrException(String tableName, Function e) - throws E { - HMSExternalTable table = getTableNullable(tableName); - if (table == null) { - throw e.apply(tableName); - } - return table; - } - - @Override - public HMSExternalTable getTableOrException(long tableId, Function e) throws E { - HMSExternalTable table = getTableNullable(tableId); - if (table == null) { - throw e.apply(tableId); - } - return table; - } - - @Override - public HMSExternalTable getTableOrMetaException(String tableName) throws MetaNotFoundException { - return getTableOrException(tableName, t -> new MetaNotFoundException("unknown table, tableName=" + t)); - } - - @Override - public HMSExternalTable getTableOrMetaException(long tableId) throws MetaNotFoundException { - return getTableOrException(tableId, t -> new MetaNotFoundException("unknown table, tableName=" + t)); - } - - @Override - public HMSExternalTable getTableOrMetaException(String tableName, TableIf.TableType tableType) - throws MetaNotFoundException { - HMSExternalTable table = getTableOrMetaException(tableName); - if (table.getType() != tableType) { - throw new MetaNotFoundException("table type is not " - + tableType + ", tableName=" + tableName + ", type=" + table.getType()); - } - return table; - } - - @Override - public HMSExternalTable getTableOrMetaException(long tableId, TableIf.TableType tableType) - throws MetaNotFoundException { - HMSExternalTable table = getTableOrMetaException(tableId); - if (table.getType() != tableType) { - throw new MetaNotFoundException("table type is not " - + tableType + ", tableId=" + tableId + ", type=" + table.getType()); - } - return table; - } - - @Override - public HMSExternalTable getTableOrDdlException(String tableName) throws DdlException { - return getTableOrException(tableName, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t))); - } - - @Override - public HMSExternalTable getTableOrDdlException(long tableId) throws DdlException { - return getTableOrException(tableId, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t))); - } - - @Override - public HMSExternalTable getTableOrAnalysisException(String tableName) throws AnalysisException { - return getTableOrException(tableName, - t -> new AnalysisException(ErrorCode.ERR_UNKNOWN_TABLE.formatErrorMsg(t, name))); - } - - @Override - public HMSExternalTable getTableOrAnalysisException(long tableId) throws AnalysisException { - return getTableOrException(tableId, - t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t))); + makeSureInitialized(); + return idToTbl.get(tableId); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java index 993a124285..49e003b904 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java @@ -26,11 +26,11 @@ import org.apache.doris.thrift.THiveTable; import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; +import com.google.common.collect.Lists; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -45,12 +45,11 @@ public class HMSExternalTable extends ExternalTable { private final HMSExternalDataSource ds; private final String dbName; private org.apache.hadoop.hive.metastore.api.Table remoteTable = null; - private DLAType dlaType = null; + private DLAType dlaType = DLAType.UNKNOWN; + private boolean initialized = false; public enum DLAType { - HIVE, - HUDI, - ICEBERG + UNKNOWN, HIVE, HUDI, ICEBERG } /** @@ -61,24 +60,59 @@ public class HMSExternalTable extends ExternalTable { * @param dbName Database name. * @param ds HMSExternalDataSource. */ - public HMSExternalTable(long id, String name, String dbName, HMSExternalDataSource ds) - throws MetaNotFoundException { + public HMSExternalTable(long id, String name, String dbName, HMSExternalDataSource ds) { super(id, name); this.dbName = dbName; this.ds = ds; this.type = TableType.HMS_EXTERNAL_TABLE; - init(); } - private void init() throws MetaNotFoundException { - getRemoteTable(); - if (remoteTable.getParameters().containsKey("table_type") - && remoteTable.getParameters().get("table_type").equalsIgnoreCase("ICEBERG")) { - dlaType = DLAType.ICEBERG; - } else if (remoteTable.getSd().getInputFormat().toLowerCase().contains("hoodie")) { - dlaType = DLAType.HUDI; + private synchronized void makeSureInitialized() { + if (!initialized) { + init(); + initialized = true; + } + } + + private void init() { + try { + getRemoteTable(); + } catch (MetaNotFoundException e) { + // CHECKSTYLE IGNORE THIS LINE + } + if (remoteTable == null) { + dlaType = DLAType.UNKNOWN; + fullSchema = Lists.newArrayList(); } else { - dlaType = DLAType.HIVE; + if (remoteTable.getParameters().containsKey("table_type") && remoteTable.getParameters().get("table_type") + .equalsIgnoreCase("ICEBERG")) { + dlaType = DLAType.ICEBERG; + } else if (remoteTable.getSd().getInputFormat().toLowerCase().contains("hoodie")) { + dlaType = DLAType.HUDI; + } else { + dlaType = DLAType.HIVE; + } + initSchema(); + } + } + + private void initSchema() { + if (fullSchema == null) { + synchronized (this) { + if (fullSchema == null) { + fullSchema = Lists.newArrayList(); + try { + for (FieldSchema field : HiveMetaStoreClientHelper.getSchema(dbName, name, + ds.getHiveMetastoreUris())) { + fullSchema.add(new Column(field.getName(), + HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType()), true, null, true, + null, field.getComment())); + } + } catch (DdlException e) { + LOG.warn("Fail to get schema of hms table {}", name, e); + } + } + } } } @@ -105,24 +139,7 @@ public class HMSExternalTable extends ExternalTable { @Override public List getFullSchema() { - if (fullSchema == null) { - synchronized (this) { - if (fullSchema == null) { - fullSchema = new ArrayList<>(); - try { - for (FieldSchema field : HiveMetaStoreClientHelper.getSchema(dbName, name, - ds.getHiveMetastoreUris())) { - fullSchema.add(new Column(field.getName(), - HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType()), true, null, true, - null, field.getComment())); - } - } catch (DdlException e) { - LOG.warn("Fail to get schema of hms table {}", name, e); - } - } - } - } - // TODO: Refresh cached fullSchema. + makeSureInitialized(); return fullSchema; } @@ -138,9 +155,7 @@ public class HMSExternalTable extends ExternalTable { @Override public Column getColumn(String name) { - if (fullSchema == null) { - getFullSchema(); - } + makeSureInitialized(); for (Column column : fullSchema) { if (name.equals(column.getName())) { return column; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java new file mode 100644 index 0000000000..5bb64062a1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.proc; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.ListComparator; +import org.apache.doris.datasource.DataSourceIf; + +import com.clearspring.analytics.util.Lists; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/* + * SHOW PROC /catalogs/ + * show all catalogs' info + */ +public class CatalogsProcDir implements ProcDirInterface { + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() + .add("CatalogIds").add("CatalogName").add("DatabaseNum") + .build(); + + private Catalog catalog; + + public CatalogsProcDir(Catalog catalog) { + this.catalog = catalog; + } + + @Override + public boolean register(String name, ProcNodeInterface node) { + return false; + } + + @Override + public ProcNodeInterface lookup(String catalogIdStr) throws AnalysisException { + if (catalog == null || Strings.isNullOrEmpty(catalogIdStr)) { + throw new AnalysisException("Catalog id is null"); + } + + long catalogId = -1L; + try { + catalogId = Long.valueOf(catalogIdStr); + } catch (NumberFormatException e) { + throw new AnalysisException("Invalid catalog id format: " + catalogIdStr); + } + + DataSourceIf ds = catalog.getDataSourceMgr().getCatalog(catalogId); + if (ds == null) { + throw new AnalysisException("Catalog " + catalogIdStr + " does not exist"); + } + + return new DbsProcDir(catalog, ds); + } + + @Override + public ProcResult fetchResult() throws AnalysisException { + Preconditions.checkNotNull(catalog); + BaseProcResult result = new BaseProcResult(); + result.setNames(TITLE_NAMES); + + List catalogIds = catalog.getDataSourceMgr().getCatalogIds(); + // get info + List> catalogInfos = Lists.newArrayList(); + for (long catalogId : catalogIds) { + DataSourceIf ds = catalog.getDataSourceMgr().getCatalog(catalogId); + if (ds == null) { + continue; + } + List catalogInfo = Lists.newArrayList(); + catalogInfo.add(ds.getId()); + catalogInfo.add(ds.getName()); + catalogInfo.add(ds.getDbNames().size()); + catalogInfos.add(catalogInfo); + } + + // order by catalogId, asc + ListComparator> comparator = new ListComparator>(0); + Collections.sort(catalogInfos, comparator); + + // set result + for (List info : catalogInfos) { + List row = new ArrayList(info.size()); + for (Comparable comparable : info) { + row.add(comparable.toString()); + } + result.addRow(row); + } + return result; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java index 8ab49edfff..50ada69d70 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java @@ -21,9 +21,11 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.datasource.DataSourceIf; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -44,9 +46,11 @@ public class DbsProcDir implements ProcDirInterface { .build(); private Catalog catalog; + private DataSourceIf ds; - public DbsProcDir(Catalog catalog) { + public DbsProcDir(Catalog catalog, DataSourceIf ds) { this.catalog = catalog; + this.ds = ds; } @Override @@ -67,7 +71,7 @@ public class DbsProcDir implements ProcDirInterface { throw new AnalysisException("Invalid db id format: " + dbIdStr); } - DatabaseIf db = catalog.getInternalDataSource().getDbNullable(dbId); + DatabaseIf db = ds.getDbNullable(dbId); if (db == null) { throw new AnalysisException("Database " + dbId + " does not exist"); } @@ -81,7 +85,7 @@ public class DbsProcDir implements ProcDirInterface { BaseProcResult result = new BaseProcResult(); result.setNames(TITLE_NAMES); - List dbNames = catalog.getInternalDataSource().getDbNames(); + List dbNames = ds.getDbNames(); if (dbNames == null || dbNames.isEmpty()) { // empty return result; @@ -90,7 +94,7 @@ public class DbsProcDir implements ProcDirInterface { // get info List> dbInfos = new ArrayList<>(); for (String dbName : dbNames) { - DatabaseIf db = catalog.getInternalDataSource().getDbNullable(dbName); + DatabaseIf db = ds.getDbNullable(dbName); if (db == null) { continue; } @@ -102,13 +106,14 @@ public class DbsProcDir implements ProcDirInterface { dbInfo.add(dbName); dbInfo.add(tableNum); - long usedDataQuota = ((Database) db).getUsedDataQuotaWithLock(); - long dataQuota = ((Database) db).getDataQuota(); + long usedDataQuota = (db instanceof Database) ? ((Database) db).getUsedDataQuotaWithLock() : 0; + long dataQuota = (db instanceof Database) ? ((Database) db).getDataQuota() : 0; String readableUsedQuota = DebugUtil.printByteWithUnit(usedDataQuota); String readableQuota = DebugUtil.printByteWithUnit(dataQuota); - String lastCheckTime = TimeUtils.longToTimeString(((Database) db).getLastCheckTime()); - long replicaCount = ((Database) db).getReplicaCountWithLock(); - long replicaQuota = ((Database) db).getReplicaQuota(); + String lastCheckTime = (db instanceof Database) ? TimeUtils.longToTimeString( + ((Database) db).getLastCheckTime()) : FeConstants.null_string; + long replicaCount = (db instanceof Database) ? ((Database) db).getReplicaCountWithLock() : 0; + long replicaQuota = (db instanceof Database) ? ((Database) db).getReplicaQuota() : 0; dbInfo.add(readableUsedQuota); dbInfo.add(readableQuota); dbInfo.add(lastCheckTime); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java index 6ab28a094b..4775ba23ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java @@ -91,7 +91,7 @@ public class IndexInfoProcDir implements ProcDirInterface { builder.toString())); } } else { - result.addRow(Lists.newArrayList("-1", table.getName(), "", "", "", "", "")); + result.addRow(Lists.newArrayList(String.valueOf(table.getId()), table.getName(), "", "", "", "", "")); } return result; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java index 59a354265b..8c21287aea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java @@ -37,7 +37,8 @@ public final class ProcService { root = new BaseProcDir(); root.register("auth", new AuthProcDir(Catalog.getCurrentCatalog().getAuth())); root.register("backends", new BackendsProcDir(Catalog.getCurrentSystemInfo())); - root.register("dbs", new DbsProcDir(Catalog.getCurrentCatalog())); + root.register("catalogs", new CatalogsProcDir(Catalog.getCurrentCatalog())); + root.register("dbs", new DbsProcDir(Catalog.getCurrentCatalog(), Catalog.getCurrentInternalCatalog())); root.register("jobs", new JobsDbProcDir(Catalog.getCurrentCatalog())); root.register("statistic", new StatisticProcNode(Catalog.getCurrentCatalog())); root.register("tasks", new TasksProcDir()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceIf.java index 93260e2746..7f0e01c2cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceIf.java @@ -20,11 +20,13 @@ package org.apache.doris.datasource; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; +import org.apache.doris.common.ErrorCode; import org.apache.doris.common.MetaNotFoundException; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; import javax.annotation.Nullable; /** @@ -50,29 +52,63 @@ public interface DataSourceIf { @Nullable T getDbNullable(long dbId); - Optional getDb(String dbName); - - Optional getDb(long dbId); - - T getDbOrException(String dbName, java.util.function.Function e) throws E; - - T getDbOrException(long dbId, java.util.function.Function e) throws E; - - T getDbOrMetaException(String dbName) throws MetaNotFoundException; - - T getDbOrMetaException(long dbId) throws MetaNotFoundException; - - T getDbOrDdlException(String dbName) throws DdlException; - - T getDbOrDdlException(long dbId) throws DdlException; - - T getDbOrAnalysisException(String dbName) throws AnalysisException; - - T getDbOrAnalysisException(long dbId) throws AnalysisException; - Map getProperties(); void modifyDatasourceName(String name); void modifyDatasourceProps(Map props); + + default Optional getDb(String dbName) { + return Optional.ofNullable(getDbNullable(dbName)); + } + + default Optional getDb(long dbId) { + return Optional.ofNullable(getDbNullable(dbId)); + } + + default T getDbOrException(String dbName, Function e) throws E { + T db = getDbNullable(dbName); + if (db == null) { + throw e.apply(dbName); + } + return db; + } + + default T getDbOrException(long dbId, Function e) throws E { + T db = getDbNullable(dbId); + if (db == null) { + throw e.apply(dbId); + } + return db; + } + + default T getDbOrMetaException(String dbName) throws MetaNotFoundException { + return getDbOrException(dbName, + s -> new MetaNotFoundException("unknown databases, dbName=" + s, ErrorCode.ERR_BAD_DB_ERROR)); + } + + default T getDbOrMetaException(long dbId) throws MetaNotFoundException { + return getDbOrException(dbId, + s -> new MetaNotFoundException("unknown databases, dbId=" + s, ErrorCode.ERR_BAD_DB_ERROR)); + } + + default T getDbOrDdlException(String dbName) throws DdlException { + return getDbOrException(dbName, + s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR)); + } + + default T getDbOrDdlException(long dbId) throws DdlException { + return getDbOrException(dbId, + s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR)); + } + + default T getDbOrAnalysisException(String dbName) throws AnalysisException { + return getDbOrException(dbName, + s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR)); + } + + default T getDbOrAnalysisException(long dbId) throws AnalysisException { + return getDbOrException(dbId, + s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR)); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceMgr.java index 45e8b00a1c..8fbe946f47 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceMgr.java @@ -116,8 +116,12 @@ public class DataSourceMgr implements Writable, GsonPostProcessable { } public DataSourceIf getCatalogOrAnalysisException(String name) throws AnalysisException { - return getCatalogOrException(name, ds -> new AnalysisException( - ErrorCode.ERR_UNKNOWN_CATALOG.formatErrorMsg(ds), ErrorCode.ERR_UNKNOWN_CATALOG)); + return getCatalogOrException(name, ds -> new AnalysisException(ErrorCode.ERR_UNKNOWN_CATALOG.formatErrorMsg(ds), + ErrorCode.ERR_UNKNOWN_CATALOG)); + } + + public List getCatalogIds() { + return Lists.newArrayList(idToCatalog.keySet()); } public DatabaseIf getDbNullable(long dbId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java index ca08ff2e67..79223498f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java @@ -19,9 +19,6 @@ package org.apache.doris.datasource; import org.apache.doris.catalog.external.ExternalDatabase; import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.persist.gson.GsonUtils; @@ -36,8 +33,6 @@ import java.io.DataOutput; import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.function.Function; /** * The abstract class for all types of external data sources. @@ -107,56 +102,6 @@ public abstract class ExternalDataSource implements DataSourceIf getDb(String dbName) { - throw new NotImplementedException(); - } - - @Override - public Optional getDb(long dbId) { - throw new NotImplementedException(); - } - - @Override - public ExternalDatabase getDbOrException(String dbName, Function e) throws E { - throw new NotImplementedException(); - } - - @Override - public ExternalDatabase getDbOrException(long dbId, Function e) throws E { - throw new NotImplementedException(); - } - - @Override - public ExternalDatabase getDbOrMetaException(String dbName) throws MetaNotFoundException { - throw new NotImplementedException(); - } - - @Override - public ExternalDatabase getDbOrMetaException(long dbId) throws MetaNotFoundException { - throw new NotImplementedException(); - } - - @Override - public ExternalDatabase getDbOrDdlException(String dbName) throws DdlException { - throw new NotImplementedException(); - } - - @Override - public ExternalDatabase getDbOrDdlException(long dbId) throws DdlException { - throw new NotImplementedException(); - } - - @Override - public ExternalDatabase getDbOrAnalysisException(String dbName) throws AnalysisException { - throw new NotImplementedException(); - } - - @Override - public ExternalDatabase getDbOrAnalysisException(long dbId) throws AnalysisException { - throw new NotImplementedException(); - } - @Override public Map getProperties() { return dsProperty.getProperties(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java index 22c5f09303..8abc4f30ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java @@ -21,10 +21,6 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.external.ExternalDatabase; import org.apache.doris.catalog.external.HMSExternalDatabase; import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.ErrorCode; -import org.apache.doris.common.MetaNotFoundException; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -38,8 +34,6 @@ import org.jetbrains.annotations.Nullable; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.function.Function; /** * External data source for hive metastore compatible data sources. @@ -48,7 +42,8 @@ public class HMSExternalDataSource extends ExternalDataSource { private static final Logger LOG = LogManager.getLogger(HMSExternalDataSource.class); // Cache of db name to db id. - private Map dbNameToId; + private Map dbNameToId = Maps.newConcurrentMap(); + private Map idToDb = Maps.newConcurrentMap(); private boolean initialized = false; protected HiveMetaStoreClient client; @@ -68,7 +63,6 @@ public class HMSExternalDataSource extends ExternalDataSource { } private void init() { - dbNameToId = Maps.newConcurrentMap(); HiveConf hiveConf = new HiveConf(); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, getHiveMetastoreUris()); try { @@ -87,8 +81,10 @@ public class HMSExternalDataSource extends ExternalDataSource { if (allDatabases == null) { return; } - for (String db : allDatabases) { - dbNameToId.put(db, Catalog.getCurrentCatalog().getNextId()); + for (String dbName : allDatabases) { + long dbId = Catalog.getCurrentCatalog().getNextId(); + dbNameToId.put(dbName, dbId); + idToDb.put(dbId, new HMSExternalDatabase(this, dbId, dbName)); } } @@ -111,7 +107,6 @@ public class HMSExternalDataSource extends ExternalDataSource { @Override public List listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); try { return client.getAllTables(getRealTableName(dbName)); } catch (MetaException e) { @@ -122,7 +117,6 @@ public class HMSExternalDataSource extends ExternalDataSource { @Override public boolean tableExist(SessionContext ctx, String dbName, String tblName) { - makeSureInitialized(); try { return client.tableExists(getRealTableName(dbName), tblName); } catch (TException e) { @@ -139,83 +133,14 @@ public class HMSExternalDataSource extends ExternalDataSource { if (!dbNameToId.containsKey(realDbName)) { return null; } - return new HMSExternalDatabase(this, dbNameToId.get(realDbName), realDbName, getHiveMetastoreUris()); + return idToDb.get(dbNameToId.get(realDbName)); } @Nullable @Override public ExternalDatabase getDbNullable(long dbId) { makeSureInitialized(); - for (Map.Entry entry : dbNameToId.entrySet()) { - if (entry.getValue() == dbId) { - return new HMSExternalDatabase(this, dbId, entry.getKey(), getHiveMetastoreUris()); - } - } - return null; - } - - @Override - public Optional getDb(String dbName) { - return Optional.ofNullable(getDbNullable(dbName)); - } - - @Override - public Optional getDb(long dbId) { - return Optional.ofNullable(getDbNullable(dbId)); - } - - @Override - public ExternalDatabase getDbOrException(String dbName, Function e) throws E { - ExternalDatabase db = getDbNullable(dbName); - if (db == null) { - throw e.apply(dbName); - } - return db; - } - - @Override - public ExternalDatabase getDbOrException(long dbId, Function e) throws E { - ExternalDatabase db = getDbNullable(dbId); - if (db == null) { - throw e.apply(dbId); - } - return db; - } - - @Override - public ExternalDatabase getDbOrMetaException(String dbName) throws MetaNotFoundException { - return getDbOrException(dbName, - s -> new MetaNotFoundException("unknown databases, dbName=" + s, ErrorCode.ERR_BAD_DB_ERROR)); - } - - @Override - public ExternalDatabase getDbOrMetaException(long dbId) throws MetaNotFoundException { - return getDbOrException(dbId, - s -> new MetaNotFoundException("unknown databases, dbId=" + s, ErrorCode.ERR_BAD_DB_ERROR)); - } - - @Override - public ExternalDatabase getDbOrDdlException(String dbName) throws DdlException { - return getDbOrException(dbName, - s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR)); - } - - @Override - public ExternalDatabase getDbOrDdlException(long dbId) throws DdlException { - return getDbOrException(dbId, - s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR)); - } - - @Override - public ExternalDatabase getDbOrAnalysisException(String dbName) throws AnalysisException { - return getDbOrException(dbName, - s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR)); - } - - @Override - public ExternalDatabase getDbOrAnalysisException(long dbId) throws AnalysisException { - return getDbOrException(dbId, - s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR)); + return idToDb.get(dbId); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java index 1e355c9e8b..90797a0d66 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java @@ -187,11 +187,9 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import java.util.function.Function; /** * The Internal data source will manage all self-managed meta object in a Doris cluster. @@ -263,70 +261,6 @@ public class InternalDataSource implements DataSourceIf { return idToDb.get(dbId); } - @Override - public Optional getDb(String dbName) { - return Optional.ofNullable(getDbNullable(dbName)); - } - - @Override - public Optional getDb(long dbId) { - return Optional.ofNullable(getDbNullable(dbId)); - } - - @Override - public Database getDbOrException(String dbName, Function e) throws E { - Database db = getDbNullable(dbName); - if (db == null) { - throw e.apply(dbName); - } - return db; - } - - @Override - public Database getDbOrException(long dbId, Function e) throws E { - Database db = getDbNullable(dbId); - if (db == null) { - throw e.apply(dbId); - } - return db; - } - - @Override - public Database getDbOrMetaException(String dbName) throws MetaNotFoundException { - return getDbOrException(dbName, - s -> new MetaNotFoundException("unknown databases, dbName=" + s, ErrorCode.ERR_BAD_DB_ERROR)); - } - - @Override - public Database getDbOrMetaException(long dbId) throws MetaNotFoundException { - return getDbOrException(dbId, - s -> new MetaNotFoundException("unknown databases, dbId=" + s, ErrorCode.ERR_BAD_DB_ERROR)); - } - - @Override - public Database getDbOrDdlException(String dbName) throws DdlException { - return getDbOrException(dbName, - s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR)); - } - - @Override - public Database getDbOrDdlException(long dbId) throws DdlException { - return getDbOrException(dbId, - s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR)); - } - - @Override - public Database getDbOrAnalysisException(String dbName) throws AnalysisException { - return getDbOrException(dbName, - s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR)); - } - - @Override - public Database getDbOrAnalysisException(long dbId) throws AnalysisException { - return getDbOrException(dbId, - s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR)); - } - @Override public Map getProperties() { return Maps.newHashMap(); @@ -734,7 +668,7 @@ public class InternalDataSource implements DataSourceIf { String dbName = recoverStmt.getDbName(); String tableName = recoverStmt.getTableName(); - Database db = (Database) getDbOrDdlException(dbName); + Database db = getDbOrDdlException(dbName); OlapTable olapTable = db.getOlapTableOrDdlException(tableName); olapTable.writeLockOrDdlException(); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 006d27008e..7f702e83af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -909,7 +909,6 @@ public class ShowExecutor { ShowIndexStmt showStmt = (ShowIndexStmt) stmt; List> rows = Lists.newArrayList(); DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName()); - OlapTable table = db.getOlapTableOrAnalysisException(showStmt.getTableName().getTbl()); table.readLock(); try { diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java index dadaa4eeeb..3f056f5d9b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java @@ -19,11 +19,13 @@ package org.apache.doris.alter; import org.apache.doris.analysis.AlterTableStmt; import org.apache.doris.analysis.CreateDbStmt; +import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.analysis.CreatePolicyStmt; import org.apache.doris.analysis.CreateResourceStmt; import org.apache.doris.analysis.CreateTableStmt; import org.apache.doris.analysis.DateLiteral; import org.apache.doris.analysis.DropResourceStmt; +import org.apache.doris.analysis.ShowCreateMaterializedViewStmt; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DataProperty; @@ -38,11 +40,14 @@ import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletInvertedIndex; import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.FeConstants; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowExecutor; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.utframe.UtFrameUtils; @@ -138,118 +143,63 @@ public class AlterTest { + " 'storage_cooldown_time' = '2999-12-31 00:00:00'\n" + ");"); - createTable("CREATE TABLE test.tbl5\n" - + "(\n" - + " k1 date,\n" - + " k2 int,\n" - + " v1 int \n" - + ") ENGINE=OLAP\n" - + "UNIQUE KEY (k1,k2)\n" - + "PARTITION BY RANGE(k1)\n" - + "(\n" + createTable("CREATE TABLE test.tbl5\n" + "(\n" + " k1 date,\n" + " k2 int,\n" + " v1 int \n" + + ") ENGINE=OLAP\n" + "UNIQUE KEY (k1,k2)\n" + "PARTITION BY RANGE(k1)\n" + "(\n" + " PARTITION p1 values less than('2020-02-01'),\n" - + " PARTITION p2 values less than('2020-03-01')\n" - + ")\n" - + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + " PARTITION p2 values less than('2020-03-01')\n" + ")\n" + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + "PROPERTIES('replication_num' = '1');"); - createTable("CREATE TABLE test.tbl6\n" - + "(\n" - + " k1 datetime(3),\n" - + " k2 time(3),\n" - + " v1 int \n," - + " v2 datetime(3)\n" - + ") ENGINE=OLAP\n" - + "UNIQUE KEY (k1,k2)\n" - + "PARTITION BY RANGE(k1)\n" - + "(\n" - + " PARTITION p1 values less than('2020-02-01 00:00:00'),\n" - + " PARTITION p2 values less than('2020-03-01 00:00:00')\n" - + ")\n" - + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" - + "PROPERTIES('replication_num' = '1');"); + createTable( + "CREATE TABLE test.tbl6\n" + "(\n" + " k1 datetime(3),\n" + " k2 time(3),\n" + " v1 int \n," + + " v2 datetime(3)\n" + ") ENGINE=OLAP\n" + "UNIQUE KEY (k1,k2)\n" + + "PARTITION BY RANGE(k1)\n" + "(\n" + + " PARTITION p1 values less than('2020-02-01 00:00:00'),\n" + + " PARTITION p2 values less than('2020-03-01 00:00:00')\n" + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + "PROPERTIES('replication_num' = '1');"); - createTable("create external table test.odbc_table\n" - + "( `k1` bigint(20) COMMENT \"\",\n" - + " `k2` datetime COMMENT \"\",\n" - + " `k3` varchar(20) COMMENT \"\",\n" - + " `k4` varchar(100) COMMENT \"\",\n" - + " `k5` float COMMENT \"\"\n" - + ")ENGINE=ODBC\n" - + "PROPERTIES (\n" - + "\"host\" = \"127.0.0.1\",\n" - + "\"port\" = \"3306\",\n" - + "\"user\" = \"root\",\n" - + "\"password\" = \"123\",\n" - + "\"database\" = \"db1\",\n" - + "\"table\" = \"tbl1\",\n" - + "\"driver\" = \"Oracle Driver\",\n" - + "\"odbc_type\" = \"oracle\"\n" - + ");"); + createTable("create external table test.odbc_table\n" + "( `k1` bigint(20) COMMENT \"\",\n" + + " `k2` datetime COMMENT \"\",\n" + " `k3` varchar(20) COMMENT \"\",\n" + + " `k4` varchar(100) COMMENT \"\",\n" + " `k5` float COMMENT \"\"\n" + ")ENGINE=ODBC\n" + + "PROPERTIES (\n" + "\"host\" = \"127.0.0.1\",\n" + "\"port\" = \"3306\",\n" + "\"user\" = \"root\",\n" + + "\"password\" = \"123\",\n" + "\"database\" = \"db1\",\n" + "\"table\" = \"tbl1\",\n" + + "\"driver\" = \"Oracle Driver\",\n" + "\"odbc_type\" = \"oracle\"\n" + ");"); // s3 resource - createRemoteStorageResource("create resource \"remote_s3\"\n" - + "properties\n" - + "(\n" - + " \"type\" = \"s3\", \n" - + " \"s3_endpoint\" = \"bj\",\n" - + " \"s3_region\" = \"bj\",\n" - + " \"s3_root_path\" = \"/path/to/root\",\n" - + " \"s3_access_key\" = \"bbb\",\n" - + " \"s3_secret_key\" = \"aaaa\",\n" - + " \"s3_max_connections\" = \"50\",\n" - + " \"s3_request_timeout_ms\" = \"3000\",\n" - + " \"s3_connection_timeout_ms\" = \"1000\"\n" - + ");"); + createRemoteStorageResource( + "create resource \"remote_s3\"\n" + "properties\n" + "(\n" + " \"type\" = \"s3\", \n" + + " \"s3_endpoint\" = \"bj\",\n" + " \"s3_region\" = \"bj\",\n" + + " \"s3_root_path\" = \"/path/to/root\",\n" + " \"s3_access_key\" = \"bbb\",\n" + + " \"s3_secret_key\" = \"aaaa\",\n" + " \"s3_max_connections\" = \"50\",\n" + + " \"s3_request_timeout_ms\" = \"3000\",\n" + " \"s3_connection_timeout_ms\" = \"1000\"\n" + + ");"); - createRemoteStorageResource("create resource \"remote_s3_1\"\n" - + "properties\n" - + "(\n" - + " \"type\" = \"s3\", \n" - + " \"s3_endpoint\" = \"bj\",\n" - + " \"s3_region\" = \"bj\",\n" - + " \"s3_root_path\" = \"/path/to/root\",\n" - + " \"s3_access_key\" = \"bbb\",\n" - + " \"s3_secret_key\" = \"aaaa\",\n" - + " \"s3_max_connections\" = \"50\",\n" - + " \"s3_request_timeout_ms\" = \"3000\",\n" - + " \"s3_connection_timeout_ms\" = \"1000\"\n" - + ");"); + createRemoteStorageResource( + "create resource \"remote_s3_1\"\n" + "properties\n" + "(\n" + " \"type\" = \"s3\", \n" + + " \"s3_endpoint\" = \"bj\",\n" + " \"s3_region\" = \"bj\",\n" + + " \"s3_root_path\" = \"/path/to/root\",\n" + " \"s3_access_key\" = \"bbb\",\n" + + " \"s3_secret_key\" = \"aaaa\",\n" + " \"s3_max_connections\" = \"50\",\n" + + " \"s3_request_timeout_ms\" = \"3000\",\n" + " \"s3_connection_timeout_ms\" = \"1000\"\n" + + ");"); - createRemoteStoragePolicy("CREATE STORAGE POLICY testPolicy\n" - + "PROPERTIES(\n" - + " \"storage_resource\" = \"remote_s3\",\n" - + " \"cooldown_datetime\" = \"2100-05-10 00:00:00\"\n" - + ");"); + createRemoteStoragePolicy( + "CREATE STORAGE POLICY testPolicy\n" + "PROPERTIES(\n" + " \"storage_resource\" = \"remote_s3\",\n" + + " \"cooldown_datetime\" = \"2100-05-10 00:00:00\"\n" + ");"); - createRemoteStoragePolicy("CREATE STORAGE POLICY testPolicy2\n" - + "PROPERTIES(\n" - + " \"storage_resource\" = \"remote_s3\",\n" - + " \"cooldown_ttl\" = \"1d\"\n" - + ");"); + createRemoteStoragePolicy( + "CREATE STORAGE POLICY testPolicy2\n" + "PROPERTIES(\n" + " \"storage_resource\" = \"remote_s3\",\n" + + " \"cooldown_ttl\" = \"1d\"\n" + ");"); - createTable("CREATE TABLE test.tbl_remote\n" - + "(\n" - + " k1 date,\n" - + " k2 int,\n" - + " v1 int sum\n" - + ")\n" - + "PARTITION BY RANGE(k1)\n" - + "(\n" - + " PARTITION p1 values less than('2020-02-01'),\n" + createTable("CREATE TABLE test.tbl_remote\n" + "(\n" + " k1 date,\n" + " k2 int,\n" + " v1 int sum\n" + + ")\n" + "PARTITION BY RANGE(k1)\n" + "(\n" + " PARTITION p1 values less than('2020-02-01'),\n" + " PARTITION p2 values less than('2020-03-01'),\n" + " PARTITION p3 values less than('2020-04-01'),\n" - + " PARTITION p4 values less than('2020-05-01')\n" - + ")\n" - + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" - + "PROPERTIES" - + "(" - + " 'replication_num' = '1',\n" - + " 'in_memory' = 'false',\n" - + " 'storage_medium' = 'SSD',\n" - + " 'storage_cooldown_time' = '2100-05-09 00:00:00',\n" - + " 'remote_storage_policy' = 'testPolicy'\n" - + ");"); + + " PARTITION p4 values less than('2020-05-01')\n" + ")\n" + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES" + "(" + " 'replication_num' = '1',\n" + " 'in_memory' = 'false',\n" + + " 'storage_medium' = 'SSD',\n" + " 'storage_cooldown_time' = '2100-05-09 00:00:00',\n" + + " 'remote_storage_policy' = 'testPolicy'\n" + ");"); + + createTable("create table test.show_test (k1 int, k2 int) distributed by hash(k1) " + + "buckets 1 properties(\"replication_num\" = \"1\");"); } @AfterClass @@ -273,7 +223,7 @@ public class AlterTest { Catalog.getCurrentCatalog().getPolicyMgr().createPolicy(stmt); } - private static void alterTable(String sql, boolean expectedException) throws Exception { + private static void alterTable(String sql, boolean expectedException) { try { AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext); Catalog.getCurrentCatalog().alterTable(alterTableStmt); @@ -288,6 +238,22 @@ public class AlterTest { } } + private static void createMV(String sql, boolean expectedException) { + try { + CreateMaterializedViewStmt createMaterializedViewStmt + = (CreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext); + Catalog.getCurrentCatalog().createMaterializedView(createMaterializedViewStmt); + if (expectedException) { + Assert.fail(); + } + } catch (Exception e) { + e.printStackTrace(); + if (!expectedException) { + Assert.fail(); + } + } + } + private static void alterTableWithExceptionMsg(String sql, String msg) throws Exception { AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext); try { @@ -1171,4 +1137,28 @@ public class AlterTest { DropResourceStmt stmt = (DropResourceStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext); Catalog.getCurrentCatalog().getResourceMgr().dropResource(stmt); } + + @Test + public void testShowMV() throws Exception { + createMV("CREATE MATERIALIZED VIEW test_mv as select k1 from test.show_test;", false); + waitSchemaChangeJobDone(true); + + String showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv on test.show_test;"; + ShowCreateMaterializedViewStmt showStmt = (ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt( + showMvSql, connectContext); + ShowExecutor executor = new ShowExecutor(connectContext, showStmt); + Assert.assertEquals(executor.execute().getResultRows().get(0).get(2), + "CREATE MATERIALIZED VIEW test_mv as select k1 from test.show_test;"); + + showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv_empty on test.show_test;"; + showStmt = (ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(showMvSql, connectContext); + executor = new ShowExecutor(connectContext, showStmt); + Assert.assertTrue(executor.execute().getResultRows().isEmpty()); + + showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv on test.table1_error;"; + showStmt = (ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(showMvSql, connectContext); + executor = new ShowExecutor(connectContext, showStmt); + ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "Unknown table 'table1_error'", + executor::execute); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateMaterializedViewStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateMaterializedViewStmtTest.java deleted file mode 100644 index 501b50b418..0000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateMaterializedViewStmtTest.java +++ /dev/null @@ -1,97 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.analysis; - -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.ExceptionChecker; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.ShowExecutor; -import org.apache.doris.utframe.DorisAssert; -import org.apache.doris.utframe.UtFrameUtils; - -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.File; -import java.util.UUID; - -/** - * test for ShowCreateMaterializedViewStmt. - **/ -public class ShowCreateMaterializedViewStmtTest { - - private static String runningDir = "fe/mocked/ShowCreateMaterializedViewStmtTest/" + UUID.randomUUID() + "/"; - - private static ConnectContext connectContext; - - private static DorisAssert dorisAssert; - - /** - * init. - **/ - @BeforeClass - public static void beforeClass() throws Exception { - UtFrameUtils.createDorisCluster(runningDir); - - // create connect context - connectContext = UtFrameUtils.createDefaultCtx(); - dorisAssert = new DorisAssert(connectContext); - dorisAssert.withDatabase("test") - .withTable("create table test.table1 (k1 int, k2 int) distributed by hash(k1) " - + "buckets 1 properties(\"replication_num\" = \"1\");") - .withMaterializedView("CREATE MATERIALIZED VIEW test_mv as select k1 from test.table1;"); - } - - @AfterClass - public static void tearDown() { - File file = new File(runningDir); - file.delete(); - } - - @Test - public void testNormal() throws Exception { - String showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv on test.table1;"; - ShowCreateMaterializedViewStmt showStmt = - (ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(showMvSql, connectContext); - ShowExecutor executor = new ShowExecutor(connectContext, showStmt); - Assert.assertEquals(executor.execute().getResultRows().get(0).get(2), - "CREATE MATERIALIZED VIEW test_mv as select k1 from test.table1;"); - } - - @Test - public void testNoView() throws Exception { - String showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv_empty on test.table1;"; - ShowCreateMaterializedViewStmt showStmt = - (ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(showMvSql, connectContext); - ShowExecutor executor = new ShowExecutor(connectContext, showStmt); - Assert.assertTrue(executor.execute().getResultRows().isEmpty()); - } - - @Test - public void testNoTable() throws Exception { - String showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv on test.table1_error;"; - ShowCreateMaterializedViewStmt showStmt = - (ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(showMvSql, connectContext); - ShowExecutor executor = new ShowExecutor(connectContext, showStmt); - ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, - "Unknown table 'table1_error' in default_cluster:test", executor::execute); - - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java index c2522aaa5c..be14837e43 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java @@ -62,7 +62,7 @@ public class DbsProcDirTest { public void testRegister() { DbsProcDir dir; - dir = new DbsProcDir(catalog); + dir = new DbsProcDir(catalog, ds); Assert.assertFalse(dir.register("db1", new BaseProcDir())); } @@ -103,7 +103,7 @@ public class DbsProcDirTest { DbsProcDir dir; ProcNodeInterface node; - dir = new DbsProcDir(catalog); + dir = new DbsProcDir(catalog, ds); try { node = dir.lookup(String.valueOf(db1.getId())); Assert.assertNotNull(node); @@ -112,7 +112,7 @@ public class DbsProcDirTest { Assert.fail(); } - dir = new DbsProcDir(catalog); + dir = new DbsProcDir(catalog, ds); try { node = dir.lookup(String.valueOf(db2.getId())); Assert.assertNotNull(node); @@ -121,7 +121,7 @@ public class DbsProcDirTest { Assert.fail(); } - dir = new DbsProcDir(catalog); + dir = new DbsProcDir(catalog, ds); node = dir.lookup("10002"); Assert.assertNull(node); } @@ -130,7 +130,7 @@ public class DbsProcDirTest { public void testLookupInvalid() { DbsProcDir dir; - dir = new DbsProcDir(catalog); + dir = new DbsProcDir(catalog, ds); try { dir.lookup(null); } catch (AnalysisException e) { @@ -187,7 +187,7 @@ public class DbsProcDirTest { DbsProcDir dir; ProcResult result; - dir = new DbsProcDir(catalog); + dir = new DbsProcDir(catalog, ds); result = dir.fetchResult(); Assert.assertNotNull(result); Assert.assertTrue(result instanceof BaseProcResult); @@ -218,14 +218,14 @@ public class DbsProcDirTest { DbsProcDir dir; ProcResult result; - dir = new DbsProcDir(null); + dir = new DbsProcDir(null, ds); try { result = dir.fetchResult(); } catch (NullPointerException e) { e.printStackTrace(); } - dir = new DbsProcDir(catalog); + dir = new DbsProcDir(catalog, ds); result = dir.fetchResult(); Assert.assertEquals(Lists.newArrayList("DbId", "DbName", "TableNum", "Size", "Quota", "LastConsistencyCheckTime", "ReplicaCount", "ReplicaQuota"), diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadManagerTest.java index c4fddfe0a5..5949b2d2bf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadManagerTest.java @@ -27,6 +27,7 @@ import org.apache.doris.analysis.StopRoutineLoadStmt; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; @@ -587,7 +588,7 @@ public class RoutineLoadManagerTest { @Test public void testPauseRoutineLoadJob(@Injectable PauseRoutineLoadStmt pauseRoutineLoadStmt, @Mocked Catalog catalog, - @Mocked InternalDataSource ds, @Mocked Database database, @Mocked PaloAuth paloAuth, + @Mocked InternalDataSource ds, @Mocked Database database, @Mocked Table tbl, @Mocked PaloAuth paloAuth, @Mocked ConnectContext connectContext) throws UserException { RoutineLoadManager routineLoadManager = new RoutineLoadManager(); Map>> dbToNameToRoutineLoadJob = Maps.newHashMap(); @@ -620,6 +621,12 @@ public class RoutineLoadManagerTest { database.getId(); minTimes = 0; result = 1L; + database.getTableOrAnalysisException(anyLong); + minTimes = 0; + result = tbl; + tbl.getName(); + minTimes = 0; + result = "tbl"; catalog.getAuth(); minTimes = 0; result = paloAuth; @@ -652,7 +659,7 @@ public class RoutineLoadManagerTest { @Test public void testResumeRoutineLoadJob(@Injectable ResumeRoutineLoadStmt resumeRoutineLoadStmt, - @Mocked Catalog catalog, @Mocked InternalDataSource ds, @Mocked Database database, + @Mocked Catalog catalog, @Mocked InternalDataSource ds, @Mocked Database database, @Mocked Table tbl, @Mocked PaloAuth paloAuth, @Mocked ConnectContext connectContext) throws UserException { RoutineLoadManager routineLoadManager = new RoutineLoadManager(); Map>> dbToNameToRoutineLoadJob = Maps.newHashMap(); @@ -681,6 +688,12 @@ public class RoutineLoadManagerTest { database.getId(); minTimes = 0; result = 1L; + database.getTableOrAnalysisException(anyLong); + minTimes = 0; + result = tbl; + tbl.getName(); + minTimes = 0; + result = "tbl"; catalog.getAuth(); minTimes = 0; result = paloAuth; @@ -697,7 +710,7 @@ public class RoutineLoadManagerTest { @Test public void testStopRoutineLoadJob(@Injectable StopRoutineLoadStmt stopRoutineLoadStmt, @Mocked Catalog catalog, - @Mocked InternalDataSource ds, @Mocked Database database, @Mocked PaloAuth paloAuth, + @Mocked InternalDataSource ds, @Mocked Database database, @Mocked Table tbl, @Mocked PaloAuth paloAuth, @Mocked ConnectContext connectContext) throws UserException { RoutineLoadManager routineLoadManager = new RoutineLoadManager(); Map>> dbToNameToRoutineLoadJob = Maps.newHashMap(); @@ -726,6 +739,12 @@ public class RoutineLoadManagerTest { database.getId(); minTimes = 0; result = 1L; + database.getTableOrAnalysisException(anyLong); + minTimes = 0; + result = tbl; + tbl.getName(); + minTimes = 0; + result = "tbl"; catalog.getAuth(); minTimes = 0; result = paloAuth; @@ -894,7 +913,7 @@ public class RoutineLoadManagerTest { @Test public void testAlterRoutineLoadJob(@Injectable StopRoutineLoadStmt stopRoutineLoadStmt, @Mocked Catalog catalog, - @Mocked InternalDataSource ds, @Mocked Database database, @Mocked PaloAuth paloAuth, + @Mocked InternalDataSource ds, @Mocked Database database, @Mocked Table tbl, @Mocked PaloAuth paloAuth, @Mocked ConnectContext connectContext) throws UserException { RoutineLoadManager routineLoadManager = new RoutineLoadManager(); Map>> dbToNameToRoutineLoadJob = Maps.newHashMap(); @@ -923,6 +942,12 @@ public class RoutineLoadManagerTest { database.getId(); minTimes = 0; result = 1L; + database.getTableOrAnalysisException(anyLong); + minTimes = 0; + result = tbl; + tbl.getName(); + minTimes = 0; + result = "tbl"; catalog.getAuth(); minTimes = 0; result = paloAuth; @@ -940,7 +965,7 @@ public class RoutineLoadManagerTest { @Test public void testPauseAndResumeAllRoutineLoadJob(@Injectable PauseRoutineLoadStmt pauseRoutineLoadStmt, @Injectable ResumeRoutineLoadStmt resumeRoutineLoadStmt, @Mocked Catalog catalog, - @Mocked InternalDataSource ds, @Mocked Database database, @Mocked PaloAuth paloAuth, + @Mocked InternalDataSource ds, @Mocked Database database, @Mocked Table tbl, @Mocked PaloAuth paloAuth, @Mocked ConnectContext connectContext) throws UserException { RoutineLoadManager routineLoadManager = new RoutineLoadManager(); Map>> dbToNameToRoutineLoadJob = Maps.newHashMap(); @@ -981,6 +1006,12 @@ public class RoutineLoadManagerTest { database.getId(); minTimes = 0; result = 1L; + database.getTableOrAnalysisException(anyLong); + minTimes = 0; + result = tbl; + tbl.getName(); + minTimes = 0; + result = "tbl"; catalog.getAuth(); minTimes = 0; result = paloAuth;