database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature-wip](multi-catalog) support "show proc 'catalogs/'" (#10596)

Browse Source
This commit is contained in:
Mingyu Chen
2022-07-05 13:40:24 +08:00
committed by GitHub
parent 680118c6b9
commit f40ae7c654
20 changed files with 507 additions and 852 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java
View File

@ -37,12 +37,12 @@ import org.apache.doris.common.proc.ProcResult;
import org.apache.doris.common.proc.ProcService;
import org.apache.doris.common.proc.TableProcDir;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.DataSourceIf;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSetMetaData;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
@ -50,10 +50,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
public class DescribeStmt extends ShowStmt {
private static final ShowResultSetMetaData DESC_OLAP_TABLE_ALL_META_DATA =
@ -79,17 +77,6 @@ public class DescribeStmt extends ShowStmt {
.addColumn(new Column("Table", ScalarType.createVarchar(30)))
.build();
// The same columns in IndexSchemaProcNode.TITLE_NAMES
private static final ShowResultSetMetaData HMS_EXTERNAL_TABLE_META_DATA =
ShowResultSetMetaData.builder()
.addColumn(new Column("Field", ScalarType.createVarchar(20)))
.addColumn(new Column("Type", ScalarType.createVarchar(20)))
.addColumn(new Column("Null", ScalarType.createVarchar(20)))
.addColumn(new Column("Key", ScalarType.createVarchar(20)))
.addColumn(new Column("Default", ScalarType.createVarchar(20)))
.addColumn(new Column("Extra", ScalarType.createVarchar(20)))
.build();
// empty col num equals to DESC_OLAP_TABLE_ALL_META_DATA.size()
private static final List<String> EMPTY_ROW = initEmptyRow();
@ -114,47 +101,35 @@ public class DescribeStmt extends ShowStmt {
}
@Override
public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
public void analyze(Analyzer analyzer) throws UserException {
dbTableName.analyze(analyzer);
if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(
ConnectContext.get(), dbTableName, PrivPredicate.SHOW)) {
if (!Catalog.getCurrentCatalog().getAuth()
.checkTblPriv(ConnectContext.get(), dbTableName, PrivPredicate.SHOW)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "DESCRIBE",
ConnectContext.get().getQualifiedUser(),
ConnectContext.get().getRemoteIP(),
dbTableName.toString());
ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(),
dbTableName.toString());
}
DatabaseIf db = Catalog.getCurrentCatalog().getDataSourceMgr()
.getCatalogOrAnalysisException(dbTableName.getCtl()).getDbOrAnalysisException(dbTableName.getDb());
DataSourceIf ds = Catalog.getCurrentCatalog().getDataSourceMgr().getCatalog(dbTableName.getCtl());
DatabaseIf db = ds.getDbOrAnalysisException(dbTableName.getDb());
TableIf table = db.getTableOrAnalysisException(dbTableName.getTbl());
table.readLock();
try {
if (!isAllTables) {
if (table.getType() == TableType.HMS_EXTERNAL_TABLE) {
hmsSchema = table.getFullSchema().stream().map(col -> Arrays.asList(
col.getName(),
col.getType().toSql().toUpperCase(Locale.ROOT),
Boolean.toString(col.isAllowNull()),
Boolean.toString(col.isKey()),
Strings.nullToEmpty(col.getDefaultValue()),
"" /* no extra field */))
.collect(Collectors.toList());
// show base table schema only
String procString = "/catalogs/" + ds.getId() + "/" + db.getId() + "/" + table.getId() + "/"
+ TableProcDir.INDEX_SCHEMA + "/";
if (table.getType() == TableType.OLAP) {
procString += ((OlapTable) table).getBaseIndexId();
} else {
// show base table schema only
String procString = "/dbs/" + db.getId() + "/" + table.getId() + "/" + TableProcDir.INDEX_SCHEMA
+ "/";
if (table.getType() == TableType.OLAP) {
procString += ((OlapTable) table).getBaseIndexId();
} else {
procString += table.getId();
}
procString += table.getId();
}
node = ProcService.getInstance().open(procString);
if (node == null) {
throw new AnalysisException("Describe table[" + dbTableName.getTbl() + "] failed");
}
node = ProcService.getInstance().open(procString);
if (node == null) {
throw new AnalysisException("Describe table[" + dbTableName.getTbl() + "] failed");
}
} else {
Util.prohibitExternalCatalog(dbTableName.getCtl(), this.getClass().getSimpleName() + " ALL");
@ -272,9 +247,6 @@ public class DescribeStmt extends ShowStmt {
@Override
public ShowResultSetMetaData getMetaData() {
if (!isAllTables) {
if (hmsSchema != null) {
return HMS_EXTERNAL_TABLE_META_DATA;
}
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
ProcResult result = null;

95
fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
View File

@ -22,7 +22,6 @@ import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.Pair;
@ -50,13 +49,11 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
/**
* Internal representation of db-related metadata. Owned by Catalog instance.
@ -457,7 +454,7 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
/**
* This is a thread-safe method when nameToTable is a concurrent hash map
*/
@Nullable
@Override
public Table getTableNullable(String tableName) {
if (Catalog.isStoredTableNamesLowerCase()) {
tableName = tableName.toLowerCase();
@ -474,99 +471,11 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
/**
* This is a thread-safe method when idToTable is a concurrent hash map
*/
@Nullable
@Override
public Table getTableNullable(long tableId) {
return idToTable.get(tableId);
}
public Optional<Table> getTable(String tableName) {
return Optional.ofNullable(getTableNullable(tableName));
}
public Optional<Table> getTable(long tableId) {
return Optional.ofNullable(getTableNullable(tableId));
}
public <E extends Exception> Table getTableOrException(
String tableName, java.util.function.Function<String, E> e) throws E {
Table table = getTableNullable(tableName);
if (table == null) {
throw e.apply(tableName);
}
return table;
}
public <E extends Exception> Table getTableOrException(
long tableId, java.util.function.Function<Long, E> e) throws E {
Table table = getTableNullable(tableId);
if (table == null) {
throw e.apply(tableId);
}
return table;
}
public Table getTableOrMetaException(String tableName) throws MetaNotFoundException {
return getTableOrException(tableName, t -> new MetaNotFoundException("unknown table, tableName=" + t));
}
public Table getTableOrMetaException(long tableId) throws MetaNotFoundException {
return getTableOrException(tableId, t -> new MetaNotFoundException("unknown table, tableId=" + t));
}
@SuppressWarnings("unchecked")
public Table getTableOrMetaException(String tableName, TableType tableType) throws MetaNotFoundException {
Table table = getTableOrMetaException(tableName);
if (table.getType() != tableType) {
throw new MetaNotFoundException("table type is not "
+ tableType + ", tableName=" + tableName + ", type=" + table.getType());
}
return table;
}
@SuppressWarnings("unchecked")
public Table getTableOrMetaException(long tableId, TableType tableType) throws MetaNotFoundException {
Table table = getTableOrMetaException(tableId);
if (table.getType() != tableType) {
throw new MetaNotFoundException("table type is not " + tableType
+ ", tableId=" + tableId + ", type=" + table.getType());
}
return table;
}
public Table getTableOrDdlException(String tableName) throws DdlException {
return getTableOrException(tableName, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
}
public OlapTable getOlapTableOrDdlException(String tableName) throws DdlException {
Table table = getTableOrDdlException(tableName);
if (!(table instanceof OlapTable)) {
throw new DdlException(ErrorCode.ERR_NOT_OLAP_TABLE.formatErrorMsg(tableName));
}
return (OlapTable) table;
}
public Table getTableOrDdlException(long tableId) throws DdlException {
return getTableOrException(tableId, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
}
public Table getTableOrAnalysisException(String tableName) throws AnalysisException {
return getTableOrException(tableName,
t -> new AnalysisException(ErrorCode.ERR_UNKNOWN_TABLE.formatErrorMsg(t, fullQualifiedName)));
}
public OlapTable getOlapTableOrAnalysisException(String tableName) throws AnalysisException {
Table table = getTableOrAnalysisException(tableName);
if (!(table instanceof OlapTable)) {
throw new AnalysisException(ErrorCode.ERR_NOT_OLAP_TABLE.formatErrorMsg(tableName));
}
return (OlapTable) table;
}
public Table getTableOrAnalysisException(long tableId) throws AnalysisException {
return getTableOrException(tableId,
t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
}
public int getMaxReplicationNum() {
int ret = 0;
readLock();

89
fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
View File

@ -19,12 +19,14 @@ package org.apache.doris.catalog;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.MetaNotFoundException;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
/**
* Database interface.
@ -76,31 +78,88 @@ public interface DatabaseIf<T extends TableIf> {
T getTableNullable(long tableId);
Optional<T> getTable(String tableName);
default Optional<T> getTable(String tableName) {
return Optional.ofNullable(getTableNullable(tableName));
}
Optional<T> getTable(long tableId);
default Optional<T> getTable(long tableId) {
return Optional.ofNullable(getTableNullable(tableId));
}
<E extends Exception> T getTableOrException(String tableName, java.util.function.Function<String, E> e) throws E;
default <E extends Exception> T getTableOrException(String tableName, java.util.function.Function<String, E> e)
throws E {
T table = getTableNullable(tableName);
if (table == null) {
throw e.apply(tableName);
}
return table;
}
<E extends Exception> T getTableOrException(long tableId, java.util.function.Function<Long, E> e) throws E;
default <E extends Exception> T getTableOrException(long tableId, Function<Long, E> e) throws E {
T table = getTableNullable(tableId);
if (table == null) {
throw e.apply(tableId);
}
return table;
}
T getTableOrMetaException(String tableName) throws MetaNotFoundException;
default T getTableOrMetaException(String tableName) throws MetaNotFoundException {
return getTableOrException(tableName, t -> new MetaNotFoundException("unknown table, tableName=" + t));
}
T getTableOrMetaException(long tableId) throws MetaNotFoundException;
default T getTableOrMetaException(long tableId) throws MetaNotFoundException {
return getTableOrException(tableId, t -> new MetaNotFoundException("unknown table, tableId=" + t));
}
@SuppressWarnings("unchecked")
T getTableOrMetaException(String tableName, Table.TableType tableType) throws MetaNotFoundException;
default T getTableOrMetaException(String tableName, TableIf.TableType tableType) throws MetaNotFoundException {
T table = getTableOrMetaException(tableName);
if (table.getType() != tableType) {
throw new MetaNotFoundException(
"table type is not " + tableType + ", tableName=" + tableName + ", type=" + table.getType());
}
return table;
}
@SuppressWarnings("unchecked")
T getTableOrMetaException(long tableId, Table.TableType tableType) throws MetaNotFoundException;
default T getTableOrMetaException(long tableId, TableIf.TableType tableType) throws MetaNotFoundException {
T table = getTableOrMetaException(tableId);
if (table.getType() != tableType) {
throw new MetaNotFoundException(
"table type is not " + tableType + ", tableId=" + tableId + ", type=" + table.getType());
}
return table;
}
T getTableOrDdlException(String tableName) throws DdlException;
default T getTableOrDdlException(String tableName) throws DdlException {
return getTableOrException(tableName, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
}
T getTableOrDdlException(long tableId) throws DdlException;
default T getTableOrDdlException(long tableId) throws DdlException {
return getTableOrException(tableId, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
}
T getTableOrAnalysisException(String tableName) throws AnalysisException;
default T getTableOrAnalysisException(String tableName) throws AnalysisException {
return getTableOrException(tableName,
t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
}
T getTableOrAnalysisException(long tableId) throws AnalysisException;
default T getTableOrAnalysisException(long tableId) throws AnalysisException {
return getTableOrException(tableId,
t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
}
OlapTable getOlapTableOrAnalysisException(String tableName) throws AnalysisException;
default OlapTable getOlapTableOrDdlException(String tableName) throws DdlException {
T table = getTableOrDdlException(tableName);
if (!(table instanceof OlapTable)) {
throw new DdlException(ErrorCode.ERR_NOT_OLAP_TABLE.formatErrorMsg(tableName));
}
return (OlapTable) table;
}
default OlapTable getOlapTableOrAnalysisException(String tableName) throws AnalysisException {
T table = getTableOrAnalysisException(tableName);
if (!(table instanceof OlapTable)) {
throw new AnalysisException(ErrorCode.ERR_NOT_OLAP_TABLE.formatErrorMsg(tableName));
}
return (OlapTable) table;
}
}

72
fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java vendored
View File

@ -19,9 +19,6 @@ package org.apache.doris.catalog.external;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.DatabaseProperty;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.datasource.ExternalDataSource;
@ -32,11 +29,9 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
/**
* Base class of external database.
@ -177,71 +172,4 @@ public class ExternalDatabase<T extends ExternalTable> implements DatabaseIf<T>
public T getTableNullable(long tableId) {
throw new NotImplementedException();
}
@Override
public Optional<T> getTable(String tableName) {
throw new NotImplementedException();
}
@Override
public Optional<T> getTable(long tableId) {
throw new NotImplementedException();
}
@Override
public <E extends Exception> T getTableOrException(String tableName, Function<String, E> e) throws E {
throw new NotImplementedException();
}
@Override
public <E extends Exception> T getTableOrException(long tableId, Function<Long, E> e) throws E {
throw new NotImplementedException();
}
@Override
public T getTableOrMetaException(String tableName) throws MetaNotFoundException {
throw new NotImplementedException();
}
@Override
public T getTableOrMetaException(long tableId) throws MetaNotFoundException {
throw new NotImplementedException();
}
@Override
public T getTableOrMetaException(String tableName, Table.TableType tableType)
throws MetaNotFoundException {
throw new NotImplementedException();
}
@Override
public T getTableOrMetaException(long tableId, Table.TableType tableType)
throws MetaNotFoundException {
throw new NotImplementedException();
}
@Override
public T getTableOrDdlException(String tableName) throws DdlException {
throw new NotImplementedException();
}
@Override
public T getTableOrDdlException(long tableId) throws DdlException {
throw new NotImplementedException();
}
@Override
public T getTableOrAnalysisException(String tableName) throws AnalysisException {
throw new NotImplementedException();
}
@Override
public T getTableOrAnalysisException(long tableId) throws AnalysisException {
throw new NotImplementedException();
}
@Override
public OlapTable getOlapTableOrAnalysisException(String tableName) throws AnalysisException {
throw new NotImplementedException();
}
}

167
fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java vendored
View File

@ -17,40 +17,33 @@
package org.apache.doris.catalog.external;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.datasource.ExternalDataSource;
import org.apache.doris.datasource.HMSExternalDataSource;
import com.clearspring.analytics.util.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* Hive metastore external database.
*/
public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
private static final Logger LOG = LogManager.getLogger(HMSExternalDatabase.class);
// Cache of table name to table id.
private ConcurrentHashMap<String, Long> tableNameToId = new ConcurrentHashMap<>();
private AtomicLong nextId = new AtomicLong(0);
private final String hiveMetastoreUri;
private Map<String, Long> tableNameToId = Maps.newConcurrentMap();
private Map<Long, HMSExternalTable> idToTbl = Maps.newHashMap();
private boolean initialized = false;
/**
* Create HMS external database.
@ -58,37 +51,33 @@ public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
* @param extDataSource External data source this database belongs to.
* @param id database id.
* @param name database name.
* @param uri Hive metastore uri.
*/
public HMSExternalDatabase(ExternalDataSource extDataSource, long id, String name, String uri) {
public HMSExternalDatabase(ExternalDataSource extDataSource, long id, String name) {
super(extDataSource, id, name);
this.hiveMetastoreUri = uri;
init();
}
private synchronized void makeSureInitialized() {
if (!initialized) {
init();
initialized = true;
}
}
private void init() {
List<String> tableNames = extDataSource.listTableNames(null, name);
if (tableNames != null) {
for (String tableName : tableNames) {
tableNameToId.put(tableName, nextId.incrementAndGet());
long tblId = Catalog.getCurrentCatalog().getNextId();
tableNameToId.put(tableName, tblId);
idToTbl.put(tblId, new HMSExternalTable(tblId, tableName, name, (HMSExternalDataSource) extDataSource));
}
}
}
@Override
public List<HMSExternalTable> getTables() {
List<HMSExternalTable> tables = new ArrayList<>();
List<String> tableNames = extDataSource.listTableNames(null, name);
for (String tableName : tableNames) {
tableNameToId.putIfAbsent(tableName, nextId.incrementAndGet());
try {
tables.add(new HMSExternalTable(tableNameToId.get(tableName), tableName, name,
(HMSExternalDataSource) extDataSource));
} catch (MetaNotFoundException e) {
LOG.warn("Table {} in db {} not found in Hive metastore.", tableName, name, e);
}
}
return tables;
makeSureInitialized();
return Lists.newArrayList(idToTbl.values());
}
@Override
@ -99,122 +88,22 @@ public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
@Override
public Set<String> getTableNamesWithLock() {
// Doesn't need to lock because everytime we call the hive metastore api to get table names.
return new HashSet<>(extDataSource.listTableNames(null, name));
makeSureInitialized();
return Sets.newHashSet(tableNameToId.keySet());
}
@Override
public HMSExternalTable getTableNullable(String tableName) {
if (extDataSource.tableExist(null, name, tableName)) {
tableNameToId.putIfAbsent(tableName, nextId.incrementAndGet());
try {
return new HMSExternalTable(tableNameToId.get(tableName), tableName, name,
(HMSExternalDataSource) extDataSource);
} catch (MetaNotFoundException e) {
LOG.warn("Table {} in db {} not found in Hive metastore.", tableName, name, e);
}
makeSureInitialized();
if (!tableNameToId.containsKey(tableName)) {
return null;
}
return null;
return idToTbl.get(tableNameToId.get(tableName));
}
@Override
public HMSExternalTable getTableNullable(long tableId) {
for (Map.Entry<String, Long> entry : tableNameToId.entrySet()) {
if (entry.getValue() == tableId) {
if (extDataSource.tableExist(null, name, entry.getKey())) {
try {
return new HMSExternalTable(tableId, entry.getKey(), name,
(HMSExternalDataSource) extDataSource);
} catch (MetaNotFoundException e) {
LOG.warn("Table {} in db {} not found in Hive metastore.", entry.getKey(), name, e);
}
}
return null;
}
}
return null;
}
@Override
public Optional<HMSExternalTable> getTable(String tableName) {
return Optional.ofNullable(getTableNullable(tableName));
}
@Override
public Optional<HMSExternalTable> getTable(long tableId) {
return Optional.ofNullable(getTableNullable(tableId));
}
@Override
public <E extends Exception> HMSExternalTable getTableOrException(String tableName, Function<String, E> e)
throws E {
HMSExternalTable table = getTableNullable(tableName);
if (table == null) {
throw e.apply(tableName);
}
return table;
}
@Override
public <E extends Exception> HMSExternalTable getTableOrException(long tableId, Function<Long, E> e) throws E {
HMSExternalTable table = getTableNullable(tableId);
if (table == null) {
throw e.apply(tableId);
}
return table;
}
@Override
public HMSExternalTable getTableOrMetaException(String tableName) throws MetaNotFoundException {
return getTableOrException(tableName, t -> new MetaNotFoundException("unknown table, tableName=" + t));
}
@Override
public HMSExternalTable getTableOrMetaException(long tableId) throws MetaNotFoundException {
return getTableOrException(tableId, t -> new MetaNotFoundException("unknown table, tableName=" + t));
}
@Override
public HMSExternalTable getTableOrMetaException(String tableName, TableIf.TableType tableType)
throws MetaNotFoundException {
HMSExternalTable table = getTableOrMetaException(tableName);
if (table.getType() != tableType) {
throw new MetaNotFoundException("table type is not "
+ tableType + ", tableName=" + tableName + ", type=" + table.getType());
}
return table;
}
@Override
public HMSExternalTable getTableOrMetaException(long tableId, TableIf.TableType tableType)
throws MetaNotFoundException {
HMSExternalTable table = getTableOrMetaException(tableId);
if (table.getType() != tableType) {
throw new MetaNotFoundException("table type is not "
+ tableType + ", tableId=" + tableId + ", type=" + table.getType());
}
return table;
}
@Override
public HMSExternalTable getTableOrDdlException(String tableName) throws DdlException {
return getTableOrException(tableName, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
}
@Override
public HMSExternalTable getTableOrDdlException(long tableId) throws DdlException {
return getTableOrException(tableId, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
}
@Override
public HMSExternalTable getTableOrAnalysisException(String tableName) throws AnalysisException {
return getTableOrException(tableName,
t -> new AnalysisException(ErrorCode.ERR_UNKNOWN_TABLE.formatErrorMsg(t, name)));
}
@Override
public HMSExternalTable getTableOrAnalysisException(long tableId) throws AnalysisException {
return getTableOrException(tableId,
t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
makeSureInitialized();
return idToTbl.get(tableId);
}
}

89
fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java vendored
View File

@ -26,11 +26,11 @@ import org.apache.doris.thrift.THiveTable;
import org.apache.doris.thrift.TTableDescriptor;
import org.apache.doris.thrift.TTableType;
import com.google.common.collect.Lists;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -45,12 +45,11 @@ public class HMSExternalTable extends ExternalTable {
private final HMSExternalDataSource ds;
private final String dbName;
private org.apache.hadoop.hive.metastore.api.Table remoteTable = null;
private DLAType dlaType = null;
private DLAType dlaType = DLAType.UNKNOWN;
private boolean initialized = false;
public enum DLAType {
HIVE,
HUDI,
ICEBERG
UNKNOWN, HIVE, HUDI, ICEBERG
}
/**
@ -61,24 +60,59 @@ public class HMSExternalTable extends ExternalTable {
* @param dbName Database name.
* @param ds HMSExternalDataSource.
*/
public HMSExternalTable(long id, String name, String dbName, HMSExternalDataSource ds)
throws MetaNotFoundException {
public HMSExternalTable(long id, String name, String dbName, HMSExternalDataSource ds) {
super(id, name);
this.dbName = dbName;
this.ds = ds;
this.type = TableType.HMS_EXTERNAL_TABLE;
init();
}
private void init() throws MetaNotFoundException {
getRemoteTable();
if (remoteTable.getParameters().containsKey("table_type")
&& remoteTable.getParameters().get("table_type").equalsIgnoreCase("ICEBERG")) {
dlaType = DLAType.ICEBERG;
} else if (remoteTable.getSd().getInputFormat().toLowerCase().contains("hoodie")) {
dlaType = DLAType.HUDI;
private synchronized void makeSureInitialized() {
if (!initialized) {
init();
initialized = true;
}
}
private void init() {
try {
getRemoteTable();
} catch (MetaNotFoundException e) {
// CHECKSTYLE IGNORE THIS LINE
}
if (remoteTable == null) {
dlaType = DLAType.UNKNOWN;
fullSchema = Lists.newArrayList();
} else {
dlaType = DLAType.HIVE;
if (remoteTable.getParameters().containsKey("table_type") && remoteTable.getParameters().get("table_type")
.equalsIgnoreCase("ICEBERG")) {
dlaType = DLAType.ICEBERG;
} else if (remoteTable.getSd().getInputFormat().toLowerCase().contains("hoodie")) {
dlaType = DLAType.HUDI;
} else {
dlaType = DLAType.HIVE;
}
initSchema();
}
}
private void initSchema() {
if (fullSchema == null) {
synchronized (this) {
if (fullSchema == null) {
fullSchema = Lists.newArrayList();
try {
for (FieldSchema field : HiveMetaStoreClientHelper.getSchema(dbName, name,
ds.getHiveMetastoreUris())) {
fullSchema.add(new Column(field.getName(),
HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType()), true, null, true,
null, field.getComment()));
}
} catch (DdlException e) {
LOG.warn("Fail to get schema of hms table {}", name, e);
}
}
}
}
}
@ -105,24 +139,7 @@ public class HMSExternalTable extends ExternalTable {
@Override
public List<Column> getFullSchema() {
if (fullSchema == null) {
synchronized (this) {
if (fullSchema == null) {
fullSchema = new ArrayList<>();
try {
for (FieldSchema field : HiveMetaStoreClientHelper.getSchema(dbName, name,
ds.getHiveMetastoreUris())) {
fullSchema.add(new Column(field.getName(),
HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType()), true, null, true,
null, field.getComment()));
}
} catch (DdlException e) {
LOG.warn("Fail to get schema of hms table {}", name, e);
}
}
}
}
// TODO: Refresh cached fullSchema.
makeSureInitialized();
return fullSchema;
}
@ -138,9 +155,7 @@ public class HMSExternalTable extends ExternalTable {
@Override
public Column getColumn(String name) {
if (fullSchema == null) {
getFullSchema();
}
makeSureInitialized();
for (Column column : fullSchema) {
if (name.equals(column.getName())) {
return column;

110
fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java Normal file
View File

@ -0,0 +1,110 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.common.proc;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.util.ListComparator;
import org.apache.doris.datasource.DataSourceIf;
import com.clearspring.analytics.util.Lists;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/*
* SHOW PROC /catalogs/
* show all catalogs' info
*/
public class CatalogsProcDir implements ProcDirInterface {
public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
.add("CatalogIds").add("CatalogName").add("DatabaseNum")
.build();
private Catalog catalog;
public CatalogsProcDir(Catalog catalog) {
this.catalog = catalog;
}
@Override
public boolean register(String name, ProcNodeInterface node) {
return false;
}
@Override
public ProcNodeInterface lookup(String catalogIdStr) throws AnalysisException {
if (catalog == null || Strings.isNullOrEmpty(catalogIdStr)) {
throw new AnalysisException("Catalog id is null");
}
long catalogId = -1L;
try {
catalogId = Long.valueOf(catalogIdStr);
} catch (NumberFormatException e) {
throw new AnalysisException("Invalid catalog id format: " + catalogIdStr);
}
DataSourceIf ds = catalog.getDataSourceMgr().getCatalog(catalogId);
if (ds == null) {
throw new AnalysisException("Catalog " + catalogIdStr + " does not exist");
}
return new DbsProcDir(catalog, ds);
}
@Override
public ProcResult fetchResult() throws AnalysisException {
Preconditions.checkNotNull(catalog);
BaseProcResult result = new BaseProcResult();
result.setNames(TITLE_NAMES);
List<Long> catalogIds = catalog.getDataSourceMgr().getCatalogIds();
// get info
List<List<Comparable>> catalogInfos = Lists.newArrayList();
for (long catalogId : catalogIds) {
DataSourceIf ds = catalog.getDataSourceMgr().getCatalog(catalogId);
if (ds == null) {
continue;
}
List<Comparable> catalogInfo = Lists.newArrayList();
catalogInfo.add(ds.getId());
catalogInfo.add(ds.getName());
catalogInfo.add(ds.getDbNames().size());
catalogInfos.add(catalogInfo);
}
// order by catalogId, asc
ListComparator<List<Comparable>> comparator = new ListComparator<List<Comparable>>(0);
Collections.sort(catalogInfos, comparator);
// set result
for (List<Comparable> info : catalogInfos) {
List<String> row = new ArrayList<String>(info.size());
for (Comparable comparable : info) {
row.add(comparable.toString());
}
result.addRow(row);
}
return result;
}
}

23
fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
View File

@ -21,9 +21,11 @@ import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.ListComparator;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.DataSourceIf;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@ -44,9 +46,11 @@ public class DbsProcDir implements ProcDirInterface {
.build();
private Catalog catalog;
private DataSourceIf ds;
public DbsProcDir(Catalog catalog) {
public DbsProcDir(Catalog catalog, DataSourceIf ds) {
this.catalog = catalog;
this.ds = ds;
}
@Override
@ -67,7 +71,7 @@ public class DbsProcDir implements ProcDirInterface {
throw new AnalysisException("Invalid db id format: " + dbIdStr);
}
DatabaseIf db = catalog.getInternalDataSource().getDbNullable(dbId);
DatabaseIf db = ds.getDbNullable(dbId);
if (db == null) {
throw new AnalysisException("Database " + dbId + " does not exist");
}
@ -81,7 +85,7 @@ public class DbsProcDir implements ProcDirInterface {
BaseProcResult result = new BaseProcResult();
result.setNames(TITLE_NAMES);
List<String> dbNames = catalog.getInternalDataSource().getDbNames();
List<String> dbNames = ds.getDbNames();
if (dbNames == null || dbNames.isEmpty()) {
// empty
return result;
@ -90,7 +94,7 @@ public class DbsProcDir implements ProcDirInterface {
// get info
List<List<Comparable>> dbInfos = new ArrayList<>();
for (String dbName : dbNames) {
DatabaseIf db = catalog.getInternalDataSource().getDbNullable(dbName);
DatabaseIf db = ds.getDbNullable(dbName);
if (db == null) {
continue;
}
@ -102,13 +106,14 @@ public class DbsProcDir implements ProcDirInterface {
dbInfo.add(dbName);
dbInfo.add(tableNum);
long usedDataQuota = ((Database) db).getUsedDataQuotaWithLock();
long dataQuota = ((Database) db).getDataQuota();
long usedDataQuota = (db instanceof Database) ? ((Database) db).getUsedDataQuotaWithLock() : 0;
long dataQuota = (db instanceof Database) ? ((Database) db).getDataQuota() : 0;
String readableUsedQuota = DebugUtil.printByteWithUnit(usedDataQuota);
String readableQuota = DebugUtil.printByteWithUnit(dataQuota);
String lastCheckTime = TimeUtils.longToTimeString(((Database) db).getLastCheckTime());
long replicaCount = ((Database) db).getReplicaCountWithLock();
long replicaQuota = ((Database) db).getReplicaQuota();
String lastCheckTime = (db instanceof Database) ? TimeUtils.longToTimeString(
((Database) db).getLastCheckTime()) : FeConstants.null_string;
long replicaCount = (db instanceof Database) ? ((Database) db).getReplicaCountWithLock() : 0;
long replicaQuota = (db instanceof Database) ? ((Database) db).getReplicaQuota() : 0;
dbInfo.add(readableUsedQuota);
dbInfo.add(readableQuota);
dbInfo.add(lastCheckTime);

2
fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java
View File

@ -91,7 +91,7 @@ public class IndexInfoProcDir implements ProcDirInterface {
builder.toString()));
}
} else {
result.addRow(Lists.newArrayList("-1", table.getName(), "", "", "", "", ""));
result.addRow(Lists.newArrayList(String.valueOf(table.getId()), table.getName(), "", "", "", "", ""));
}
return result;

3
fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java
View File

@ -37,7 +37,8 @@ public final class ProcService {
root = new BaseProcDir();
root.register("auth", new AuthProcDir(Catalog.getCurrentCatalog().getAuth()));
root.register("backends", new BackendsProcDir(Catalog.getCurrentSystemInfo()));
root.register("dbs", new DbsProcDir(Catalog.getCurrentCatalog()));
root.register("catalogs", new CatalogsProcDir(Catalog.getCurrentCatalog()));
root.register("dbs", new DbsProcDir(Catalog.getCurrentCatalog(), Catalog.getCurrentInternalCatalog()));
root.register("jobs", new JobsDbProcDir(Catalog.getCurrentCatalog()));
root.register("statistic", new StatisticProcNode(Catalog.getCurrentCatalog()));
root.register("tasks", new TasksProcDir());

76
fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceIf.java
View File

@ -20,11 +20,13 @@ package org.apache.doris.datasource;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.MetaNotFoundException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import javax.annotation.Nullable;
/**
@ -50,29 +52,63 @@ public interface DataSourceIf<T extends DatabaseIf> {
@Nullable
T getDbNullable(long dbId);
Optional<T> getDb(String dbName);
Optional<T> getDb(long dbId);
<E extends Exception> T getDbOrException(String dbName, java.util.function.Function<String, E> e) throws E;
<E extends Exception> T getDbOrException(long dbId, java.util.function.Function<Long, E> e) throws E;
T getDbOrMetaException(String dbName) throws MetaNotFoundException;
T getDbOrMetaException(long dbId) throws MetaNotFoundException;
T getDbOrDdlException(String dbName) throws DdlException;
T getDbOrDdlException(long dbId) throws DdlException;
T getDbOrAnalysisException(String dbName) throws AnalysisException;
T getDbOrAnalysisException(long dbId) throws AnalysisException;
Map<String, String> getProperties();
void modifyDatasourceName(String name);
void modifyDatasourceProps(Map<String, String> props);
default Optional<T> getDb(String dbName) {
return Optional.ofNullable(getDbNullable(dbName));
}
default Optional<T> getDb(long dbId) {
return Optional.ofNullable(getDbNullable(dbId));
}
default <E extends Exception> T getDbOrException(String dbName, Function<String, E> e) throws E {
T db = getDbNullable(dbName);
if (db == null) {
throw e.apply(dbName);
}
return db;
}
default <E extends Exception> T getDbOrException(long dbId, Function<Long, E> e) throws E {
T db = getDbNullable(dbId);
if (db == null) {
throw e.apply(dbId);
}
return db;
}
default T getDbOrMetaException(String dbName) throws MetaNotFoundException {
return getDbOrException(dbName,
s -> new MetaNotFoundException("unknown databases, dbName=" + s, ErrorCode.ERR_BAD_DB_ERROR));
}
default T getDbOrMetaException(long dbId) throws MetaNotFoundException {
return getDbOrException(dbId,
s -> new MetaNotFoundException("unknown databases, dbId=" + s, ErrorCode.ERR_BAD_DB_ERROR));
}
default T getDbOrDdlException(String dbName) throws DdlException {
return getDbOrException(dbName,
s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
}
default T getDbOrDdlException(long dbId) throws DdlException {
return getDbOrException(dbId,
s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
}
default T getDbOrAnalysisException(String dbName) throws AnalysisException {
return getDbOrException(dbName,
s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
}
default T getDbOrAnalysisException(long dbId) throws AnalysisException {
return getDbOrException(dbId,
s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
}
}

8
fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceMgr.java
View File

@ -116,8 +116,12 @@ public class DataSourceMgr implements Writable, GsonPostProcessable {
}
public DataSourceIf getCatalogOrAnalysisException(String name) throws AnalysisException {
return getCatalogOrException(name, ds -> new AnalysisException(
ErrorCode.ERR_UNKNOWN_CATALOG.formatErrorMsg(ds), ErrorCode.ERR_UNKNOWN_CATALOG));
return getCatalogOrException(name, ds -> new AnalysisException(ErrorCode.ERR_UNKNOWN_CATALOG.formatErrorMsg(ds),
ErrorCode.ERR_UNKNOWN_CATALOG));
}
public List<Long> getCatalogIds() {
return Lists.newArrayList(idToCatalog.keySet());
}
public DatabaseIf getDbNullable(long dbId) {

55
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java
View File

@ -19,9 +19,6 @@ package org.apache.doris.datasource;
import org.apache.doris.catalog.external.ExternalDatabase;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.persist.gson.GsonUtils;
@ -36,8 +33,6 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
/**
* The abstract class for all types of external data sources.
@ -107,56 +102,6 @@ public abstract class ExternalDataSource implements DataSourceIf<ExternalDatabas
throw new NotImplementedException();
}
@Override
public Optional<ExternalDatabase> getDb(String dbName) {
throw new NotImplementedException();
}
@Override
public Optional<ExternalDatabase> getDb(long dbId) {
throw new NotImplementedException();
}
@Override
public <E extends Exception> ExternalDatabase getDbOrException(String dbName, Function<String, E> e) throws E {
throw new NotImplementedException();
}
@Override
public <E extends Exception> ExternalDatabase getDbOrException(long dbId, Function<Long, E> e) throws E {
throw new NotImplementedException();
}
@Override
public ExternalDatabase getDbOrMetaException(String dbName) throws MetaNotFoundException {
throw new NotImplementedException();
}
@Override
public ExternalDatabase getDbOrMetaException(long dbId) throws MetaNotFoundException {
throw new NotImplementedException();
}
@Override
public ExternalDatabase getDbOrDdlException(String dbName) throws DdlException {
throw new NotImplementedException();
}
@Override
public ExternalDatabase getDbOrDdlException(long dbId) throws DdlException {
throw new NotImplementedException();
}
@Override
public ExternalDatabase getDbOrAnalysisException(String dbName) throws AnalysisException {
throw new NotImplementedException();
}
@Override
public ExternalDatabase getDbOrAnalysisException(long dbId) throws AnalysisException {
throw new NotImplementedException();
}
@Override
public Map<String, String> getProperties() {
return dsProperty.getProperties();

91
fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java
View File

@ -21,10 +21,6 @@ import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.external.ExternalDatabase;
import org.apache.doris.catalog.external.HMSExternalDatabase;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.MetaNotFoundException;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@ -38,8 +34,6 @@ import org.jetbrains.annotations.Nullable;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
/**
* External data source for hive metastore compatible data sources.
@ -48,7 +42,8 @@ public class HMSExternalDataSource extends ExternalDataSource {
private static final Logger LOG = LogManager.getLogger(HMSExternalDataSource.class);
// Cache of db name to db id.
private Map<String, Long> dbNameToId;
private Map<String, Long> dbNameToId = Maps.newConcurrentMap();
private Map<Long, HMSExternalDatabase> idToDb = Maps.newConcurrentMap();
private boolean initialized = false;
protected HiveMetaStoreClient client;
@ -68,7 +63,6 @@ public class HMSExternalDataSource extends ExternalDataSource {
}
private void init() {
dbNameToId = Maps.newConcurrentMap();
HiveConf hiveConf = new HiveConf();
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, getHiveMetastoreUris());
try {
@ -87,8 +81,10 @@ public class HMSExternalDataSource extends ExternalDataSource {
if (allDatabases == null) {
return;
}
for (String db : allDatabases) {
dbNameToId.put(db, Catalog.getCurrentCatalog().getNextId());
for (String dbName : allDatabases) {
long dbId = Catalog.getCurrentCatalog().getNextId();
dbNameToId.put(dbName, dbId);
idToDb.put(dbId, new HMSExternalDatabase(this, dbId, dbName));
}
}
@ -111,7 +107,6 @@ public class HMSExternalDataSource extends ExternalDataSource {
@Override
public List<String> listTableNames(SessionContext ctx, String dbName) {
makeSureInitialized();
try {
return client.getAllTables(getRealTableName(dbName));
} catch (MetaException e) {
@ -122,7 +117,6 @@ public class HMSExternalDataSource extends ExternalDataSource {
@Override
public boolean tableExist(SessionContext ctx, String dbName, String tblName) {
makeSureInitialized();
try {
return client.tableExists(getRealTableName(dbName), tblName);
} catch (TException e) {
@ -139,83 +133,14 @@ public class HMSExternalDataSource extends ExternalDataSource {
if (!dbNameToId.containsKey(realDbName)) {
return null;
}
return new HMSExternalDatabase(this, dbNameToId.get(realDbName), realDbName, getHiveMetastoreUris());
return idToDb.get(dbNameToId.get(realDbName));
}
@Nullable
@Override
public ExternalDatabase getDbNullable(long dbId) {
makeSureInitialized();
for (Map.Entry<String, Long> entry : dbNameToId.entrySet()) {
if (entry.getValue() == dbId) {
return new HMSExternalDatabase(this, dbId, entry.getKey(), getHiveMetastoreUris());
}
}
return null;
}
@Override
public Optional<ExternalDatabase> getDb(String dbName) {
return Optional.ofNullable(getDbNullable(dbName));
}
@Override
public Optional<ExternalDatabase> getDb(long dbId) {
return Optional.ofNullable(getDbNullable(dbId));
}
@Override
public <E extends Exception> ExternalDatabase getDbOrException(String dbName, Function<String, E> e) throws E {
ExternalDatabase db = getDbNullable(dbName);
if (db == null) {
throw e.apply(dbName);
}
return db;
}
@Override
public <E extends Exception> ExternalDatabase getDbOrException(long dbId, Function<Long, E> e) throws E {
ExternalDatabase db = getDbNullable(dbId);
if (db == null) {
throw e.apply(dbId);
}
return db;
}
@Override
public ExternalDatabase getDbOrMetaException(String dbName) throws MetaNotFoundException {
return getDbOrException(dbName,
s -> new MetaNotFoundException("unknown databases, dbName=" + s, ErrorCode.ERR_BAD_DB_ERROR));
}
@Override
public ExternalDatabase getDbOrMetaException(long dbId) throws MetaNotFoundException {
return getDbOrException(dbId,
s -> new MetaNotFoundException("unknown databases, dbId=" + s, ErrorCode.ERR_BAD_DB_ERROR));
}
@Override
public ExternalDatabase getDbOrDdlException(String dbName) throws DdlException {
return getDbOrException(dbName,
s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
}
@Override
public ExternalDatabase getDbOrDdlException(long dbId) throws DdlException {
return getDbOrException(dbId,
s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
}
@Override
public ExternalDatabase getDbOrAnalysisException(String dbName) throws AnalysisException {
return getDbOrException(dbName,
s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
}
@Override
public ExternalDatabase getDbOrAnalysisException(long dbId) throws AnalysisException {
return getDbOrException(dbId,
s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
return idToDb.get(dbId);
}
@Override

68
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java
View File

@ -187,11 +187,9 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
/**
* The Internal data source will manage all self-managed meta object in a Doris cluster.
@ -263,70 +261,6 @@ public class InternalDataSource implements DataSourceIf<Database> {
return idToDb.get(dbId);
}
@Override
public Optional<Database> getDb(String dbName) {
return Optional.ofNullable(getDbNullable(dbName));
}
@Override
public Optional<Database> getDb(long dbId) {
return Optional.ofNullable(getDbNullable(dbId));
}
@Override
public <E extends Exception> Database getDbOrException(String dbName, Function<String, E> e) throws E {
Database db = getDbNullable(dbName);
if (db == null) {
throw e.apply(dbName);
}
return db;
}
@Override
public <E extends Exception> Database getDbOrException(long dbId, Function<Long, E> e) throws E {
Database db = getDbNullable(dbId);
if (db == null) {
throw e.apply(dbId);
}
return db;
}
@Override
public Database getDbOrMetaException(String dbName) throws MetaNotFoundException {
return getDbOrException(dbName,
s -> new MetaNotFoundException("unknown databases, dbName=" + s, ErrorCode.ERR_BAD_DB_ERROR));
}
@Override
public Database getDbOrMetaException(long dbId) throws MetaNotFoundException {
return getDbOrException(dbId,
s -> new MetaNotFoundException("unknown databases, dbId=" + s, ErrorCode.ERR_BAD_DB_ERROR));
}
@Override
public Database getDbOrDdlException(String dbName) throws DdlException {
return getDbOrException(dbName,
s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
}
@Override
public Database getDbOrDdlException(long dbId) throws DdlException {
return getDbOrException(dbId,
s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
}
@Override
public Database getDbOrAnalysisException(String dbName) throws AnalysisException {
return getDbOrException(dbName,
s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
}
@Override
public Database getDbOrAnalysisException(long dbId) throws AnalysisException {
return getDbOrException(dbId,
s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
}
@Override
public Map<String, String> getProperties() {
return Maps.newHashMap();
@ -734,7 +668,7 @@ public class InternalDataSource implements DataSourceIf<Database> {
String dbName = recoverStmt.getDbName();
String tableName = recoverStmt.getTableName();
Database db = (Database) getDbOrDdlException(dbName);
Database db = getDbOrDdlException(dbName);
OlapTable olapTable = db.getOlapTableOrDdlException(tableName);
olapTable.writeLockOrDdlException();
try {

1
fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
View File

@ -909,7 +909,6 @@ public class ShowExecutor {
ShowIndexStmt showStmt = (ShowIndexStmt) stmt;
List<List<String>> rows = Lists.newArrayList();
DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName());
OlapTable table = db.getOlapTableOrAnalysisException(showStmt.getTableName().getTbl());
table.readLock();
try {

192
fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
View File

@ -19,11 +19,13 @@ package org.apache.doris.alter;
import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.CreatePolicyStmt;
import org.apache.doris.analysis.CreateResourceStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DateLiteral;
import org.apache.doris.analysis.DropResourceStmt;
import org.apache.doris.analysis.ShowCreateMaterializedViewStmt;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DataProperty;
@ -38,11 +40,14 @@ import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletInvertedIndex;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowExecutor;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.utframe.UtFrameUtils;
@ -138,118 +143,63 @@ public class AlterTest {
+ " 'storage_cooldown_time' = '2999-12-31 00:00:00'\n"
+ ");");
createTable("CREATE TABLE test.tbl5\n"
+ "(\n"
+ " k1 date,\n"
+ " k2 int,\n"
+ " v1 int \n"
+ ") ENGINE=OLAP\n"
+ "UNIQUE KEY (k1,k2)\n"
+ "PARTITION BY RANGE(k1)\n"
+ "(\n"
createTable("CREATE TABLE test.tbl5\n" + "(\n" + " k1 date,\n" + " k2 int,\n" + " v1 int \n"
+ ") ENGINE=OLAP\n" + "UNIQUE KEY (k1,k2)\n" + "PARTITION BY RANGE(k1)\n" + "(\n"
+ " PARTITION p1 values less than('2020-02-01'),\n"
+ " PARTITION p2 values less than('2020-03-01')\n"
+ ")\n"
+ "DISTRIBUTED BY HASH(k2) BUCKETS 3\n"
+ " PARTITION p2 values less than('2020-03-01')\n" + ")\n" + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n"
+ "PROPERTIES('replication_num' = '1');");
createTable("CREATE TABLE test.tbl6\n"
+ "(\n"
+ " k1 datetime(3),\n"
+ " k2 time(3),\n"
+ " v1 int \n,"
+ " v2 datetime(3)\n"
+ ") ENGINE=OLAP\n"
+ "UNIQUE KEY (k1,k2)\n"
+ "PARTITION BY RANGE(k1)\n"
+ "(\n"
+ " PARTITION p1 values less than('2020-02-01 00:00:00'),\n"
+ " PARTITION p2 values less than('2020-03-01 00:00:00')\n"
+ ")\n"
+ "DISTRIBUTED BY HASH(k2) BUCKETS 3\n"
+ "PROPERTIES('replication_num' = '1');");
createTable(
"CREATE TABLE test.tbl6\n" + "(\n" + " k1 datetime(3),\n" + " k2 time(3),\n" + " v1 int \n,"
+ " v2 datetime(3)\n" + ") ENGINE=OLAP\n" + "UNIQUE KEY (k1,k2)\n"
+ "PARTITION BY RANGE(k1)\n" + "(\n"
+ " PARTITION p1 values less than('2020-02-01 00:00:00'),\n"
+ " PARTITION p2 values less than('2020-03-01 00:00:00')\n" + ")\n"
+ "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + "PROPERTIES('replication_num' = '1');");
createTable("create external table test.odbc_table\n"
+ "( `k1` bigint(20) COMMENT \"\",\n"
+ " `k2` datetime COMMENT \"\",\n"
+ " `k3` varchar(20) COMMENT \"\",\n"
+ " `k4` varchar(100) COMMENT \"\",\n"
+ " `k5` float COMMENT \"\"\n"
+ ")ENGINE=ODBC\n"
+ "PROPERTIES (\n"
+ "\"host\" = \"127.0.0.1\",\n"
+ "\"port\" = \"3306\",\n"
+ "\"user\" = \"root\",\n"
+ "\"password\" = \"123\",\n"
+ "\"database\" = \"db1\",\n"
+ "\"table\" = \"tbl1\",\n"
+ "\"driver\" = \"Oracle Driver\",\n"
+ "\"odbc_type\" = \"oracle\"\n"
+ ");");
createTable("create external table test.odbc_table\n" + "( `k1` bigint(20) COMMENT \"\",\n"
+ " `k2` datetime COMMENT \"\",\n" + " `k3` varchar(20) COMMENT \"\",\n"
+ " `k4` varchar(100) COMMENT \"\",\n" + " `k5` float COMMENT \"\"\n" + ")ENGINE=ODBC\n"
+ "PROPERTIES (\n" + "\"host\" = \"127.0.0.1\",\n" + "\"port\" = \"3306\",\n" + "\"user\" = \"root\",\n"
+ "\"password\" = \"123\",\n" + "\"database\" = \"db1\",\n" + "\"table\" = \"tbl1\",\n"
+ "\"driver\" = \"Oracle Driver\",\n" + "\"odbc_type\" = \"oracle\"\n" + ");");
// s3 resource
createRemoteStorageResource("create resource \"remote_s3\"\n"
+ "properties\n"
+ "(\n"
+ " \"type\" = \"s3\", \n"
+ " \"s3_endpoint\" = \"bj\",\n"
+ " \"s3_region\" = \"bj\",\n"
+ " \"s3_root_path\" = \"/path/to/root\",\n"
+ " \"s3_access_key\" = \"bbb\",\n"
+ " \"s3_secret_key\" = \"aaaa\",\n"
+ " \"s3_max_connections\" = \"50\",\n"
+ " \"s3_request_timeout_ms\" = \"3000\",\n"
+ " \"s3_connection_timeout_ms\" = \"1000\"\n"
+ ");");
createRemoteStorageResource(
"create resource \"remote_s3\"\n" + "properties\n" + "(\n" + " \"type\" = \"s3\", \n"
+ " \"s3_endpoint\" = \"bj\",\n" + " \"s3_region\" = \"bj\",\n"
+ " \"s3_root_path\" = \"/path/to/root\",\n" + " \"s3_access_key\" = \"bbb\",\n"
+ " \"s3_secret_key\" = \"aaaa\",\n" + " \"s3_max_connections\" = \"50\",\n"
+ " \"s3_request_timeout_ms\" = \"3000\",\n" + " \"s3_connection_timeout_ms\" = \"1000\"\n"
+ ");");
createRemoteStorageResource("create resource \"remote_s3_1\"\n"
+ "properties\n"
+ "(\n"
+ " \"type\" = \"s3\", \n"
+ " \"s3_endpoint\" = \"bj\",\n"
+ " \"s3_region\" = \"bj\",\n"
+ " \"s3_root_path\" = \"/path/to/root\",\n"
+ " \"s3_access_key\" = \"bbb\",\n"
+ " \"s3_secret_key\" = \"aaaa\",\n"
+ " \"s3_max_connections\" = \"50\",\n"
+ " \"s3_request_timeout_ms\" = \"3000\",\n"
+ " \"s3_connection_timeout_ms\" = \"1000\"\n"
+ ");");
createRemoteStorageResource(
"create resource \"remote_s3_1\"\n" + "properties\n" + "(\n" + " \"type\" = \"s3\", \n"
+ " \"s3_endpoint\" = \"bj\",\n" + " \"s3_region\" = \"bj\",\n"
+ " \"s3_root_path\" = \"/path/to/root\",\n" + " \"s3_access_key\" = \"bbb\",\n"
+ " \"s3_secret_key\" = \"aaaa\",\n" + " \"s3_max_connections\" = \"50\",\n"
+ " \"s3_request_timeout_ms\" = \"3000\",\n" + " \"s3_connection_timeout_ms\" = \"1000\"\n"
+ ");");
createRemoteStoragePolicy("CREATE STORAGE POLICY testPolicy\n"
+ "PROPERTIES(\n"
+ " \"storage_resource\" = \"remote_s3\",\n"
+ " \"cooldown_datetime\" = \"2100-05-10 00:00:00\"\n"
+ ");");
createRemoteStoragePolicy(
"CREATE STORAGE POLICY testPolicy\n" + "PROPERTIES(\n" + " \"storage_resource\" = \"remote_s3\",\n"
+ " \"cooldown_datetime\" = \"2100-05-10 00:00:00\"\n" + ");");
createRemoteStoragePolicy("CREATE STORAGE POLICY testPolicy2\n"
+ "PROPERTIES(\n"
+ " \"storage_resource\" = \"remote_s3\",\n"
+ " \"cooldown_ttl\" = \"1d\"\n"
+ ");");
createRemoteStoragePolicy(
"CREATE STORAGE POLICY testPolicy2\n" + "PROPERTIES(\n" + " \"storage_resource\" = \"remote_s3\",\n"
+ " \"cooldown_ttl\" = \"1d\"\n" + ");");
createTable("CREATE TABLE test.tbl_remote\n"
+ "(\n"
+ " k1 date,\n"
+ " k2 int,\n"
+ " v1 int sum\n"
+ ")\n"
+ "PARTITION BY RANGE(k1)\n"
+ "(\n"
+ " PARTITION p1 values less than('2020-02-01'),\n"
createTable("CREATE TABLE test.tbl_remote\n" + "(\n" + " k1 date,\n" + " k2 int,\n" + " v1 int sum\n"
+ ")\n" + "PARTITION BY RANGE(k1)\n" + "(\n" + " PARTITION p1 values less than('2020-02-01'),\n"
+ " PARTITION p2 values less than('2020-03-01'),\n"
+ " PARTITION p3 values less than('2020-04-01'),\n"
+ " PARTITION p4 values less than('2020-05-01')\n"
+ ")\n"
+ "DISTRIBUTED BY HASH(k2) BUCKETS 3\n"
+ "PROPERTIES"
+ "("
+ " 'replication_num' = '1',\n"
+ " 'in_memory' = 'false',\n"
+ " 'storage_medium' = 'SSD',\n"
+ " 'storage_cooldown_time' = '2100-05-09 00:00:00',\n"
+ " 'remote_storage_policy' = 'testPolicy'\n"
+ ");");
+ " PARTITION p4 values less than('2020-05-01')\n" + ")\n" + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n"
+ "PROPERTIES" + "(" + " 'replication_num' = '1',\n" + " 'in_memory' = 'false',\n"
+ " 'storage_medium' = 'SSD',\n" + " 'storage_cooldown_time' = '2100-05-09 00:00:00',\n"
+ " 'remote_storage_policy' = 'testPolicy'\n" + ");");
createTable("create table test.show_test (k1 int, k2 int) distributed by hash(k1) "
+ "buckets 1 properties(\"replication_num\" = \"1\");");
}
@AfterClass
@ -273,7 +223,7 @@ public class AlterTest {
Catalog.getCurrentCatalog().getPolicyMgr().createPolicy(stmt);
}
private static void alterTable(String sql, boolean expectedException) throws Exception {
private static void alterTable(String sql, boolean expectedException) {
try {
AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
Catalog.getCurrentCatalog().alterTable(alterTableStmt);
@ -288,6 +238,22 @@ public class AlterTest {
}
}
private static void createMV(String sql, boolean expectedException) {
try {
CreateMaterializedViewStmt createMaterializedViewStmt
= (CreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
Catalog.getCurrentCatalog().createMaterializedView(createMaterializedViewStmt);
if (expectedException) {
Assert.fail();
}
} catch (Exception e) {
e.printStackTrace();
if (!expectedException) {
Assert.fail();
}
}
}
private static void alterTableWithExceptionMsg(String sql, String msg) throws Exception {
AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
try {
@ -1171,4 +1137,28 @@ public class AlterTest {
DropResourceStmt stmt = (DropResourceStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
Catalog.getCurrentCatalog().getResourceMgr().dropResource(stmt);
}
@Test
public void testShowMV() throws Exception {
createMV("CREATE MATERIALIZED VIEW test_mv as select k1 from test.show_test;", false);
waitSchemaChangeJobDone(true);
String showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv on test.show_test;";
ShowCreateMaterializedViewStmt showStmt = (ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(
showMvSql, connectContext);
ShowExecutor executor = new ShowExecutor(connectContext, showStmt);
Assert.assertEquals(executor.execute().getResultRows().get(0).get(2),
"CREATE MATERIALIZED VIEW test_mv as select k1 from test.show_test;");
showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv_empty on test.show_test;";
showStmt = (ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(showMvSql, connectContext);
executor = new ShowExecutor(connectContext, showStmt);
Assert.assertTrue(executor.execute().getResultRows().isEmpty());
showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv on test.table1_error;";
showStmt = (ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(showMvSql, connectContext);
executor = new ShowExecutor(connectContext, showStmt);
ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "Unknown table 'table1_error'",
executor::execute);
}
}

97
fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateMaterializedViewStmtTest.java
View File

@ -1,97 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.analysis;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowExecutor;
import org.apache.doris.utframe.DorisAssert;
import org.apache.doris.utframe.UtFrameUtils;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
import java.util.UUID;
/**
* test for ShowCreateMaterializedViewStmt.
**/
public class ShowCreateMaterializedViewStmtTest {
private static String runningDir = "fe/mocked/ShowCreateMaterializedViewStmtTest/" + UUID.randomUUID() + "/";
private static ConnectContext connectContext;
private static DorisAssert dorisAssert;
/**
* init.
**/
@BeforeClass
public static void beforeClass() throws Exception {
UtFrameUtils.createDorisCluster(runningDir);
// create connect context
connectContext = UtFrameUtils.createDefaultCtx();
dorisAssert = new DorisAssert(connectContext);
dorisAssert.withDatabase("test")
.withTable("create table test.table1 (k1 int, k2 int) distributed by hash(k1) "
+ "buckets 1 properties(\"replication_num\" = \"1\");")
.withMaterializedView("CREATE MATERIALIZED VIEW test_mv as select k1 from test.table1;");
}
@AfterClass
public static void tearDown() {
File file = new File(runningDir);
file.delete();
}
@Test
public void testNormal() throws Exception {
String showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv on test.table1;";
ShowCreateMaterializedViewStmt showStmt =
(ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(showMvSql, connectContext);
ShowExecutor executor = new ShowExecutor(connectContext, showStmt);
Assert.assertEquals(executor.execute().getResultRows().get(0).get(2),
"CREATE MATERIALIZED VIEW test_mv as select k1 from test.table1;");
}
@Test
public void testNoView() throws Exception {
String showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv_empty on test.table1;";
ShowCreateMaterializedViewStmt showStmt =
(ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(showMvSql, connectContext);
ShowExecutor executor = new ShowExecutor(connectContext, showStmt);
Assert.assertTrue(executor.execute().getResultRows().isEmpty());
}
@Test
public void testNoTable() throws Exception {
String showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv on test.table1_error;";
ShowCreateMaterializedViewStmt showStmt =
(ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(showMvSql, connectContext);
ShowExecutor executor = new ShowExecutor(connectContext, showStmt);
ExceptionChecker.expectThrowsWithMsg(AnalysisException.class,
"Unknown table 'table1_error' in default_cluster:test", executor::execute);
}
}

16
fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java
View File

@ -62,7 +62,7 @@ public class DbsProcDirTest {
public void testRegister() {
DbsProcDir dir;
dir = new DbsProcDir(catalog);
dir = new DbsProcDir(catalog, ds);
Assert.assertFalse(dir.register("db1", new BaseProcDir()));
}
@ -103,7 +103,7 @@ public class DbsProcDirTest {
DbsProcDir dir;
ProcNodeInterface node;
dir = new DbsProcDir(catalog);
dir = new DbsProcDir(catalog, ds);
try {
node = dir.lookup(String.valueOf(db1.getId()));
Assert.assertNotNull(node);
@ -112,7 +112,7 @@ public class DbsProcDirTest {
Assert.fail();
}
dir = new DbsProcDir(catalog);
dir = new DbsProcDir(catalog, ds);
try {
node = dir.lookup(String.valueOf(db2.getId()));
Assert.assertNotNull(node);
@ -121,7 +121,7 @@ public class DbsProcDirTest {
Assert.fail();
}
dir = new DbsProcDir(catalog);
dir = new DbsProcDir(catalog, ds);
node = dir.lookup("10002");
Assert.assertNull(node);
}
@ -130,7 +130,7 @@ public class DbsProcDirTest {
public void testLookupInvalid() {
DbsProcDir dir;
dir = new DbsProcDir(catalog);
dir = new DbsProcDir(catalog, ds);
try {
dir.lookup(null);
} catch (AnalysisException e) {
@ -187,7 +187,7 @@ public class DbsProcDirTest {
DbsProcDir dir;
ProcResult result;
dir = new DbsProcDir(catalog);
dir = new DbsProcDir(catalog, ds);
result = dir.fetchResult();
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BaseProcResult);
@ -218,14 +218,14 @@ public class DbsProcDirTest {
DbsProcDir dir;
ProcResult result;
dir = new DbsProcDir(null);
dir = new DbsProcDir(null, ds);
try {
result = dir.fetchResult();
} catch (NullPointerException e) {
e.printStackTrace();
}
dir = new DbsProcDir(catalog);
dir = new DbsProcDir(catalog, ds);
result = dir.fetchResult();
Assert.assertEquals(Lists.newArrayList("DbId", "DbName", "TableNum", "Size", "Quota",
"LastConsistencyCheckTime", "ReplicaCount", "ReplicaQuota"),

41
fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadManagerTest.java
View File

@ -27,6 +27,7 @@ import org.apache.doris.analysis.StopRoutineLoadStmt;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
@ -587,7 +588,7 @@ public class RoutineLoadManagerTest {
@Test
public void testPauseRoutineLoadJob(@Injectable PauseRoutineLoadStmt pauseRoutineLoadStmt, @Mocked Catalog catalog,
@Mocked InternalDataSource ds, @Mocked Database database, @Mocked PaloAuth paloAuth,
@Mocked InternalDataSource ds, @Mocked Database database, @Mocked Table tbl, @Mocked PaloAuth paloAuth,
@Mocked ConnectContext connectContext) throws UserException {
RoutineLoadManager routineLoadManager = new RoutineLoadManager();
Map<Long, Map<String, List<RoutineLoadJob>>> dbToNameToRoutineLoadJob = Maps.newHashMap();
@ -620,6 +621,12 @@ public class RoutineLoadManagerTest {
database.getId();
minTimes = 0;
result = 1L;
database.getTableOrAnalysisException(anyLong);
minTimes = 0;
result = tbl;
tbl.getName();
minTimes = 0;
result = "tbl";
catalog.getAuth();
minTimes = 0;
result = paloAuth;
@ -652,7 +659,7 @@ public class RoutineLoadManagerTest {
@Test
public void testResumeRoutineLoadJob(@Injectable ResumeRoutineLoadStmt resumeRoutineLoadStmt,
@Mocked Catalog catalog, @Mocked InternalDataSource ds, @Mocked Database database,
@Mocked Catalog catalog, @Mocked InternalDataSource ds, @Mocked Database database, @Mocked Table tbl,
@Mocked PaloAuth paloAuth, @Mocked ConnectContext connectContext) throws UserException {
RoutineLoadManager routineLoadManager = new RoutineLoadManager();
Map<Long, Map<String, List<RoutineLoadJob>>> dbToNameToRoutineLoadJob = Maps.newHashMap();
@ -681,6 +688,12 @@ public class RoutineLoadManagerTest {
database.getId();
minTimes = 0;
result = 1L;
database.getTableOrAnalysisException(anyLong);
minTimes = 0;
result = tbl;
tbl.getName();
minTimes = 0;
result = "tbl";
catalog.getAuth();
minTimes = 0;
result = paloAuth;
@ -697,7 +710,7 @@ public class RoutineLoadManagerTest {
@Test
public void testStopRoutineLoadJob(@Injectable StopRoutineLoadStmt stopRoutineLoadStmt, @Mocked Catalog catalog,
@Mocked InternalDataSource ds, @Mocked Database database, @Mocked PaloAuth paloAuth,
@Mocked InternalDataSource ds, @Mocked Database database, @Mocked Table tbl, @Mocked PaloAuth paloAuth,
@Mocked ConnectContext connectContext) throws UserException {
RoutineLoadManager routineLoadManager = new RoutineLoadManager();
Map<Long, Map<String, List<RoutineLoadJob>>> dbToNameToRoutineLoadJob = Maps.newHashMap();
@ -726,6 +739,12 @@ public class RoutineLoadManagerTest {
database.getId();
minTimes = 0;
result = 1L;
database.getTableOrAnalysisException(anyLong);
minTimes = 0;
result = tbl;
tbl.getName();
minTimes = 0;
result = "tbl";
catalog.getAuth();
minTimes = 0;
result = paloAuth;
@ -894,7 +913,7 @@ public class RoutineLoadManagerTest {
@Test
public void testAlterRoutineLoadJob(@Injectable StopRoutineLoadStmt stopRoutineLoadStmt, @Mocked Catalog catalog,
@Mocked InternalDataSource ds, @Mocked Database database, @Mocked PaloAuth paloAuth,
@Mocked InternalDataSource ds, @Mocked Database database, @Mocked Table tbl, @Mocked PaloAuth paloAuth,
@Mocked ConnectContext connectContext) throws UserException {
RoutineLoadManager routineLoadManager = new RoutineLoadManager();
Map<Long, Map<String, List<RoutineLoadJob>>> dbToNameToRoutineLoadJob = Maps.newHashMap();
@ -923,6 +942,12 @@ public class RoutineLoadManagerTest {
database.getId();
minTimes = 0;
result = 1L;
database.getTableOrAnalysisException(anyLong);
minTimes = 0;
result = tbl;
tbl.getName();
minTimes = 0;
result = "tbl";
catalog.getAuth();
minTimes = 0;
result = paloAuth;
@ -940,7 +965,7 @@ public class RoutineLoadManagerTest {
@Test
public void testPauseAndResumeAllRoutineLoadJob(@Injectable PauseRoutineLoadStmt pauseRoutineLoadStmt,
@Injectable ResumeRoutineLoadStmt resumeRoutineLoadStmt, @Mocked Catalog catalog,
@Mocked InternalDataSource ds, @Mocked Database database, @Mocked PaloAuth paloAuth,
@Mocked InternalDataSource ds, @Mocked Database database, @Mocked Table tbl, @Mocked PaloAuth paloAuth,
@Mocked ConnectContext connectContext) throws UserException {
RoutineLoadManager routineLoadManager = new RoutineLoadManager();
Map<Long, Map<String, List<RoutineLoadJob>>> dbToNameToRoutineLoadJob = Maps.newHashMap();
@ -981,6 +1006,12 @@ public class RoutineLoadManagerTest {
database.getId();
minTimes = 0;
result = 1L;
database.getTableOrAnalysisException(anyLong);
minTimes = 0;
result = tbl;
tbl.getName();
minTimes = 0;
result = "tbl";
catalog.getAuth();
minTimes = 0;
result = paloAuth;