database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature-wip](multi-catalog) end to end to support multi-catalog (#10521)

Browse Source 
Get through the previous pull requests that support multi-catalog, and end to end to achieve multi-catalog.
This commit is contained in:
Ashin Gau
2022-07-02 20:43:10 +08:00
committed by GitHub
parent c9f86bc7e2
commit 078cb3b4db
33 changed files with 303 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
fe/fe-core/src/main/java/org/apache/doris/analysis/AdminCancelRepairTableStmt.java
View File

@ -18,15 +18,14 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.Util;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import java.util.List;
@ -48,17 +47,8 @@ public class AdminCancelRepairTableStmt extends DdlStmt {
ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
}
String dbName = null;
if (Strings.isNullOrEmpty(tblRef.getName().getDb())) {
dbName = analyzer.getDefaultDb();
if (Strings.isNullOrEmpty(dbName)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR);
}
} else {
dbName = ClusterNamespace.getFullName(getClusterName(), tblRef.getName().getDb());
}
tblRef.getName().setDb(dbName);
tblRef.getName().analyze(analyzer);
Util.prohibitExternalCatalog(tblRef.getName().getCtl(), this.getClass().getSimpleName());
PartitionNames partitionNames = tblRef.getPartitionNames();
if (partitionNames != null) {

16
fe/fe-core/src/main/java/org/apache/doris/analysis/AdminCompactTableStmt.java
View File

@ -18,15 +18,14 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.Util;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import java.util.List;
@ -59,17 +58,8 @@ public class AdminCompactTableStmt extends DdlStmt {
ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
}
String dbName = null;
if (Strings.isNullOrEmpty(tblRef.getName().getDb())) {
dbName = analyzer.getDefaultDb();
if (Strings.isNullOrEmpty(dbName)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR);
}
} else {
dbName = ClusterNamespace.getFullName(getClusterName(), tblRef.getName().getDb());
}
tblRef.getName().setDb(dbName);
tblRef.getName().analyze(analyzer);
Util.prohibitExternalCatalog(tblRef.getName().getCtl(), this.getClass().getSimpleName());
PartitionNames partitionNames = tblRef.getPartitionNames();
if (partitionNames != null) {

16
fe/fe-core/src/main/java/org/apache/doris/analysis/AdminRepairTableStmt.java
View File

@ -18,15 +18,14 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.Util;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import java.util.List;
@ -51,17 +50,8 @@ public class AdminRepairTableStmt extends DdlStmt {
ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
}
String dbName = null;
if (Strings.isNullOrEmpty(tblRef.getName().getDb())) {
dbName = analyzer.getDefaultDb();
if (Strings.isNullOrEmpty(dbName)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR);
}
} else {
dbName = ClusterNamespace.getFullName(getClusterName(), tblRef.getName().getDb());
}
tblRef.getName().setDb(dbName);
tblRef.getName().analyze(analyzer);
Util.prohibitExternalCatalog(tblRef.getName().getCtl(), this.getClass().getSimpleName());
PartitionNames partitionNames = tblRef.getPartitionNames();
if (partitionNames != null) {

16
fe/fe-core/src/main/java/org/apache/doris/analysis/AdminShowReplicaDistributionStmt.java
View File

@ -20,16 +20,15 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.Util;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSetMetaData;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
// admin show replica distribution from tbl [partition(p1, p2, ...)]
@ -55,17 +54,8 @@ public class AdminShowReplicaDistributionStmt extends ShowStmt {
ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
}
String dbName = null;
if (Strings.isNullOrEmpty(tblRef.getName().getDb())) {
dbName = analyzer.getDefaultDb();
if (Strings.isNullOrEmpty(dbName)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR);
}
} else {
dbName = ClusterNamespace.getFullName(getClusterName(), tblRef.getName().getDb());
}
tblRef.getName().setDb(dbName);
tblRef.getName().analyze(analyzer);
Util.prohibitExternalCatalog(tblRef.getName().getCtl(), this.getClass().getSimpleName());
}
public String getDbName() {

16
fe/fe-core/src/main/java/org/apache/doris/analysis/AdminShowReplicaStatusStmt.java
View File

@ -22,16 +22,15 @@ import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Replica.ReplicaStatus;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.Util;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSetMetaData;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@ -65,17 +64,8 @@ public class AdminShowReplicaStatusStmt extends ShowStmt {
ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
}
String dbName = null;
if (Strings.isNullOrEmpty(tblRef.getName().getDb())) {
dbName = analyzer.getDefaultDb();
if (Strings.isNullOrEmpty(dbName)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR);
}
} else {
dbName = ClusterNamespace.getFullName(getClusterName(), tblRef.getName().getDb());
}
tblRef.getName().setDb(dbName);
tblRef.getName().analyze(analyzer);
Util.prohibitExternalCatalog(tblRef.getName().getCtl(), this.getClass().getSimpleName());
PartitionNames partitionNames = tblRef.getPartitionNames();
if (partitionNames != null) {

19
fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
View File

@ -31,7 +31,6 @@ import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Type;
import org.apache.doris.catalog.View;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
@ -649,17 +648,10 @@ public class Analyzer {
// Resolve the table ref's path and determine what resolved table ref
// to replace it with.
String dbName = tableName.getDb();
if (Strings.isNullOrEmpty(dbName)) {
dbName = getDefaultDb();
} else {
dbName = ClusterNamespace.getFullName(getClusterName(), tableName.getDb());
}
if (Strings.isNullOrEmpty(dbName)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR);
}
tableName.analyze(this);
DatabaseIf database = globalState.catalog.getCurrentDataSource().getDbOrAnalysisException(dbName);
DatabaseIf database = globalState.catalog.getDataSourceMgr().getCatalogOrAnalysisException(tableName.getCtl())
.getDbOrAnalysisException(tableName.getDb());
TableIf table = database.getTableOrAnalysisException(tableName.getTbl());
if (table.getType() == TableType.OLAP && (((OlapTable) table).getState() == OlapTableState.RESTORE
@ -688,7 +680,7 @@ public class Analyzer {
// which may not be the same as the user's reference to the table name, causing the table name not to be found
// in registerColumnRef(). So here the tblName is constructed using tableName.getTbl()
// instead of table.getName().
TableName tblName = new TableName(dbName, tableName.getTbl());
TableName tblName = new TableName(tableName.getDb(), tableName.getTbl());
if (table instanceof View) {
return new InlineViewRef((View) table, tableRef);
} else {
@ -698,7 +690,8 @@ public class Analyzer {
}
public TableIf getTableOrAnalysisException(TableName tblName) throws AnalysisException {
DatabaseIf db = globalState.catalog.getCurrentDataSource().getDbOrAnalysisException(tblName.getDb());
DatabaseIf db = globalState.catalog.getDataSourceMgr().getCatalogOrAnalysisException(tblName.getCtl())
.getDbOrAnalysisException(tblName.getDb());
return db.getTableOrAnalysisException(tblName.getTbl());
}

64
fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java
View File

@ -19,13 +19,13 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.MysqlTable;
import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
@ -36,11 +36,13 @@ import org.apache.doris.common.proc.ProcNodeInterface;
import org.apache.doris.common.proc.ProcResult;
import org.apache.doris.common.proc.ProcService;
import org.apache.doris.common.proc.TableProcDir;
import org.apache.doris.common.util.Util;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSetMetaData;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
@ -48,8 +50,10 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
public class DescribeStmt extends ShowStmt {
private static final ShowResultSetMetaData DESC_OLAP_TABLE_ALL_META_DATA =
@ -75,6 +79,17 @@ public class DescribeStmt extends ShowStmt {
.addColumn(new Column("Table", ScalarType.createVarchar(30)))
.build();
// The same columns in IndexSchemaProcNode.TITLE_NAMES
private static final ShowResultSetMetaData HMS_EXTERNAL_TABLE_META_DATA =
ShowResultSetMetaData.builder()
.addColumn(new Column("Field", ScalarType.createVarchar(20)))
.addColumn(new Column("Type", ScalarType.createVarchar(20)))
.addColumn(new Column("Null", ScalarType.createVarchar(20)))
.addColumn(new Column("Key", ScalarType.createVarchar(20)))
.addColumn(new Column("Default", ScalarType.createVarchar(20)))
.addColumn(new Column("Extra", ScalarType.createVarchar(20)))
.build();
// empty col num equals to DESC_OLAP_TABLE_ALL_META_DATA.size()
private static final List<String> EMPTY_ROW = initEmptyRow();
@ -86,6 +101,8 @@ public class DescribeStmt extends ShowStmt {
private boolean isAllTables;
private boolean isOlapTable;
private List<List<String>> hmsSchema = null;
public DescribeStmt(TableName dbTableName, boolean isAllTables) {
this.dbTableName = dbTableName;
this.totalRows = new LinkedList<List<String>>();
@ -108,26 +125,39 @@ public class DescribeStmt extends ShowStmt {
dbTableName.toString());
}
Database db = Catalog.getCurrentInternalCatalog().getDbOrAnalysisException(dbTableName.getDb());
Table table = db.getTableOrAnalysisException(dbTableName.getTbl());
DatabaseIf db = Catalog.getCurrentCatalog().getDataSourceMgr()
.getCatalogOrAnalysisException(dbTableName.getCtl()).getDbOrAnalysisException(dbTableName.getDb());
TableIf table = db.getTableOrAnalysisException(dbTableName.getTbl());
table.readLock();
try {
if (!isAllTables) {
// show base table schema only
String procString = "/dbs/" + db.getId() + "/" + table.getId() + "/" + TableProcDir.INDEX_SCHEMA
+ "/";
if (table.getType() == TableType.OLAP) {
procString += ((OlapTable) table).getBaseIndexId();
if (table.getType() == TableType.HMS_EXTERNAL_TABLE) {
hmsSchema = table.getFullSchema().stream().map(col -> Arrays.asList(
col.getName(),
col.getType().toSql().toUpperCase(Locale.ROOT),
Boolean.toString(col.isAllowNull()),
Boolean.toString(col.isKey()),
Strings.nullToEmpty(col.getDefaultValue()),
"" /* no extra field */))
.collect(Collectors.toList());
} else {
procString += table.getId();
}
// show base table schema only
String procString = "/dbs/" + db.getId() + "/" + table.getId() + "/" + TableProcDir.INDEX_SCHEMA
+ "/";
if (table.getType() == TableType.OLAP) {
procString += ((OlapTable) table).getBaseIndexId();
} else {
procString += table.getId();
}
node = ProcService.getInstance().open(procString);
if (node == null) {
throw new AnalysisException("Describe table[" + dbTableName.getTbl() + "] failed");
node = ProcService.getInstance().open(procString);
if (node == null) {
throw new AnalysisException("Describe table[" + dbTableName.getTbl() + "] failed");
}
}
} else {
Util.prohibitExternalCatalog(dbTableName.getCtl(), this.getClass().getSimpleName() + " ALL");
if (table.getType() == TableType.OLAP) {
isOlapTable = true;
OlapTable olapTable = (OlapTable) table;
@ -231,6 +261,9 @@ public class DescribeStmt extends ShowStmt {
if (isAllTables) {
return totalRows;
} else {
if (hmsSchema != null) {
return hmsSchema;
}
Preconditions.checkNotNull(node);
return node.fetchResult().getRows();
}
@ -239,6 +272,9 @@ public class DescribeStmt extends ShowStmt {
@Override
public ShowResultSetMetaData getMetaData() {
if (!isAllTables) {
if (hmsSchema != null) {
return HMS_EXTERNAL_TABLE_META_DATA;
}
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
ProcResult result = null;

32
fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
View File

@ -20,16 +20,10 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Database;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import java.util.ArrayList;
@ -70,29 +64,6 @@ public class FromClause implements ParseNode, Iterable<TableRef> {
this.needToSql = needToSql;
}
private void checkFromHiveTable(Analyzer analyzer) throws AnalysisException {
for (TableRef tblRef : tablerefs) {
if (!(tblRef instanceof BaseTableRef)) {
continue;
}
TableName tableName = tblRef.getName();
String dbName = tableName.getDb();
if (Strings.isNullOrEmpty(dbName)) {
dbName = analyzer.getDefaultDb();
} else {
dbName = ClusterNamespace.getFullName(analyzer.getClusterName(), tblRef.getName().getDb());
}
if (Strings.isNullOrEmpty(dbName)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR);
}
Database db = analyzer.getCatalog().getInternalDataSource().getDbOrAnalysisException(dbName);
String tblName = tableName.getTbl();
db.getTableOrAnalysisException(tblName);
}
}
/**
* In some cases, the reorder method of select stmt will incorrectly sort the tableRef with on clause.
* The meaning of this function is to reset those tableRefs with on clauses.
@ -155,9 +126,6 @@ public class FromClause implements ParseNode, Iterable<TableRef> {
leftTblRef = tblRef;
}
// TODO: remove when query from hive table is supported
checkFromHiveTable(analyzer);
analyzed = true;
}

3
fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
View File

@ -936,7 +936,8 @@ public class FunctionCallExpr extends Expr {
.checkDbPriv(ConnectContext.get(), dbName, PrivPredicate.SELECT)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "SELECT");
}
DatabaseIf db = Catalog.getCurrentCatalog().getCurrentDataSource().getDbNullable(dbName);
// TODO(gaoxin): ExternalDatabase not implement udf yet.
DatabaseIf db = Catalog.getCurrentCatalog().getInternalDataSource().getDbNullable(dbName);
if (db != null && (db instanceof Database)) {
Function searchDesc =
new Function(fnName, Arrays.asList(collectChildReturnTypes()), Type.INVALID, false);

4
fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
View File

@ -23,7 +23,6 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ColumnStats;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Type;
import org.apache.doris.thrift.TSlotDescriptor;
@ -329,7 +328,6 @@ public class SlotDescriptor {
}
public boolean isScanSlot() {
Table table = (Table) parent.getTable();
return table instanceof OlapTable;
return parent.getTable() instanceof OlapTable;
}
}

8
fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
View File

@ -21,7 +21,7 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.io.Text;
@ -373,7 +373,7 @@ public class SlotRef extends Expr {
expr.getTableIdToColumnNames(tableIdToColumnNames);
}
} else {
Table table = (Table) desc.getParent().getTable();
TableIf table = desc.getParent().getTable();
if (table == null) {
// Maybe this column comes from inline view.
return;
@ -388,9 +388,9 @@ public class SlotRef extends Expr {
}
}
public Table getTable() {
public TableIf getTable() {
Preconditions.checkState(desc != null);
return (Table) desc.getParent().getTable();
return desc.getParent().getTable();
}
public void setLabel(String label) {

10
fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
View File

@ -479,9 +479,11 @@ public class Catalog {
}
public DataSourceIf getCurrentDataSource() {
// TODO: this should be got from connect context.
// Will be fixed later.
return dataSourceMgr.getInternalDataSource();
ConnectContext ctx = ConnectContext.get();
if (ctx == null) {
return dataSourceMgr.getInternalDataSource();
}
return ctx.getCurrentDataSource();
}
public InternalDataSource getInternalDataSource() {
@ -4163,7 +4165,7 @@ public class Catalog {
ErrorReport.reportDdlException(ErrorCode.ERR_DBACCESS_DENIED_ERROR, ctx.getQualifiedUser(), qualifiedDb);
}
getInternalDataSource().getDbOrDdlException(qualifiedDb);
ctx.getCurrentDataSource().getDbOrDdlException(qualifiedDb);
ctx.setDatabase(qualifiedDb);
}

14
fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceMgr.java
View File

@ -48,6 +48,7 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
@ -84,6 +85,19 @@ public class DataSourceMgr implements Writable {
return nameToCatalogs.get(name);
}
public <E extends Exception> DataSourceIf getCatalogOrException(String name, Function<String, E> e) throws E {
DataSourceIf ds = nameToCatalogs.get(name);
if (ds == null) {
throw e.apply(name);
}
return ds;
}
public DataSourceIf getCatalogOrAnalysisException(String name) throws AnalysisException {
return getCatalogOrException(name, ds -> new AnalysisException(
ErrorCode.ERR_UNKNOWN_CATALOG.formatErrorMsg(ds), ErrorCode.ERR_UNKNOWN_CATALOG));
}
public DatabaseIf getDbNullable(long dbId) {
DatabaseIf db = internalDataSource.getDbNullable(dbId);
if (db != null) {

8
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java
View File

@ -18,6 +18,7 @@
package org.apache.doris.datasource;
import org.apache.doris.catalog.external.ExternalDatabase;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
@ -176,6 +177,13 @@ public abstract class ExternalDataSource implements DataSourceIf<ExternalDatabas
Text.writeString(out, GsonUtils.GSON.toJson(this));
}
/**
* External catalog has no cluster semantics.
*/
protected static String getRealTableName(String tableName) {
return ClusterNamespace.getNameFromFullName(tableName);
}
public static ExternalDataSource read(DataInput in) throws IOException {
String json = Text.readString(in);
return GsonUtils.GSON.fromJson(json, ExternalDataSource.class);

5
fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java
View File

@ -121,7 +121,7 @@ public class HMSExternalDataSource extends ExternalDataSource {
public List<String> listTableNames(SessionContext ctx, String dbName) {
makeSureInitialized();
try {
return client.getAllTables(dbName);
return client.getAllTables(getRealTableName(dbName));
} catch (MetaException e) {
LOG.warn("List Table Names failed. {}", e.getMessage());
}
@ -132,7 +132,7 @@ public class HMSExternalDataSource extends ExternalDataSource {
public boolean tableExist(SessionContext ctx, String dbName, String tblName) {
makeSureInitialized();
try {
return client.tableExists(dbName, tblName);
return client.tableExists(getRealTableName(dbName), tblName);
} catch (TException e) {
LOG.warn("Check table exist failed. {}", e.getMessage());
}
@ -143,6 +143,7 @@ public class HMSExternalDataSource extends ExternalDataSource {
@Override
public ExternalDatabase getDbNullable(String dbName) {
makeSureInitialized();
dbName = getRealTableName(dbName);
try {
client.getDatabase(dbName);
} catch (TException e) {

13
fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaInfoAction.java
View File

@ -30,6 +30,8 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.proc.ProcNodeInterface;
import org.apache.doris.common.proc.ProcResult;
import org.apache.doris.common.proc.ProcService;
import org.apache.doris.datasource.DataSourceIf;
import org.apache.doris.datasource.InternalDataSource;
import org.apache.doris.httpv2.entity.ResponseEntityBuilder;
import org.apache.doris.httpv2.exception.BadRequestException;
import org.apache.doris.mysql.privilege.PrivPredicate;
@ -86,12 +88,17 @@ public class MetaInfoAction extends RestBaseController {
HttpServletRequest request, HttpServletResponse response) {
checkWithCookie(request, response, false);
if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) {
return ResponseEntityBuilder.badRequest("Only support 'default_cluster' now");
// use NS_KEY as catalog, but NS_KEY's default value is 'default_cluster'.
if (ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) {
ns = InternalDataSource.INTERNAL_DS_NAME;
}
// 1. get all database with privilege
List<String> dbNames = Catalog.getCurrentCatalog().getCurrentDataSource().getDbNames();
DataSourceIf ds = Catalog.getCurrentCatalog().getDataSourceMgr().getCatalog(ns);
if (ds == null) {
return ResponseEntityBuilder.badRequest("Unknown catalog " + ns);
}
List<String> dbNames = ds.getDbNames();
List<String> dbNameSet = Lists.newArrayList();
for (String fullName : dbNames) {
final String db = ClusterNamespace.getNameFromFullName(fullName);

10
fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
View File

@ -32,7 +32,7 @@ import org.apache.doris.analysis.TableRef;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.catalog.ColumnStats;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.CheckedMath;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.Pair;
@ -363,15 +363,15 @@ public class HashJoinNode extends PlanNode {
}
public double lhsNumRows() {
Table table = (Table) lhs.getParent().getTable();
TableIf table = lhs.getParent().getTable();
Preconditions.checkState(table instanceof OlapTable);
return ((OlapTable) (table)).getRowCount();
return table.getRowCount();
}
public double rhsNumRows() {
Table table = (Table) rhs.getParent().getTable();
TableIf table = rhs.getParent().getTable();
Preconditions.checkState(table instanceof OlapTable);
return ((OlapTable) (table)).getRowCount();
return table.getRowCount();
}
public TupleId lhsTid() {

9
fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java
View File

@ -84,11 +84,10 @@ public abstract class Policy implements Writable, GsonPostProcessable {
return storagePolicy;
case ROW:
default:
String curDb = stmt.getTableName().getDb();
if (curDb == null) {
curDb = ConnectContext.get().getDatabase();
}
DatabaseIf db = Catalog.getCurrentCatalog().getCurrentDataSource().getDbOrAnalysisException(curDb);
// stmt must be analyzed.
DatabaseIf db = Catalog.getCurrentCatalog().getDataSourceMgr()
.getCatalogOrAnalysisException(stmt.getTableName().getCtl())
.getDbOrAnalysisException(stmt.getTableName().getDb());
UserIdentity userIdent = stmt.getUser();
userIdent.analyze(ConnectContext.get().getClusterName());
TableIf table = db.getTableOrAnalysisException(stmt.getTableName().getTbl());

11
fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.datasource.DataSourceIf;
import org.apache.doris.datasource.InternalDataSource;
import org.apache.doris.datasource.SessionContext;
import org.apache.doris.mysql.MysqlCapability;
@ -418,6 +419,14 @@ public class ConnectContext {
return defaultCatalog;
}
public DataSourceIf getCurrentDataSource() {
// defaultCatalog is switched by SwitchStmt, so we don't need to check to exist of catalog.
if (catalog == null) {
return Catalog.getCurrentCatalog().getDataSourceMgr().getCatalog(defaultCatalog);
}
return catalog.getDataSourceMgr().getCatalog(defaultCatalog);
}
public void changeDefaultCatalog(String catalogName) {
defaultCatalog = catalogName;
}
@ -428,7 +437,7 @@ public class ConnectContext {
public void setDatabase(String db) {
currentDb = db;
Optional<DatabaseIf> dbInstance = Catalog.getCurrentCatalog().getCurrentDataSource().getDb(db);
Optional<DatabaseIf> dbInstance = getCurrentDataSource().getDb(db);
currentDbId = dbInstance.isPresent() ? dbInstance.get().getId() : -1;
}

2
fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
View File

@ -311,7 +311,7 @@ public class ConnectProcessor {
ctx.getState().setError(ErrorCode.ERR_UNKNOWN_TABLE, "Empty tableName");
return;
}
DatabaseIf db = ctx.getCatalog().getCurrentDataSource().getDbNullable(ctx.getDatabase());
DatabaseIf db = ctx.getCurrentDataSource().getDbNullable(ctx.getDatabase());
if (db == null) {
ctx.getState().setError(ErrorCode.ERR_BAD_DB_ERROR, "Unknown database(" + ctx.getDatabase() + ")");
return;

37
fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
View File

@ -151,6 +151,7 @@ import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.common.util.ProfileManager;
import org.apache.doris.common.util.RuntimeProfile;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.common.util.Util;
import org.apache.doris.external.iceberg.IcebergTableCreationRecord;
import org.apache.doris.load.DeleteHandler;
import org.apache.doris.load.ExportJob;
@ -413,7 +414,8 @@ public class ShowExecutor {
// Handle show functions
private void handleShowFunctions() throws AnalysisException {
ShowFunctionsStmt showStmt = (ShowFunctionsStmt) stmt;
DatabaseIf db = ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName());
Util.prohibitExternalCatalog(ctx.getDefaultCatalog(), stmt.getClass().getSimpleName());
DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName());
List<List<String>> resultRowSet = Lists.newArrayList();
if (db instanceof Database) {
@ -460,8 +462,8 @@ public class ShowExecutor {
// Handle show create function
private void handleShowCreateFunction() throws AnalysisException {
ShowCreateFunctionStmt showCreateFunctionStmt = (ShowCreateFunctionStmt) stmt;
DatabaseIf db =
ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(showCreateFunctionStmt.getDbName());
Util.prohibitExternalCatalog(ctx.getDefaultCatalog(), stmt.getClass().getSimpleName());
DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showCreateFunctionStmt.getDbName());
List<List<String>> resultRowSet = Lists.newArrayList();
if (db instanceof Database) {
Function function = ((Database) db).getFunction(showCreateFunctionStmt.getFunction());
@ -476,7 +478,8 @@ public class ShowExecutor {
// Handle show encryptkeys
private void handleShowEncryptKeys() throws AnalysisException {
ShowEncryptKeysStmt showStmt = (ShowEncryptKeysStmt) stmt;
DatabaseIf db = ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName());
Util.prohibitExternalCatalog(ctx.getDefaultCatalog(), stmt.getClass().getSimpleName());
DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName());
List<List<String>> resultRowSet = Lists.newArrayList();
if (db instanceof Database) {
List<EncryptKey> encryptKeys = ((Database) db).getEncryptKeys();
@ -571,8 +574,7 @@ public class ShowExecutor {
ShowDbIdStmt showStmt = (ShowDbIdStmt) stmt;
long dbId = showStmt.getDbId();
List<List<String>> rows = Lists.newArrayList();
Catalog catalog = ctx.getCatalog();
DatabaseIf database = catalog.getCurrentDataSource().getDbNullable(dbId);
DatabaseIf database = ctx.getCurrentDataSource().getDbNullable(dbId);
if (database != null) {
List<String> row = new ArrayList<>();
row.add(database.getFullName());
@ -647,7 +649,8 @@ public class ShowExecutor {
private void handleShowDb() throws AnalysisException {
ShowDbStmt showDbStmt = (ShowDbStmt) stmt;
List<List<String>> rows = Lists.newArrayList();
List<String> dbNames = ctx.getCatalog().getInternalDataSource().getClusterDbNames(ctx.getClusterName());
// cluster feature is deprecated.
List<String> dbNames = ctx.getCurrentDataSource().getDbNames();
PatternMatcher matcher = null;
if (showDbStmt.getPattern() != null) {
matcher = PatternMatcher.createMysqlPattern(showDbStmt.getPattern(),
@ -682,8 +685,7 @@ public class ShowExecutor {
List<List<String>> rows = Lists.newArrayList();
// TODO(gaoxin): Whether to support "show tables from `ctl.db`" syntax in show statement?
String catalogName = ctx.getDefaultCatalog();
DatabaseIf<TableIf> db =
ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(showTableStmt.getDb());
DatabaseIf<TableIf> db = ctx.getCurrentDataSource().getDbOrAnalysisException(showTableStmt.getDb());
PatternMatcher matcher = null;
if (showTableStmt.getPattern() != null) {
matcher = PatternMatcher.createMysqlPattern(showTableStmt.getPattern(),
@ -715,7 +717,7 @@ public class ShowExecutor {
private void handleShowTableStatus() throws AnalysisException {
ShowTableStatusStmt showStmt = (ShowTableStatusStmt) stmt;
List<List<String>> rows = Lists.newArrayList();
DatabaseIf<TableIf> db = ctx.getCatalog().getCurrentDataSource().getDbNullable(showStmt.getDb());
DatabaseIf<TableIf> db = ctx.getCurrentDataSource().getDbNullable(showStmt.getDb());
if (db != null) {
PatternMatcher matcher = null;
if (showStmt.getPattern() != null) {
@ -799,7 +801,7 @@ public class ShowExecutor {
private void handleShowCreateDb() throws AnalysisException {
ShowCreateDbStmt showStmt = (ShowCreateDbStmt) stmt;
List<List<String>> rows = Lists.newArrayList();
DatabaseIf db = ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(showStmt.getDb());
DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDb());
StringBuilder sb = new StringBuilder();
sb.append("CREATE DATABASE `").append(ClusterNamespace.getNameFromFullName(showStmt.getDb())).append("`");
if (db.getDbProperties().getProperties().size() > 0) {
@ -815,7 +817,7 @@ public class ShowExecutor {
// Show create table
private void handleShowCreateTable() throws AnalysisException {
ShowCreateTableStmt showStmt = (ShowCreateTableStmt) stmt;
DatabaseIf db = ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(showStmt.getDb());
DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDb());
TableIf table = db.getTableOrAnalysisException(showStmt.getTable());
List<List<String>> rows = Lists.newArrayList();
@ -854,7 +856,7 @@ public class ShowExecutor {
private void handleShowColumn() throws AnalysisException {
ShowColumnStmt showStmt = (ShowColumnStmt) stmt;
List<List<String>> rows = Lists.newArrayList();
DatabaseIf db = ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(showStmt.getDb());
DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDb());
TableIf table = db.getTableOrAnalysisException(showStmt.getTable());
PatternMatcher matcher = null;
if (showStmt.getPattern() != null) {
@ -906,7 +908,7 @@ public class ShowExecutor {
private void handleShowIndex() throws AnalysisException {
ShowIndexStmt showStmt = (ShowIndexStmt) stmt;
List<List<String>> rows = Lists.newArrayList();
DatabaseIf db = ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName());
DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName());
OlapTable table = db.getOlapTableOrAnalysisException(showStmt.getTableName().getTbl());
table.readLock();
@ -1005,8 +1007,9 @@ public class ShowExecutor {
private void handleShowLoad() throws AnalysisException {
ShowLoadStmt showStmt = (ShowLoadStmt) stmt;
Catalog catalog = Catalog.getCurrentCatalog();
DatabaseIf db = catalog.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName());
Util.prohibitExternalCatalog(ctx.getDefaultCatalog(), stmt.getClass().getSimpleName());
Catalog catalog = ctx.getCatalog();
DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName());
long dbId = db.getId();
// combine the List<LoadInfo> of load(v1) and loadManager(v2)
@ -2125,7 +2128,7 @@ public class ShowExecutor {
private void handleShowTableCreation() throws AnalysisException {
ShowTableCreationStmt showStmt = (ShowTableCreationStmt) stmt;
String dbName = showStmt.getDbName();
DatabaseIf db = ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(dbName);
DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(dbName);
List<IcebergTableCreationRecord> records =
ctx.getCatalog().getIcebergTableCreationRecordMgr().getTableCreationRecordByDbId(db.getId());

4
fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java
View File

@ -27,7 +27,7 @@ import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.rewrite.ExprRewriteRule;
import org.apache.doris.rewrite.ExprRewriter;
@ -69,7 +69,7 @@ public class CountDistinctToBitmap implements ExprRewriteRule {
}
SlotRef fnChild0 = (SlotRef) fnExpr.getChild(0);
Column column = fnChild0.getColumn();
Table table = fnChild0.getTable();
TableIf table = fnChild0.getTable();
if (column == null || table == null || !(table instanceof OlapTable)) {
return expr;
}

4
fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java
View File

@ -25,7 +25,7 @@ import org.apache.doris.analysis.SlotRef;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.rewrite.ExprRewriteRule;
import org.apache.doris.rewrite.ExprRewriter;
@ -65,7 +65,7 @@ public class CountFieldToSum implements ExprRewriteRule {
}
SlotRef fnChild0 = (SlotRef) fnExpr.getChild(0);
Column column = fnChild0.getColumn();
Table table = fnChild0.getTable();
TableIf table = fnChild0.getTable();
if (column == null || table == null || !(table instanceof OlapTable)) {
return expr;
}

4
fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java
View File

@ -27,7 +27,7 @@ import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.rewrite.ExprRewriteRule;
import org.apache.doris.rewrite.ExprRewriter;
@ -81,7 +81,7 @@ public class HLLHashToSlotRefRule implements ExprRewriteRule {
return expr;
}
Column column = queryColumnSlotRef.getColumn();
Table table = queryColumnSlotRef.getTable();
TableIf table = queryColumnSlotRef.getTable();
if (column == null || table == null || !(table instanceof OlapTable)) {
return expr;
}

4
fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java
View File

@ -26,7 +26,7 @@ import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.rewrite.ExprRewriteRule;
import org.apache.doris.rewrite.ExprRewriter;
@ -66,7 +66,7 @@ public class NDVToHll implements ExprRewriteRule {
}
SlotRef fnChild0 = (SlotRef) fnExpr.getChild(0);
Column column = fnChild0.getColumn();
Table table = fnChild0.getTable();
TableIf table = fnChild0.getTable();
if (column == null || table == null || !(table instanceof OlapTable)) {
return expr;
}

4
fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java
View File

@ -28,7 +28,7 @@ import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.rewrite.ExprRewriteRule;
import org.apache.doris.rewrite.ExprRewriter;
@ -80,7 +80,7 @@ public class ToBitmapToSlotRefRule implements ExprRewriteRule {
return expr;
}
Column column = queryColumnSlotRef.getColumn();
Table table = queryColumnSlotRef.getTable();
TableIf table = queryColumnSlotRef.getTable();
if (column == null || table == null || !(table instanceof OlapTable)) {
return expr;
}

4
fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
View File

@ -204,7 +204,9 @@ public class FrontendServiceImpl implements FrontendService.Iface {
} else {
currentUser = UserIdentity.createAnalyzedUserIdentWithIp(params.user, params.user_ip);
}
DatabaseIf<TableIf> db = Catalog.getCurrentCatalog().getCurrentDataSource().getDbNullable(params.db);
String catalog = Strings.isNullOrEmpty(params.catalog) ? InternalDataSource.INTERNAL_DS_NAME : params.catalog;
DatabaseIf<TableIf> db = Catalog.getCurrentCatalog().getDataSourceMgr()
.getCatalogOrException(catalog, ds -> new TException("Unknown catalog " + ds)).getDbNullable(params.db);
if (db != null) {
for (String tableName : db.getTableNamesWithLock()) {
LOG.debug("get table: {}, wait to check", tableName);

44
fe/fe-core/src/test/java/org/apache/doris/analysis/AccessTestUtil.java
View File

@ -33,6 +33,7 @@ import org.apache.doris.catalog.SinglePartitionInfo;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.datasource.DataSourceMgr;
import org.apache.doris.datasource.InternalDataSource;
import org.apache.doris.load.Load;
import org.apache.doris.mysql.privilege.PaloAuth;
@ -48,6 +49,7 @@ import mockit.Expectations;
import java.util.LinkedList;
import java.util.List;
import java.util.function.Function;
public class AccessTestUtil {
private static FakeEditLog fakeEditLog;
@ -142,6 +144,23 @@ public class AccessTestUtil {
}
};
DataSourceMgr dsMgr = new DataSourceMgr();
new Expectations(dsMgr) {
{
dsMgr.getCatalog((String) any);
minTimes = 0;
result = ds;
dsMgr.getCatalogOrException((String) any, (Function) any);
minTimes = 0;
result = ds;
dsMgr.getCatalogOrAnalysisException((String) any);
minTimes = 0;
result = ds;
}
};
new Expectations(catalog, ds) {
{
catalog.getAuth();
@ -178,6 +197,10 @@ public class AccessTestUtil {
catalog.getBrokerMgr();
minTimes = 0;
result = new BrokerMgr();
catalog.getDataSourceMgr();
minTimes = 0;
result = dsMgr;
}
};
return catalog;
@ -353,6 +376,23 @@ public class AccessTestUtil {
}
};
DataSourceMgr dsMgr = new DataSourceMgr();
new Expectations(dsMgr) {
{
dsMgr.getCatalog((String) any);
minTimes = 0;
result = ds;
dsMgr.getCatalogOrException((String) any, (Function) any);
minTimes = 0;
result = ds;
dsMgr.getCatalogOrAnalysisException((String) any);
minTimes = 0;
result = ds;
}
};
new Expectations(catalog) {
{
catalog.getAuth();
@ -370,6 +410,10 @@ public class AccessTestUtil {
catalog.getCurrentDataSource();
minTimes = 0;
result = ds;
catalog.getDataSourceMgr();
minTimes = 0;
result = dsMgr;
}
};
return catalog;

4
fe/fe-core/src/test/java/org/apache/doris/analysis/VirtualSlotRefTest.java
View File

@ -67,7 +67,9 @@ public class VirtualSlotRefTest {
f.setAccessible(true);
Multimap<String, TupleDescriptor> tupleByAlias = ArrayListMultimap.create();
TupleDescriptor td = new TupleDescriptor(new TupleId(0));
td.setTable(analyzerBase.getTableOrAnalysisException(new TableName("testdb", "t")));
TableName tableName = new TableName("testdb", "t");
tableName.analyze(analyzerBase);
td.setTable(analyzerBase.getTableOrAnalysisException(tableName));
tupleByAlias.put("testdb.t", td);
f.set(analyzer, tupleByAlias);
} catch (NoSuchFieldException | IllegalAccessException e) {

23
fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java
View File

@ -42,6 +42,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker.ThrowingRunnable;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.datasource.DataSourceMgr;
import org.apache.doris.datasource.InternalDataSource;
import org.apache.doris.httpv2.HttpServer;
import org.apache.doris.httpv2.IllegalArgException;
@ -78,6 +79,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
public abstract class DorisHttpTestCase {
@ -245,6 +247,23 @@ public abstract class DorisHttpTestCase {
}
};
DataSourceMgr dsMgr = new DataSourceMgr();
new Expectations(dsMgr) {
{
dsMgr.getCatalog((String) any);
minTimes = 0;
result = internalDataSource;
dsMgr.getCatalogOrException((String) any, (Function) any);
minTimes = 0;
result = internalDataSource;
dsMgr.getCatalogOrAnalysisException((String) any);
minTimes = 0;
result = internalDataSource;
}
};
new Expectations(catalog) {
{
catalog.getAuth();
@ -279,6 +298,10 @@ public abstract class DorisHttpTestCase {
catalog.initDefaultCluster();
minTimes = 0;
catalog.getDataSourceMgr();
minTimes = 0;
result = dsMgr;
}
};
return catalog;

7
fe/fe-core/src/test/java/org/apache/doris/qe/ConnectProcessorTest.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.qe;
import org.apache.doris.analysis.AccessTestUtil;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.datasource.DataSourceIf;
import org.apache.doris.metric.MetricRepo;
import org.apache.doris.mysql.MysqlChannel;
import org.apache.doris.mysql.MysqlCommand;
@ -191,6 +192,8 @@ public class ConnectProcessorTest {
}
};
DataSourceIf ds = catalog.getCurrentDataSource();
new Expectations(context) {
{
context.getMysqlChannel();
@ -236,6 +239,10 @@ public class ConnectProcessorTest {
context.queryId();
minTimes = 0;
result = new TUniqueId();
context.getCurrentDataSource();
minTimes = 0;
result = ds;
}
};

39
fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java
View File

@ -47,6 +47,7 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.DataSourceMgr;
import org.apache.doris.datasource.InternalDataSource;
import org.apache.doris.metric.MetricRepo;
import org.apache.doris.mysql.MysqlChannel;
@ -90,6 +91,7 @@ import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.function.Function;
public class PartitionCacheTest {
private static final Logger LOG = LogManager.getLogger(PartitionCacheTest.class);
@ -191,6 +193,23 @@ public class PartitionCacheTest {
}
};
DataSourceMgr dsMgr = new DataSourceMgr();
new Expectations(dsMgr) {
{
dsMgr.getCatalog((String) any);
minTimes = 0;
result = ds;
dsMgr.getCatalogOrException((String) any, (Function) any);
minTimes = 0;
result = ds;
dsMgr.getCatalogOrAnalysisException((String) any);
minTimes = 0;
result = ds;
}
};
new Expectations(catalog) {
{
catalog.getAuth();
@ -204,6 +223,10 @@ public class PartitionCacheTest {
catalog.getInternalDataSource();
minTimes = 0;
result = ds;
catalog.getDataSourceMgr();
minTimes = 0;
result = dsMgr;
}
};
FunctionSet fs = new FunctionSet();
@ -1067,8 +1090,8 @@ public class PartitionCacheTest {
SqlCache sqlCache = (SqlCache) ca.getCache();
String cacheKey = sqlCache.getSqlWithViewStmt();
Assert.assertEquals(cacheKey, "SELECT `testDb`.`view1`.`eventdate` AS `eventdate`, `testDb`.`view1`."
+ "`count(`userid`)` AS `count(``userid``)` FROM `testDb`.`view1`|select eventdate, COUNT(userid) "
Assert.assertEquals(cacheKey, "SELECT `testCluster:testDb`.`view1`.`eventdate` AS `eventdate`, `testCluster:testDb`.`view1`."
+ "`count(`userid`)` AS `count(``userid``)` FROM `testCluster:testDb`.`view1`|select eventdate, COUNT(userid) "
+ "FROM appevent WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate");
}
@ -1094,7 +1117,7 @@ public class PartitionCacheTest {
String cacheKey = sqlCache.getSqlWithViewStmt();
Assert.assertEquals(cacheKey, "SELECT `origin`.`eventdate` AS `eventdate`, `origin`.`userid` AS "
+ "`userid` FROM (SELECT `view2`.`eventdate` AS `eventdate`, `view2`.`userid` AS `userid` FROM "
+ "`testDb`.`view2` view2 WHERE `view2`.`eventdate` >= '2020-01-12 00:00:00' AND `view2`.`eventdate`"
+ "`testCluster:testDb`.`view2` view2 WHERE `view2`.`eventdate` >= '2020-01-12 00:00:00' AND `view2`.`eventdate`"
+ " <= '2020-01-14 00:00:00') origin|select eventdate, userid FROM appevent");
}
@ -1114,9 +1137,9 @@ public class PartitionCacheTest {
cache.rewriteSelectStmt(null);
Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null);
Assert.assertEquals(cache.getSqlWithViewStmt(), "SELECT `testDb`.`view3`.`eventdate` AS "
+ "`eventdate`, `testDb`.`view3`.`count(`userid`)` AS `count(``userid``)` FROM "
+ "`testDb`.`view3`|select eventdate, COUNT(userid) FROM appevent WHERE eventdate>="
Assert.assertEquals(cache.getSqlWithViewStmt(), "SELECT `testCluster:testDb`.`view3`.`eventdate` AS "
+ "`eventdate`, `testCluster:testDb`.`view3`.`count(`userid`)` AS `count(``userid``)` FROM "
+ "`testCluster:testDb`.`view3`|select eventdate, COUNT(userid) FROM appevent WHERE eventdate>="
+ "\"2020-01-12\" and eventdate<=\"2020-01-15\" GROUP BY eventdate");
} catch (Exception e) {
LOG.warn("ex={}", e);
@ -1170,8 +1193,8 @@ public class PartitionCacheTest {
SqlCache sqlCache = (SqlCache) ca.getCache();
String cacheKey = sqlCache.getSqlWithViewStmt();
Assert.assertEquals(cacheKey, "SELECT `testDb`.`view4`.`eventdate` AS `eventdate`, "
+ "`testDb`.`view4`.`count(`userid`)` AS `count(``userid``)` FROM `testDb`.`view4`|select "
Assert.assertEquals(cacheKey, "SELECT `testCluster:testDb`.`view4`.`eventdate` AS `eventdate`, "
+ "`testCluster:testDb`.`view4`.`count(`userid`)` AS `count(``userid``)` FROM `testCluster:testDb`.`view4`|select "
+ "eventdate, COUNT(userid) FROM view2 WHERE eventdate>=\"2020-01-12\" and "
+ "eventdate<=\"2020-01-14\" GROUP BY eventdate|select eventdate, userid FROM appevent");
}

23
fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java
View File

@ -50,6 +50,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.PatternMatcher;
import org.apache.doris.common.UserException;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.datasource.DataSourceMgr;
import org.apache.doris.datasource.InternalDataSource;
import org.apache.doris.mysql.MysqlCommand;
import org.apache.doris.mysql.privilege.PaloAuth;
@ -70,6 +71,7 @@ import org.junit.rules.ExpectedException;
import java.io.IOException;
import java.net.URL;
import java.util.List;
import java.util.function.Function;
public class ShowExecutorTest {
private ConnectContext ctx;
@ -189,6 +191,23 @@ public class ShowExecutorTest {
}
};
DataSourceMgr dsMgr = new DataSourceMgr();
new Expectations(dsMgr) {
{
dsMgr.getCatalog((String) any);
minTimes = 0;
result = ds;
dsMgr.getCatalogOrException((String) any, (Function) any);
minTimes = 0;
result = ds;
dsMgr.getCatalogOrAnalysisException((String) any);
minTimes = 0;
result = ds;
}
};
// mock catalog.
catalog = Deencapsulation.newInstance(Catalog.class);
new Expectations(catalog) {
@ -218,6 +237,10 @@ public class ShowExecutorTest {
Catalog.getDdlStmt((Table) any, (List) any, null, null, anyBoolean, anyBoolean);
minTimes = 0;
catalog.getDataSourceMgr();
minTimes = 0;
result = dsMgr;
}
};