database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Feature](multi-catalog) support query hive views. (#18815)

Browse Source 
A very simple implementation to query hive views, it is an EXPERIMENTAL feature.
We can try to parse the ddl of hive views and try to execute the query relies on the fact that HiveQL
is very similar to Doris SQL. But if the ddl of hive views use some complicated or incompatible grammar,
the query might fail.
This commit is contained in:
Xiangyu Wang
2023-04-24 08:49:26 +08:00
committed by GitHub
parent 0dd45ce158
commit 2d7903e2bd
7 changed files with 513 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
View File

@ -34,6 +34,7 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.catalog.View;
import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.IdGenerator;
@ -81,6 +82,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -319,6 +321,11 @@ public class Analyzer {
// True if at least one of the analyzers belongs to a subquery.
public boolean containsSubquery = false;
// When parsing a ddl of hive view, it does not contains any catalog info,
// so we need to record it in Analyzer
// otherwise some error will occurs when resolving TableRef later.
public String externalCtl;
// all registered conjuncts (map from id to Predicate)
private final Map<ExprId, Expr> conjuncts = Maps.newHashMap();
@ -541,6 +548,14 @@ public class Analyzer {
return new Analyzer(parentAnalyzer, globalState, new InferPredicateState());
}
public void setExternalCtl(String externalCtl) {
globalState.externalCtl = externalCtl;
}
public String getExternalCtl() {
return globalState.externalCtl;
}
public void setIsExplain() {
globalState.isExplain = true;
}
@ -757,6 +772,10 @@ public class Analyzer {
}
// Try to find a matching local view.
TableName tableName = tableRef.getName();
if (StringUtils.isNotEmpty(this.globalState.externalCtl)
&& StringUtils.isEmpty(tableName.getCtl())) {
tableName.setCtl(this.globalState.externalCtl);
}
if (!tableName.isFullyQualified()) {
// Searches the hierarchy of analyzers bottom-up for a registered local view with
// a matching alias.
@ -801,11 +820,26 @@ public class Analyzer {
// Now hms table only support a bit of table kinds in the whole hive system.
// So Add this strong checker here to avoid some undefine behaviour in doris.
if (table.getType() == TableType.HMS_EXTERNAL_TABLE && !((HMSExternalTable) table).isSupportedHmsTable()) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_NONSUPPORT_HMS_TABLE,
table.getName(),
((HMSExternalTable) table).getDbName(),
tableName.getCtl());
if (table.getType() == TableType.HMS_EXTERNAL_TABLE) {
if (!((HMSExternalTable) table).isSupportedHmsTable()) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_NONSUPPORT_HMS_TABLE,
table.getName(),
((HMSExternalTable) table).getDbName(),
tableName.getCtl());
}
if (Config.enable_query_hive_views) {
if (((HMSExternalTable) table).isView()
&& StringUtils.isNotEmpty(((HMSExternalTable) table).getViewText())) {
View hmsView = new View(table.getId(), table.getName(), table.getFullSchema());
hmsView.setInlineViewDefWithSqlMode(((HMSExternalTable) table).getViewText(),
ConnectContext.get().getSessionVariable().getSqlMode());
InlineViewRef inlineViewRef = new InlineViewRef(hmsView, tableRef);
if (StringUtils.isNotEmpty(tableName.getCtl())) {
inlineViewRef.setExternalCtl(tableName.getCtl());
}
return inlineViewRef;
}
}
}
// tableName.getTbl() stores the table name specified by the user in the from statement.

12
fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
View File

@ -25,6 +25,7 @@ import org.apache.doris.common.UserException;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.Collections;
@ -131,10 +132,21 @@ public class FromClause implements ParseNode, Iterable<TableRef> {
tblRef = analyzer.resolveTableRef(tblRef);
tablerefs.set(i, Preconditions.checkNotNull(tblRef));
tblRef.setLeftTblRef(leftTblRef);
boolean setExternalCtl = false;
String preExternalCtl = null;
if (tblRef instanceof InlineViewRef) {
((InlineViewRef) tblRef).setNeedToSql(needToSql);
String externalCtl = ((InlineViewRef) tblRef).getExternalCtl();
if (StringUtils.isNotEmpty(externalCtl)) {
preExternalCtl = analyzer.getExternalCtl();
analyzer.setExternalCtl(externalCtl);
setExternalCtl = true;
}
}
tblRef.analyze(analyzer);
if (setExternalCtl) {
analyzer.setExternalCtl(preExternalCtl);
}
leftTblRef = tblRef;
Expr clause = tblRef.getOnClause();
if (clause != null && clause.contains(Subquery.class)) {

13
fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
View File

@ -75,6 +75,11 @@ public class InlineViewRef extends TableRef {
// Map inline view's output slots to the corresponding baseTblResultExpr of queryStmt.
protected final ExprSubstitutionMap baseTblSmap;
// When parsing a ddl of hive view, it does not contains any catalog info,
// so we need to record it in Analyzer
// otherwise some error will occurs when resolving TableRef later.
protected String externalCtl;
// END: Members that need to be reset()
// ///////////////////////////////////////
@ -448,6 +453,14 @@ public class InlineViewRef extends TableRef {
return queryStmt;
}
public void setExternalCtl(String externalCtl) {
this.externalCtl = externalCtl;
}
public String getExternalCtl() {
return this.externalCtl;
}
@Override
public String tableNameToSql() {
// Enclose the alias in quotes if Hive cannot parse it without quotes.

21
fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java vendored
View File

@ -32,6 +32,7 @@ import org.apache.doris.thrift.TTableType;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Partition;
@ -253,6 +254,26 @@ public class HMSExternalTable extends ExternalTable {
}
}
public String getViewText() {
String viewText = getViewExpandedText();
if (StringUtils.isNotEmpty(viewText)) {
return viewText;
}
return getViewOriginalText();
}
public String getViewExpandedText() {
LOG.debug("View expanded text of hms table [{}.{}.{}] : {}",
this.getCatalog().getName(), this.getDbName(), this.getName(), remoteTable.getViewExpandedText());
return remoteTable.getViewExpandedText();
}
public String getViewOriginalText() {
LOG.debug("View original text of hms table [{}.{}.{}] : {}",
this.getCatalog().getName(), this.getDbName(), this.getName(), remoteTable.getViewOriginalText());
return remoteTable.getViewOriginalText();
}
public String getMetastoreUri() {
return ((HMSExternalCatalog) catalog).getHiveMetastoreUris();
}

18
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
View File

@ -40,6 +40,7 @@ public class AlterTableEvent extends MetastoreTableEvent {
// true if this alter event was due to a rename operation
private final boolean isRename;
private final boolean isView;
private AlterTableEvent(NotificationEvent event, String catalogName) {
super(event, catalogName);
@ -59,7 +60,7 @@ public class AlterTableEvent extends MetastoreTableEvent {
// this is a rename event if either dbName or tblName of before and after object changed
isRename = !tableBefore.getDbName().equalsIgnoreCase(tableAfter.getDbName())
|| !tableBefore.getTableName().equalsIgnoreCase(tableAfter.getTableName());
isView = tableBefore.isSetViewExpandedText() || tableBefore.isSetViewOriginalText();
}
public static List<MetastoreEvent> getEvents(NotificationEvent event,
@ -67,6 +68,15 @@ public class AlterTableEvent extends MetastoreTableEvent {
return Lists.newArrayList(new AlterTableEvent(event, catalogName));
}
private void processRecreateTable() throws DdlException {
if (!isView) {
return;
}
Env.getCurrentEnv().getCatalogMgr()
.dropExternalTable(tableBefore.getDbName(), tableBefore.getTableName(), catalogName);
Env.getCurrentEnv().getCatalogMgr()
.createExternalTable(tableAfter.getDbName(), tableAfter.getTableName(), catalogName);
}
private void processRename() throws DdlException {
if (!isRename) {
@ -100,6 +110,12 @@ public class AlterTableEvent extends MetastoreTableEvent {
processRename();
return;
}
if (isView) {
// if this table is a view, `viewExpandedText/viewOriginalText` of this table may be changed,
// so we need to recreate the table to make sure `remoteTable` will be rebuild
processRecreateTable();
return;
}
//The scope of refresh can be narrowed in the future
Env.getCurrentEnv().getCatalogMgr()
.refreshExternalTable(tableBefore.getDbName(), tableBefore.getTableName(), catalogName);