database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](Nereids) support query on ES table in internal catalog (#40449) (#40754)

Browse Source 
pick from master #40449
This commit is contained in:
morrySnow
2024-09-12 20:01:48 +08:00
committed by GitHub
parent fedadbba6e
commit 967801ca66
6 changed files with 16 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View File

@ -48,6 +48,7 @@ import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.FileQueryScanNode;
import org.apache.doris.datasource.es.EsExternalTable;
import org.apache.doris.datasource.es.source.EsScanNode;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable.DLAType;
@ -624,9 +625,10 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
@Override
public PlanFragment visitPhysicalEsScan(PhysicalEsScan esScan, PlanTranslatorContext context) {
List<Slot> slots = esScan.getOutput();
ExternalTable table = esScan.getTable();
TableIf table = esScan.getTable();
TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table, context);
EsScanNode esScanNode = new EsScanNode(context.nextPlanNodeId(), tupleDescriptor, true);
EsScanNode esScanNode = new EsScanNode(context.nextPlanNodeId(), tupleDescriptor,
table instanceof EsExternalTable);
esScanNode.setNereidsId(esScan.getId());
esScanNode.addConjuncts(translateToLegacyConjuncts(esScan.getConjuncts()));
Utils.execWithUncheckedException(esScanNode::init);

11
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
View File

@ -33,7 +33,6 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.es.EsExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable.DLAType;
import org.apache.doris.nereids.CTEContext;
@ -416,17 +415,11 @@ public class BindRelation extends OneAnalysisRuleFactory {
case ODBC:
return new LogicalOdbcScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName);
case ES_EXTERNAL_TABLE:
return new LogicalEsScan(unboundRelation.getRelationId(), (EsExternalTable) table,
qualifierWithoutTableName);
case ELASTICSEARCH:
return new LogicalEsScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName);
case TEST_EXTERNAL_TABLE:
return new LogicalTestScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName);
default:
try {
// TODO: support other type table, such as ELASTICSEARCH
cascadesContext.getConnectContext().getSessionVariable().enableFallbackToOriginalPlannerOnce();
} catch (Exception e) {
// ignore
}
throw new AnalysisException("Unsupported tableType " + table.getType());
}
} finally {

21
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java
View File

@ -17,7 +17,7 @@
package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
@ -27,7 +27,6 @@ import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import java.util.List;
@ -42,22 +41,16 @@ public class LogicalEsScan extends LogicalExternalRelation {
/**
* Constructor for LogicalEsScan.
*/
public LogicalEsScan(RelationId id, ExternalTable table, List<String> qualifier,
public LogicalEsScan(RelationId id, TableIf table, List<String> qualifier,
Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, Set<Expression> conjuncts) {
super(id, PlanType.LOGICAL_ES_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties);
}
public LogicalEsScan(RelationId id, ExternalTable table, List<String> qualifier) {
public LogicalEsScan(RelationId id, TableIf table, List<String> qualifier) {
this(id, table, qualifier, Optional.empty(), Optional.empty(), ImmutableSet.of());
}
@Override
public ExternalTable getTable() {
Preconditions.checkArgument(table instanceof ExternalTable);
return (ExternalTable) table;
}
@Override
public String toString() {
return Utils.toSqlString("LogicalEsScan",
@ -68,26 +61,26 @@ public class LogicalEsScan extends LogicalExternalRelation {
@Override
public LogicalEsScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalEsScan(relationId, (ExternalTable) table, qualifier, groupExpression,
return new LogicalEsScan(relationId, table, qualifier, groupExpression,
Optional.of(getLogicalProperties()), conjuncts);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new LogicalEsScan(relationId, (ExternalTable) table, qualifier, groupExpression, logicalProperties,
return new LogicalEsScan(relationId, table, qualifier, groupExpression, logicalProperties,
conjuncts);
}
@Override
public LogicalEsScan withConjuncts(Set<Expression> conjuncts) {
return new LogicalEsScan(relationId, (ExternalTable) table, qualifier, Optional.empty(),
return new LogicalEsScan(relationId, table, qualifier, Optional.empty(),
Optional.of(getLogicalProperties()), conjuncts);
}
@Override
public LogicalEsScan withRelationId(RelationId relationId) {
return new LogicalEsScan(relationId, (ExternalTable) table, qualifier, Optional.empty(),
return new LogicalEsScan(relationId, table, qualifier, Optional.empty(),
Optional.empty(), conjuncts);
}

11
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java
View File

@ -17,7 +17,7 @@
package org.apache.doris.nereids.trees.plans.physical;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.DistributionSpec;
import org.apache.doris.nereids.properties.LogicalProperties;
@ -48,7 +48,7 @@ public class PhysicalEsScan extends PhysicalCatalogRelation {
/**
* Constructor for PhysicalEsScan.
*/
public PhysicalEsScan(RelationId id, ExternalTable table, List<String> qualifier,
public PhysicalEsScan(RelationId id, TableIf table, List<String> qualifier,
DistributionSpec distributionSpec, Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties, Set<Expression> conjuncts) {
super(id, PlanType.PHYSICAL_ES_SCAN, table, qualifier, groupExpression, logicalProperties);
@ -59,7 +59,7 @@ public class PhysicalEsScan extends PhysicalCatalogRelation {
/**
* Constructor for PhysicalEsScan.
*/
public PhysicalEsScan(RelationId id, ExternalTable table, List<String> qualifier,
public PhysicalEsScan(RelationId id, TableIf table, List<String> qualifier,
DistributionSpec distributionSpec, Optional<GroupExpression> groupExpression,
LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics,
Set<Expression> conjuncts) {
@ -96,11 +96,6 @@ public class PhysicalEsScan extends PhysicalCatalogRelation {
groupExpression, logicalProperties.get(), conjuncts);
}
@Override
public ExternalTable getTable() {
return (ExternalTable) table;
}
@Override
public PhysicalEsScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties,
Statistics statsDeriveResult) {