From 967801ca666f84a60010bef1f6ca5ca777d5b901 Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Thu, 12 Sep 2024 20:01:48 +0800 Subject: [PATCH] [feature](Nereids) support query on ES table in internal catalog (#40449) (#40754) pick from master #40449 --- .../translator/PhysicalPlanTranslator.java | 6 +- .../nereids/rules/analysis/BindRelation.java | 11 +-- .../trees/plans/logical/LogicalEsScan.java | 21 ++---- .../trees/plans/physical/PhysicalEsScan.java | 11 +-- .../doris/planner/StatisticDeriveTest.java | 27 -------- .../test_push_conjunct_external_table.groovy | 67 ------------------- 6 files changed, 16 insertions(+), 127 deletions(-) delete mode 100644 regression-test/suites/correctness_p0/test_push_conjunct_external_table.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 129854b7ff..78a6a08309 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -48,6 +48,7 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.FileQueryScanNode; +import org.apache.doris.datasource.es.EsExternalTable; import org.apache.doris.datasource.es.source.EsScanNode; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; @@ -624,9 +625,10 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor slots = esScan.getOutput(); - ExternalTable table = esScan.getTable(); + TableIf table = esScan.getTable(); TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table, context); - EsScanNode esScanNode = new EsScanNode(context.nextPlanNodeId(), tupleDescriptor, true); + EsScanNode esScanNode = new EsScanNode(context.nextPlanNodeId(), tupleDescriptor, + table instanceof EsExternalTable); esScanNode.setNereidsId(esScan.getId()); esScanNode.addConjuncts(translateToLegacyConjuncts(esScan.getConjuncts())); Utils.execWithUncheckedException(esScanNode::init); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 67000b3fee..4e18039c4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -33,7 +33,6 @@ import org.apache.doris.common.Config; import org.apache.doris.common.Pair; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.ExternalTable; -import org.apache.doris.datasource.es.EsExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; import org.apache.doris.nereids.CTEContext; @@ -416,17 +415,11 @@ public class BindRelation extends OneAnalysisRuleFactory { case ODBC: return new LogicalOdbcScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); case ES_EXTERNAL_TABLE: - return new LogicalEsScan(unboundRelation.getRelationId(), (EsExternalTable) table, - qualifierWithoutTableName); + case ELASTICSEARCH: + return new LogicalEsScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); case TEST_EXTERNAL_TABLE: return new LogicalTestScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); default: - try { - // TODO: support other type table, such as ELASTICSEARCH - cascadesContext.getConnectContext().getSessionVariable().enableFallbackToOriginalPlannerOnce(); - } catch (Exception e) { - // ignore - } throw new AnalysisException("Unsupported tableType " + table.getType()); } } finally { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java index ea278aa203..5895469442 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java @@ -17,7 +17,7 @@ package org.apache.doris.nereids.trees.plans.logical; -import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; @@ -27,7 +27,6 @@ import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; import java.util.List; @@ -42,22 +41,16 @@ public class LogicalEsScan extends LogicalExternalRelation { /** * Constructor for LogicalEsScan. */ - public LogicalEsScan(RelationId id, ExternalTable table, List qualifier, + public LogicalEsScan(RelationId id, TableIf table, List qualifier, Optional groupExpression, Optional logicalProperties, Set conjuncts) { super(id, PlanType.LOGICAL_ES_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties); } - public LogicalEsScan(RelationId id, ExternalTable table, List qualifier) { + public LogicalEsScan(RelationId id, TableIf table, List qualifier) { this(id, table, qualifier, Optional.empty(), Optional.empty(), ImmutableSet.of()); } - @Override - public ExternalTable getTable() { - Preconditions.checkArgument(table instanceof ExternalTable); - return (ExternalTable) table; - } - @Override public String toString() { return Utils.toSqlString("LogicalEsScan", @@ -68,26 +61,26 @@ public class LogicalEsScan extends LogicalExternalRelation { @Override public LogicalEsScan withGroupExpression(Optional groupExpression) { - return new LogicalEsScan(relationId, (ExternalTable) table, qualifier, groupExpression, + return new LogicalEsScan(relationId, table, qualifier, groupExpression, Optional.of(getLogicalProperties()), conjuncts); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalEsScan(relationId, (ExternalTable) table, qualifier, groupExpression, logicalProperties, + return new LogicalEsScan(relationId, table, qualifier, groupExpression, logicalProperties, conjuncts); } @Override public LogicalEsScan withConjuncts(Set conjuncts) { - return new LogicalEsScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), + return new LogicalEsScan(relationId, table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), conjuncts); } @Override public LogicalEsScan withRelationId(RelationId relationId) { - return new LogicalEsScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), + return new LogicalEsScan(relationId, table, qualifier, Optional.empty(), Optional.empty(), conjuncts); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java index 2f3a399ee2..474436f1db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java @@ -17,7 +17,7 @@ package org.apache.doris.nereids.trees.plans.physical; -import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.LogicalProperties; @@ -48,7 +48,7 @@ public class PhysicalEsScan extends PhysicalCatalogRelation { /** * Constructor for PhysicalEsScan. */ - public PhysicalEsScan(RelationId id, ExternalTable table, List qualifier, + public PhysicalEsScan(RelationId id, TableIf table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, LogicalProperties logicalProperties, Set conjuncts) { super(id, PlanType.PHYSICAL_ES_SCAN, table, qualifier, groupExpression, logicalProperties); @@ -59,7 +59,7 @@ public class PhysicalEsScan extends PhysicalCatalogRelation { /** * Constructor for PhysicalEsScan. */ - public PhysicalEsScan(RelationId id, ExternalTable table, List qualifier, + public PhysicalEsScan(RelationId id, TableIf table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics, Set conjuncts) { @@ -96,11 +96,6 @@ public class PhysicalEsScan extends PhysicalCatalogRelation { groupExpression, logicalProperties.get(), conjuncts); } - @Override - public ExternalTable getTable() { - return (ExternalTable) table; - } - @Override public PhysicalEsScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statsDeriveResult) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/StatisticDeriveTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/StatisticDeriveTest.java index 0771508d49..f66f9cb077 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/StatisticDeriveTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/StatisticDeriveTest.java @@ -61,18 +61,6 @@ public class StatisticDeriveTest extends TestWithFeService { + " \"replication_num\" = \"1\"\n" + ");"); - createTable("create external table test.mysql_table\n" - + "(k1 int, k2 int)\n" - + "ENGINE=MYSQL\n" - + "PROPERTIES (\n" - + "\"host\" = \"127.0.0.1\",\n" - + "\"port\" = \"3306\",\n" - + "\"user\" = \"root\",\n" - + "\"password\" = \"123\",\n" - + "\"database\" = \"db1\",\n" - + "\"table\" = \"tbl1\"\n" - + ");"); - createTable("create external table test.odbc_oracle\n" + "(k1 float, k2 int)\n" + "ENGINE=ODBC\n" @@ -216,21 +204,6 @@ public class StatisticDeriveTest extends TestWithFeService { assertSQLPlanOrErrorMsgContains(sql, "HASH JOIN"); } - @Test - public void testMysqlScanStatsDerive() throws Exception { - String sql = "select * from test.mysql_table"; - SessionVariable sessionVariable = connectContext.getSessionVariable(); - sessionVariable.setEnableJoinReorderBasedCost(true); - sessionVariable.setDisableJoinReorder(false); - StmtExecutor stmtExecutor = new StmtExecutor(connectContext, sql); - stmtExecutor.execute(); - Assert.assertNotNull(stmtExecutor.planner()); - Assert.assertNotNull(stmtExecutor.planner().getFragments()); - Assert.assertNotEquals(0, stmtExecutor.planner().getFragments().size()); - System.out.println(getSQLPlanOrErrorMsg("explain " + sql)); - assertSQLPlanOrErrorMsgContains(sql, "SCAN MYSQL"); - } - @Test public void testOdbcScanStatsDerive() throws Exception { String sql = "select * from test.odbc_mysql"; diff --git a/regression-test/suites/correctness_p0/test_push_conjunct_external_table.groovy b/regression-test/suites/correctness_p0/test_push_conjunct_external_table.groovy deleted file mode 100644 index 008be4f11b..0000000000 --- a/regression-test/suites/correctness_p0/test_push_conjunct_external_table.groovy +++ /dev/null @@ -1,67 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one - // or more contributor license agreements. See the NOTICE file - // distributed with this work for additional information - // regarding copyright ownership. The ASF licenses this file - // to you under the Apache License, Version 2.0 (the - // "License"); you may not use this file except in compliance - // with the License. You may obtain a copy of the License at - // - // http://www.apache.org/licenses/LICENSE-2.0 - // - // Unless required by applicable law or agreed to in writing, - // software distributed under the License is distributed on an - // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - // KIND, either express or implied. See the License for the - // specific language governing permissions and limitations - // under the License. - -suite("test_push_conjunct_external_table") { - sql """ DROP TABLE IF EXISTS dim_server; """ - sql """ admin set frontend config("enable_odbc_mysql_broker_table" = "true")""" - - sql """ - CREATE EXTERNAL TABLE `dim_server` ( - `col1` varchar(50) NOT NULL, - `col2` varchar(50) NOT NULL - ) ENGINE=mysql - PROPERTIES - ( - "host" = "127.0.0.1", - "port" = "8239", - "user" = "mysql_user", - "password" = "mysql_passwd", - "database" = "mysql_db_test", - "table" = "mysql_table_test", - "charset" = "utf8mb4" - ); - """ - - sql """ - DROP view if exists ads_oreo_sid_report; - """ - - sql """ - create view ads_oreo_sid_report - ( - `col1` , - `col2` - ) - AS - select - tmp.col1,tmp.col2 - from ( - select 'abc' as col1,'def' as col2 - ) tmp - inner join dim_server ds on tmp.col1 = ds.col1 and tmp.col2 = ds.col2; - """ - - explain { - sql """select * from ads_oreo_sid_report where col1='abc' and col2='def';""" - contains "`ds`.`col1` = 'abc'" - contains "`ds`.`col2` = 'def'" - } - - sql """ DROP TABLE IF EXISTS dim_server; """ - sql """ DROP view if exists ads_oreo_sid_report; """ -} -