[Improvement](multi catalog) Support Iceberg, Paimon and MaxCompute table in nereids. (#22338)
This commit is contained in:
Jibing-Li
@ -48,6 +48,7 @@ import org.apache.doris.catalog.external.ExternalTable;
|
||||
import org.apache.doris.catalog.external.HMSExternalTable;
|
||||
import org.apache.doris.catalog.external.IcebergExternalTable;
|
||||
import org.apache.doris.catalog.external.JdbcExternalTable;
|
||||
import org.apache.doris.catalog.external.MaxComputeExternalTable;
|
||||
import org.apache.doris.catalog.external.PaimonExternalTable;
|
||||
import org.apache.doris.common.UserException;
|
||||
import org.apache.doris.common.util.Util;
|
||||
@ -158,6 +159,7 @@ import org.apache.doris.planner.SortNode;
|
||||
import org.apache.doris.planner.TableFunctionNode;
|
||||
import org.apache.doris.planner.UnionNode;
|
||||
import org.apache.doris.planner.external.HiveScanNode;
|
||||
import org.apache.doris.planner.external.MaxComputeScanNode;
|
||||
import org.apache.doris.planner.external.hudi.HudiScanNode;
|
||||
import org.apache.doris.planner.external.iceberg.IcebergScanNode;
|
||||
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
|
||||
@ -419,6 +421,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
|
||||
scanNode = new IcebergScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
|
||||
} else if (table instanceof PaimonExternalTable) {
|
||||
scanNode = new PaimonScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
|
||||
} else if (table instanceof MaxComputeExternalTable) {
|
||||
scanNode = new MaxComputeScanNode(context.nextPlanNodeId(), tupleDescriptor, false);
|
||||
} else {
|
||||
throw new RuntimeException("do not support table type " + table.getType());
|
||||
}
|
||||
|
||||
@ -23,6 +23,7 @@ import org.apache.doris.catalog.Partition;
|
||||
import org.apache.doris.catalog.TableIf;
|
||||
import org.apache.doris.catalog.View;
|
||||
import org.apache.doris.catalog.external.EsExternalTable;
|
||||
import org.apache.doris.catalog.external.ExternalTable;
|
||||
import org.apache.doris.catalog.external.HMSExternalTable;
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.common.util.Util;
|
||||
@ -213,6 +214,10 @@ public class BindRelation extends OneAnalysisRuleFactory {
|
||||
}
|
||||
}
|
||||
return new LogicalFileScan(unboundRelation.getRelationId(), (HMSExternalTable) table, tableQualifier);
|
||||
case ICEBERG_EXTERNAL_TABLE:
|
||||
case PAIMON_EXTERNAL_TABLE:
|
||||
case MAX_COMPUTE_EXTERNAL_TABLE:
|
||||
return new LogicalFileScan(unboundRelation.getRelationId(), (ExternalTable) table, tableQualifier);
|
||||
case SCHEMA:
|
||||
return new LogicalSchemaScan(unboundRelation.getRelationId(), table, tableQualifier);
|
||||
case JDBC_EXTERNAL_TABLE:
|
||||
|
||||
@ -44,6 +44,10 @@ public class MaxComputeScanNode extends FileQueryScanNode {
|
||||
private final MaxComputeExternalCatalog catalog;
|
||||
public static final int MIN_SPLIT_SIZE = 4096;
|
||||
|
||||
public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) {
|
||||
this(id, desc, "MCScanNode", StatisticalType.MAX_COMPUTE_SCAN_NODE, needCheckColumnPriv);
|
||||
}
|
||||
|
||||
public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName,
|
||||
StatisticalType statisticalType, boolean needCheckColumnPriv) {
|
||||
super(id, desc, planNodeName, statisticalType, needCheckColumnPriv);
|
||||
|
||||
@ -0,0 +1,74 @@
|
||||
-- This file is automatically generated. You should know what you did if you want to edit this
|
||||
-- !q01 --
|
||||
149988
|
||||
|
||||
-- !q02 --
|
||||
1
|
||||
3
|
||||
4
|
||||
7
|
||||
|
||||
-- !q03 --
|
||||
8242263
|
||||
|
||||
-- !q04 --
|
||||
0
|
||||
|
||||
-- !q05 --
|
||||
1 Customer#000000001 IVhzIApeRb ot,c,E 15 25-989-741-2988 711.56 BUILDING to the even, regular platelets. regular, ironic epitaphs nag e
|
||||
3 Customer#000000003 MG9kdTD2WBHm 1 11-719-748-3364 7498.12 AUTOMOBILE deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov
|
||||
4 Customer#000000004 XxVSJsLAGtn 4 14-128-190-5944 2866.83 MACHINERY requests. final, regular ideas sleep final accou
|
||||
|
||||
-- !q06 --
|
||||
604519555
|
||||
604519557
|
||||
604519558
|
||||
|
||||
-- !q07 --
|
||||
12979.65
|
||||
219204.52
|
||||
5908.20
|
||||
|
||||
-- !q08 --
|
||||
120001848
|
||||
|
||||
-- !q09 --
|
||||
1
|
||||
2
|
||||
3
|
||||
|
||||
-- !q10 --
|
||||
150000000
|
||||
149999999
|
||||
149999996
|
||||
|
||||
-- !q11 --
|
||||
1
|
||||
2
|
||||
3
|
||||
|
||||
-- !q12 --
|
||||
150000000
|
||||
149999999
|
||||
149999996
|
||||
|
||||
-- !q13 --
|
||||
1
|
||||
4
|
||||
7
|
||||
|
||||
-- !q14 --
|
||||
Customer#000000004
|
||||
Customer#000000007
|
||||
|
||||
-- !q15 --
|
||||
150000
|
||||
|
||||
-- !q16 --
|
||||
150000
|
||||
|
||||
-- !q17 --
|
||||
150000
|
||||
|
||||
-- !q18 --
|
||||
150000
|
||||
@ -0,0 +1,84 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
suite("test_external_catalog_icebergv2_nereids", "p2") {
|
||||
String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
|
||||
if (enabled != null && enabled.equalsIgnoreCase("true")) {
|
||||
String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost")
|
||||
String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort")
|
||||
String hms_catalog_name = "test_external_hms_catalog_iceberg"
|
||||
String iceberg_catalog_name = "test_external_iceberg_catalog_nereids"
|
||||
|
||||
sql """drop catalog if exists ${hms_catalog_name};"""
|
||||
sql """
|
||||
create catalog if not exists ${hms_catalog_name} properties (
|
||||
'type'='hms',
|
||||
'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
|
||||
);
|
||||
"""
|
||||
|
||||
sql """drop catalog if exists ${iceberg_catalog_name};"""
|
||||
sql """
|
||||
create catalog if not exists ${iceberg_catalog_name} properties (
|
||||
'type'='iceberg',
|
||||
'iceberg.catalog.type'='hms',
|
||||
'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
|
||||
);
|
||||
"""
|
||||
sql """set enable_nereids_planner=true;"""
|
||||
sql """set enable_fallback_to_original_planner=false;"""
|
||||
|
||||
sql """switch ${hms_catalog_name};"""
|
||||
// test parquet format format
|
||||
def q01 = {
|
||||
qt_q01 """ select count(1) as c from customer_small """
|
||||
qt_q02 """ select c_custkey from customer_small group by c_custkey order by c_custkey limit 4 """
|
||||
qt_q03 """ select count(1) from orders_small """
|
||||
qt_q04 """ select count(1) from customer_small where c_name = 'Customer#000000005' or c_name = 'Customer#000000006' """
|
||||
qt_q05 """ select * from customer_small order by c_custkey limit 3 """
|
||||
qt_q06 """ select o_orderkey from orders_small where o_orderkey > 652566 order by o_orderkey limit 3 """
|
||||
qt_q07 """ select o_totalprice from orders_small where o_custkey < 3357 order by o_custkey limit 3 """
|
||||
qt_q08 """ select count(1) as c from customer """
|
||||
}
|
||||
// test time travel stmt
|
||||
def q02 = {
|
||||
qt_q09 """ select c_custkey from customer for time as of '2022-12-27 10:21:36' order by c_custkey limit 3 """
|
||||
qt_q10 """ select c_custkey from customer for time as of '2022-12-28 10:21:36' order by c_custkey desc limit 3 """
|
||||
qt_q11 """ select c_custkey from customer for version as of 906874575350293177 order by c_custkey limit 3 """
|
||||
qt_q12 """ select c_custkey from customer for version as of 6352416983354893547 order by c_custkey desc limit 3 """
|
||||
}
|
||||
// in predicate
|
||||
def q03 = {
|
||||
qt_q13 """ select c_custkey from customer_small where c_custkey in (1, 2, 4, 7) order by c_custkey """
|
||||
qt_q14 """ select c_name from customer_small where c_name in ('Customer#000000004', 'Customer#000000007') order by c_custkey """
|
||||
}
|
||||
|
||||
// test for 'FOR TIME AS OF' and 'FOR VERSION AS OF'
|
||||
def q04 = {
|
||||
qt_q15 """ select count(*) from ${hms_catalog_name}.tpch_1000_icebergv2.customer_small FOR TIME AS OF '2022-12-22 02:29:30' """
|
||||
qt_q16 """ select count(*) from ${hms_catalog_name}.tpch_1000_icebergv2.customer_small FOR VERSION AS OF 6113938156088124425 """
|
||||
qt_q17 """ select count(*) from ${iceberg_catalog_name}.tpch_1000_icebergv2.customer_small FOR TIME AS OF '2022-12-22 02:29:30' """
|
||||
qt_q18 """ select count(*) from ${iceberg_catalog_name}.tpch_1000_icebergv2.customer_small FOR VERSION AS OF 6113938156088124425 """
|
||||
}
|
||||
|
||||
sql """ use `tpch_1000_icebergv2`; """
|
||||
q01()
|
||||
q02()
|
||||
q03()
|
||||
q04()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user