From cc727cfc324dcfe52a8dff8242a9d9e66764e90d Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Sat, 17 Feb 2024 22:25:38 +0800 Subject: [PATCH] [refactor](catalog) move some package related to catalog (#31022) Move some package and files related to multi-catalog. No logic is changed. --- .../doris/analysis/CreateTableStmt.java | 2 +- .../org/apache/doris/analysis/TableRef.java | 2 +- .../java/org/apache/doris/catalog/Env.java | 2 +- .../org/apache/doris/catalog/EsResource.java | 2 +- .../org/apache/doris/catalog/EsTable.java | 8 +- .../apache/doris/catalog/IcebergProperty.java | 107 --------- .../catalog/external/EsExternalTable.java | 4 +- .../catalog/external/HMSExternalTable.java | 4 +- .../external/IcebergExternalTable.java | 2 +- .../common/proc/EsPartitionsProcDir.java | 2 +- .../doris/common/proc/EsShardProcDir.java | 4 +- .../doris/datasource/EsExternalCatalog.java | 4 +- .../doris/datasource/InternalCatalog.java | 2 +- .../es}/DorisEsException.java | 2 +- .../es}/EsMajorVersion.java | 2 +- .../es}/EsMetaStateTracker.java | 2 +- .../es}/EsNodeInfo.java | 2 +- .../es}/EsRepository.java | 2 +- .../es}/EsRestClient.java | 2 +- .../es}/EsShardPartitions.java | 2 +- .../es}/EsShardRouting.java | 2 +- .../es}/EsTablePartitions.java | 2 +- .../es}/EsUrls.java | 2 +- .../es}/EsUtil.java | 2 +- .../es}/MappingPhase.java | 2 +- .../es}/PartitionPhase.java | 2 +- .../es}/QueryBuilders.java | 38 ++-- .../es}/SearchContext.java | 2 +- .../es}/SearchPhase.java | 2 +- .../datasource/hive/HiveMetaStoreCache.java | 1 - .../doris/datasource/hive/HiveUtil.java | 103 +++++++++ .../hudi}/HudiUtils.java | 9 +- .../iceberg}/IcebergUtils.java | 2 +- .../util => datasource/kafka}/KafkaUtil.java | 2 +- .../doris/external/hive/util/HiveUtil.java | 208 ------------------ .../iceberg/util/DorisTypeToType.java | 116 ---------- .../iceberg/util/DorisTypeVisitor.java | 79 ------- .../iceberg/util/TypeToDorisType.java | 98 --------- .../load/routineload/KafkaRoutineLoadJob.java | 2 +- .../plans/commands/info/CreateTableInfo.java | 2 +- .../org/apache/doris/planner/EsScanNode.java | 14 +- .../planner/external/hudi/HudiScanNode.java | 2 +- .../external/iceberg/IcebergScanNode.java | 2 +- .../doris/statistics/HMSAnalysisTask.java | 2 +- .../elasticsearch/EsNodeInfoTest.java | 2 + .../elasticsearch/EsShardPartitionsTest.java | 2 + .../external/elasticsearch/EsTestCase.java | 1 + .../external/elasticsearch/EsUtilTest.java | 5 + .../elasticsearch/PartitionPhaseTest.java | 5 + .../elasticsearch/QueryBuildersTest.java | 3 +- .../routineload/KafkaRoutineLoadJobTest.java | 2 +- .../load/routineload/RoutineLoadJobTest.java | 2 +- 52 files changed, 197 insertions(+), 680 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/catalog/IcebergProperty.java rename fe/fe-core/src/main/java/org/apache/doris/{external/elasticsearch => datasource/es}/DorisEsException.java (95%) rename fe/fe-core/src/main/java/org/apache/doris/{external/elasticsearch => datasource/es}/EsMajorVersion.java (98%) rename fe/fe-core/src/main/java/org/apache/doris/{external/elasticsearch => datasource/es}/EsMetaStateTracker.java (97%) rename fe/fe-core/src/main/java/org/apache/doris/{external/elasticsearch => datasource/es}/EsNodeInfo.java (99%) rename fe/fe-core/src/main/java/org/apache/doris/{external/elasticsearch => datasource/es}/EsRepository.java (98%) rename fe/fe-core/src/main/java/org/apache/doris/{external/elasticsearch => datasource/es}/EsRestClient.java (99%) rename fe/fe-core/src/main/java/org/apache/doris/{external/elasticsearch => datasource/es}/EsShardPartitions.java (99%) rename fe/fe-core/src/main/java/org/apache/doris/{external/elasticsearch => datasource/es}/EsShardRouting.java (97%) rename fe/fe-core/src/main/java/org/apache/doris/{external/elasticsearch => datasource/es}/EsTablePartitions.java (99%) rename fe/fe-core/src/main/java/org/apache/doris/{external/elasticsearch => datasource/es}/EsUrls.java (95%) rename fe/fe-core/src/main/java/org/apache/doris/{external/elasticsearch => datasource/es}/EsUtil.java (99%) rename fe/fe-core/src/main/java/org/apache/doris/{external/elasticsearch => datasource/es}/MappingPhase.java (99%) rename fe/fe-core/src/main/java/org/apache/doris/{external/elasticsearch => datasource/es}/PartitionPhase.java (97%) rename fe/fe-core/src/main/java/org/apache/doris/{external/elasticsearch => datasource/es}/QueryBuilders.java (95%) rename fe/fe-core/src/main/java/org/apache/doris/{external/elasticsearch => datasource/es}/SearchContext.java (99%) rename fe/fe-core/src/main/java/org/apache/doris/{external/elasticsearch => datasource/es}/SearchPhase.java (96%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveUtil.java rename fe/fe-core/src/main/java/org/apache/doris/{catalog => datasource/hudi}/HudiUtils.java (96%) rename fe/fe-core/src/main/java/org/apache/doris/{external/iceberg/util => datasource/iceberg}/IcebergUtils.java (99%) rename fe/fe-core/src/main/java/org/apache/doris/{common/util => datasource/kafka}/KafkaUtil.java (99%) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/DorisTypeToType.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/DorisTypeVisitor.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/TypeToDorisType.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index ae6958a70c..cca3946f3d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -40,7 +40,7 @@ import org.apache.doris.common.util.ParseUtil; import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.Util; -import org.apache.doris.external.elasticsearch.EsUtil; +import org.apache.doris.datasource.es.EsUtil; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java index e4b5caa446..4b58ab8ea4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java @@ -21,7 +21,6 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.HudiUtils; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; import org.apache.doris.catalog.external.HMSExternalTable; @@ -33,6 +32,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.datasource.hudi.HudiUtils; import org.apache.doris.rewrite.ExprRewriter; import org.apache.doris.rewrite.ExprRewriter.ClauseType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 8bd6cfdb2f..793dcb1f76 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -133,13 +133,13 @@ import org.apache.doris.datasource.EsExternalCatalog; import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.ExternalMetaIdMgr; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.es.EsRepository; import org.apache.doris.datasource.hive.HiveTransactionMgr; import org.apache.doris.datasource.hive.event.MetastoreEventsProcessor; import org.apache.doris.deploy.DeployManager; import org.apache.doris.deploy.impl.AmbariDeployManager; import org.apache.doris.deploy.impl.K8sDeployManager; import org.apache.doris.deploy.impl.LocalFileDeployManager; -import org.apache.doris.external.elasticsearch.EsRepository; import org.apache.doris.ha.BDBHA; import org.apache.doris.ha.FrontendNodeType; import org.apache.doris.ha.HAProtocol; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java index 17fddb8cde..fa1a4a1182 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java @@ -19,7 +19,7 @@ package org.apache.doris.catalog; import org.apache.doris.common.DdlException; import org.apache.doris.common.proc.BaseProcResult; -import org.apache.doris.external.elasticsearch.EsUtil; +import org.apache.doris.datasource.es.EsUtil; import com.google.common.collect.Lists; import com.google.common.collect.Maps; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java index a29c547fa9..be517c7114 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java @@ -20,10 +20,10 @@ package org.apache.doris.catalog; import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; -import org.apache.doris.external.elasticsearch.EsMetaStateTracker; -import org.apache.doris.external.elasticsearch.EsRestClient; -import org.apache.doris.external.elasticsearch.EsTablePartitions; -import org.apache.doris.external.elasticsearch.EsUtil; +import org.apache.doris.datasource.es.EsMetaStateTracker; +import org.apache.doris.datasource.es.EsRestClient; +import org.apache.doris.datasource.es.EsTablePartitions; +import org.apache.doris.datasource.es.EsUtil; import org.apache.doris.thrift.TEsTable; import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/IcebergProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/IcebergProperty.java deleted file mode 100644 index 882524a575..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/IcebergProperty.java +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.catalog; - -import com.google.common.collect.Maps; - -import java.util.Iterator; -import java.util.Map; - -/** - * Iceberg property contains information to connect a remote iceberg db or table. - */ -public class IcebergProperty { - public static final String ICEBERG_DATABASE = "iceberg.database"; - public static final String ICEBERG_TABLE = "iceberg.table"; - public static final String ICEBERG_HIVE_METASTORE_URIS = "iceberg.hive.metastore.uris"; - public static final String ICEBERG_CATALOG_TYPE = "iceberg.catalog.type"; - public static final String ICEBERG_HDFS_PREFIX = "dfs"; - - private boolean exist; - - private String database; - private String table; - private String hiveMetastoreUris; - private String catalogType; - private Map dfsProperties = Maps.newHashMap(); - - private void initDfsProperties(Map properties) { - Iterator> iterator = properties.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - if (entry.getKey().startsWith(ICEBERG_HDFS_PREFIX)) { - dfsProperties.put(entry.getKey(), entry.getValue()); - } - } - } - - public IcebergProperty(Map properties) { - if (properties != null && !properties.isEmpty()) { - this.exist = true; - this.database = properties.get(ICEBERG_DATABASE); - this.table = properties.get(ICEBERG_TABLE); - this.hiveMetastoreUris = properties.get(ICEBERG_HIVE_METASTORE_URIS); - this.catalogType = properties.get(ICEBERG_CATALOG_TYPE); - initDfsProperties(properties); - } else { - this.exist = false; - } - } - - // Create a new Iceberg property from other property - public IcebergProperty(IcebergProperty otherProperty) { - this.exist = otherProperty.exist; - this.database = otherProperty.database; - this.table = otherProperty.table; - this.hiveMetastoreUris = otherProperty.hiveMetastoreUris; - this.catalogType = otherProperty.catalogType; - this.dfsProperties = otherProperty.dfsProperties; - } - - public boolean isExist() { - return exist; - } - - public String getDatabase() { - return database; - } - - public String getTable() { - return table; - } - - public String getHiveMetastoreUris() { - return hiveMetastoreUris; - } - - public String getCatalogType() { - return catalogType; - } - - public String getProperties() { - return ""; - } - - public void setTable(String table) { - this.table = table; - } - - public Map getDfsProperties() { - return dfsProperties; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java index 8516bccc1d..f25a6610cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java @@ -20,8 +20,8 @@ package org.apache.doris.catalog.external; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.EsTable; import org.apache.doris.datasource.EsExternalCatalog; -import org.apache.doris.external.elasticsearch.EsRestClient; -import org.apache.doris.external.elasticsearch.EsUtil; +import org.apache.doris.datasource.es.EsRestClient; +import org.apache.doris.datasource.es.EsUtil; import org.apache.doris.thrift.TEsTable; import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java index c31ba11a0f..c2c302a889 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java @@ -20,7 +20,6 @@ package org.apache.doris.catalog.external; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.HiveMetaStoreClientHelper; -import org.apache.doris.catalog.HudiUtils; import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionType; @@ -32,7 +31,8 @@ import org.apache.doris.datasource.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSCachedClient; import org.apache.doris.datasource.hive.HiveMetaStoreCache; import org.apache.doris.datasource.hive.HivePartition; -import org.apache.doris.external.iceberg.util.IcebergUtils; +import org.apache.doris.datasource.hudi.HudiUtils; +import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.mtmv.MTMVMaxTimestampSnapshot; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.mtmv.MTMVSnapshotIf; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java index be320fc926..b6c899b640 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/IcebergExternalTable.java @@ -20,7 +20,7 @@ package org.apache.doris.catalog.external; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.HiveMetaStoreClientHelper; import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; -import org.apache.doris.external.iceberg.util.IcebergUtils; +import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.BaseAnalysisTask; import org.apache.doris.statistics.ColumnStatistic; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java index 5919855723..87e7fa449f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java @@ -24,7 +24,7 @@ import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.AnalysisException; -import org.apache.doris.external.elasticsearch.EsShardPartitions; +import org.apache.doris.datasource.es.EsShardPartitions; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsShardProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsShardProcDir.java index 8d4d3bf89c..a82b6db13c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsShardProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsShardProcDir.java @@ -21,8 +21,8 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.EsTable; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.util.ListComparator; -import org.apache.doris.external.elasticsearch.EsShardPartitions; -import org.apache.doris.external.elasticsearch.EsShardRouting; +import org.apache.doris.datasource.es.EsShardPartitions; +import org.apache.doris.datasource.es.EsShardRouting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java index 847b49cf8a..7588817a56 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java @@ -20,8 +20,8 @@ package org.apache.doris.datasource; import org.apache.doris.catalog.EsResource; import org.apache.doris.catalog.external.EsExternalDatabase; import org.apache.doris.common.DdlException; -import org.apache.doris.external.elasticsearch.DorisEsException; -import org.apache.doris.external.elasticsearch.EsRestClient; +import org.apache.doris.datasource.es.DorisEsException; +import org.apache.doris.datasource.es.EsRestClient; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index e6b4052134..44cbbc8cfa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -132,10 +132,10 @@ import org.apache.doris.common.util.QueryableReentrantLock; import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.es.EsRepository; import org.apache.doris.datasource.hive.HMSCachedClient; import org.apache.doris.datasource.hive.HMSCachedClientFactory; import org.apache.doris.datasource.property.constants.HMSProperties; -import org.apache.doris.external.elasticsearch.EsRepository; import org.apache.doris.nereids.trees.plans.commands.info.DropMTMVInfo; import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; import org.apache.doris.persist.AlterDatabasePropertyInfo; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/DorisEsException.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/DorisEsException.java similarity index 95% rename from fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/DorisEsException.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/es/DorisEsException.java index dd7964d462..9d46e86acd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/DorisEsException.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/DorisEsException.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.elasticsearch; +package org.apache.doris.datasource.es; public class DorisEsException extends RuntimeException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsMajorVersion.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMajorVersion.java similarity index 98% rename from fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsMajorVersion.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMajorVersion.java index 9300fd3605..ccca6e1ee6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsMajorVersion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMajorVersion.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.elasticsearch; +package org.apache.doris.datasource.es; /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsMetaStateTracker.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMetaStateTracker.java similarity index 97% rename from fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsMetaStateTracker.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMetaStateTracker.java index d3b1539df6..43c112a8f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsMetaStateTracker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsMetaStateTracker.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.elasticsearch; +package org.apache.doris.datasource.es; import org.apache.doris.catalog.EsTable; import org.apache.doris.common.UserException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsNodeInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsNodeInfo.java similarity index 99% rename from fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsNodeInfo.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsNodeInfo.java index 706024c2ed..04ffa4b460 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsNodeInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsNodeInfo.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.elasticsearch; +package org.apache.doris.datasource.es; import org.apache.doris.thrift.TNetworkAddress; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsRepository.java similarity index 98% rename from fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsRepository.java index aa591cfe6a..b67c36e72e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsRepository.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.elasticsearch; +package org.apache.doris.datasource.es; import org.apache.doris.catalog.Database; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsRestClient.java similarity index 99% rename from fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsRestClient.java index 0604926527..a57814f69a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsRestClient.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.elasticsearch; +package org.apache.doris.datasource.es; import org.apache.doris.common.util.JsonUtil; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardPartitions.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsShardPartitions.java similarity index 99% rename from fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardPartitions.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsShardPartitions.java index 1ff6202160..ee32b1103f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardPartitions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsShardPartitions.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.elasticsearch; +package org.apache.doris.datasource.es; import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.catalog.PartitionKey; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardRouting.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsShardRouting.java similarity index 97% rename from fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardRouting.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsShardRouting.java index b3a5a8d463..0ea626d3a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardRouting.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsShardRouting.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.elasticsearch; +package org.apache.doris.datasource.es; import org.apache.doris.thrift.TNetworkAddress; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsTablePartitions.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsTablePartitions.java similarity index 99% rename from fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsTablePartitions.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsTablePartitions.java index b493338e43..cbda7338f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsTablePartitions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsTablePartitions.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.elasticsearch; +package org.apache.doris.datasource.es; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.EsTable; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUrls.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUrls.java similarity index 95% rename from fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUrls.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUrls.java index 0df1effa4b..193035740a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUrls.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUrls.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.elasticsearch; +package org.apache.doris.datasource.es; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUtil.java similarity index 99% rename from fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUtil.java index 021e92260b..115fca897d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsUtil.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.elasticsearch; +package org.apache.doris.datasource.es; import org.apache.doris.analysis.DistributionDesc; import org.apache.doris.analysis.PartitionDesc; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/MappingPhase.java similarity index 99% rename from fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/es/MappingPhase.java index 26d1bc1dcc..70941397e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/MappingPhase.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.elasticsearch; +package org.apache.doris.datasource.es; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.EsTable; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/PartitionPhase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/PartitionPhase.java similarity index 97% rename from fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/PartitionPhase.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/es/PartitionPhase.java index f13db98fb5..d1118fdab6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/PartitionPhase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/PartitionPhase.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.elasticsearch; +package org.apache.doris.datasource.es; import java.util.HashMap; import java.util.Map; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java similarity index 95% rename from fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java index 350fd3857e..241f9e4ba4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/QueryBuilders.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.elasticsearch; +package org.apache.doris.datasource.es; import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.BoolLiteral; @@ -477,7 +477,7 @@ public final class QueryBuilders { * @param out used to generate JSON elements * @throws IOException if IO error occurred */ - abstract void toJson(JsonGenerator out) throws IOException; + public abstract void toJson(JsonGenerator out) throws IOException; /** * Convert query to JSON format and catch error. @@ -509,7 +509,7 @@ public final class QueryBuilders { } @Override - void toJson(JsonGenerator out) throws IOException { + public void toJson(JsonGenerator out) throws IOException { JsonNode jsonNode = mapper.readTree(value); out.writeStartObject(); Iterator> values = jsonNode.fields(); @@ -541,26 +541,26 @@ public final class QueryBuilders { return this; } - BoolQueryBuilder filter(QueryBuilder queryBuilder) { + public BoolQueryBuilder filter(QueryBuilder queryBuilder) { Objects.requireNonNull(queryBuilder); filterClauses.add(queryBuilder); return this; } - BoolQueryBuilder mustNot(QueryBuilder queryBuilder) { + public BoolQueryBuilder mustNot(QueryBuilder queryBuilder) { Objects.requireNonNull(queryBuilder); mustNotClauses.add(queryBuilder); return this; } - BoolQueryBuilder should(QueryBuilder queryBuilder) { + public BoolQueryBuilder should(QueryBuilder queryBuilder) { Objects.requireNonNull(queryBuilder); shouldClauses.add(queryBuilder); return this; } @Override - protected void toJson(JsonGenerator out) throws IOException { + public void toJson(JsonGenerator out) throws IOException { out.writeStartObject(); out.writeFieldName("bool"); out.writeStartObject(); @@ -603,7 +603,7 @@ public final class QueryBuilders { } @Override - void toJson(final JsonGenerator out) throws IOException { + public void toJson(final JsonGenerator out) throws IOException { out.writeStartObject(); out.writeFieldName("term"); out.writeStartObject(); @@ -627,7 +627,7 @@ public final class QueryBuilders { } @Override - void toJson(final JsonGenerator out) throws IOException { + public void toJson(final JsonGenerator out) throws IOException { out.writeStartObject(); out.writeFieldName("terms"); out.writeStartObject(); @@ -645,7 +645,7 @@ public final class QueryBuilders { /** * A Query that matches documents within an range of terms */ - static class RangeQueryBuilder extends QueryBuilder { + public static class RangeQueryBuilder extends QueryBuilder { private final String field; @@ -672,29 +672,29 @@ public final class QueryBuilders { return this; } - RangeQueryBuilder lt(Object value) { + public RangeQueryBuilder lt(Object value) { return to(value, false); } - RangeQueryBuilder lte(Object value) { + public RangeQueryBuilder lte(Object value) { return to(value, true); } - RangeQueryBuilder gt(Object value) { + public RangeQueryBuilder gt(Object value) { return from(value, false); } - RangeQueryBuilder gte(Object value) { + public RangeQueryBuilder gte(Object value) { return from(value, true); } - RangeQueryBuilder format(String format) { + public RangeQueryBuilder format(String format) { this.format = format; return this; } @Override - void toJson(final JsonGenerator out) throws IOException { + public void toJson(final JsonGenerator out) throws IOException { if (lt == null && gt == null) { throw new IllegalStateException("Either lower or upper bound should be provided"); } @@ -745,7 +745,7 @@ public final class QueryBuilders { } @Override - void toJson(JsonGenerator out) throws IOException { + public void toJson(JsonGenerator out) throws IOException { out.writeStartObject(); out.writeFieldName("wildcard"); out.writeStartObject(); @@ -768,7 +768,7 @@ public final class QueryBuilders { } @Override - void toJson(JsonGenerator out) throws IOException { + public void toJson(JsonGenerator out) throws IOException { out.writeStartObject(); out.writeFieldName("exists"); out.writeStartObject(); @@ -788,7 +788,7 @@ public final class QueryBuilders { } @Override - void toJson(final JsonGenerator out) throws IOException { + public void toJson(final JsonGenerator out) throws IOException { out.writeStartObject(); out.writeFieldName("match_all"); out.writeStartObject(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/SearchContext.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/SearchContext.java similarity index 99% rename from fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/SearchContext.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/es/SearchContext.java index bb842ea567..857d14551a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/SearchContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/SearchContext.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.elasticsearch; +package org.apache.doris.datasource.es; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.EsTable; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/SearchPhase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/SearchPhase.java similarity index 96% rename from fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/SearchPhase.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/es/SearchPhase.java index 928524da58..ebcd78b046 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/SearchPhase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/SearchPhase.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.elasticsearch; +package org.apache.doris.datasource.es; /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index 84e0587441..b65594060a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -39,7 +39,6 @@ import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.HMSExternalCatalog; import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo; import org.apache.doris.datasource.property.PropertyConverter; -import org.apache.doris.external.hive.util.HiveUtil; import org.apache.doris.fs.FileSystemCache; import org.apache.doris.fs.RemoteFiles; import org.apache.doris.fs.remote.RemoteFile; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveUtil.java new file mode 100644 index 0000000000..4387eb2162 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveUtil.java @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.catalog.external.HMSExternalTable; +import org.apache.doris.common.UserException; +import org.apache.doris.fs.remote.BrokerFileSystem; +import org.apache.doris.fs.remote.RemoteFileSystem; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat; +import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.TextInputFormat; +import org.apache.hadoop.util.ReflectionUtils; + +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; + +/** + * Hive util for create or query hive table. + */ +public final class HiveUtil { + + private HiveUtil() { + } + + /** + * get input format class from inputFormatName. + * + * @param jobConf jobConf used when getInputFormatClass + * @param inputFormatName inputFormat class name + * @param symlinkTarget use target inputFormat class when inputFormat is SymlinkTextInputFormat + * @return a class of inputFormat. + * @throws UserException when class not found. + */ + public static InputFormat getInputFormat(JobConf jobConf, + String inputFormatName, boolean symlinkTarget) throws UserException { + try { + Class> inputFormatClass = getInputFormatClass(jobConf, inputFormatName); + if (symlinkTarget && (inputFormatClass == SymlinkTextInputFormat.class)) { + // symlink targets are always TextInputFormat + inputFormatClass = TextInputFormat.class; + } + + return ReflectionUtils.newInstance(inputFormatClass, jobConf); + } catch (ClassNotFoundException | RuntimeException e) { + throw new UserException("Unable to create input format " + inputFormatName, e); + } + } + + @SuppressWarnings({"unchecked", "RedundantCast"}) + private static Class> getInputFormatClass(JobConf conf, String inputFormatName) + throws ClassNotFoundException { + // CDH uses different names for Parquet + if ("parquet.hive.DeprecatedParquetInputFormat".equals(inputFormatName) + || "parquet.hive.MapredParquetInputFormat".equals(inputFormatName)) { + return MapredParquetInputFormat.class; + } + + Class clazz = conf.getClassByName(inputFormatName); + return (Class>) clazz.asSubclass(InputFormat.class); + } + + public static boolean isSplittable(RemoteFileSystem remoteFileSystem, String inputFormat, + String location, JobConf jobConf) throws UserException { + if (remoteFileSystem instanceof BrokerFileSystem) { + return ((BrokerFileSystem) remoteFileSystem).isSplittable(location, inputFormat); + } + + // All supported hive input format are splittable + return HMSExternalTable.SUPPORTED_HIVE_FILE_FORMATS.contains(inputFormat); + } + + public static String getHivePartitionValue(String part) { + String[] kv = part.split("="); + Preconditions.checkState(kv.length == 2, String.format("Malformed partition name %s", part)); + try { + // hive partition value maybe contains special characters like '=' and '/' + return URLDecoder.decode(kv[1], StandardCharsets.UTF_8.name()); + } catch (UnsupportedEncodingException e) { + // It should not be here + throw new RuntimeException(e); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HudiUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java similarity index 96% rename from fe/fe-core/src/main/java/org/apache/doris/catalog/HudiUtils.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java index e52b52ab8e..3885f1de3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HudiUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java @@ -15,7 +15,14 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.catalog; +package org.apache.doris.datasource.hudi; + +import org.apache.doris.catalog.ArrayType; +import org.apache.doris.catalog.MapType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.StructField; +import org.apache.doris.catalog.StructType; +import org.apache.doris.catalog.Type; import com.google.common.base.Preconditions; import org.apache.avro.LogicalType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java similarity index 99% rename from fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/IcebergUtils.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 8a6864aba3..5499f8c080 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.external.iceberg.util; +package org.apache.doris.datasource.iceberg; import org.apache.doris.analysis.BinaryPredicate; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/KafkaUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/kafka/KafkaUtil.java similarity index 99% rename from fe/fe-core/src/main/java/org/apache/doris/common/util/KafkaUtil.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/kafka/KafkaUtil.java index 40041502ca..efcf56d358 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/KafkaUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/kafka/KafkaUtil.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.common.util; +package org.apache.doris.datasource.kafka; import org.apache.doris.catalog.Env; import org.apache.doris.common.LoadException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java b/fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java deleted file mode 100644 index 7ed44620a3..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/external/hive/util/HiveUtil.java +++ /dev/null @@ -1,208 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.external.hive.util; - -import org.apache.doris.catalog.ArrayType; -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.ScalarType; -import org.apache.doris.catalog.Type; -import org.apache.doris.catalog.external.HMSExternalTable; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.UserException; -import org.apache.doris.fs.remote.BrokerFileSystem; -import org.apache.doris.fs.remote.RemoteFileSystem; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat; -import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.mapred.InputFormat; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.TextInputFormat; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; -import java.util.List; - -/** - * Hive util for create or query hive table. - */ -public final class HiveUtil { - private static final Logger LOG = LogManager.getLogger(HiveUtil.class); - - private HiveUtil() { - } - - /** - * get input format class from inputFormatName. - * - * @param jobConf jobConf used when getInputFormatClass - * @param inputFormatName inputFormat class name - * @param symlinkTarget use target inputFormat class when inputFormat is SymlinkTextInputFormat - * @return a class of inputFormat. - * @throws UserException when class not found. - */ - public static InputFormat getInputFormat(JobConf jobConf, - String inputFormatName, boolean symlinkTarget) throws UserException { - try { - Class> inputFormatClass = getInputFormatClass(jobConf, inputFormatName); - if (symlinkTarget && (inputFormatClass == SymlinkTextInputFormat.class)) { - // symlink targets are always TextInputFormat - inputFormatClass = TextInputFormat.class; - } - - return ReflectionUtils.newInstance(inputFormatClass, jobConf); - } catch (ClassNotFoundException | RuntimeException e) { - throw new UserException("Unable to create input format " + inputFormatName, e); - } - } - - @SuppressWarnings({"unchecked", "RedundantCast"}) - private static Class> getInputFormatClass(JobConf conf, String inputFormatName) - throws ClassNotFoundException { - // CDH uses different names for Parquet - if ("parquet.hive.DeprecatedParquetInputFormat".equals(inputFormatName) - || "parquet.hive.MapredParquetInputFormat".equals(inputFormatName)) { - return MapredParquetInputFormat.class; - } - - Class clazz = conf.getClassByName(inputFormatName); - return (Class>) clazz.asSubclass(InputFormat.class); - } - - /** - * transform hiveSchema to Doris schema. - * - * @param hiveSchema hive schema - * @return doris schema - * @throws AnalysisException when transform failed. - */ - public static List transformHiveSchema(List hiveSchema) throws AnalysisException { - List newSchema = Lists.newArrayList(); - for (FieldSchema hiveColumn : hiveSchema) { - try { - newSchema.add(HiveUtil.transformHiveField(hiveColumn)); - } catch (UnsupportedOperationException e) { - LOG.warn("Unsupported data type in Doris, ignore column[{}], with error: {}", - hiveColumn.getName(), e.getMessage()); - throw e; - } - } - return newSchema; - } - - /** - * tranform hive field to doris column. - * - * @param field hive field to be transformed - * @return doris column - */ - public static Column transformHiveField(FieldSchema field) { - TypeInfo hiveTypeInfo = TypeInfoUtils.getTypeInfoFromTypeString(field.getType()); - Type type = convertHiveTypeToiveDoris(hiveTypeInfo); - return new Column(field.getName(), type, false, null, true, null, field.getComment()); - } - - private static Type convertHiveTypeToiveDoris(TypeInfo hiveTypeInfo) { - switch (hiveTypeInfo.getCategory()) { - case PRIMITIVE: { - PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) hiveTypeInfo; - switch (primitiveTypeInfo.getPrimitiveCategory()) { - case VOID: - return Type.NULL; - case BOOLEAN: - return Type.BOOLEAN; - case BYTE: - return Type.TINYINT; - case SHORT: - return Type.SMALLINT; - case INT: - return Type.INT; - case LONG: - return Type.BIGINT; - case FLOAT: - return Type.FLOAT; - case DOUBLE: - return Type.DOUBLE; - case STRING: - return Type.STRING; - case CHAR: - return Type.CHAR; - case VARCHAR: - return Type.VARCHAR; - case DATE: - return ScalarType.getDefaultDateType(Type.DATE); - case TIMESTAMP: - return ScalarType.getDefaultDateType(Type.DATETIME); - case DECIMAL: - return Type.DECIMALV2; - default: - throw new UnsupportedOperationException("Unsupported type: " - + primitiveTypeInfo.getPrimitiveCategory()); - } - } - case LIST: - TypeInfo elementTypeInfo = ((ListTypeInfo) hiveTypeInfo) - .getListElementTypeInfo(); - Type newType = null; - if (elementTypeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { - newType = convertHiveTypeToiveDoris(elementTypeInfo); - } else { - throw new UnsupportedOperationException("Unsupported type: " + hiveTypeInfo.toString()); - } - return new ArrayType(newType); - case MAP: - case STRUCT: - case UNION: - default: - throw new UnsupportedOperationException("Unsupported type: " + hiveTypeInfo.toString()); - } - } - - public static boolean isSplittable(RemoteFileSystem remoteFileSystem, String inputFormat, - String location, JobConf jobConf) throws UserException { - if (remoteFileSystem instanceof BrokerFileSystem) { - return ((BrokerFileSystem) remoteFileSystem).isSplittable(location, inputFormat); - } - - // All supported hive input format are splittable - return HMSExternalTable.SUPPORTED_HIVE_FILE_FORMATS.contains(inputFormat); - } - - public static String getHivePartitionValue(String part) { - String[] kv = part.split("="); - Preconditions.checkState(kv.length == 2, String.format("Malformed partition name %s", part)); - try { - // hive partition value maybe contains special characters like '=' and '/' - return URLDecoder.decode(kv[1], StandardCharsets.UTF_8.name()); - } catch (UnsupportedEncodingException e) { - // It should not be here - throw new RuntimeException(e); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/DorisTypeToType.java b/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/DorisTypeToType.java deleted file mode 100644 index 531de582ea..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/DorisTypeToType.java +++ /dev/null @@ -1,116 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.external.iceberg.util; - -import org.apache.doris.catalog.ArrayType; -import org.apache.doris.catalog.MapType; -import org.apache.doris.catalog.PrimitiveType; -import org.apache.doris.catalog.ScalarType; -import org.apache.doris.catalog.StructField; -import org.apache.doris.catalog.StructType; - -import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; - -import java.util.List; - -/** - * Convert Doris type to Iceberg type - */ -public class DorisTypeToType extends DorisTypeVisitor { - private final StructType root; - private int nextId = 0; - - public DorisTypeToType() { - this.root = null; - } - - public DorisTypeToType(StructType root) { - this.root = root; - // the root struct's fields use the first ids - this.nextId = root.getFields().size(); - } - - private int getNextId() { - int next = nextId; - nextId += 1; - return next; - } - - @Override - public Type struct(StructType struct, List types) { - throw new UnsupportedOperationException( - "Not a supported type: " + struct.toSql(0)); - } - - @Override - public Type field(StructField field, Type typeResult) { - return typeResult; - } - - @Override - public Type array(ArrayType array, Type elementType) { - throw new UnsupportedOperationException( - "Not a supported type: " + array.toSql(0)); - } - - @Override - public Type map(MapType map, Type keyType, Type valueType) { - throw new UnsupportedOperationException( - "Not a supported type: " + map.toSql(0)); - } - - @Override - public Type atomic(org.apache.doris.catalog.Type atomic) { - if (atomic.getPrimitiveType().equals(PrimitiveType.BOOLEAN)) { - return Types.BooleanType.get(); - } else if (atomic.getPrimitiveType().equals(PrimitiveType.TINYINT) - || atomic.getPrimitiveType().equals(PrimitiveType.SMALLINT) - || atomic.getPrimitiveType().equals(PrimitiveType.INT)) { - return Types.IntegerType.get(); - } else if (atomic.getPrimitiveType().equals(PrimitiveType.BIGINT) - || atomic.getPrimitiveType().equals(PrimitiveType.LARGEINT)) { - return Types.LongType.get(); - } else if (atomic.getPrimitiveType().equals(PrimitiveType.FLOAT)) { - return Types.FloatType.get(); - } else if (atomic.getPrimitiveType().equals(PrimitiveType.DOUBLE)) { - return Types.DoubleType.get(); - } else if (atomic.getPrimitiveType().equals(PrimitiveType.CHAR) - || atomic.getPrimitiveType().equals(PrimitiveType.VARCHAR)) { - return Types.StringType.get(); - } else if (atomic.getPrimitiveType().equals(PrimitiveType.DATE) - || atomic.getPrimitiveType().equals(PrimitiveType.DATEV2)) { - return Types.DateType.get(); - } else if (atomic.getPrimitiveType().equals(PrimitiveType.TIME) - || atomic.getPrimitiveType().equals(PrimitiveType.TIMEV2)) { - return Types.TimeType.get(); - } else if (atomic.getPrimitiveType().equals(PrimitiveType.DECIMALV2) - || atomic.getPrimitiveType().isDecimalV3Type()) { - return Types.DecimalType.of( - ((ScalarType) atomic).getScalarPrecision(), - ((ScalarType) atomic).getScalarScale()); - } else if (atomic.getPrimitiveType().equals(PrimitiveType.DATETIME) - || atomic.getPrimitiveType().equals(PrimitiveType.DATETIMEV2)) { - return Types.TimestampType.withZone(); - } - // unsupported type: PrimitiveType.HLL BITMAP BINARY - - throw new UnsupportedOperationException( - "Not a supported type: " + atomic.getPrimitiveType()); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/DorisTypeVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/DorisTypeVisitor.java deleted file mode 100644 index 8393672bb9..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/DorisTypeVisitor.java +++ /dev/null @@ -1,79 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.external.iceberg.util; - -import org.apache.doris.catalog.ArrayType; -import org.apache.doris.catalog.MapType; -import org.apache.doris.catalog.StructField; -import org.apache.doris.catalog.StructType; -import org.apache.doris.catalog.Type; - -import com.google.common.collect.Lists; - -import java.util.List; - -/** - * Utils to visit doris and iceberg type - * @param - */ -public class DorisTypeVisitor { - public static T visit(Type type, DorisTypeVisitor visitor) { - if (type instanceof StructType) { - List fields = ((StructType) type).getFields(); - List fieldResults = Lists.newArrayListWithExpectedSize(fields.size()); - - for (StructField field : fields) { - fieldResults.add(visitor.field( - field, - visit(field.getType(), visitor))); - } - - return visitor.struct((StructType) type, fieldResults); - } else if (type instanceof MapType) { - return visitor.map((MapType) type, - visit(((MapType) type).getKeyType(), visitor), - visit(((MapType) type).getValueType(), visitor)); - } else if (type instanceof ArrayType) { - return visitor.array( - (ArrayType) type, - visit(((ArrayType) type).getItemType(), visitor)); - } else { - return visitor.atomic(type); - } - } - - public T struct(StructType struct, List fieldResults) { - return null; - } - - public T field(StructField field, T typeResult) { - return null; - } - - public T array(ArrayType array, T elementResult) { - return null; - } - - public T map(MapType map, T keyResult, T valueResult) { - return null; - } - - public T atomic(Type atomic) { - return null; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/TypeToDorisType.java b/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/TypeToDorisType.java deleted file mode 100644 index 2fc05d96b5..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/TypeToDorisType.java +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.external.iceberg.util; - -import org.apache.doris.catalog.ScalarType; -import org.apache.doris.catalog.Type; - -import org.apache.iceberg.Schema; -import org.apache.iceberg.types.TypeUtil; -import org.apache.iceberg.types.Types; - -import java.util.List; - -/** - * Convert Iceberg types to Doris type - */ -public class TypeToDorisType extends TypeUtil.SchemaVisitor { - public TypeToDorisType() { - } - - @Override - public Type schema(Schema schema, Type structType) { - return structType; - } - - @Override - public Type struct(Types.StructType struct, List fieldResults) { - throw new UnsupportedOperationException( - String.format("Cannot convert Iceberg type[%s] to Doris type.", struct)); - } - - @Override - public Type field(Types.NestedField field, Type fieldResult) { - return fieldResult; - } - - @Override - public Type list(Types.ListType list, Type elementResult) { - throw new UnsupportedOperationException( - String.format("Cannot convert Iceberg type[%s] to Doris type.", list)); - } - - @Override - public Type map(Types.MapType map, Type keyResult, Type valueResult) { - throw new UnsupportedOperationException( - String.format("Cannot convert Iceberg type[%s] to Doris type.", map)); - } - - @Override - public Type primitive(org.apache.iceberg.types.Type.PrimitiveType primitive) { - switch (primitive.typeId()) { - case BOOLEAN: - return Type.BOOLEAN; - case INTEGER: - return Type.INT; - case LONG: - return Type.BIGINT; - case FLOAT: - return Type.FLOAT; - case DOUBLE: - return Type.DOUBLE; - case DECIMAL: - Types.DecimalType decimal = (Types.DecimalType) primitive; - return ScalarType.createDecimalType(decimal.precision(), decimal.scale()); - case DATE: - return ScalarType.getDefaultDateType(Type.DATE); - case TIMESTAMP: - return ScalarType.getDefaultDateType(Type.DATETIME); - case STRING: - return Type.STRING; - // use varchar - case UUID: - return Type.VARCHAR; - // unsupported primitive type - case TIME: - case FIXED: - case BINARY: - default: - throw new UnsupportedOperationException(String.format("Cannot convert Iceberg type[%s] to Doris type.", - primitive)); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java index 2f4d250930..7d41194d93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaRoutineLoadJob.java @@ -31,12 +31,12 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.util.DebugUtil; -import org.apache.doris.common.util.KafkaUtil; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; import org.apache.doris.common.util.SmallFileMgr; import org.apache.doris.common.util.SmallFileMgr.SmallFile; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.datasource.kafka.KafkaUtil; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index 296a5c71ec..f669466b13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -51,7 +51,7 @@ import org.apache.doris.common.util.ParseUtil; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.external.elasticsearch.EsUtil; +import org.apache.doris.datasource.es.EsUtil; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.analyzer.UnboundSlot; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java index 16e6ed040b..720c6b68df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java @@ -30,13 +30,13 @@ import org.apache.doris.catalog.RangePartitionInfo; import org.apache.doris.catalog.external.EsExternalTable; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; -import org.apache.doris.external.elasticsearch.EsShardPartitions; -import org.apache.doris.external.elasticsearch.EsShardRouting; -import org.apache.doris.external.elasticsearch.EsTablePartitions; -import org.apache.doris.external.elasticsearch.QueryBuilders; -import org.apache.doris.external.elasticsearch.QueryBuilders.BoolQueryBuilder; -import org.apache.doris.external.elasticsearch.QueryBuilders.BuilderOptions; -import org.apache.doris.external.elasticsearch.QueryBuilders.QueryBuilder; +import org.apache.doris.datasource.es.EsShardPartitions; +import org.apache.doris.datasource.es.EsShardRouting; +import org.apache.doris.datasource.es.EsTablePartitions; +import org.apache.doris.datasource.es.QueryBuilders; +import org.apache.doris.datasource.es.QueryBuilders.BoolQueryBuilder; +import org.apache.doris.datasource.es.QueryBuilders.BuilderOptions; +import org.apache.doris.datasource.es.QueryBuilders.QueryBuilder; import org.apache.doris.planner.external.ExternalScanNode; import org.apache.doris.planner.external.FederationBackendPolicy; import org.apache.doris.statistics.StatisticalType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java index f3c8bdf594..64e11024af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java @@ -20,7 +20,6 @@ package org.apache.doris.planner.external.hudi; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.HiveMetaStoreClientHelper; -import org.apache.doris.catalog.HudiUtils; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.Type; import org.apache.doris.catalog.external.ExternalTable; @@ -28,6 +27,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.common.util.LocationPath; import org.apache.doris.datasource.hive.HivePartition; +import org.apache.doris.datasource.hudi.HudiUtils; import org.apache.doris.planner.ListPartitionPrunerV2; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.external.FileSplit; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java index 2a0de32acb..4bedf2f75b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java @@ -33,7 +33,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.LocationPath; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; -import org.apache.doris.external.iceberg.util.IcebergUtils; +import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.external.FileQueryScanNode; import org.apache.doris.planner.external.TableFormatType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java index 9e8be62282..2ab671dd7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java @@ -22,7 +22,7 @@ import org.apache.doris.catalog.external.ExternalTable; import org.apache.doris.catalog.external.HMSExternalTable; import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.hive.HiveMetaStoreCache; -import org.apache.doris.external.hive.util.HiveUtil; +import org.apache.doris.datasource.hive.HiveUtil; import org.apache.doris.statistics.util.StatisticsUtil; import com.google.common.collect.Sets; diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsNodeInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsNodeInfoTest.java index 9350521b62..45c415f6db 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsNodeInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsNodeInfoTest.java @@ -17,6 +17,8 @@ package org.apache.doris.external.elasticsearch; +import org.apache.doris.datasource.es.EsNodeInfo; + import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Assert; diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsShardPartitionsTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsShardPartitionsTest.java index 180147cf1b..58c4002086 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsShardPartitionsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsShardPartitionsTest.java @@ -20,6 +20,8 @@ package org.apache.doris.external.elasticsearch; import org.apache.doris.catalog.CatalogTestUtil; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.EsTable; +import org.apache.doris.datasource.es.EsShardPartitions; +import org.apache.doris.datasource.es.EsTablePartitions; import org.junit.Assert; import org.junit.Test; diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsTestCase.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsTestCase.java index fcfcd11d77..e1d9c2670d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsTestCase.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsTestCase.java @@ -26,6 +26,7 @@ import org.apache.doris.catalog.FakeEditLog; import org.apache.doris.catalog.FakeEnv; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.datasource.es.EsUtil; import org.apache.doris.meta.MetaContext; import org.junit.BeforeClass; diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java index 025876fd06..5ec5151532 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java @@ -21,6 +21,11 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.EsTable; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.ExceptionChecker; +import org.apache.doris.datasource.es.DorisEsException; +import org.apache.doris.datasource.es.EsRestClient; +import org.apache.doris.datasource.es.EsUtil; +import org.apache.doris.datasource.es.MappingPhase; +import org.apache.doris.datasource.es.SearchContext; import com.fasterxml.jackson.databind.node.ObjectNode; import mockit.Expectations; diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/PartitionPhaseTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/PartitionPhaseTest.java index b700938727..59b852f2a8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/PartitionPhaseTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/PartitionPhaseTest.java @@ -21,6 +21,11 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.EsTable; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.ExceptionChecker; +import org.apache.doris.datasource.es.EsNodeInfo; +import org.apache.doris.datasource.es.EsRestClient; +import org.apache.doris.datasource.es.EsShardPartitions; +import org.apache.doris.datasource.es.PartitionPhase; +import org.apache.doris.datasource.es.SearchContext; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java index d0cc111bef..9829c1705c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/QueryBuildersTest.java @@ -31,7 +31,8 @@ import org.apache.doris.analysis.LikePredicate; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StringLiteral; import org.apache.doris.catalog.Type; -import org.apache.doris.external.elasticsearch.QueryBuilders.BuilderOptions; +import org.apache.doris.datasource.es.QueryBuilders; +import org.apache.doris.datasource.es.QueryBuilders.BuilderOptions; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index 3b892e1a89..79786fe8be 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -35,7 +35,7 @@ import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.common.util.KafkaUtil; +import org.apache.doris.datasource.kafka.KafkaUtil; import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java index 14c58c2e56..7ac8820c92 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobTest.java @@ -26,8 +26,8 @@ import org.apache.doris.catalog.Table; import org.apache.doris.common.InternalErrorCode; import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.common.util.KafkaUtil; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.kafka.KafkaUtil; import org.apache.doris.persist.EditLog; import org.apache.doris.thrift.TKafkaRLTaskProgress; import org.apache.doris.transaction.TransactionException;