database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](multi-catalog) support hive 1.x (#15886)

Browse Source 
The inferface of hive metastore changes from version to version.
Currently, Doris use hive 2.3.7 as hms client version.
When using to connect hive 1.x, some interface such as get_table_req does not exist
in hive 1.x. So we can't get metadata from hive 1.x.

In this PR, I copied the HiveMetastoreClient from hive 2.3.7 release, and modify some of interface's
implementation, so that it will use old interface to connect to hive 1.x.

And when creating hms catalog, you can specify the hive version, eg:

CREATE CATALOG `hive` PROPERTIES (
  "hive.metastore.uris" = "thrift://127.0.0.1:9083",
  "type" = "hms",
  "hive.version" = "1.1"
);
If hive.version does not specified, Doris will use hive 2.3.x compatible interface to visit hms.
This commit is contained in:
Mingyu Chen
2023-01-13 18:32:12 +08:00
committed by GitHub
parent 1489e3cfbf
commit e979cc444a
9 changed files with 2844 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java
View File

@ -47,6 +47,7 @@ public class HMSResource extends Resource {
private static final Logger LOG = LogManager.getLogger(HMSResource.class);
public static final String HIVE_METASTORE_TYPE = "hive.metastore.type";
public static final String DLF_TYPE = "dlf";
public static final String HIVE_VERSION = "hive.version";
// required
public static final String HIVE_METASTORE_URIS = "hive.metastore.uris";

2
fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java vendored
View File

@ -21,7 +21,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.datasource.HMSExternalCatalog;
import org.apache.doris.datasource.PooledHiveMetaStoreClient;
import org.apache.doris.datasource.hive.PooledHiveMetaStoreClient;
import org.apache.doris.statistics.AnalysisTaskInfo;
import org.apache.doris.statistics.AnalysisTaskScheduler;
import org.apache.doris.statistics.BaseAnalysisTask;

5
fe/fe-core/src/main/java/org/apache/doris/datasource/HMSClientException.java
View File

@ -17,8 +17,11 @@
package org.apache.doris.datasource;
import org.apache.doris.common.util.Util;
public class HMSClientException extends RuntimeException {
public HMSClientException(String format, Throwable cause, Object... msg) {
super(String.format(format, msg), cause);
super(String.format(format, msg) + (cause == null ? "" : ". reason: " + Util.getRootCauseMessage(cause)),
cause);
}
}

1
fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
View File

@ -26,6 +26,7 @@ import org.apache.doris.catalog.HiveMetaStoreClientHelper;
import org.apache.doris.catalog.external.ExternalDatabase;
import org.apache.doris.catalog.external.HMSExternalDatabase;
import org.apache.doris.common.Config;
import org.apache.doris.datasource.hive.PooledHiveMetaStoreClient;
import org.apache.doris.datasource.hive.event.MetastoreNotificationFetchException;
import com.google.common.collect.Lists;

1
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -137,6 +137,7 @@ import org.apache.doris.common.util.QueryableReentrantLock;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.hive.PooledHiveMetaStoreClient;
import org.apache.doris.external.elasticsearch.EsRepository;
import org.apache.doris.external.hudi.HudiProperty;
import org.apache.doris.external.hudi.HudiTable;

2758
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClient.java Normal file
View File

File diff suppressed because it is too large Load Diff

75
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveVersionUtil.java Normal file
View File

@ -0,0 +1,75 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.datasource.hive;
import com.google.common.base.Strings;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/**
* For getting a compatible version of hive
* if user specified the version, it will parse it and return the compatible HiveVersion,
* otherwise, use DEFAULT_HIVE_VERSION
*/
public class HiveVersionUtil {
private static final Logger LOG = LogManager.getLogger(HiveVersionUtil.class);
private static final HiveVersion DEFAULT_HIVE_VERSION = HiveVersion.V2_3;
public enum HiveVersion {
V1_0, // [1.0.0 - 1.2.2]
V2_0, // [2.0.0 - 2.2.0]
V2_3, // [2.3.0 - 2.3.6]
V3_0 // [3.0.0 - 3.1.2]
}
public static HiveVersion getVersion(String version) {
if (Strings.isNullOrEmpty(version)) {
return DEFAULT_HIVE_VERSION;
}
String[] parts = version.split("\\.");
if (parts.length < 2) {
LOG.warn("invalid hive version: " + version);
return DEFAULT_HIVE_VERSION;
}
try {
int major = Integer.parseInt(parts[0]);
int minor = Integer.parseInt(parts[1]);
if (major == 1) {
return HiveVersion.V1_0;
} else if (major == 2) {
if (minor >= 0 && minor <= 2) {
return HiveVersion.V1_0;
} else if (minor >= 3) {
return HiveVersion.V2_3;
} else {
LOG.warn("invalid hive version: " + version);
return DEFAULT_HIVE_VERSION;
}
} else if (major >= 3) {
return HiveVersion.V2_3;
} else {
LOG.warn("invalid hive version: " + version);
return DEFAULT_HIVE_VERSION;
}
} catch (NumberFormatException e) {
LOG.warn("invalid hive version: " + version);
return DEFAULT_HIVE_VERSION;
}
}
}

4
fe/fe-core/src/main/java/org/apache/doris/datasource/PooledHiveMetaStoreClient.java → fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java
View File

@ -15,10 +15,11 @@
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.datasource;
package org.apache.doris.datasource.hive;
import org.apache.doris.catalog.HMSResource;
import org.apache.doris.common.Config;
import org.apache.doris.datasource.HMSClientException;
import org.apache.doris.datasource.hive.event.MetastoreNotificationFetchException;
import com.aliyun.datalake.metastore.hive2.ProxyMetaStoreClient;
@ -26,7 +27,6 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;