From c08ab9edc73915f39755d093a1908a1761c4803b Mon Sep 17 00:00:00 2001 From: Tiewei Fang <43782773+BePPPower@users.noreply.github.com> Date: Thu, 14 Dec 2023 17:50:17 +0800 Subject: [PATCH] [feature](HiveCatalog) Support for getting hive meta data from relational databases under HMS (#28188) --- .../catalog/external/HMSExternalTable.java | 10 +- .../doris/datasource/HMSClientException.java | 4 + .../doris/datasource/HMSExternalCatalog.java | 69 ++- .../doris/datasource/InternalCatalog.java | 5 +- .../datasource/hive/HMSCachedClient.java | 81 +++ .../hive/HMSCachedClientFactory.java | 42 ++ .../datasource/hive/HiveTransaction.java | 2 +- .../datasource/hive/JdbcHMSCachedClient.java | 37 ++ .../hive/PostgreSQLJdbcHMSCachedClient.java | 477 ++++++++++++++++++ ...Client.java => ThriftHMSCachedClient.java} | 125 +++-- .../datasource/hive/event/MetastoreEvent.java | 4 +- .../datasource/jdbc/client/JdbcClient.java | 2 +- .../jdbc/client/JdbcClientConfig.java | 31 +- regression-test/conf/regression-conf.groovy | 1 + .../jdbc_hive_meta/test_jdbc_hive_orc.out | 103 ++++ .../external/conf/regression-conf.groovy | 1 + .../pipeline/p0/conf/regression-conf.groovy | 1 + .../jdbc_hive_meta/test_jdbc_hive_orc.groovy | 116 +++++ 18 files changed, 1005 insertions(+), 106 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClient.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClientFactory.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hive/JdbcHMSCachedClient.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java rename fe/fe-core/src/main/java/org/apache/doris/datasource/hive/{PooledHiveMetaStoreClient.java => ThriftHMSCachedClient.java} (84%) create mode 100644 regression-test/data/external_table_p0/hive/jdbc_hive_meta/test_jdbc_hive_orc.out create mode 100644 regression-test/suites/external_table_p0/hive/jdbc_hive_meta/test_jdbc_hive_orc.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java index b284b4d60e..e4c12252c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java @@ -26,8 +26,8 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HMSCachedClient; import org.apache.doris.datasource.hive.HiveMetaStoreCache; -import org.apache.doris.datasource.hive.PooledHiveMetaStoreClient; import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.BaseAnalysisTask; import org.apache.doris.statistics.ColumnStatistic; @@ -382,25 +382,25 @@ public class HMSExternalTable extends ExternalTable { } public List getHiveTableColumnStats(List columns) { - PooledHiveMetaStoreClient client = ((HMSExternalCatalog) catalog).getClient(); + HMSCachedClient client = ((HMSExternalCatalog) catalog).getClient(); return client.getTableColumnStatistics(dbName, name, columns); } public Map> getHivePartitionColumnStats( List partNames, List columns) { - PooledHiveMetaStoreClient client = ((HMSExternalCatalog) catalog).getClient(); + HMSCachedClient client = ((HMSExternalCatalog) catalog).getClient(); return client.getPartitionColumnStatistics(dbName, name, partNames, columns); } public Partition getPartition(List partitionValues) { - PooledHiveMetaStoreClient client = ((HMSExternalCatalog) catalog).getClient(); + HMSCachedClient client = ((HMSExternalCatalog) catalog).getClient(); return client.getPartition(dbName, name, partitionValues); } @Override public Set getPartitionNames() { makeSureInitialized(); - PooledHiveMetaStoreClient client = ((HMSExternalCatalog) catalog).getClient(); + HMSCachedClient client = ((HMSExternalCatalog) catalog).getClient(); List names = client.listPartitionNames(dbName, name); return new HashSet<>(names); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSClientException.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSClientException.java index 818967ce02..8e1a4fd33a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSClientException.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSClientException.java @@ -24,4 +24,8 @@ public class HMSClientException extends RuntimeException { super(String.format(format, msg) + (cause == null ? "" : ". reason: " + Util.getRootCauseMessage(cause)), cause); } + + public HMSClientException(String format, Object... msg) { + super(String.format(format, msg)); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java index 6e3543dfcc..ae6b5f0473 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java @@ -25,8 +25,10 @@ import org.apache.doris.catalog.external.ExternalTable; import org.apache.doris.catalog.external.HMSExternalDatabase; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; -import org.apache.doris.datasource.hive.PooledHiveMetaStoreClient; +import org.apache.doris.datasource.hive.HMSCachedClient; +import org.apache.doris.datasource.hive.HMSCachedClientFactory; import org.apache.doris.datasource.hive.event.MetastoreNotificationFetchException; +import org.apache.doris.datasource.jdbc.client.JdbcClientConfig; import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.property.constants.HMSProperties; @@ -54,7 +56,7 @@ public class HMSExternalCatalog extends ExternalCatalog { private static final Logger LOG = LogManager.getLogger(HMSExternalCatalog.class); private static final int MIN_CLIENT_POOL_SIZE = 8; - protected PooledHiveMetaStoreClient client; + protected HMSCachedClient client; // Record the latest synced event id when processing hive events // Must set to -1 otherwise client.getNextNotification will throw exception // Reference to https://github.com/apDdlache/doris/issues/18251 @@ -144,33 +146,44 @@ public class HMSExternalCatalog extends ExternalCatalog { @Override protected void initLocalObjectsImpl() { - HiveConf hiveConf = new HiveConf(); - for (Map.Entry kv : catalogProperty.getHadoopProperties().entrySet()) { - hiveConf.set(kv.getKey(), kv.getValue()); - } - hiveConf.set(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT.name(), - String.valueOf(Config.hive_metastore_client_timeout_second)); - String authentication = catalogProperty.getOrDefault( - HdfsResource.HADOOP_SECURITY_AUTHENTICATION, ""); - if (AuthType.KERBEROS.getDesc().equals(authentication)) { - hiveConf.set(HdfsResource.HADOOP_SECURITY_AUTHENTICATION, authentication); - UserGroupInformation.setConfiguration(hiveConf); - try { - /** - * Because metastore client is created by using - * {@link org.apache.hadoop.hive.metastore.RetryingMetaStoreClient#getProxy} - * it will relogin when TGT is expired, so we don't need to relogin manually. - */ - UserGroupInformation.loginUserFromKeytab( - catalogProperty.getOrDefault(HdfsResource.HADOOP_KERBEROS_PRINCIPAL, ""), - catalogProperty.getOrDefault(HdfsResource.HADOOP_KERBEROS_KEYTAB, "")); - } catch (IOException e) { - throw new HMSClientException("login with kerberos auth failed for catalog %s", e, this.getName()); + HiveConf hiveConf = null; + JdbcClientConfig jdbcClientConfig = null; + String hiveMetastoreType = catalogProperty.getOrDefault(HMSProperties.HIVE_METASTORE_TYPE, ""); + if (hiveMetastoreType.equalsIgnoreCase("jdbc")) { + jdbcClientConfig = new JdbcClientConfig(); + jdbcClientConfig.setUser(catalogProperty.getOrDefault("user", "")); + jdbcClientConfig.setPassword(catalogProperty.getOrDefault("password", "")); + jdbcClientConfig.setJdbcUrl(catalogProperty.getOrDefault("jdbc_url", "")); + jdbcClientConfig.setDriverUrl(catalogProperty.getOrDefault("driver_url", "")); + jdbcClientConfig.setDriverClass(catalogProperty.getOrDefault("driver_class", "")); + } else { + hiveConf = new HiveConf(); + for (Map.Entry kv : catalogProperty.getHadoopProperties().entrySet()) { + hiveConf.set(kv.getKey(), kv.getValue()); + } + hiveConf.set(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT.name(), + String.valueOf(Config.hive_metastore_client_timeout_second)); + String authentication = catalogProperty.getOrDefault( + HdfsResource.HADOOP_SECURITY_AUTHENTICATION, ""); + if (AuthType.KERBEROS.getDesc().equals(authentication)) { + hiveConf.set(HdfsResource.HADOOP_SECURITY_AUTHENTICATION, authentication); + UserGroupInformation.setConfiguration(hiveConf); + try { + /** + * Because metastore client is created by using + * {@link org.apache.hadoop.hive.metastore.RetryingMetaStoreClient#getProxy} + * it will relogin when TGT is expired, so we don't need to relogin manually. + */ + UserGroupInformation.loginUserFromKeytab( + catalogProperty.getOrDefault(HdfsResource.HADOOP_KERBEROS_PRINCIPAL, ""), + catalogProperty.getOrDefault(HdfsResource.HADOOP_KERBEROS_KEYTAB, "")); + } catch (IOException e) { + throw new HMSClientException("login with kerberos auth failed for catalog %s", e, this.getName()); + } } } - - client = new PooledHiveMetaStoreClient(hiveConf, - Math.max(MIN_CLIENT_POOL_SIZE, Config.max_external_cache_loader_thread_pool_size)); + client = HMSCachedClientFactory.createCachedClient(hiveConf, + Math.max(MIN_CLIENT_POOL_SIZE, Config.max_external_cache_loader_thread_pool_size), jdbcClientConfig); } @Override @@ -201,7 +214,7 @@ public class HMSExternalCatalog extends ExternalCatalog { return hmsExternalDatabase.getTable(getRealTableName(tblName)).isPresent(); } - public PooledHiveMetaStoreClient getClient() { + public HMSCachedClient getClient() { makeSureInitialized(); return client; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 9b96dfa4c3..70f7b2cb9c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -131,7 +131,8 @@ import org.apache.doris.common.util.QueryableReentrantLock; import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; -import org.apache.doris.datasource.hive.PooledHiveMetaStoreClient; +import org.apache.doris.datasource.hive.HMSCachedClient; +import org.apache.doris.datasource.hive.HMSCachedClientFactory; import org.apache.doris.datasource.property.constants.HMSProperties; import org.apache.doris.external.elasticsearch.EsRepository; import org.apache.doris.persist.AlterDatabasePropertyInfo; @@ -2700,7 +2701,7 @@ public class InternalCatalog implements CatalogIf { if (!Strings.isNullOrEmpty(hiveTable.getHiveProperties().get(HMSProperties.HIVE_VERSION))) { hiveConf.set(HMSProperties.HIVE_VERSION, hiveTable.getHiveProperties().get(HMSProperties.HIVE_VERSION)); } - PooledHiveMetaStoreClient client = new PooledHiveMetaStoreClient(hiveConf, 1); + HMSCachedClient client = new HMSCachedClientFactory().createCachedClient(hiveConf, 1, null); if (!client.tableExists(hiveTable.getHiveDb(), hiveTable.getHiveTable())) { throw new DdlException(String.format("Table [%s] dose not exist in Hive.", hiveTable.getHiveDbTable())); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClient.java new file mode 100644 index 0000000000..6e7f45aaa4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClient.java @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.analysis.TableName; +import org.apache.doris.datasource.hive.event.MetastoreNotificationFetchException; + +import org.apache.hadoop.hive.common.ValidWriteIdList; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; + +import java.util.List; +import java.util.Map; + +/** + * A hive metastore client pool for a specific catalog with hive configuration. + * Currently, we support obtain hive metadata from thrift protocol and JDBC protocol. + */ +public interface HMSCachedClient { + Database getDatabase(String dbName); + + List getAllDatabases(); + + List getAllTables(String dbName); + + boolean tableExists(String dbName, String tblName); + + List listPartitionNames(String dbName, String tblName); + + List listPartitionNames(String dbName, String tblName, long maxListPartitionNum); + + Partition getPartition(String dbName, String tblName, List partitionValues); + + List getPartitions(String dbName, String tblName, List partitionNames); + + Table getTable(String dbName, String tblName); + + List getSchema(String dbName, String tblName); + + List getTableColumnStatistics(String dbName, String tblName, + List columns); + + Map> getPartitionColumnStatistics( + String dbName, String tblName, List partNames, List columns); + + CurrentNotificationEventId getCurrentNotificationEventId(); + + NotificationEventResponse getNextNotification(long lastEventId, + int maxEvents, + IMetaStoreClient.NotificationFilter filter) throws MetastoreNotificationFetchException; + + long openTxn(String user); + + void commitTxn(long txnId); + + ValidWriteIdList getValidWriteIds(String fullTableName, long currentTransactionId); + + void acquireSharedLock(String queryId, long txnId, String user, TableName tblName, + List partitionNames, long timeoutMs); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClientFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClientFactory.java new file mode 100644 index 0000000000..10e81993d0 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClientFactory.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.catalog.JdbcResource; +import org.apache.doris.datasource.jdbc.client.JdbcClient; +import org.apache.doris.datasource.jdbc.client.JdbcClientConfig; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.hive.conf.HiveConf; + +public class HMSCachedClientFactory { + public static HMSCachedClient createCachedClient(HiveConf hiveConf, int thriftClientPoolSize, + JdbcClientConfig jdbcClientConfig) { + if (hiveConf != null) { + return new ThriftHMSCachedClient(hiveConf, thriftClientPoolSize); + } + Preconditions.checkNotNull(jdbcClientConfig, "hiveConf and jdbcClientConfig are both null"); + String dbType = JdbcClient.parseDbType(jdbcClientConfig.getJdbcUrl()); + switch (dbType) { + case JdbcResource.POSTGRESQL: + return new PostgreSQLJdbcHMSCachedClient(jdbcClientConfig); + default: + throw new IllegalArgumentException("Unsupported DB type: " + dbType); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveTransaction.java index 3f3c2f91d7..f6bae24723 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveTransaction.java @@ -63,7 +63,7 @@ public class HiveTransaction { return isFullAcid; } - public ValidWriteIdList getValidWriteIds(PooledHiveMetaStoreClient client) { + public ValidWriteIdList getValidWriteIds(HMSCachedClient client) { if (validWriteIdList == null) { TableName tableName = new TableName(hiveTable.getCatalog().getName(), hiveTable.getDbName(), hiveTable.getName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/JdbcHMSCachedClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/JdbcHMSCachedClient.java new file mode 100644 index 0000000000..e384657d84 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/JdbcHMSCachedClient.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.datasource.jdbc.client.JdbcClient; +import org.apache.doris.datasource.jdbc.client.JdbcClientConfig; + +import com.google.common.base.Preconditions; + +/** + * This class uses the JDBC protocol to directly access the relational databases under HMS + * to obtain Hive metadata information + */ +public abstract class JdbcHMSCachedClient extends JdbcClient implements HMSCachedClient { + protected JdbcClientConfig jdbcClientConfig; + + protected JdbcHMSCachedClient(JdbcClientConfig jdbcClientConfig) { + super(jdbcClientConfig); + Preconditions.checkNotNull(jdbcClientConfig, "JdbcClientConfig can not be null"); + this.jdbcClientConfig = jdbcClientConfig; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java new file mode 100644 index 0000000000..afbd929c5a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java @@ -0,0 +1,477 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.analysis.TableName; +import org.apache.doris.catalog.JdbcTable; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.HMSClientException; +import org.apache.doris.datasource.hive.event.MetastoreNotificationFetchException; +import org.apache.doris.datasource.jdbc.client.JdbcClientConfig; +import org.apache.doris.thrift.TOdbcTableType; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import org.apache.hadoop.hive.common.ValidWriteIdList; +import org.apache.hadoop.hive.metastore.IMetaStoreClient.NotificationFilter; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class PostgreSQLJdbcHMSCachedClient extends JdbcHMSCachedClient { + private static final Logger LOG = LogManager.getLogger(PostgreSQLJdbcHMSCachedClient.class); + + public PostgreSQLJdbcHMSCachedClient(JdbcClientConfig jdbcClientConfig) { + super(jdbcClientConfig); + } + + @Override + public Database getDatabase(String dbName) { + throw new HMSClientException("Do not support in PostgreSQLJdbcHMSCachedClient."); + } + + @Override + public List getAllDatabases() { + String nameFiled = JdbcTable.databaseProperName(TOdbcTableType.POSTGRESQL, "NAME"); + String tableName = JdbcTable.databaseProperName(TOdbcTableType.POSTGRESQL, "DBS"); + String sql = String.format("SELECT %s FROM %s;", nameFiled, tableName); + LOG.debug("getAllDatabases exec sql: {}", sql); + try (Connection conn = getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql); + ResultSet rs = stmt.executeQuery()) { + Builder builder = ImmutableList.builder(); + while (rs.next()) { + String hiveDatabaseName = rs.getString("NAME"); + builder.add(hiveDatabaseName); + } + return builder.build(); + } catch (Exception e) { + throw new HMSClientException("failed to get all database from hms client", e); + } + } + + @Override + public List getAllTables(String dbName) { + String sql = "SELECT \"TBL_NAME\" FROM \"TBLS\" join \"DBS\" on \"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\"" + + " WHERE \"DBS\".\"NAME\" = '" + dbName + "';"; + LOG.debug("getAllTables exec sql: {}", sql); + + try (Connection conn = getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql); + ResultSet rs = stmt.executeQuery()) { + Builder builder = ImmutableList.builder(); + while (rs.next()) { + String hiveDatabaseName = rs.getString("TBL_NAME"); + builder.add(hiveDatabaseName); + } + return builder.build(); + } catch (Exception e) { + throw new HMSClientException("failed to get all tables for db %s", e, dbName); + } + } + + @Override + public boolean tableExists(String dbName, String tblName) { + List allTables = getAllTables(dbName); + return allTables.contains(tblName); + } + + @Override + public List listPartitionNames(String dbName, String tblName) { + return listPartitionNames(dbName, tblName, (long) -1); + } + + @Override + public List listPartitionNames(String dbName, String tblName, long maxListPartitionNum) { + String sql = String.format("SELECT \"PART_NAME\" from \"PARTITIONS\" WHERE \"TBL_ID\" = (" + + "SELECT \"TBL_ID\" FROM \"TBLS\" join \"DBS\" on \"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\"" + + " WHERE \"DBS\".\"NAME\" = '%s' AND \"TBLS\".\"TBL_NAME\"='%s');", dbName, tblName); + LOG.debug("listPartitionNames exec sql: {}", sql); + + try (Connection conn = getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql); + ResultSet rs = stmt.executeQuery()) { + Builder builder = ImmutableList.builder(); + while (rs.next()) { + String hivePartitionName = rs.getString("PART_NAME"); + builder.add(hivePartitionName); + } + return builder.build(); + } catch (Exception e) { + throw new HMSClientException("failed to list partition names for table %s in db %s", e, tblName, dbName); + } + } + + // not used + @Override + public Partition getPartition(String dbName, String tblName, List partitionValues) { + LOG.debug("getPartition partitionValues: {}", partitionValues); + String partitionName = Joiner.on("/").join(partitionValues); + ImmutableList partitionNames = ImmutableList.of(partitionName); + LOG.debug("getPartition partitionNames: {}", partitionNames); + List partitions = getPartitionsByNames(dbName, tblName, partitionNames); + if (!partitions.isEmpty()) { + return partitions.get(0); + } + throw new HMSClientException("Can not get partition of partitionName = " + partitionName + + ", from " + dbName + "." + tblName); + } + + @Override + public List getPartitions(String dbName, String tblName, List partitionNames) { + return getPartitionsByNames(dbName, tblName, partitionNames); + } + + private List getPartitionsByNames(String dbName, String tblName, List partitionNames) { + List partitionNamesWithQuote = partitionNames.stream().map(partitionName -> "'" + partitionName + "'") + .collect(Collectors.toList()); + String partitionNamesString = Joiner.on(", ").join(partitionNamesWithQuote); + String sql = String.format("SELECT \"PART_ID\", \"PARTITIONS\".\"CREATE_TIME\"," + + " \"PARTITIONS\".\"LAST_ACCESS_TIME\"," + + " \"PART_NAME\", \"PARTITIONS\".\"SD_ID\" FROM \"PARTITIONS\"" + + " join \"TBLS\" on \"TBLS\".\"TBL_ID\" = \"PARTITIONS\".\"TBL_ID\"" + + " join \"DBS\" on \"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\"" + + " WHERE \"DBS\".\"NAME\" = '%s' AND \"TBLS\".\"TBL_NAME\"='%s'" + + " AND \"PART_NAME\" in (%s);", + dbName, tblName, partitionNamesString); + LOG.debug("getPartitionsByNames exec sql: {}", sql); + + try (Connection conn = getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql); + ResultSet rs = stmt.executeQuery()) { + Builder builder = ImmutableList.builder(); + while (rs.next()) { + Partition partition = new Partition(); + partition.setDbName(dbName); + partition.setTableName(tblName); + partition.setCreateTime(rs.getInt("CREATE_TIME")); + partition.setLastAccessTime(rs.getInt("LAST_ACCESS_TIME")); + + // set partition values + partition.setValues(getPartitionValues(rs.getInt("PART_ID"))); + + // set SD + StorageDescriptor storageDescriptor = getStorageDescriptor(rs.getInt("SD_ID")); + partition.setSd(storageDescriptor); + + builder.add(partition); + } + return builder.build(); + } catch (Exception e) { + throw new HMSClientException("failed to get partition for table %s in db %s with value %s", e, tblName, + dbName, partitionNames); + } + } + + private List getPartitionValues(int partitionId) { + String sql = String.format("SELECT \"PART_KEY_VAL\" FROM \"PARTITION_KEY_VALS\"" + + " WHERE \"PART_ID\" = " + partitionId); + LOG.debug("getPartitionValues exec sql: {}", sql); + try (Connection conn = getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql); + ResultSet rs = stmt.executeQuery()) { + Builder builder = ImmutableList.builder(); + while (rs.next()) { + builder.add(rs.getString("PART_KEY_VAL")); + } + return builder.build(); + } catch (Exception e) { + throw new HMSClientException("failed to get partition Value for partitionId %s", e, partitionId); + } + } + + @Override + public Table getTable(String dbName, String tblName) { + String sql = "SELECT \"TBL_ID\", \"TBL_NAME\", \"DBS\".\"NAME\", \"OWNER\", \"CREATE_TIME\"," + + " \"LAST_ACCESS_TIME\", \"LAST_ACCESS_TIME\", \"RETENTION\", \"TBLS\".\"SD_ID\", " + + " \"IS_REWRITE_ENABLED\", \"VIEW_EXPANDED_TEXT\", \"VIEW_ORIGINAL_TEXT\", \"DBS\".\"OWNER_TYPE\"" + + " FROM \"TBLS\" join \"DBS\" on \"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\" " + + " WHERE \"DBS\".\"NAME\" = '" + dbName + "' AND \"TBLS\".\"TBL_NAME\"='" + tblName + "';"; + LOG.debug("getTable exec sql: {}", sql); + + try (Connection conn = getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql); + ResultSet rs = stmt.executeQuery()) { + Table hiveTable = new Table(); + if (rs.next()) { + hiveTable.setTableName(rs.getString("TBL_NAME")); + hiveTable.setDbName(rs.getString("NAME")); + hiveTable.setOwner(rs.getString("OWNER")); + hiveTable.setCreateTime(rs.getInt("CREATE_TIME")); + hiveTable.setLastAccessTime(rs.getInt("LAST_ACCESS_TIME")); + hiveTable.setRetention(rs.getInt("RETENTION")); + hiveTable.setOwnerType(getOwnerType(rs.getString("OWNER_TYPE"))); + hiveTable.setRewriteEnabled(rs.getBoolean("IS_REWRITE_ENABLED")); + hiveTable.setViewExpandedText(rs.getString("VIEW_EXPANDED_TEXT")); + hiveTable.setViewOriginalText(rs.getString("VIEW_ORIGINAL_TEXT")); + + hiveTable.setSd(getStorageDescriptor(rs.getInt("SD_ID"))); + hiveTable.setParameters(getTableParameters(rs.getInt("TBL_ID"))); + hiveTable.setPartitionKeys(getTablePartitionKeys(rs.getInt("TBL_ID"))); + return hiveTable; + } + throw new HMSClientException("Can not get Table from PG databases. dbName = " + dbName + + ", tblName = " + tblName); + } catch (Exception e) { + throw new HMSClientException("failed to get table %s in db %s from hms client", e, tblName, dbName); + } + } + + private StorageDescriptor getStorageDescriptor(int sdId) { + String sql = "SELECT * from \"SDS\" WHERE \"SD_ID\" = " + sdId; + LOG.debug("getStorageDescriptorByDbAndTable exec sql: {}", sql); + + StorageDescriptor sd = new StorageDescriptor(); + sd.setCols(getSchemaExcludePartitionKeys(sdId)); + + try (Connection conn = getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql); + ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + sd.setSerdeInfo(getSerdeInfo(rs.getInt("SERDE_ID"))); + sd.setInputFormat(rs.getString("INPUT_FORMAT")); + sd.setCompressed(rs.getBoolean("IS_COMPRESSED")); + sd.setLocation(rs.getString("LOCATION")); + sd.setNumBuckets(rs.getInt("NUM_BUCKETS")); + sd.setOutputFormat(rs.getString("OUTPUT_FORMAT")); + sd.setStoredAsSubDirectories(rs.getBoolean("IS_STOREDASSUBDIRECTORIES")); + return sd; + } + throw new HMSClientException("Can not get StorageDescriptor from PG, SD_ID = " + sdId); + } catch (Exception e) { + throw new HMSClientException("failed to get StorageDescriptor in sdId %s", e, sdId); + } + } + + private SerDeInfo getSerdeInfo(int serdeId) { + String sql = "SELECT * FROM \"SERDES\" WHERE \"SERDE_ID\" = " + serdeId; + LOG.debug("getSerdeInfo exec sql: {}", sql); + + SerDeInfo serDeInfo = new SerDeInfo(); + serDeInfo.setParameters(getSerdeInfoParameters(serdeId)); + + try (Connection conn = getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql); + ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + serDeInfo.setName(rs.getString("NAME")); + serDeInfo.setSerializationLib(rs.getString("SLIB")); + return serDeInfo; + } + throw new HMSClientException("Can not get SerDeInfo from PG databases, serdeId = " + serdeId + "."); + } catch (Exception e) { + throw new HMSClientException("failed to get SerdeInfo in serdeId %s", e, serdeId); + } + } + + private Map getSerdeInfoParameters(int serdeId) { + String sql = "SELECT \"PARAM_KEY\", \"PARAM_VALUE\" from \"SERDE_PARAMS\" WHERE \"SERDE_ID\" = " + serdeId; + LOG.debug("getSerdeInfoParameters exec sql: {}", sql); + + try (Connection conn = getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql); + ResultSet rs = stmt.executeQuery()) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + while (rs.next()) { + builder.put(rs.getString("PARAM_KEY"), rs.getString("PARAM_VALUE")); + } + return builder.build(); + } catch (Exception e) { + throw new HMSClientException("failed to get SerdeInfoParameters in serdeId %s", e, serdeId); + } + } + + private List getTablePartitionKeys(int tableId) { + String sql = "SELECT \"PKEY_NAME\", \"PKEY_TYPE\", \"PKEY_COMMENT\" from \"PARTITION_KEYS\"" + + " WHERE \"TBL_ID\"= " + tableId; + LOG.debug("getTablePartitionKeys exec sql: {}", sql); + + try (Connection conn = getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql); + ResultSet rs = stmt.executeQuery()) { + Builder builder = ImmutableList.builder(); + while (rs.next()) { + FieldSchema fieldSchema = new FieldSchema(rs.getString("PKEY_NAME"), + rs.getString("PKEY_TYPE"), rs.getString("PKEY_COMMENT")); + builder.add(fieldSchema); + } + + List fieldSchemas = builder.build(); + // must reverse fields + List reversedFieldSchemas = Lists.newArrayList(fieldSchemas); + Collections.reverse(reversedFieldSchemas); + return reversedFieldSchemas; + } catch (Exception e) { + throw new HMSClientException("failed to get TablePartitionKeys in tableId %s", e, tableId); + } + } + + private Map getTableParameters(int tableId) { + String sql = "SELECT \"PARAM_KEY\", \"PARAM_VALUE\" from \"TABLE_PARAMS\" WHERE \"TBL_ID\" = " + tableId; + LOG.debug("getParameters exec sql: {}", sql); + + try (Connection conn = getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql); + ResultSet rs = stmt.executeQuery()) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + while (rs.next()) { + builder.put(rs.getString("PARAM_KEY"), rs.getString("PARAM_VALUE")); + } + return builder.build(); + } catch (Exception e) { + throw new HMSClientException("failed to get TableParameters in tableId %s", e, tableId); + } + } + + private PrincipalType getOwnerType(String ownerTypeString) { + switch (ownerTypeString) { + case "USER": + return PrincipalType.findByValue(1); + case "ROLE": + return PrincipalType.findByValue(2); + case "GROUP": + return PrincipalType.findByValue(3); + default: + throw new HMSClientException("Unknown owner type of this table"); + } + } + + @Override + public List getSchema(String dbName, String tblName) { + String sql = "SELECT \"COLUMN_NAME\", \"TYPE_NAME\", \"COMMENT\", \"TBLS\".\"TBL_ID\"" + + " FROM \"TBLS\" join \"DBS\" on \"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\"" + + " join \"SDS\" on \"SDS\".\"SD_ID\" = \"TBLS\".\"SD_ID\"" + + " join \"COLUMNS_V2\" on \"COLUMNS_V2\".\"CD_ID\" = \"SDS\".\"CD_ID\"" + + " WHERE \"DBS\".\"NAME\" = '" + dbName + "' AND \"TBLS\".\"TBL_NAME\"='" + tblName + "';"; + LOG.debug("getSchema exec sql: {}", sql); + + Builder builder = ImmutableList.builder(); + int tableId = -1; + try (Connection conn = getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql); + ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + FieldSchema fieldSchema = new FieldSchema(rs.getString("COLUMN_NAME"), + rs.getString("TYPE_NAME"), rs.getString("COMMENT")); + builder.add(fieldSchema); + // actually, all resultSets have the same TBL_ID. + tableId = rs.getInt("TBL_ID"); + } + } catch (Exception e) { + throw new HMSClientException("Can not get schema of db = " + dbName + ", table = " + tblName); + } + + // add partition columns + getTablePartitionKeys(tableId).stream().forEach(field -> builder.add(field)); + return builder.build(); + } + + private List getSchemaExcludePartitionKeys(int sdId) { + String sql = "SELECT \"COLUMN_NAME\", \"TYPE_NAME\", \"COMMENT\"" + + " FROM \"SDS\" join \"COLUMNS_V2\" on \"COLUMNS_V2\".\"CD_ID\" = \"SDS\".\"CD_ID\"" + + " WHERE \"SDS\".\"SD_ID\" = " + sdId; + LOG.debug("getSchema exec sql: {}", sql); + + Builder colsExcludePartitionKeys = ImmutableList.builder(); + try (Connection conn = getConnection(); + PreparedStatement stmt = conn.prepareStatement(sql); + ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + FieldSchema fieldSchema = new FieldSchema(rs.getString("COLUMN_NAME"), + rs.getString("TYPE_NAME"), rs.getString("COMMENT")); + colsExcludePartitionKeys.add(fieldSchema); + } + } catch (Exception e) { + throw new HMSClientException("Can not get schema of SD_ID = " + sdId); + } + return colsExcludePartitionKeys.build(); + } + + @Override + public List getTableColumnStatistics(String dbName, String tblName, List columns) { + throw new HMSClientException("Do not support in PostgreSQLJdbcHMSCachedClient."); + } + + // no use + @Override + public Map> getPartitionColumnStatistics(String dbName, String tblName, + List partNames, List columns) { + throw new HMSClientException("Do not support in PostgreSQLJdbcHMSCachedClient."); + } + + @Override + public CurrentNotificationEventId getCurrentNotificationEventId() { + throw new MetastoreNotificationFetchException("Do not support in PostgreSQLJdbcHMSCachedClient."); + } + + @Override + public NotificationEventResponse getNextNotification(long lastEventId, int maxEvents, NotificationFilter filter) { + throw new HMSClientException("Do not support in PostgreSQLJdbcHMSCachedClient."); + } + + @Override + public long openTxn(String user) { + throw new HMSClientException("Do not support in PostgreSQLJdbcHMSCachedClient."); + } + + @Override + public void commitTxn(long txnId) { + throw new HMSClientException("Do not support in PostgreSQLJdbcHMSCachedClient."); + } + + @Override + public ValidWriteIdList getValidWriteIds(String fullTableName, long currentTransactionId) { + throw new HMSClientException("Do not support in PostgreSQLJdbcHMSCachedClient."); + } + + @Override + public void acquireSharedLock(String queryId, long txnId, String user, TableName tblName, + List partitionNames, long timeoutMs) { + throw new HMSClientException("Do not support in PostgreSQLJdbcHMSCachedClient."); + } + + @Override + protected String getDatabaseQuery() { + throw new HMSClientException("Do not support in PostgreSQLJdbcHMSCachedClient."); + } + + @Override + protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { + throw new HMSClientException("Do not support in PostgreSQLJdbcHMSCachedClient."); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java similarity index 84% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java index 4328ddc4da..abb3fda24b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java @@ -65,29 +65,33 @@ import java.util.Optional; import java.util.Queue; /** - * A hive metastore client pool for a specific catalog with hive configuration. + * This class uses the thrift protocol to directly access the HiveMetaStore service + * to obtain Hive metadata information */ -public class PooledHiveMetaStoreClient { - private static final Logger LOG = LogManager.getLogger(PooledHiveMetaStoreClient.class); +public class ThriftHMSCachedClient implements HMSCachedClient { + private static final Logger LOG = LogManager.getLogger(ThriftHMSCachedClient.class); private static final HiveMetaHookLoader DUMMY_HOOK_LOADER = t -> null; // -1 means no limit on the partitions returned. private static final short MAX_LIST_PARTITION_NUM = Config.max_hive_list_partition_num; - private Queue clientPool = new LinkedList<>(); + private Queue clientPool = new LinkedList<>(); private final int poolSize; private final HiveConf hiveConf; - public PooledHiveMetaStoreClient(HiveConf hiveConf, int pooSize) { - Preconditions.checkArgument(pooSize > 0, pooSize); - hiveConf.set(ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT.name(), - String.valueOf(Config.hive_metastore_client_timeout_second)); + public ThriftHMSCachedClient(HiveConf hiveConf, int poolSize) { + Preconditions.checkArgument(poolSize > 0, poolSize); + if (hiveConf != null) { + hiveConf.set(ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT.name(), + String.valueOf(Config.hive_metastore_client_timeout_second)); + } this.hiveConf = hiveConf; - this.poolSize = pooSize; + this.poolSize = poolSize; } + @Override public List getAllDatabases() { - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { try { return client.client.getAllDatabases(); } catch (Exception e) { @@ -99,8 +103,9 @@ public class PooledHiveMetaStoreClient { } } + @Override public List getAllTables(String dbName) { - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { try { return client.client.getAllTables(dbName); } catch (Exception e) { @@ -112,8 +117,9 @@ public class PooledHiveMetaStoreClient { } } + @Override public boolean tableExists(String dbName, String tblName) { - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { try { return client.client.tableExists(dbName, tblName); } catch (Exception e) { @@ -125,14 +131,16 @@ public class PooledHiveMetaStoreClient { } } + @Override public List listPartitionNames(String dbName, String tblName) { return listPartitionNames(dbName, tblName, MAX_LIST_PARTITION_NUM); } - public List listPartitionNames(String dbName, String tblName, long max) { + @Override + public List listPartitionNames(String dbName, String tblName, long maxListPartitionNum) { // list all parts when the limit is greater than the short maximum - short limited = max <= Short.MAX_VALUE ? (short) max : MAX_LIST_PARTITION_NUM; - try (CachedClient client = getClient()) { + short limited = maxListPartitionNum <= Short.MAX_VALUE ? (short) maxListPartitionNum : MAX_LIST_PARTITION_NUM; + try (ThriftHMSClient client = getClient()) { try { return client.client.listPartitionNames(dbName, tblName, limited); } catch (Exception e) { @@ -144,8 +152,9 @@ public class PooledHiveMetaStoreClient { } } + @Override public Partition getPartition(String dbName, String tblName, List partitionValues) { - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { try { return client.client.getPartition(dbName, tblName, partitionValues); } catch (Exception e) { @@ -158,8 +167,9 @@ public class PooledHiveMetaStoreClient { } } + @Override public List getPartitions(String dbName, String tblName, List partitionNames) { - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { try { return client.client.getPartitionsByNames(dbName, tblName, partitionNames); } catch (Exception e) { @@ -168,26 +178,13 @@ public class PooledHiveMetaStoreClient { } } catch (Exception e) { throw new HMSClientException("failed to get partition for table %s in db %s with value %s", e, tblName, - dbName, partitionNames); - } - } - - public List getPartitionsByFilter(String dbName, String tblName, String filter) { - try (CachedClient client = getClient()) { - try { - return client.client.listPartitionsByFilter(dbName, tblName, filter, MAX_LIST_PARTITION_NUM); - } catch (Exception e) { - client.setThrowable(e); - throw e; - } - } catch (Exception e) { - throw new HMSClientException("failed to get partition by filter for table %s in db %s", e, tblName, - dbName); + dbName, partitionNames); } } + @Override public Database getDatabase(String dbName) { - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { try { return client.client.getDatabase(dbName); } catch (Exception e) { @@ -199,8 +196,9 @@ public class PooledHiveMetaStoreClient { } } + @Override public Table getTable(String dbName, String tblName) { - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { try { return client.client.getTable(dbName, tblName); } catch (Exception e) { @@ -212,8 +210,9 @@ public class PooledHiveMetaStoreClient { } } + @Override public List getSchema(String dbName, String tblName) { - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { try { return client.client.getSchema(dbName, tblName); } catch (Exception e) { @@ -225,8 +224,9 @@ public class PooledHiveMetaStoreClient { } } + @Override public List getTableColumnStatistics(String dbName, String tblName, List columns) { - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { try { return client.client.getTableColumnStatistics(dbName, tblName, columns); } catch (Exception e) { @@ -238,9 +238,10 @@ public class PooledHiveMetaStoreClient { } } + @Override public Map> getPartitionColumnStatistics( String dbName, String tblName, List partNames, List columns) { - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { try { return client.client.getPartitionColumnStatistics(dbName, tblName, partNames, columns); } catch (Exception e) { @@ -252,8 +253,9 @@ public class PooledHiveMetaStoreClient { } } + @Override public CurrentNotificationEventId getCurrentNotificationEventId() { - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { try { return client.client.getCurrentNotificationEventId(); } catch (Exception e) { @@ -267,11 +269,12 @@ public class PooledHiveMetaStoreClient { } } + @Override public NotificationEventResponse getNextNotification(long lastEventId, int maxEvents, IMetaStoreClient.NotificationFilter filter) throws MetastoreNotificationFetchException { - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { try { return client.client.getNextNotification(lastEventId, maxEvents, filter); } catch (Exception e) { @@ -286,8 +289,9 @@ public class PooledHiveMetaStoreClient { } } + @Override public long openTxn(String user) { - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { try { return client.client.openTxn(user); } catch (Exception e) { @@ -299,8 +303,9 @@ public class PooledHiveMetaStoreClient { } } + @Override public void commitTxn(long txnId) { - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { try { client.client.commitTxn(txnId); } catch (Exception e) { @@ -312,6 +317,7 @@ public class PooledHiveMetaStoreClient { } } + @Override public void acquireSharedLock(String queryId, long txnId, String user, TableName tblName, List partitionNames, long timeoutMs) { LockRequestBuilder request = new LockRequestBuilder(queryId).setTransactionId(txnId).setUser(user); @@ -319,7 +325,7 @@ public class PooledHiveMetaStoreClient { for (LockComponent component : lockComponents) { request.addLockComponent(component); } - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { LockResponse response; try { response = client.client.lock(request.build()); @@ -346,11 +352,13 @@ public class PooledHiveMetaStoreClient { } } + @Override public ValidWriteIdList getValidWriteIds(String fullTableName, long currentTransactionId) { - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { try { // Pass currentTxn as 0L to get the recent snapshot of valid transactions in Hive - // Do not pass currentTransactionId instead as it will break Hive's listing of delta directories if major compaction + // Do not pass currentTransactionId instead as + // it will break Hive's listing of delta directories if major compaction // deletes delta directories for valid transactions that existed at the time transaction is opened ValidTxnList validTransactions = client.client.getValidTxns(); List tableValidWriteIdsList = client.client.getValidWriteIds( @@ -362,11 +370,11 @@ public class PooledHiveMetaStoreClient { tableValidWriteIdsList); ValidWriteIdList writeIdList = validTxnWriteIdList.getTableValidWriteIdList(fullTableName); return writeIdList; - } catch (Exception e) { + } catch (Exception e) { client.setThrowable(e); throw e; } - } catch (Exception e) { + } catch (Exception e) { // Ignore this exception when the version of hive is not compatible with these apis. // Currently, the workaround is using a max watermark. LOG.warn("failed to get valid write ids for {}, transaction {}", fullTableName, currentTransactionId, e); @@ -375,7 +383,7 @@ public class PooledHiveMetaStoreClient { } private LockResponse checkLock(long lockId) { - try (CachedClient client = getClient()) { + try (ThriftHMSClient client = getClient()) { try { return client.client.checkLock(lockId); } catch (Exception e) { @@ -411,24 +419,11 @@ public class PooledHiveMetaStoreClient { return builder.build(); } - public void heartbeatForTxn(long txnId) { - try (CachedClient client = getClient()) { - try { - client.client.heartbeat(txnId, txnId); - } catch (Exception e) { - client.setThrowable(e); - throw e; - } - } catch (Exception e) { - throw new RuntimeException("failed to do heartbeat for transaction " + txnId, e); - } - } - - private class CachedClient implements AutoCloseable { + private class ThriftHMSClient implements AutoCloseable { private final IMetaStoreClient client; private volatile Throwable throwable; - private CachedClient(HiveConf hiveConf) throws MetaException { + private ThriftHMSClient(HiveConf hiveConf) throws MetaException { String type = hiveConf.get(HMSProperties.HIVE_METASTORE_TYPE); if (HMSProperties.DLF_TYPE.equalsIgnoreCase(type)) { client = RetryingMetaStoreClient.getProxy(hiveConf, DUMMY_HOOK_LOADER, @@ -458,14 +453,14 @@ public class PooledHiveMetaStoreClient { } } - private CachedClient getClient() throws MetaException { + private ThriftHMSClient getClient() throws MetaException { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(ClassLoader.getSystemClassLoader()); synchronized (clientPool) { - CachedClient client = clientPool.poll(); + ThriftHMSClient client = clientPool.poll(); if (client == null) { - return new CachedClient(hiveConf); + return new ThriftHMSClient(hiveConf); } return client; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java index 0a6e830316..9dd4e79741 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java @@ -18,6 +18,8 @@ package org.apache.doris.datasource.hive.event; +import org.apache.doris.datasource.hive.HMSCachedClient; + import org.apache.hadoop.hive.metastore.api.NotificationEvent; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -144,7 +146,7 @@ public abstract class MetastoreEvent { /** * Process the information available in the NotificationEvent. - * Better not to call (direct/indirect) apis of {@link org.apache.doris.datasource.hive.PooledHiveMetaStoreClient} + * Better not to call (direct/indirect) apis of {@link HMSCachedClient} * during handling hms events (Reference to https://github.com/apache/doris/pull/19120). * Try to add some fallback strategies if it is highly necessary. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java index b7f4fd4955..29f361df1c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java @@ -154,7 +154,7 @@ public abstract class JdbcClient { } } - private static String parseDbType(String jdbcUrl) { + public static String parseDbType(String jdbcUrl) { try { return JdbcResource.parseDbType(jdbcUrl); } catch (DdlException e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java index 69dedd16d7..77beef9459 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java @@ -18,9 +18,11 @@ package org.apache.doris.datasource.jdbc.client; +import com.google.common.collect.Maps; + import java.util.Map; -public class JdbcClientConfig { +public class JdbcClientConfig implements Cloneable { private String catalog; private String user; private String password; @@ -29,8 +31,23 @@ public class JdbcClientConfig { private String driverClass; private String onlySpecifiedDatabase; private String isLowerCaseTableNames; - private Map includeDatabaseMap; - private Map excludeDatabaseMap; + private Map includeDatabaseMap = Maps.newHashMap(); + private Map excludeDatabaseMap = Maps.newHashMap(); + private Map customizedProperties = Maps.newHashMap(); + + @Override + public JdbcClientConfig clone() { + try { + JdbcClientConfig cloned = (JdbcClientConfig) super.clone(); + + cloned.includeDatabaseMap = Maps.newHashMap(includeDatabaseMap); + cloned.excludeDatabaseMap = Maps.newHashMap(excludeDatabaseMap); + cloned.customizedProperties = Maps.newHashMap(customizedProperties); + return cloned; + } catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } public String getCatalog() { return catalog; @@ -121,4 +138,12 @@ public class JdbcClientConfig { this.excludeDatabaseMap = excludeDatabaseMap; return this; } + + public void setCustomizedProperties(Map customizedProperties) { + this.customizedProperties = customizedProperties; + } + + public Map getCustomizedProperties() { + return customizedProperties; + } } diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index 482403e000..39c41f7c11 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -122,6 +122,7 @@ enablePaimonTest=false hms_port=9183 hdfs_port=8120 hiveServerPort=10000 +hive_pg_port=5432 // kafka test config // to enable kafka test, you need firstly to start kafka container diff --git a/regression-test/data/external_table_p0/hive/jdbc_hive_meta/test_jdbc_hive_orc.out b/regression-test/data/external_table_p0/hive/jdbc_hive_meta/test_jdbc_hive_orc.out new file mode 100644 index 0000000000..ca74b31cb2 --- /dev/null +++ b/regression-test/data/external_table_p0/hive/jdbc_hive_meta/test_jdbc_hive_orc.out @@ -0,0 +1,103 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_top50 -- +400899305488827731 base tennis pit vertical friday false tablets 2019-02-07 \N 7.8723304616937395E17 6.5976813E8 999742610 [7.53124931825377e+17] ["NbSSBtwzpxNSkkwga"] 55 \N 2022-08-19T07:29:58 4 smallint_col tablets smallint_col +105493714032727452 how literary \N tablets 2020-01-12 481707.1065 9.8642324410240179E17 6.3322381E8 999613702 [] ["HoMrAnn", "wteEFvIwoZsVpVQdscMb", null, "zcGFmv", "kGEBBckbMtX", "hrEtCGFdPWZK"] 49 Unveil bright recruit participate. Suspect impression camera mathematical revelation. Fault live2 elbow debt west hydrogen current. 2022-09-03T17:20:21 2 boolean_col tablets boolean_col +575009259015135910 \N \N desktops 2013-03-17 453810.9551 5.4512431766617229E17 9.0493901E8 998979717 [5.5663403615529414e+17, 63330505233342272, 3.1009537653752474e+17] ["VDwZhzTah", "AEGcQbFCNyemU", null, "UfYoRU", "lRNGmphqUUwsVQYVGU", "mVdEZ", "LSzhkkcrAmQxP", "BslhNs"] 95 Authority assassination annoy skiing. Pleased employ broad crime fortunate punish apartment. \N 5 bigint_col desktops bigint_col +603880418542146480 subtle gym farm interval included false phones 2020-01-18 671308.8364 7.0963347087235728E16 2.72213472E8 998719010 [] ["irMZIDUiOq", null, "IuDJPPTIGDMdQ", null, "XmxFGOWLc", "ULHMLLulC", "vqXhvfdFH"] 99 Revival lifelong crawl purpose. Regulator scary elbow. Economist mine league bone launch. 2022-09-03T02:00:12 8 tinyint_col phones tinyint_col +754459379266079664 \N false tablets 2019-07-11 733652.7631 3.2850645569567008E17 5.08661376E8 998617581 [3.5693025295728045e+17, 7.1714103590147226e+17] ["lEWHB", "nrjkSFzMMSBPLwiT"] 10 Born fundraising sharp cult isolated race. Pension1 used opponent edition trustee. 2022-08-30T19:28:59 1 bigint_col tablets bigint_col +636848138056884948 thank issue assassination fork true desktops 2022-08-14 170099.5226 1.3031709761648916E16 9.56856E8 998442612 [662084635579153.12, 9.5202508425028557e+17, 6.8547874796067738e+17, 2009792464461647, 7.0130044568537254e+17] ["vjEpySFbMUMyag", "BZTPdpKLiAbk", "LeRAXjdsgLa", "ftuYwfHsqPNXpgyTHo", "XzCIkxluwWqKjyhwjD", "OmkjC", "xNOOEfAJyUWUWlNDjGDO", "taHiDvYdekOigld"] 32 Distribute sunday beneath especially crystal mentor. Enterprise unemployed snow depression green. Sweet screening and habit limited report. 2022-08-15T21:04:19 8 float_col desktops float_col +732069608470626517 massive convincing entity precisely invade \N desktops 2015-12-06 979252.0764 8.3877805956411546E17 5.9623251E8 997001325 [8.9364082315504947e+17] \N 48 Costume rock extensive wooden placement play. Neutral roof domain entertainment unveil someone. Beside correctly gallon. Classify meditation jail complaint please abolish. 2022-08-26T09:42:16 1 float_col desktops float_col +768862777706807153 allegedly evil lovely false tablets \N 554241.1621 6.074364826499328E17 3.49622816E8 996952097 [] ["rlYNubjGewb", "GlibPXItkVXfCqHIzfF", "ifmwTRnYxOiIy", "oVZTvGtkfiCbjh", "HztHxOXyIl"] 58 Strategy steam branch confer. Outsider radiation click boot season beach communicate. 2022-08-22T08:11:40 4 float_col tablets float_col +420927942708721583 training solidarity similarity false desktops 2018-08-26 988855.7929 5.83945915640288E17 9.0464557E8 996555483 [8.14988510525504e+17] ["ZFEqYVLAUjEczpPn", "NVTIw", "wCnYBqSTiuFozOmxfafI", null] 81 Flood appreciation offering diary. Cake comment celebrity processor boy recommend. 2022-08-10T04:45:43 1 boolean_col desktops boolean_col +496640756596372260 authentic efficiently shake predator true phones 2018-12-03 719156.1038 8.2488184598939226E17 7.7654016E8 995559445 [3.0370154940373338e+17, 7.1058600146591091e+17, 5.523870697300384e+17, 4.6299085569885766e+17, 6.7001514539109235e+17] ["knnkRrThAETWlKDtn", null, "bhWydMwoQuqgWFyqia", "LmXUNZpGhIOYBZQgz", "DPhGDqtqAHDFJlmCbLAn", "yFghgTlk", null] 60 Opinion soccer regulation sum firm prejudice. Largely strong educated organize stand. \N 3 float_col phones float_col +623416117915767990 \N false desktops 2021-01-13 189213.3634 8.5693376228360768E17 2.83810944E8 995548120 [8.13564051647795e+17] [] \N Library partly argument evaluation pace breast. Criminal thereafter dense educator arms. 2022-09-05T15:00:05 3 float_col desktops float_col +39850937084984262 gap investigate instruct can2 false tablets 2022-07-20 719982.8368 \N 2.86086336E8 995547119 [4.7556700332533555e+17, 3.6441248240169235e+17, 6.92717278190866e+17, 8.5068031969243622e+17, 3.8655969043505446e+17] ["rwPIlzMLGzVxezOzg", "CLUplJ", "thXF", "tNPD", "qxtAZcbONyrVdtSVshf", "mgjKvIBHPTOkpx", "rKrnVxk", "FYbrxCQUGO"] 67 Lane decide illegal banana chamber. Capital pressure medal. 2022-09-01T02:09:15 1 tinyint_col tablets tinyint_col +676410314096484340 clause flee log machine number \N desktops 2015-06-25 148885.7960 5.4213585819745126E17 8.2933288E7 995328433 [7.9229087257987763e+17, 8.2703650833379558e+17] ["VEAAL", "tMWGSZkSwjzL", "rBaICQpyFqiEKYAJP", "JIHiNfWZEOZbaijWcoz", "HXUZXapTct", "jUcqSS", "DWgSFfuDoYvbb"] 55 Comfort allegedly noble equation. Philosophy improve divide dancer breakdown. 2022-09-03T20:36:17 7 boolean_col desktops boolean_col +329139323170285912 murder rebel such ear front true tablets \N \N 2.460622878030516E16 4.191344E8 995168790 [5.0322602630748339e+17] ["xKlRXvLcsdgEiYEHf"] 97 Gift discovery yesterday weather condemn damage. Realm consume robbery december surprised plan league. Smoking useful viewer delicate methodology gathering pair. Regardless throughout how port. Pencil swallow valuable logical adopt maybe statistic. 2022-08-31T18:00:52 8 float_col tablets float_col +64458235379408511 \N true tablets 2021-12-14 178924.0501 \N 2.78464512E8 994976337 [6.7366244590619533e+17, 8.5591107171774963e+17, 6.8121166912315174e+17] ["OsqHo", "FSsynpWBSaEFQQt", "LNYBxPkA", "mqCnnbNiAUCutU", "XprhHSwYvKvlmZaQfIgO", "GtHgElaIdmLLsndMoJ", "WvPBq", null] 37 Cost span my garden unity resign. Carve last1 anything brutal programming whereas plain. Contact plain website language odds drain deadly. Evening newsletter steel elsewhere long-standing figure methodology. 2022-08-16T08:01:21 2 float_col tablets float_col +663343287411312115 yet relax true desktops 2022-06-19 371344.1451 4.8859864838042835E17 9.9466253E8 994723799 [5.3569236084611917e+17, 9.503022354445399e+17, 3.7718724009399072e+17, 1.1026878203904256e+17] ["KCfZsewMFUVBvJZKscU", "ztIJBHMAA", "hfLz", null, "MSEomldskPu", "pwCR", null, "CwIfrZdWp"] 41 Regardless old-fashioned penny congregation. Access palace drain credit neck. Receive middle teens reason number. Check dollar lung. \N 7 int_col desktops int_col +\N moreover class \N tablets 2015-09-08 855610.6494 \N \N 994639378 [] [null, "LVlPuTu", "BJRcLDsjyPBgiM", "tYXgzGMr", "mafPcmX", "gXsGcqhqIH", "szFDxBGcvML", "UwVJugQgKURgYchUDx"] 82 \N 2022-08-26T00:16:25 8 boolean_col tablets boolean_col +275056213033724461 arrive access false desktops 2018-10-03 252619.5125 3.9353102442374336E16 8.8335142E8 994534648 [4.9706313273106061e+17, 6.2041444684110042e+17, 7.9479428357356211e+17, 3.3045269424080678e+17, 3.0402739456761965e+17] ["QWm", "MXYZizChOLneBymZSGou", "LqNwFXvZ", "YKBsSvnhJY", "coUcb", "eLOYYXRSz", "ikWSBjo"] \N Bishop different used. Globalization trio absolute people smell. 2022-09-02T08:42:57 \N boolean_col desktops boolean_col +348028031439811917 consultation believe false tablets 2016-08-22 415339.5028 4.2382370154214624E17 6.8747546E8 994382962 [3.0086734676548032e+17, 4.8735516725405146e+17, 9.3596536631516083e+17, 9.1663816035208179e+17] ["tpiRs", "kKEN", "wpClDVNjnxwe", "vHGHkxjqXHIpmevovl"] 65 Many negotiate goodbye have. Associated cruise ban philosophy reckon inspection. Mature possess treatment mixed. Affordable insert primary expect rock. Disturb pity accident year cute hatred. 2022-08-07T11:16:09 7 float_col tablets float_col +338219472721526956 \N false desktops \N 689199.0771 7.9466221089006246E17 2.59046112E8 994133872 [9.0070495056671181e+17, 1.7996775601617808e+17, 20298765261513864, 180060680264327.75, 2.6825749730073821e+17] ["tZIpUDUW", "rrHPe", null, "fgWhtNZ", "uMA", "JuOyGRSsiji", "VkCsMDMDYP", null, "hRO"] 46 Disability nevertheless old-fashioned word. Influence commissioner invent adjust accordance underwear ceiling. Wander comfort disrupt enquire scholarship breathing reputation. Significant monk dissolve flaw fraud. 2022-08-15T05:28:19 5 bigint_col desktops bigint_col +977588954291513457 detention fighting false phones 2021-09-07 817473.6937 1.63251635114689536E17 6.9995917E8 993507340 [4.946917661433303e+17, 1.3273031316668304e+17, 7.5244240424299174e+17, 8.7644354739655078e+17] ["LVHOZIzE", "dugMZkZFydCTBh"] 46 Heavily decorate profession beloved monster compelling bedroom. Collective publicity discount copper throat. Analyst constantly proceedings ultimate advertising. Ambition discovery internal angry dirty strongly. 2022-08-15T00:38:02 4 boolean_col phones boolean_col +137993334783196500 emergency gaming pump metal slam \N phones 2017-02-13 577513.4432 \N 1.53571424E8 993355824 [8.1179390476499021e+17, 7.4148037798744269e+17, 7938246732863808] ["ErSbGzXmodRzRie", "cHSzuVLhEMAf"] \N Prosperity printer vitamin equipment vision height twin. Traveller stereotype circulate personal little cemetery tide. 2022-08-27T18:50:50 7 bigint_col phones bigint_col +449488952514291370 factor confuse produce nasty true desktops 2021-08-10 560319.8951 8.3623994541438477E17 8.0099328E8 993292527 [7.083063145462272e+17, 7.5303869026635738e+17, 5.4587446217583654e+17, 2802338805376948, 6.7811500872898483e+17] ["EaUMfRbwPP", "ExFBaRK", null, "duOefOehEXtw", "OHtYtKEj", "fvgg"] 17 Isolated coast body. Bridge initiate until philosophical. 2022-08-30T06:11:29 3 int_col desktops int_col +312372245617770804 formal eight true desktops 2021-04-29 842330.4588 9.2371528807596749E17 2.62728E8 993012822 [9.83715950101926e+17, 7.529749459101504e+17, 1.31686491596813e+16, 48034895183293072, 2.1326477477269613e+17] [] 95 Passing consultant lecture blast. Let conclude compelling indirect his. Gear sixteen harbour lost legislature primary substantially. Aim wear heel coat divert. Enjoyable devise arrest. 2022-08-11T12:16:06 2 tinyint_col desktops tinyint_col +669369990298511963 violence map camping based true phones 2018-05-24 224570.1521 2.85465663177764576E17 7.7835168E8 992799913 [6.943143080562025e+17, 18479449383053016] [null, "sQupQUoYfeq", "SAkyHKdxXgWZaWac", "dyXYkHocgXXoAh", "TFCuzviyolrwyq", "pKsXHIM"] \N Distribute following handle. Arms impress threshold mild. Export spot connect. Mainstream prosecution fundamental estimate clinical. Mess ideological advertising museum fish. 2022-08-24T09:18:53 8 bigint_col phones bigint_col +326055487254498364 nowadays pass civic label false tablets 2014-03-07 781120.8462 1.42732445000672288E17 5.5347853E8 992668782 [] ["nyg"] \N Make-up every apparently addition compute. Long1 official controversy well money devastate lesson. 2022-08-10T22:14:52 9 bigint_col tablets bigint_col +43139399012470413 listen tremendous gut true tablets 2014-05-01 242771.1208 6.0874795006894605E17 6.4972723E8 992075287 [1.4888694889510467e+17, 1.1624496877076141e+17, 4.9838309536411174e+17, 9.1944861374577472e+17] ["zCWydqSmY", "gzGhWxHbzjvucuRMzq", "GAPMPbYcQIFxS", "hszicJZUzDPUlCWadWtN", "AOSJfjeJiR", "hfjkZngkLBJxM"] 67 Cream feat weekly mystery freedom. Extend absurd beer sleep athlete project. Enthusiasm unit disorder. Capable whisper hook communicate. 2022-08-11T18:00:29 9 bigint_col tablets bigint_col +911949425639198306 o’clock sentiment theme drop true desktops 2013-05-05 302133.7746 \N \N 991827409 [] ["TRpHvUaJq", "SnQTsquVURqSDQYctA", "wyKsKxmeKlbVxpWMQTAw", "cEYG", "cEqMOPrPfg", "PmhostVgjdH", "kFmrCegIrOl", "XJTNmVfdD", "DZUeOco", "jKKmrbrqnuKheN"] \N Work revenue chase. Farming suite cut adverse campus. Disposal tune dull landmark rejection ally. Event grace voluntary acknowledge fit distress. Delight treasure grocery string evoke born. 2022-08-22T16:00:25 7 smallint_col desktops smallint_col +396070856231408452 ignorance experience couple crush true phones 2015-05-13 812360.3430 9.979261569945737E17 9.1057043E8 991236722 [6.9043852169663718e+17, 1.074495099529511e+17, 9.58145113249252e+17, 1.353462997183208e+17, 5.3607139995088506e+17] \N 59 Rehabilitation percentage pride tender. Benchmark solve probe crystal architect conflict question. Protocol wipe ancient reference trail. Exotic traveller identical talented fake load. Entire draw administrator creation affection. 2022-09-05T03:28:17 3 tinyint_col phones tinyint_col +887145496628965173 between racist fibre false tablets 2021-11-11 \N 2.33327143660900224E17 \N 991074207 [9.146044424068343e+17, 4.4065851245243014e+17, 2.714994762482793e+17] ["bzrpJOEM", "PsGUTmchpXsszTlPzOaN", "rivXLjIGLuttFR", "dOeeHNOnca", "ZTsPccW", "zgDNEzdEoTYWJOLqcW", "TGrlCBYuzgJ", "lFTJOTWZeuSuaBeIbT", "bCKG", "uqYlh"] 54 Shake donate therapist scene. Eighteen violence resist next. Courage sympathy temperature tide kidnap engage. Together estimate hockey racing mobile. Creative incur feed economic detection greet inherent. \N 6 int_col tablets int_col +257952123158816158 treatment music tour ratio true phones 2020-06-07 \N 7.5205980925500122E17 1.24285968E8 990973551 [3.9083767066960326e+17] [] 92 Lens version fault. Bare estimate live1 journal demonstrate. Coordinator openly lady without white animation organization. Drown unhappy basket editorial caution prestigious. 2022-08-15T02:55:27 7 bigint_col phones bigint_col +264342532771407294 busy exam extremely qualification true desktops 2020-11-05 689457.0794 2.83202945069486848E17 9.6825094E8 990055297 [8.7740186466710413e+17, 7.3699880175397581e+17, 7.2891532622949082e+17] ["NNbqHuhMJWqQyJY", "auMcHgSfmcTjPFsWk", "EPLgRynwJtyfbDD", "JXpuIGA", "hlgBrRXukCJenxEoYnuS", "IHLvsB"] 13 Gang infant consist pull. Pencil aircraft recycle exceed involve sand diverse. 2022-08-24T23:20:23 8 int_col desktops int_col +212691360844244176 furniture expected conceal duo false tablets 2015-10-27 173936.2991 5.5305254019680784E16 2.91214592E8 989737678 [1.8740700830673674e+17, 8.26144642072196e+17, 7.0254358170509069e+17, 7.2090724096775821e+17, 8.2513780555372224e+17] ["EDAk", "EtwcuRqIZpS", "qZzi", "EneMBfJeHBsIkxfCFxuv", "BouCYAoOW", "LvUzxAZBzaYiXBfkECI", "egvweRwmIfCZ"] 22 Enter line canal typical archive absolutely care. Come straightforward removal up fix. Weak broad immune lyric publish. Ourselves can2 disabled out permit. Text wedding add being compassion personnel consciousness. \N 3 float_col tablets float_col +861148107399538236 steep operate download framework bean true tablets 2017-09-11 \N 3.9487522718177128E16 8.2829504E8 989556776 [1.1291426654032322e+17, 1.1282105208845827e+17, 8.6101953368477517e+17, 4.4123420285917715e+17, 4.7036855003688262e+17] ["TSEojLqfgoUoyYeO", "PmjmvtuoHt", "VYatlwquiRjpqJRiG", "wBWMxkHb", null] 33 Foot breakfast regular retreat nasty smash other. Self down high topic practice resignation administrative. 2022-09-01T14:37:11 3 tinyint_col tablets tinyint_col +372415973165915438 tradition badly nurse satellite false tablets 2014-07-16 191555.6732 7.7552886519944E17 6.7393088E8 989544553 [2.114505290225689e+17, 1.9787310867757923e+17, 2.9478388153066368e+17] ["dYaDWpPMCF", "lOsnKKidKGmJeBdiN", "gCWasnndId", null, "FecdDiCjSloyfcPYABqJ", "VRoxxXO", "PnUPJhuoUOJRTMlqmxo", "SaJdoCiY"] 33 Kid composer expected rental split end. Emotionally wonderful counterpart metre. Start division drug gang cope naval supermarket. Software lean editorial multiple habitat. Side promising gentle her compromise. 2022-08-30T19:31 3 float_col tablets float_col +448462623900311272 \N true tablets 2020-02-12 928095.5454 6.2670659738407949E17 7.4245421E8 989286745 [1.155409471089488e+17, 2.0161022044884957e+17, 7.1934979717784243e+17] ["uPmfLkUPP", "AFLoOHT", "pDuBnZAIeiTYNbBOu"] 78 Spot sink blame other charge. Cheek have thousand beautiful. 2022-08-15T03:51:42 4 float_col tablets float_col +321336188038349244 next contemplate compromise true tablets 2013-03-15 807203.9008 \N 5.05806176E8 988129246 [7.4733005320873293e+17] \N \N \N 2022-08-20T14:38:27 \N smallint_col tablets smallint_col +598162529085799203 ugly smash \N phones 2014-08-20 550275.2557 5.977672985795881E17 \N 987478218 [7.74535081898372e+17, 1.815794304143369e+17] ["CCWplUlBg", "QWcgjHzWhG", "sWhqlI", "ESGYsWpUCA"] 70 Occurrence shower double. Interactive duty positive assess attain circumstance. Regime unhappy genre sheep marketing assert. Flag persuade district acknowledge regulate encouraging criterion. Voting engage break multiply. 2022-08-10T20:25:03 \N boolean_col phones boolean_col +163333247606351459 \N true tablets 2015-02-02 829923.8961 5.2763199707000384E17 9.1720755E8 987470977 [5.9909779852457741e+17, 7.2692919376306035e+17, 3.6941342971354982e+17, 5.862745930336576e+17] ["Ilo", "iyltFdsgVLds", "bNn", "YvENgCixRZVoeWUabk"] 42 Pair backing competitor dig. Whenever sketch programming outlet cease sack standing. Monster response protocol worship scared. Classical govern female. 2022-08-13T23:11:22 8 tinyint_col tablets tinyint_col +129268823222240972 mile determination false tablets 2019-01-06 604701.7592 8.6075266796586701E17 \N 987329394 [3.1078810412278368e+17, 7.3542127312740134e+17] \N 20 Despite bail including. Conventional circumstance fence story step. Ceremony belt humour reputation. Infrastructure easy toilet my dollar premise supermarket. Sword mass restaurant yesterday interference spring exit. 2022-09-02T01:03:22 \N boolean_col tablets boolean_col +840144213771116088 coincidence overall consideration ingredient true tablets 2015-02-07 930874.4553 3.5566725998016077E17 2.20893872E8 987289606 [6.498773865161129e+17, 4.8237070793227654e+17] ["aOVKyZKSfDIwlYH", "gbNUFCO", "bPEvtAYjXdKXmd"] 79 Shore parental powerful. Protection foreign firm sanction pressure. Current mechanism guilty choose instruction delegation prepare. Rival sugar kidney hit lab admit proceed. 2022-08-20T20:08:01 1 int_col tablets int_col +868449561086137157 staff reasonably platform scholar circulate true desktops 2016-08-28 326501.8514 4.8678626719734816E17 5.7386643E8 987241408 [2.2132500254258102e+17, 1.6095855062087562e+17, 7.395204232644439e+17, 4.6493367683013286e+17] ["KPKOmTVFMnSbWfx", "YtavgHAkPskCc", "uiyXC", "PAuDfkJdhO", "ATRGGAWNGJ"] 46 Find careless however invention adoption. Arbitrary open neglect. Friend embarrassment amid density possess operational qualified. Angle novel mean. Inflation depth reporting consumption spot poverty. \N 1 boolean_col desktops boolean_col +882759368495896161 grace fair hidden trail true phones 2017-01-08 591251.5881 9.1284571965376384E17 8258863.5 987011098 [3.553412039578169e+17] ["sUBpYeEjXLku", "KoDTVscoRkVO", "qhNYa", "EBXmZ", null, "HCvQtRQRCOzIzjO", "YnHK", "JAge", "FKlfYoFQzUiZYap"] 68 \N 2022-08-15T19:05:15 5 tinyint_col phones tinyint_col +498867943145833747 transportation absurd warfare false desktops 2017-05-23 850871.2292 6.8335729823235059E17 \N 986830519 [4.226899791754336e+17, 4.7095064399080653e+17] ["xNECmu", "uwyfpikWGfxM", "TVZNMxVfiSXJTWsvZS", "UqXajVrdjqdPzOGz", "RCivZjW", "vXMVwjkMiO"] 25 Conventional investigate thanks instance rhetoric. Limb firstly resident fantastic employee tall. 2022-08-09T13:57:55 8 bigint_col desktops bigint_col +518479261183178454 raise cope true phones \N 467983.1687 1.32328604302364368E17 1.0965992E7 986812276 [6.8375952419669158e+17] ["licAlaF"] 12 Metaphor brush fraud needle confidence never hook. Test tent objection connection governance similar born. Threshold superior alarm aftermath. Poison all laughter. 2022-08-15T13:41:09 4 int_col phones int_col +966965240896174962 \N false phones 2015-11-10 753918.3430 1.9757531555380768E17 3.12841216E8 986681839 [2.2531448753722371e+17, 68934654326404336] ["pPnGKupbrjtZUZMjCB", "ywuw", "rBhhKpTErtoep", "zswxiCFDXslBJGoG", "VoGkC"] 72 Pack aim rip engage fixture. Broadcast roughly smooth. Acute establish hurricane architectural correction chairman weather. 2022-08-09T05:35:56 4 bigint_col phones bigint_col +882634950713305211 extent gas hint coordinate false phones 2016-07-14 295070.2219 7.0741662914685082E17 1.8774816E7 986641435 [8.0321866928608115e+17, 7.4137236509302656e+17, 1.5073113715907139e+17, 7.8313495728948659e+17] [] 77 Guideline religious vice. Ghost excuse county derive. Glance military panel unveil. Joke lost survival most as. 2022-08-08T00:40:09 \N boolean_col phones boolean_col +86980633188107783 email senator speculate anything swim true tablets 2015-06-26 760440.5681 7.5548400246108992E17 6.3851021E8 986349337 [] ["RJGkpBfoNE", "jpQoLfYrI", "gOgkBQUg", "QBs", "IrHb", "AazarmoPsX", "AAxTPdvLHpP", "JPx"] 72 Relation ninety know cloud hate terminal egg. Extreme ticket detail domain acknowledge complain live1. Dedicated hair escape. 2022-08-07T19:11:47 5 int_col tablets int_col +88457348565826264 \N true phones 2022-08-22 140717.2626 5.177242499015833E17 3.8514276E7 986333570 [] ["PWCCGPfT"] 71 Football stumble result taste pleased midst. Mirror loyal divide. Ultimately injury chip lawyer. Leadership teacher belong. 2022-08-26T19:19:31 4 smallint_col phones smallint_col +683875991736989008 split terrify \N tablets 2015-11-04 390407.1015 \N 6.0774349E8 986131998 [6.6301721738494848e+17] [null, "lSQFYzUG", "vMVMwfZzpl", "QRFiYUUefBc", "VdtTHy", "YrPtPPzynqXCCzm", "LfIgQvGimBBzlgn"] 21 Weird period none. Assertion coincide college. Subscriber fridge craft. Poisonous donation ordinary. Explode village debt. 2022-08-27T01:05:20 3 tinyint_col tablets tinyint_col + +-- !count_all -- +desktops bigint_col 188 186 181 193 182 192 180 187 174 180 184 206 206 180 512 936 +desktops boolean_col 192 190 198 194 192 198 187 197 193 194 180 212 212 186 555 932 +desktops float_col 186 189 183 178 188 187 179 186 180 184 186 205 205 188 460 952 +desktops int_col 182 182 182 178 184 171 181 181 175 180 183 200 200 172 465 915 +desktops smallint_col 170 166 177 178 174 181 181 174 166 172 176 195 195 172 508 939 +desktops tinyint_col 191 196 191 184 197 196 191 187 196 198 194 213 213 200 599 964 +phones bigint_col 185 181 176 177 174 180 182 180 173 174 177 196 196 176 535 890 +phones boolean_col 155 160 161 171 166 170 163 168 166 167 160 183 183 164 422 770 +phones float_col 169 169 169 166 172 173 172 166 183 172 180 193 193 177 456 871 +phones int_col 185 193 190 180 188 196 187 186 193 190 194 210 210 182 510 1031 +phones smallint_col 175 167 174 171 172 172 173 172 174 178 178 194 194 175 480 888 +phones tinyint_col 166 162 167 156 167 158 165 163 164 167 165 182 182 156 480 833 +tablets bigint_col 186 188 182 192 178 183 181 191 194 189 188 208 208 187 531 897 +tablets boolean_col 184 181 182 186 182 177 174 183 182 182 182 201 201 182 485 908 +tablets float_col 183 184 177 178 176 181 174 174 179 184 171 196 196 169 514 874 +tablets int_col 178 185 185 172 174 181 183 187 180 182 181 201 201 184 512 931 +tablets smallint_col 180 180 182 182 182 183 175 188 189 181 171 203 203 186 566 927 +tablets tinyint_col 179 182 182 187 183 181 177 183 177 187 183 202 202 186 528 974 + +-- !search_lg_int -- +1 + +-- !search_in_int -- +2 + +-- !search_mix -- +996555483 988855.7929 2018-08-26 + +-- !only_partition_col -- +3600 3600 + +-- !decimals1 -- +1.1234 12.123456 123.123456789876 12 1234.123456789 123 1 +1234.1234 123456789123.123456 12345678912345678912345678.123456789876 123456789 123456789123456789.123456780 987654321 2 +1234.0000 123456789123.000000 12345678912345678912345678.000000000000 123456789 123456789123456789.000000000 987654321 3 +0.0000 0.000000 123.123456789876 12 0E-9 123 4 +1.1234 12.123456 0E-12 0 1234.123456789 0 5 + +-- !decimals2 -- +1234.0000 123456789123.000000 12345678912345678912345678.000000000000 123456789 123456789123456789.000000000 987654321 3 + +-- !decimals3 -- +1.1234 12.123456 123.123456789876 12 1234.123456789 123 1 +1234.1234 123456789123.123456 12345678912345678912345678.123456789876 123456789 123456789123456789.123456780 987654321 2 + +-- !decimals4 -- +0.0000 0.000000 123.123456789876 12 0E-9 123 4 +1.1234 12.123456 0E-12 0 1234.123456789 0 5 + diff --git a/regression-test/pipeline/external/conf/regression-conf.groovy b/regression-test/pipeline/external/conf/regression-conf.groovy index d114ed598f..99a8666fbc 100644 --- a/regression-test/pipeline/external/conf/regression-conf.groovy +++ b/regression-test/pipeline/external/conf/regression-conf.groovy @@ -86,6 +86,7 @@ clickhouse_22_port=8123 // See `docker/thirdparties/start-thirdparties-docker.sh` enableHiveTest=true hms_port=7141 +hive_pg_port=5432 enableEsTest=true es_6_port="19200/" diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy b/regression-test/pipeline/p0/conf/regression-conf.groovy index 3994d68a94..db3b31637a 100644 --- a/regression-test/pipeline/p0/conf/regression-conf.groovy +++ b/regression-test/pipeline/p0/conf/regression-conf.groovy @@ -85,6 +85,7 @@ mariadb_10_port=3326 enableHiveTest=false hms_port=7141 hiveServerPort=10000 +hive_pg_port=5432 // kafka test config // to enable kafka test, you need firstly to start kafka container diff --git a/regression-test/suites/external_table_p0/hive/jdbc_hive_meta/test_jdbc_hive_orc.groovy b/regression-test/suites/external_table_p0/hive/jdbc_hive_meta/test_jdbc_hive_orc.groovy new file mode 100644 index 0000000000..b9fa912093 --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/jdbc_hive_meta/test_jdbc_hive_orc.groovy @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_jdbc_hive_orc", "all_types,p0,external,hive,external_docker,external_docker_hive") { + // Ensure that all types are parsed correctly + def select_top50 = { + qt_select_top50 """select * from orc_all_types order by int_col desc limit 50;""" + } + + // Ensure that the null map of all types are parsed correctly + def count_all = { + qt_count_all """ + select p1_col, p2_col, + count(tinyint_col), + count(smallint_col), + count(int_col), + count(bigint_col), + count(boolean_col), + count(float_col), + count(double_col), + count(string_col), + count(binary_col), + count(timestamp_col), + count(decimal_col), + count(char_col), + count(varchar_col), + count(date_col), + sum(size(list_double_col)), + sum(size(list_string_col)) + from orc_all_types group by p1_col, p2_col + order by p1_col, p2_col; + """ + } + + // Ensure that the SearchArgument works well: LG + def search_lg_int = { + qt_search_lg_int """select count(int_col) from orc_all_types where int_col > 999613702;""" + } + + // Ensure that the SearchArgument works well: IN + def search_in_int = { + qt_search_in_int """select count(int_col) from orc_all_types where int_col in (999742610, 999613702);""" + } + + // Ensure that the SearchArgument works well: MIX + def search_mix = { + qt_search_mix """select int_col, decimal_col, date_col from orc_all_types where int_col > 995328433 and decimal_col > 988850.7929 and date_col > date '2018-08-25';""" + } + + // only partition column selected + def only_partition_col = { + qt_only_partition_col """select count(p1_col), count(p2_col) from orc_all_types;""" + } + + // decimals + def decimals = { + qt_decimals1 """select * from orc_decimal_table order by id;""" + qt_decimals2 """select * from orc_decimal_table where id = 3 order by id;""" + qt_decimals3 """select * from orc_decimal_table where id < 3 order by id;""" + qt_decimals4 """select * from orc_decimal_table where id > 3 order by id;""" + } + + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + try { + String hms_port = context.config.otherConfigs.get("hms_port") + String catalog_name = "test_jdbc_hive_orc" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String hive_pg_port = context.config.otherConfigs.get("hive_pg_port") + + + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/postgresql-42.5.0.jar" + + sql """drop catalog if exists ${catalog_name}""" + sql """create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hive.metastore.type' = 'jdbc', + "user"="hive", + "password"="hive", + "jdbc_url" = "jdbc:postgresql://${externalEnvIp}:${hive_pg_port}/metastore?useSSL=false", + "driver_url" = "${driver_url}", + "driver_class" = "org.postgresql.Driver" + ); + """ + sql """use `${catalog_name}`.`default`""" + + select_top50() + count_all() + search_lg_int() + search_in_int() + search_mix() + only_partition_col() + decimals() + + sql """drop catalog if exists ${catalog_name}""" + } finally { + } + } +} +