diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java index d883b9dc78..bd0d2315c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java @@ -48,6 +48,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import io.airlift.concurrent.MoreFutures; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; @@ -88,6 +89,7 @@ public class HMSTransaction implements Transaction { private final Map> tableActions = new HashMap<>(); private final Map, Action>> partitionActions = new HashMap<>(); + private final Map> tableColumns = new HashMap<>(); private final Executor fileSystemExecutor; private HmsCommitter hmsCommitter; @@ -123,7 +125,7 @@ public class HMSTransaction implements Transaction { } } - private Set uncompletedMpuPendingUploads = new HashSet<>(); + private final Set uncompletedMpuPendingUploads = new HashSet<>(); public HMSTransaction(HiveMetadataOps hiveOps, FileSystemProvider fileSystemProvider, Executor fileSystemExecutor) { this.hiveOps = hiveOps; @@ -241,7 +243,7 @@ public class HMSTransaction implements Transaction { Maps.newHashMap(), sd.getOutputFormat(), sd.getSerdeInfo().getSerializationLib(), - hiveOps.getClient().getSchema(dbName, tbName) + getTableColumns(dbName, tbName) ); if (updateMode == TUpdateMode.OVERWRITE) { dropPartition(dbName, tbName, hivePartition.getPartitionValues(), true); @@ -396,7 +398,7 @@ public class HMSTransaction implements Transaction { partition.getParameters(), sd.getOutputFormat(), sd.getSerdeInfo().getSerializationLib(), - hiveOps.getClient().getSchema(dbName, tbName) + getTableColumns(dbName, tbName) ); partitionActionsForTable.put( @@ -913,6 +915,11 @@ public class HMSTransaction implements Transaction { throw new RuntimeException("Not Found table: " + databaseName + "." + tableName); } + public synchronized List getTableColumns(String databaseName, String tableName) { + return tableColumns.computeIfAbsent(new DatabaseTableName(databaseName, tableName), + key -> hiveOps.getClient().getSchema(dbName, tbName)); + } + public synchronized void finishChangingExistingTable( ActionType actionType, String databaseName, @@ -1276,7 +1283,7 @@ public class HMSTransaction implements Transaction { Maps.newHashMap(), sd.getOutputFormat(), sd.getSerdeInfo().getSerializationLib(), - hiveOps.getClient().getSchema(dbName, tbName) + getTableColumns(dbName, tbName) ); HivePartitionWithStatistics partitionWithStats =