From c016eb49c56acb22498e890dc4a3af0abba7ee89 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Fri, 10 Jan 2025 10:46:09 +0800 Subject: [PATCH] =?UTF-8?q?[enhance](mtmv)When=20obtaining=20the=20partiti?= =?UTF-8?q?on=20list=20fails,=20treat=20the=20pai=E2=80=A6=20(#46708)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …mon table as an unpartitioned table (#46641) pick: https://github.com/apache/doris/pull/46641 --- .../paimon/run01.sql | 47 ++++++++++++++++++- .../paimon/PaimonExternalTable.java | 8 ++++ .../paimon/PaimonPartitionInfo.java | 5 ++ .../doris/datasource/paimon/PaimonUtil.java | 11 ++++- 4 files changed, 68 insertions(+), 3 deletions(-) diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql index 7aa4170eab..7722d09463 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql @@ -22,4 +22,49 @@ insert into test_tb_mix_format values (1,1,'b'),(2,1,'b'),(3,1,'b'),(4,1,'b'),(5 -- update some data, these splits will be readed by jni insert into test_tb_mix_format values (1,2,'b'),(2,2,'b'),(3,2,'b'),(4,2,'b'),(5,2,'b'); -- delete foramt in table properties, doris should get format by file name -alter table test_tb_mix_format unset TBLPROPERTIES ('file.format'); \ No newline at end of file +alter table test_tb_mix_format unset TBLPROPERTIES ('file.format'); + +drop table if exists two_partition; +CREATE TABLE two_partition ( + id BIGINT, + create_date STRING, + region STRING +) PARTITIONED BY (create_date,region) TBLPROPERTIES ( + 'primary-key' = 'create_date,region,id', + 'bucket'=10, + 'file.format'='orc' +); + +insert into two_partition values(1,'2020-01-01','bj'); +insert into two_partition values(2,'2020-01-01','sh'); +insert into two_partition values(3,'2038-01-01','bj'); +insert into two_partition values(4,'2038-01-01','sh'); +insert into two_partition values(5,'2038-01-02','bj'); + +drop table if exists null_partition; +CREATE TABLE null_partition ( + id BIGINT, + region STRING +) PARTITIONED BY (region) TBLPROPERTIES ( + 'primary-key' = 'region,id', + 'bucket'=10, + 'file.format'='orc' +); +-- null NULL "null" all will be in partition [null] +insert into null_partition values(1,'bj'); +insert into null_partition values(2,null); +insert into null_partition values(3,NULL); +insert into null_partition values(4,'null'); +insert into null_partition values(5,'NULL'); + +drop table if exists date_partition; +CREATE TABLE date_partition ( + id BIGINT, + create_date DATE +) PARTITIONED BY (create_date) TBLPROPERTIES ( + 'primary-key' = 'create_date,id', + 'bucket'=10, + 'file.format'='orc' +); + +insert into date_partition values(1,date '2020-01-01'); \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 3f22ce4c46..efd8e6bb7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -144,9 +144,17 @@ public class PaimonExternalTable extends ExternalTable implements MvccTable { @Override public List getPartitionColumns(Optional snapshot) { + if (isPartitionInvalid(snapshot)) { + return Collections.emptyList(); + } return getPaimonSchemaCacheValue(snapshot).getPartitionColumns(); } + private boolean isPartitionInvalid(Optional snapshot) { + PaimonSnapshotCacheValue paimonSnapshotCacheValue = getOrFetchSnapshotCacheValue(snapshot); + return paimonSnapshotCacheValue.getPartitionInfo().isPartitionInvalid(); + } + @Override public MvccSnapshot loadSnapshot() { return new PaimonMvccSnapshot(getPaimonSnapshotCacheValue()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java index 4d3326f8e4..88515a2510 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java @@ -45,4 +45,9 @@ public class PaimonPartitionInfo { public Map getNameToPartition() { return nameToPartition; } + + public boolean isPartitionInvalid() { + // when transfer to partitionItem failed, will not equal + return nameToPartitionItem.size() != nameToPartition.size(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 1f7576dca5..b3df41bc5c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -117,7 +117,7 @@ public class PaimonUtil { } public static PaimonPartitionInfo generatePartitionInfo(List partitionColumns, - List paimonPartitions) throws AnalysisException { + List paimonPartitions) { Map nameToPartitionItem = Maps.newHashMap(); Map nameToPartition = Maps.newHashMap(); PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo(nameToPartitionItem, nameToPartition); @@ -127,7 +127,14 @@ public class PaimonUtil { for (PaimonPartition paimonPartition : paimonPartitions) { String partitionName = getPartitionName(partitionColumns, paimonPartition.getPartitionValues()); nameToPartition.put(partitionName, paimonPartition); - nameToPartitionItem.put(partitionName, toListPartitionItem(partitionName, partitionColumns)); + try { + // partition values return by paimon api, may have problem, + // to avoid affecting the query, we catch exceptions here + nameToPartitionItem.put(partitionName, toListPartitionItem(partitionName, partitionColumns)); + } catch (Exception e) { + LOG.warn("toListPartitionItem failed, partitionColumns: {}, partitionValues: {}", partitionColumns, + paimonPartition.getPartitionValues(), e); + } } return partitionInfo; }