diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java index c9faaf4217..3ba1db23d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java @@ -58,7 +58,7 @@ public class ShowPartitionsStmt extends ShowStmt { private static final Logger LOG = LogManager.getLogger(ShowPartitionsStmt.class); private static final String FILTER_PARTITION_ID = "PartitionId"; - private static final String FILTER_PARTITION_NAME = "PartitionName"; + public static final String FILTER_PARTITION_NAME = "PartitionName"; private static final String FILTER_STATE = "State"; private static final String FILTER_BUCKETS = "Buckets"; private static final String FILTER_REPLICATION_NUM = "ReplicationNum"; @@ -198,6 +198,11 @@ public class ShowPartitionsStmt extends ShowStmt { throw new AnalysisException("Should order by column"); } SlotRef slotRef = (SlotRef) orderByElement.getExpr(); + if (catalog instanceof HMSExternalCatalog + && !slotRef.getColumnName().equalsIgnoreCase(FILTER_PARTITION_NAME)) { + throw new AnalysisException("External table only support Order By on PartitionName"); + } + int index = PartitionsProcDir.analyzeColumn(slotRef.getColumnName()); OrderByPair orderByPair = new OrderByPair(index, !orderByElement.getIsAsc()); orderByPairs.add(orderByPair); @@ -228,6 +233,10 @@ public class ShowPartitionsStmt extends ShowStmt { } String leftKey = ((SlotRef) subExpr.getChild(0)).getColumnName(); + if (catalog instanceof HMSExternalCatalog && !leftKey.equalsIgnoreCase(FILTER_PARTITION_NAME)) { + throw new AnalysisException(String.format("Only %s column supported in where clause for this catalog", + FILTER_PARTITION_NAME)); + } if (subExpr instanceof BinaryPredicate) { BinaryPredicate binaryPredicate = (BinaryPredicate) subExpr; if (leftKey.equalsIgnoreCase(FILTER_PARTITION_NAME) || leftKey.equalsIgnoreCase(FILTER_STATE)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java index 11dd8d4c18..9319c96e04 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java @@ -81,7 +81,8 @@ public class PartitionsProcDir implements ProcDirInterface { this.isTempPartition = isTempPartition; } - public boolean filter(String columnName, Comparable element, Map filterMap) throws AnalysisException { + public static boolean filter(String columnName, Comparable element, Map filterMap) + throws AnalysisException { if (filterMap == null) { return true; } @@ -142,7 +143,7 @@ public class PartitionsProcDir implements ProcDirInterface { return true; } - public boolean like(String str, String expr) { + public static boolean like(String str, String expr) { expr = expr.toLowerCase(); expr = expr.replace(".", "\\."); expr = expr.replace("?", "."); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index ffc8e2652f..df3ddc3151 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -24,6 +24,7 @@ import org.apache.doris.analysis.AdminShowReplicaDistributionStmt; import org.apache.doris.analysis.AdminShowReplicaStatusStmt; import org.apache.doris.analysis.AdminShowTabletStorageFormatStmt; import org.apache.doris.analysis.DescribeStmt; +import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.HelpStmt; import org.apache.doris.analysis.LimitElement; import org.apache.doris.analysis.PartitionNames; @@ -1655,28 +1656,53 @@ public class ShowExecutor { resultSet = new ShowResultSet(showStmt.getMetaData(), rows); } - private void handleShowHMSTablePartitions(ShowPartitionsStmt showStmt) { + private void handleShowHMSTablePartitions(ShowPartitionsStmt showStmt) throws AnalysisException { HMSExternalCatalog catalog = (HMSExternalCatalog) (showStmt.getCatalog()); List> rows = new ArrayList<>(); String dbName = ClusterNamespace.getNameFromFullName(showStmt.getTableName().getDb()); List partitionNames; LimitElement limit = showStmt.getLimitElement(); - if (limit != null && limit.hasLimit()) { - // only limit is valid on Hive + Map filterMap = showStmt.getFilterMap(); + List orderByPairs = showStmt.getOrderByPairs(); + + if (limit != null && limit.hasLimit() && limit.getOffset() == 0 + && (orderByPairs == null || !orderByPairs.get(0).isDesc())) { + // hmsClient returns unordered partition list, hence if offset > 0 cannot pass limit partitionNames = catalog.getClient() - .listPartitionNames(dbName, showStmt.getTableName().getTbl(), limit.getLimit()); + .listPartitionNames(dbName, showStmt.getTableName().getTbl(), limit.getLimit()); } else { partitionNames = catalog.getClient().listPartitionNames(dbName, showStmt.getTableName().getTbl()); } + + /* Filter add rows */ for (String partition : partitionNames) { List list = new ArrayList<>(); + + if (filterMap != null && !filterMap.isEmpty()) { + if (!PartitionsProcDir.filter(ShowPartitionsStmt.FILTER_PARTITION_NAME, partition, filterMap)) { + continue; + } + } list.add(partition); rows.add(list); } // sort by partition name - rows.sort(Comparator.comparing(x -> x.get(0))); + if (orderByPairs != null && orderByPairs.get(0).isDesc()) { + rows.sort(Comparator.comparing(x -> x.get(0), Comparator.reverseOrder())); + } else { + rows.sort(Comparator.comparing(x -> x.get(0))); + } + + if (limit != null && limit.hasLimit()) { + int beginIndex = (int) limit.getOffset(); + int endIndex = (int) (beginIndex + limit.getLimit()); + if (endIndex > rows.size()) { + endIndex = rows.size(); + } + rows = rows.subList(beginIndex, endIndex); + } resultSet = new ShowResultSet(showStmt.getMetaData(), rows); } diff --git a/regression-test/data/external_table_p0/hive/test_hive_partitions.out b/regression-test/data/external_table_p0/hive/test_hive_partitions.out index f5f44b2368..646cc560fb 100644 --- a/regression-test/data/external_table_p0/hive/test_hive_partitions.out +++ b/regression-test/data/external_table_p0/hive/test_hive_partitions.out @@ -71,3 +71,52 @@ 55 1.22zxy 56 1.22ZXY +-- !q11 -- +nation=cn/city=beijing +nation=cn/city=shanghai +nation=jp/city=tokyo +nation=rus/city=moscow +nation=us/city=chicago +nation=us/city=washington + +-- !q12 -- +nation=cn/city=beijing + +-- !q13 -- +nation=us/city=chicago +nation=us/city=washington + +-- !q14 -- +nation=rus/city=moscow +nation=us/city=chicago +nation=us/city=washington + +-- !q16 -- +nation=cn/city=beijing +nation=cn/city=shanghai +nation=jp/city=tokyo + +-- !q17 -- +nation=jp/city=tokyo +nation=rus/city=moscow +nation=us/city=chicago + +-- !q18 -- +nation=us/city=chicago +nation=us/city=washington + +-- !q19 -- +nation=rus/city=moscow +nation=jp/city=tokyo +nation=cn/city=shanghai + +-- !q20 -- +nation=cn/city=beijing +nation=cn/city=shanghai +nation=jp/city=tokyo +nation=rus/city=moscow +nation=us/city=chicago +nation=us/city=washington + +-- !q21 -- + diff --git a/regression-test/suites/external_table_p0/hive/test_hive_partitions.groovy b/regression-test/suites/external_table_p0/hive/test_hive_partitions.groovy index 8eae4e82e6..09bcc1b860 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_partitions.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_partitions.groovy @@ -35,6 +35,39 @@ suite("test_hive_partitions", "p0,external,hive,external_docker,external_docker_ select id, data from table_with_pars where dt_par = '2023-02-01' and time_par = '2023-02-01 01:30:00' and decimal_par1 = '1' and decimal_par2 = '1.2' and decimal_par3 = '1.22' order by id; """ + qt_q11 """ + show partitions from partition_table; + """ + qt_q12 """ + show partitions from partition_table WHERE partitionName='nation=cn/city=beijing'; + """ + qt_q13 """ + show partitions from partition_table WHERE partitionName like 'nation=us/%'; + """ + qt_q14 """ + show partitions from partition_table WHERE partitionName like 'nation=%us%'; + """ + qt_q16 """ + show partitions from partition_table LIMIT 3; + """ + qt_q17 """ + show partitions from partition_table LIMIT 3 OFFSET 2; + """ + qt_q18 """ + show partitions from partition_table LIMIT 3 OFFSET 4; + """ + qt_q19 """ + show partitions from partition_table ORDER BY partitionName desc LIMIT 3 OFFSET 2; + """ + qt_q20 """ + show partitions from partition_table ORDER BY partitionName asc; + """ + qt_q21 """ + show partitions from partition_table + WHERE partitionName like '%X%' + ORDER BY partitionName DESC + LIMIT 1; + """ } String enabled = context.config.otherConfigs.get("enableHiveTest")