database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](meta) show partitions with Limit for external HMS tables (27835) (#27835)

Browse Source 
This enhancement shall extend existing logic for SHOW PARTITIONS FROM to include: -

Limit/Offset
Where [partition name only] [equal operator and like operator]
Order by [partition name only]
Issue Number: close #27834
This commit is contained in:
Nitin-Kashyap
2023-12-08 23:14:45 +05:30
committed by GitHub
parent 99b38ddca7
commit 07336980f9
5 changed files with 126 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java
View File

@ -58,7 +58,7 @@ public class ShowPartitionsStmt extends ShowStmt {
private static final Logger LOG = LogManager.getLogger(ShowPartitionsStmt.class);
private static final String FILTER_PARTITION_ID = "PartitionId";
private static final String FILTER_PARTITION_NAME = "PartitionName";
public static final String FILTER_PARTITION_NAME = "PartitionName";
private static final String FILTER_STATE = "State";
private static final String FILTER_BUCKETS = "Buckets";
private static final String FILTER_REPLICATION_NUM = "ReplicationNum";
@ -198,6 +198,11 @@ public class ShowPartitionsStmt extends ShowStmt {
throw new AnalysisException("Should order by column");
}
SlotRef slotRef = (SlotRef) orderByElement.getExpr();
if (catalog instanceof HMSExternalCatalog
&& !slotRef.getColumnName().equalsIgnoreCase(FILTER_PARTITION_NAME)) {
throw new AnalysisException("External table only support Order By on PartitionName");
}
int index = PartitionsProcDir.analyzeColumn(slotRef.getColumnName());
OrderByPair orderByPair = new OrderByPair(index, !orderByElement.getIsAsc());
orderByPairs.add(orderByPair);
@ -228,6 +233,10 @@ public class ShowPartitionsStmt extends ShowStmt {
}
String leftKey = ((SlotRef) subExpr.getChild(0)).getColumnName();
if (catalog instanceof HMSExternalCatalog && !leftKey.equalsIgnoreCase(FILTER_PARTITION_NAME)) {
throw new AnalysisException(String.format("Only %s column supported in where clause for this catalog",
FILTER_PARTITION_NAME));
}
if (subExpr instanceof BinaryPredicate) {
BinaryPredicate binaryPredicate = (BinaryPredicate) subExpr;
if (leftKey.equalsIgnoreCase(FILTER_PARTITION_NAME) || leftKey.equalsIgnoreCase(FILTER_STATE)) {

5
fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
View File

@ -81,7 +81,8 @@ public class PartitionsProcDir implements ProcDirInterface {
this.isTempPartition = isTempPartition;
}
public boolean filter(String columnName, Comparable element, Map<String, Expr> filterMap) throws AnalysisException {
public static boolean filter(String columnName, Comparable element, Map<String, Expr> filterMap)
throws AnalysisException {
if (filterMap == null) {
return true;
}
@ -142,7 +143,7 @@ public class PartitionsProcDir implements ProcDirInterface {
return true;
}
public boolean like(String str, String expr) {
public static boolean like(String str, String expr) {
expr = expr.toLowerCase();
expr = expr.replace(".", "\\.");
expr = expr.replace("?", ".");

36
fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.analysis.AdminShowReplicaDistributionStmt;
import org.apache.doris.analysis.AdminShowReplicaStatusStmt;
import org.apache.doris.analysis.AdminShowTabletStorageFormatStmt;
import org.apache.doris.analysis.DescribeStmt;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.HelpStmt;
import org.apache.doris.analysis.LimitElement;
import org.apache.doris.analysis.PartitionNames;
@ -1655,28 +1656,53 @@ public class ShowExecutor {
resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
}
private void handleShowHMSTablePartitions(ShowPartitionsStmt showStmt) {
private void handleShowHMSTablePartitions(ShowPartitionsStmt showStmt) throws AnalysisException {
HMSExternalCatalog catalog = (HMSExternalCatalog) (showStmt.getCatalog());
List<List<String>> rows = new ArrayList<>();
String dbName = ClusterNamespace.getNameFromFullName(showStmt.getTableName().getDb());
List<String> partitionNames;
LimitElement limit = showStmt.getLimitElement();
if (limit != null && limit.hasLimit()) {
// only limit is valid on Hive
Map<String, Expr> filterMap = showStmt.getFilterMap();
List<OrderByPair> orderByPairs = showStmt.getOrderByPairs();
if (limit != null && limit.hasLimit() && limit.getOffset() == 0
&& (orderByPairs == null || !orderByPairs.get(0).isDesc())) {
// hmsClient returns unordered partition list, hence if offset > 0 cannot pass limit
partitionNames = catalog.getClient()
.listPartitionNames(dbName, showStmt.getTableName().getTbl(), limit.getLimit());
.listPartitionNames(dbName, showStmt.getTableName().getTbl(), limit.getLimit());
} else {
partitionNames = catalog.getClient().listPartitionNames(dbName, showStmt.getTableName().getTbl());
}
/* Filter add rows */
for (String partition : partitionNames) {
List<String> list = new ArrayList<>();
if (filterMap != null && !filterMap.isEmpty()) {
if (!PartitionsProcDir.filter(ShowPartitionsStmt.FILTER_PARTITION_NAME, partition, filterMap)) {
continue;
}
}
list.add(partition);
rows.add(list);
}
// sort by partition name
rows.sort(Comparator.comparing(x -> x.get(0)));
if (orderByPairs != null && orderByPairs.get(0).isDesc()) {
rows.sort(Comparator.comparing(x -> x.get(0), Comparator.reverseOrder()));
} else {
rows.sort(Comparator.comparing(x -> x.get(0)));
}
if (limit != null && limit.hasLimit()) {
int beginIndex = (int) limit.getOffset();
int endIndex = (int) (beginIndex + limit.getLimit());
if (endIndex > rows.size()) {
endIndex = rows.size();
}
rows = rows.subList(beginIndex, endIndex);
}
resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
}

49
regression-test/data/external_table_p0/hive/test_hive_partitions.out
View File

@ -71,3 +71,52 @@
55 1.22zxy
56 1.22ZXY
-- !q11 --
nation=cn/city=beijing
nation=cn/city=shanghai
nation=jp/city=tokyo
nation=rus/city=moscow
nation=us/city=chicago
nation=us/city=washington
-- !q12 --
nation=cn/city=beijing
-- !q13 --
nation=us/city=chicago
nation=us/city=washington
-- !q14 --
nation=rus/city=moscow
nation=us/city=chicago
nation=us/city=washington
-- !q16 --
nation=cn/city=beijing
nation=cn/city=shanghai
nation=jp/city=tokyo
-- !q17 --
nation=jp/city=tokyo
nation=rus/city=moscow
nation=us/city=chicago
-- !q18 --
nation=us/city=chicago
nation=us/city=washington
-- !q19 --
nation=rus/city=moscow
nation=jp/city=tokyo
nation=cn/city=shanghai
-- !q20 --
nation=cn/city=beijing
nation=cn/city=shanghai
nation=jp/city=tokyo
nation=rus/city=moscow
nation=us/city=chicago
nation=us/city=washington
-- !q21 --

33
regression-test/suites/external_table_p0/hive/test_hive_partitions.groovy
View File

@ -35,6 +35,39 @@ suite("test_hive_partitions", "p0,external,hive,external_docker,external_docker_
select id, data from table_with_pars where dt_par = '2023-02-01' and time_par = '2023-02-01 01:30:00'
and decimal_par1 = '1' and decimal_par2 = '1.2' and decimal_par3 = '1.22' order by id;
"""
qt_q11 """
show partitions from partition_table;
"""
qt_q12 """
show partitions from partition_table WHERE partitionName='nation=cn/city=beijing';
"""
qt_q13 """
show partitions from partition_table WHERE partitionName like 'nation=us/%';
"""
qt_q14 """
show partitions from partition_table WHERE partitionName like 'nation=%us%';
"""
qt_q16 """
show partitions from partition_table LIMIT 3;
"""
qt_q17 """
show partitions from partition_table LIMIT 3 OFFSET 2;
"""
qt_q18 """
show partitions from partition_table LIMIT 3 OFFSET 4;
"""
qt_q19 """
show partitions from partition_table ORDER BY partitionName desc LIMIT 3 OFFSET 2;
"""
qt_q20 """
show partitions from partition_table ORDER BY partitionName asc;
"""
qt_q21 """
show partitions from partition_table
WHERE partitionName like '%X%'
ORDER BY partitionName DESC
LIMIT 1;
"""
}
String enabled = context.config.otherConfigs.get("enableHiveTest")