database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improve](partition) support auto list partition with more columns (#27817)

Browse Source 
before the partition by column only have one column.
now remove those limit, could have more columns.
This commit is contained in:
zhangstar333
2023-12-04 11:33:18 +08:00
committed by GitHub
parent 80f528bf26
commit e62d19d90d
10 changed files with 228 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java
View File

@ -128,13 +128,12 @@ public class PartitionDesc {
+ expr.toSql());
}
} else if (expr instanceof SlotRef) {
if (colNames.isEmpty()) {
colNames.add(((SlotRef) expr).getColumnName());
} else {
if (!colNames.isEmpty() && !isListPartition) {
throw new AnalysisException(
"auto create partition only support one slotRef in expr. "
"auto create partition only support one slotRef in expr of RANGE partition. "
+ expr.toSql());
}
colNames.add(((SlotRef) expr).getColumnName());
} else {
if (!isListPartition) {
throw new AnalysisException(

34
fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java
View File

@ -34,6 +34,7 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
public class PartitionExprUtil {
@ -114,7 +115,7 @@ public class PartitionExprUtil {
}
public static Map<String, AddPartitionClause> getAddPartitionClauseFromPartitionValues(OlapTable olapTable,
ArrayList<TStringLiteral> partitionValues, PartitionInfo partitionInfo)
ArrayList<List<TStringLiteral>> partitionValues, PartitionInfo partitionInfo)
throws AnalysisException {
Map<String, AddPartitionClause> result = Maps.newHashMap();
ArrayList<Expr> partitionExprs = partitionInfo.getPartitionExprs();
@ -124,16 +125,22 @@ public class PartitionExprUtil {
FunctionIntervalInfo intervalInfo = getFunctionIntervalInfo(partitionExprs, partitionType);
Set<String> filterPartitionValues = new HashSet<String>();
for (TStringLiteral partitionValue : partitionValues) {
for (List<TStringLiteral> partitionValueList : partitionValues) {
PartitionKeyDesc partitionKeyDesc = null;
String partitionName = "p";
String value = partitionValue.value;
if (filterPartitionValues.contains(value)) {
ArrayList<String> curPartitionValues = new ArrayList<>();
for (TStringLiteral tStringLiteral : partitionValueList) {
curPartitionValues.add(tStringLiteral.value);
}
String filterStr = curPartitionValues.stream()
.map(s -> s + s.length()) // Concatenate each string with its length
.reduce("", (s1, s2) -> s1 + s2);
if (filterPartitionValues.contains(filterStr)) {
continue;
}
filterPartitionValues.add(value);
filterPartitionValues.add(filterStr);
if (partitionType == PartitionType.RANGE) {
String beginTime = value;
String beginTime = curPartitionValues.get(0); // have check range type size must be 1
DateLiteral beginDateTime = new DateLiteral(beginTime, partitionColumnType);
partitionName += String.format(DATETIME_NAME_FORMATTER,
beginDateTime.getYear(), beginDateTime.getMonth(), beginDateTime.getDay(),
@ -142,14 +149,19 @@ public class PartitionExprUtil {
partitionKeyDesc = createPartitionKeyDescWithRange(beginDateTime, endDateTime, partitionColumnType);
} else if (partitionType == PartitionType.LIST) {
List<List<PartitionValue>> listValues = new ArrayList<>();
String pointValue = value;
PartitionValue lowerValue = new PartitionValue(pointValue);
listValues.add(Collections.singletonList(lowerValue));
List<PartitionValue> inValues = new ArrayList<>();
for (String value : curPartitionValues) {
inValues.add(new PartitionValue(value));
}
listValues.add(inValues);
partitionKeyDesc = PartitionKeyDesc.createIn(
listValues);
partitionName += getFormatPartitionValue(lowerValue.getStringValue());
partitionName += getFormatPartitionValue(filterStr);
if (partitionColumnType.isStringType()) {
partitionName += "_" + System.currentTimeMillis();
if (partitionName.length() > 50) {
partitionName = partitionName.substring(40) + Objects.hash(partitionName)
+ "_" + System.currentTimeMillis();
}
}
} else {
throw new AnalysisException("now only support range and list partition");

10
fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
View File

@ -42,6 +42,7 @@ import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
@ -3152,15 +3153,16 @@ public class FrontendServiceImpl implements FrontendService.Iface {
OlapTable olapTable = (OlapTable) table;
PartitionInfo partitionInfo = olapTable.getPartitionInfo();
ArrayList<TStringLiteral> partitionValues = new ArrayList<TStringLiteral>();
ArrayList<List<TStringLiteral>> partitionValues = new ArrayList<>();
for (int i = 0; i < request.partitionValues.size(); i++) {
if (request.partitionValues.get(i).size() != 1) {
if (partitionInfo.getType() == PartitionType.RANGE && request.partitionValues.get(i).size() != 1) {
errorStatus.setErrorMsgs(
Lists.newArrayList("Only support single partition, partitionValues size should equal 1."));
Lists.newArrayList(
"Only support single partition of RANGE, partitionValues size should equal 1."));
result.setStatus(errorStatus);
return result;
}
partitionValues.add(request.partitionValues.get(i).get(0));
partitionValues.add(request.partitionValues.get(i));
}
Map<String, AddPartitionClause> addPartitionClauseMap;
try {