database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](nereids)support null partition for list partition (#31613)

Browse Source
This commit is contained in:
starocean999
2024-03-04 16:31:34 +08:00
committed by yiguolei
parent 33c356e6d8
commit 3777ffb43f
13 changed files with 191 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java
View File

@ -208,9 +208,6 @@ public class PartitionDesc {
throw new AnalysisException(
"The partition column must be NOT NULL with allow_partition_column_nullable OFF");
}
if (this instanceof ListPartitionDesc && columnDef.isAllowNull()) {
throw new AnalysisException("The list partition column must be NOT NULL");
}
if (this instanceof RangePartitionDesc && partitionExprs != null) {
if (partitionExprs.get(0) instanceof FunctionCallExpr) {
if (!columnDef.getType().isDateType()) {

15
fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionValue.java
View File

@ -27,6 +27,7 @@ public class PartitionValue {
private String value;
private boolean isHiveDefaultPartition;
private boolean isNullPartition;
private PartitionValue() {
@ -45,6 +46,12 @@ public class PartitionValue {
this.isHiveDefaultPartition = isHiveDefaultPartition;
}
public PartitionValue(String value, boolean isNullPartition, boolean isHiveDefaultPartition) {
this.value = value;
this.isNullPartition = isNullPartition;
this.isHiveDefaultPartition = isHiveDefaultPartition;
}
public LiteralExpr getValue(Type type) throws AnalysisException {
if (isHiveDefaultPartition) {
return new StringLiteral(value);
@ -81,12 +88,16 @@ public class PartitionValue {
return false;
}
PartitionValue that = (PartitionValue) o;
return isHiveDefaultPartition == that.isHiveDefaultPartition
return isHiveDefaultPartition == that.isHiveDefaultPartition && isNullPartition == that.isNullPartition
&& Objects.equal(value, that.value);
}
@Override
public int hashCode() {
return Objects.hashCode(value, isHiveDefaultPartition);
return Objects.hashCode(value, isHiveDefaultPartition, isNullPartition);
}
public boolean isNullPartition() {
return isNullPartition;
}
}

119
fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.analysis.LargeIntLiteral;
import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.MaxLiteral;
import org.apache.doris.analysis.NullLiteral;
import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.common.AnalysisException;
@ -137,7 +138,12 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable {
PartitionKey partitionKey = new PartitionKey();
for (int i = 0; i < values.size(); i++) {
partitionKey.keys.add(values.get(i).getValue(types.get(i)));
if (values.get(i).isNullPartition()) {
partitionKey.keys.add(NullLiteral.create(types.get(i)));
} else {
partitionKey.keys.add(values.get(i).getValue(types.get(i)));
}
if (isHive) {
partitionKey.originHiveKeys.add(values.get(i).getStringValue());
}
@ -267,7 +273,7 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable {
int i = 0;
for (LiteralExpr expr : keys) {
Object value = null;
if (expr == MaxLiteral.MAX_VALUE) {
if (expr == MaxLiteral.MAX_VALUE || expr.isNullLiteral()) {
value = expr.toSql();
sb.append(value);
continue;
@ -312,7 +318,7 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable {
int i = 0;
for (LiteralExpr expr : keys) {
Object value = null;
if (expr == MaxLiteral.MAX_VALUE) {
if (expr == MaxLiteral.MAX_VALUE || expr.isNullLiteral()) {
value = expr.toSql();
} else {
value = expr.getRealValue();
@ -341,9 +347,17 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable {
out.writeInt(count);
for (int i = 0; i < count; i++) {
PrimitiveType type = types.get(i);
Text.writeString(out, type.toString());
if (keys.get(i).isNullLiteral()) {
Text.writeString(out, Type.NULL.toString());
} else {
Text.writeString(out, type.toString());
}
if (keys.get(i) == MaxLiteral.MAX_VALUE) {
out.writeBoolean(true);
} else if (keys.get(i).isNullLiteral()) {
out.writeBoolean(false);
Text.writeString(out, type.toString());
} else {
out.writeBoolean(false);
keys.get(i).write(out);
@ -355,10 +369,16 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable {
int count = in.readInt();
for (int i = 0; i < count; i++) {
PrimitiveType type = PrimitiveType.valueOf(Text.readString(in));
types.add(type);
LiteralExpr literal = null;
boolean isMax = in.readBoolean();
if (type == PrimitiveType.NULL_TYPE) {
String realType = StringLiteral.read(in).getStringValue();
type = PrimitiveType.valueOf(realType);
types.add(type);
keys.add(NullLiteral.create(Type.fromPrimitiveType(type)));
continue;
}
LiteralExpr literal = null;
types.add(type);
if (isMax) {
literal = MaxLiteral.MAX_VALUE;
} else {
@ -480,47 +500,54 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable {
for (int i = 0; i < count; i++) {
JsonArray typeAndKey = new JsonArray();
PrimitiveType type = types.get(i);
typeAndKey.add(new JsonPrimitive(type.toString()));
if (keys.get(i) == MaxLiteral.MAX_VALUE) {
typeAndKey.add(new JsonPrimitive("MAX_VALUE"));
if (keys.get(i).isNullLiteral()) {
// save NULL_TYPE as type and real type as key
typeAndKey.add(new JsonPrimitive(PrimitiveType.NULL_TYPE.toString()));
typeAndKey.add(new JsonPrimitive(type.toString()));
} else {
switch (type) {
case TINYINT:
case SMALLINT:
case INT:
case BIGINT: {
IntLiteral key = (IntLiteral) keys.get(i);
typeAndKey.add(new JsonPrimitive(key.getLongValue()));
typeAndKey.add(new JsonPrimitive(type.toString()));
if (keys.get(i) == MaxLiteral.MAX_VALUE) {
typeAndKey.add(new JsonPrimitive("MAX_VALUE"));
} else {
switch (type) {
case TINYINT:
case SMALLINT:
case INT:
case BIGINT: {
IntLiteral key = (IntLiteral) keys.get(i);
typeAndKey.add(new JsonPrimitive(key.getLongValue()));
}
break;
case LARGEINT: {
LargeIntLiteral key = (LargeIntLiteral) keys.get(i);
typeAndKey.add(new JsonPrimitive(key.getRealValue().toString()));
}
break;
case DATE:
case DATETIME:
case DATEV2:
case DATETIMEV2: {
DateLiteral key = (DateLiteral) keys.get(i);
typeAndKey.add(new JsonPrimitive(key.convertToString(type)));
}
break;
case CHAR:
case VARCHAR:
case STRING: {
StringLiteral key = (StringLiteral) keys.get(i);
typeAndKey.add(new JsonPrimitive(key.getValue()));
}
break;
case BOOLEAN: {
BoolLiteral key = (BoolLiteral) keys.get(i);
typeAndKey.add(new JsonPrimitive(key.getValue()));
}
break;
default:
throw new JsonParseException(
"type[" + type.name() + "] not supported: ");
}
break;
case LARGEINT: {
LargeIntLiteral key = (LargeIntLiteral) keys.get(i);
typeAndKey.add(new JsonPrimitive(key.getRealValue().toString()));
}
break;
case DATE:
case DATETIME:
case DATEV2:
case DATETIMEV2: {
DateLiteral key = (DateLiteral) keys.get(i);
typeAndKey.add(new JsonPrimitive(key.convertToString(type)));
}
break;
case CHAR:
case VARCHAR:
case STRING: {
StringLiteral key = (StringLiteral) keys.get(i);
typeAndKey.add(new JsonPrimitive(key.getValue()));
}
break;
case BOOLEAN: {
BoolLiteral key = (BoolLiteral) keys.get(i);
typeAndKey.add(new JsonPrimitive(key.getValue()));
}
break;
default:
throw new JsonParseException("type[" + type.name() + "] not supported: ");
}
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -2669,6 +2669,8 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
return Literal.of(toStringValue(ctx.STRING_LITERAL().getText()));
} else if (ctx.MAXVALUE() != null) {
return MaxValue.INSTANCE;
} else if (ctx.NULL() != null) {
return Literal.of(null);
}
throw new AnalysisException("Unsupported partition value: " + ctx.getText());
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
View File

@ -291,6 +291,8 @@ public abstract class Literal extends Expression implements LeafExpression, Comp
DataType dataType = DataType.fromCatalogType(type);
if (literalExpr instanceof org.apache.doris.analysis.MaxLiteral) {
return new MaxLiteral(dataType);
} else if (literalExpr instanceof org.apache.doris.analysis.NullLiteral) {
return new NullLiteral(dataType);
}
String stringValue = literalExpr.getStringValue();
if (dataType.isBooleanType()) {

3
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
View File

@ -667,9 +667,6 @@ public class CreateTableInfo {
throw new AnalysisException(
"The partition column must be NOT NULL with allow_partition_column_nullable OFF");
}
if (partitionType.equalsIgnoreCase(PartitionType.LIST.name()) && column.isNullable()) {
throw new AnalysisException("The list partition column must be NOT NULL");
}
}
// if auto bucket auto bucket enable, rewrite distribution bucket num &&

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java
View File

@ -161,7 +161,7 @@ public abstract class PartitionDefinition {
*/
protected PartitionValue toLegacyPartitionValueStmt(Expression e) {
if (e.isLiteral()) {
return new PartitionValue(((Literal) e).getStringValue());
return new PartitionValue(((Literal) e).getStringValue(), e.isNullLiteral(), false);
} else if (e instanceof MaxValue) {
return PartitionValue.MAX_VALUE;
}

9
fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
View File

@ -20,6 +20,8 @@ package org.apache.doris.planner;
import org.apache.doris.alter.SchemaChangeHandler;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.NullLiteral;
import org.apache.doris.analysis.SlotDescriptor;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Column;
@ -473,7 +475,12 @@ public class OlapTableSink extends DataSink {
for (PartitionKey partitionKey : partitionKeys) {
List<TExprNode> tExprNodes = new ArrayList<>();
for (int i = 0; i < partColNum; i++) {
tExprNodes.add(partitionKey.getKeys().get(i).treeToThrift().getNodes().get(0));
LiteralExpr literalExpr = partitionKey.getKeys().get(i);
if (literalExpr.isNullLiteral()) {
tExprNodes.add(NullLiteral.create(literalExpr.getType()).treeToThrift().getNodes().get(0));
} else {
tExprNodes.add(literalExpr.treeToThrift().getNodes().get(0));
}
}
tPartition.addToInKeys(tExprNodes);
tPartition.setIsDefaultPartition(partitionItem.isDefaultPartition());