database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](nereids)support partition property in nereids (#28982)

Browse Source
This commit is contained in:
starocean999
2023-12-26 11:19:04 +08:00
committed by GitHub
parent 92660bb1b2
commit 065eb9a72b
27 changed files with 444 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
View File

@ -502,7 +502,7 @@ partitionsDef
;
partitionDef
: (lessThanPartitionDef | fixedPartitionDef | stepPartitionDef | inPartitionDef) properties=propertyClause?
: (lessThanPartitionDef | fixedPartitionDef | stepPartitionDef | inPartitionDef) (LEFT_PAREN partitionProperties=propertyItemList RIGHT_PAREN)?
;
lessThanPartitionDef

26
fe/fe-core/src/main/cup/sql_parser.cup
View File

@ -1800,19 +1800,6 @@ create_stmt ::=
RESULT = new CreateTableStmt(ifNotExists, isExternal, name, columns, engineName, keys, partition,
distribution, tblProperties, extProperties, tableComment, index);
:}
| KW_CREATE opt_external:isExternal KW_TABLE opt_if_not_exists:ifNotExists table_name:name
LPAREN column_definition_list:columns COMMA RPAREN opt_engine:engineName
opt_keys:keys
opt_comment:tableComment
opt_partition:partition
opt_distribution:distribution
opt_rollup:index
opt_properties:tblProperties
opt_ext_properties:extProperties
{:
RESULT = new CreateTableStmt(ifNotExists, isExternal, name, columns, null, engineName, keys, partition,
distribution, tblProperties, extProperties, tableComment, index);
:}
| KW_CREATE opt_external:isExternal KW_TABLE opt_if_not_exists:ifNotExists table_name:name
LPAREN column_definition_list:columns COMMA index_definition_list:indexes RPAREN opt_engine:engineName
opt_keys:keys
@ -1826,19 +1813,6 @@ create_stmt ::=
RESULT = new CreateTableStmt(ifNotExists, isExternal, name, columns, indexes, engineName, keys, partition,
distribution, tblProperties, extProperties, tableComment, index);
:}
| KW_CREATE opt_external:isExternal KW_TABLE opt_if_not_exists:ifNotExists table_name:name
LPAREN column_definition_list:columns COMMA index_definition_list:indexes COMMA RPAREN opt_engine:engineName
opt_keys:keys
opt_comment:tableComment
opt_partition:partition
opt_distribution:distribution
opt_rollup:index
opt_properties:tblProperties
opt_ext_properties:extProperties
{:
RESULT = new CreateTableStmt(ifNotExists, isExternal, name, columns, indexes, engineName, keys, partition,
distribution, tblProperties, extProperties, tableComment, index);
:}
| KW_CREATE opt_external:isExternal KW_TABLE opt_if_not_exists:ifNotExists table_name:name
opt_col_list:columns
opt_engine:engineName

19
fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java
View File

@ -68,19 +68,22 @@ public class SinglePartitionDesc implements AllPartitionDesc {
/**
* for Nereids
*/
public SinglePartitionDesc(boolean ifNotExists, String partName, PartitionKeyDesc partitionKeyDesc,
ReplicaAllocation replicaAlloc, Map<String, String> properties) {
this.ifNotExists = ifNotExists;
public SinglePartitionDesc(boolean ifNotExists, String partName,
PartitionKeyDesc partitionKeyDesc, ReplicaAllocation replicaAlloc,
Map<String, String> properties, DataProperty partitionDataProperty, boolean isInMemory,
TTabletType tabletType, Long versionInfo, String storagePolicy, boolean isMutable) {
this.isAnalyzed = true;
this.ifNotExists = ifNotExists;
this.partName = partName;
this.partitionKeyDesc = partitionKeyDesc;
this.properties = properties;
this.partitionDataProperty = new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM);
this.partitionDataProperty = partitionDataProperty;
this.replicaAlloc = replicaAlloc;
this.storagePolicy = "";
this.isInMemory = isInMemory;
this.tabletType = tabletType;
this.versionInfo = versionInfo;
this.storagePolicy = storagePolicy;
this.isMutable = isMutable;
}
public boolean isSetIfNotExists() {

27
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -2430,7 +2430,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
String indexName = ctx.indexName.getText();
List<String> indexCols = visitIdentifierList(ctx.cols);
Map<String, String> properties = visitPropertyItemList(ctx.properties);
String indexType = ctx.indexType.getText();
String indexType = ctx.indexType != null ? ctx.indexType.getText() : null;
String comment = ctx.comment != null ? ctx.comment.getText() : "";
return new IndexDefinition(indexName, indexCols, indexType, properties, comment);
}
@ -2438,8 +2438,16 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
@Override
public List<PartitionDefinition> visitPartitionsDef(PartitionsDefContext ctx) {
return ctx.partitions.stream()
.map(p -> ((PartitionDefinition) visit(p)).withProperties(visitPropertyClause(p.properties)))
.collect(Collectors.toList());
.map(p -> ((PartitionDefinition) visit(p))).collect(Collectors.toList());
}
@Override
public PartitionDefinition visitPartitionDef(DorisParser.PartitionDefContext ctx) {
PartitionDefinition partitionDefinition = (PartitionDefinition) visit(ctx.getChild(0));
if (ctx.partitionProperties != null) {
partitionDefinition.withProperties(visitPropertyItemList(ctx.partitionProperties));
}
return partitionDefinition;
}
@Override
@ -2447,9 +2455,10 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
String partitionName = ctx.partitionName.getText();
if (ctx.MAXVALUE() == null) {
List<Expression> lessThanValues = visitConstantSeq(ctx.constantSeq());
return new LessThanPartition(partitionName, lessThanValues);
return new LessThanPartition(ctx.EXISTS() != null, partitionName, lessThanValues);
} else {
return new LessThanPartition(partitionName, ImmutableList.of(MaxValue.INSTANCE));
return new LessThanPartition(ctx.EXISTS() != null, partitionName,
ImmutableList.of(MaxValue.INSTANCE));
}
}
@ -2458,15 +2467,15 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
String partitionName = ctx.partitionName.getText();
List<Expression> lowerBounds = visitConstantSeq(ctx.lower);
List<Expression> upperBounds = visitConstantSeq(ctx.upper);
return new FixedRangePartition(partitionName, lowerBounds, upperBounds);
return new FixedRangePartition(ctx.EXISTS() != null, partitionName, lowerBounds, upperBounds);
}
@Override
public PartitionDefinition visitStepPartitionDef(StepPartitionDefContext ctx) {
List<Expression> fromExpression = visitConstantSeq(ctx.from);
List<Expression> toExpression = visitConstantSeq(ctx.to);
return new StepPartition(fromExpression, toExpression, Long.parseLong(ctx.unitsAmount.getText()),
ctx.unit != null ? ctx.unit.getText() : null);
return new StepPartition(false, null, fromExpression, toExpression,
Long.parseLong(ctx.unitsAmount.getText()), ctx.unit != null ? ctx.unit.getText() : null);
}
@Override
@ -2479,7 +2488,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
values = visitConstantSeq(ctx.constants).stream().map(ImmutableList::of)
.collect(Collectors.toList());
}
return new InPartition(ctx.partitionName.getText(), values);
return new InPartition(ctx.EXISTS() != null, ctx.partitionName.getText(), values);
}
@Override

260
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
View File

@ -21,6 +21,9 @@ import org.apache.doris.analysis.ColumnDef;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.Config;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.types.ArrayType;
@ -33,10 +36,13 @@ import org.apache.doris.nereids.types.StructField;
import org.apache.doris.nereids.types.StructType;
import org.apache.doris.nereids.types.TinyIntType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.qe.SessionVariable;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@ -176,9 +182,7 @@ public class ColumnDefinition {
* validate column definition and analyze
*/
public void validate(boolean isOlap, Set<String> keysSet, boolean isEnableMergeOnWrite, KeysType keysType) {
if (Config.disable_nested_complex_type && isNestedComplexType(type)) {
throw new AnalysisException("Unsupported data type: " + type.toSql());
}
validateDataType(type.toCatalogDataType());
type = updateCharacterTypeLength(type);
if (type.isArrayType()) {
int depth = 0;
@ -369,6 +373,256 @@ public class ColumnDefinition {
return false;
}
// from TypeDef.java analyze()
private void validateDataType(Type catalogType) {
if (catalogType.exceedsMaxNestingDepth()) {
throw new AnalysisException(
String.format("Type exceeds the maximum nesting depth of %s:\n%s",
Type.MAX_NESTING_DEPTH, catalogType.toSql()));
}
if (!catalogType.isSupported()) {
throw new AnalysisException("Unsupported data type: " + catalogType.toSql());
}
if (catalogType.isScalarType()) {
validateScalarType((ScalarType) catalogType);
} else if (catalogType.isComplexType()) {
// now we not support array / map / struct nesting complex type
if (catalogType.isArrayType()) {
Type itemType = ((org.apache.doris.catalog.ArrayType) catalogType).getItemType();
if (itemType instanceof ScalarType) {
validateNestedType(catalogType, (ScalarType) itemType);
} else if (Config.disable_nested_complex_type
&& !(itemType instanceof org.apache.doris.catalog.ArrayType)) {
// now we can array nesting array
throw new AnalysisException(
"Unsupported data type: ARRAY<" + itemType.toSql() + ">");
}
}
if (catalogType.isMapType()) {
org.apache.doris.catalog.MapType mt =
(org.apache.doris.catalog.MapType) catalogType;
if (Config.disable_nested_complex_type && (!(mt.getKeyType() instanceof ScalarType)
|| !(mt.getValueType() instanceof ScalarType))) {
throw new AnalysisException("Unsupported data type: MAP<"
+ mt.getKeyType().toSql() + "," + mt.getValueType().toSql() + ">");
}
if (mt.getKeyType() instanceof ScalarType) {
validateNestedType(catalogType, (ScalarType) mt.getKeyType());
}
if (mt.getValueType() instanceof ScalarType) {
validateNestedType(catalogType, (ScalarType) mt.getValueType());
}
}
if (catalogType.isStructType()) {
ArrayList<org.apache.doris.catalog.StructField> fields =
((org.apache.doris.catalog.StructType) catalogType).getFields();
Set<String> fieldNames = new HashSet<>();
for (org.apache.doris.catalog.StructField field : fields) {
Type fieldType = field.getType();
if (fieldType instanceof ScalarType) {
validateNestedType(catalogType, (ScalarType) fieldType);
if (!fieldNames.add(field.getName())) {
throw new AnalysisException("Duplicate field name " + field.getName()
+ " in struct " + catalogType.toSql());
}
} else if (Config.disable_nested_complex_type) {
throw new AnalysisException(
"Unsupported field type: " + fieldType.toSql() + " for STRUCT");
}
}
}
}
}
private void validateScalarType(ScalarType scalarType) {
PrimitiveType type = scalarType.getPrimitiveType();
// When string type length is not assigned, it needs to be assigned to 1.
if (scalarType.getPrimitiveType().isStringType() && !scalarType.isLengthSet()) {
if (scalarType.getPrimitiveType() == PrimitiveType.VARCHAR) {
// always set varchar length MAX_VARCHAR_LENGTH
scalarType.setLength(ScalarType.MAX_VARCHAR_LENGTH);
} else if (scalarType.getPrimitiveType() == PrimitiveType.STRING) {
// always set text length MAX_STRING_LENGTH
scalarType.setLength(ScalarType.MAX_STRING_LENGTH);
} else {
scalarType.setLength(1);
}
}
switch (type) {
case CHAR:
case VARCHAR: {
String name;
int maxLen;
if (type == PrimitiveType.VARCHAR) {
name = "VARCHAR";
maxLen = ScalarType.MAX_VARCHAR_LENGTH;
} else {
name = "CHAR";
maxLen = ScalarType.MAX_CHAR_LENGTH;
return;
}
int len = scalarType.getLength();
// len is decided by child, when it is -1.
if (len <= 0) {
throw new AnalysisException(name + " size must be > 0: " + len);
}
if (scalarType.getLength() > maxLen) {
throw new AnalysisException(name + " size must be <= " + maxLen + ": " + len);
}
break;
}
case DECIMALV2: {
int precision = scalarType.decimalPrecision();
int scale = scalarType.decimalScale();
// precision: [1, 27]
if (precision < 1 || precision > ScalarType.MAX_DECIMALV2_PRECISION) {
throw new AnalysisException("Precision of decimal must between 1 and 27."
+ " Precision was set to: " + precision + ".");
}
// scale: [0, 9]
if (scale < 0 || scale > ScalarType.MAX_DECIMALV2_SCALE) {
throw new AnalysisException("Scale of decimal must between 0 and 9."
+ " Scale was set to: " + scale + ".");
}
if (precision - scale > ScalarType.MAX_DECIMALV2_PRECISION
- ScalarType.MAX_DECIMALV2_SCALE) {
throw new AnalysisException("Invalid decimal type with precision = " + precision
+ ", scale = " + scale);
}
// scale < precision
if (scale > precision) {
throw new AnalysisException("Scale of decimal must be smaller than precision."
+ " Scale is " + scale + " and precision is " + precision);
}
break;
}
case DECIMAL32: {
int decimal32Precision = scalarType.decimalPrecision();
int decimal32Scale = scalarType.decimalScale();
if (decimal32Precision < 1
|| decimal32Precision > ScalarType.MAX_DECIMAL32_PRECISION) {
throw new AnalysisException("Precision of decimal must between 1 and 9."
+ " Precision was set to: " + decimal32Precision + ".");
}
// scale >= 0
if (decimal32Scale < 0) {
throw new AnalysisException("Scale of decimal must not be less than 0."
+ " Scale was set to: " + decimal32Scale + ".");
}
// scale < precision
if (decimal32Scale > decimal32Precision) {
throw new AnalysisException(
"Scale of decimal must be smaller than precision." + " Scale is "
+ decimal32Scale + " and precision is " + decimal32Precision);
}
break;
}
case DECIMAL64: {
int decimal64Precision = scalarType.decimalPrecision();
int decimal64Scale = scalarType.decimalScale();
if (decimal64Precision < 1
|| decimal64Precision > ScalarType.MAX_DECIMAL64_PRECISION) {
throw new AnalysisException("Precision of decimal64 must between 1 and 18."
+ " Precision was set to: " + decimal64Precision + ".");
}
// scale >= 0
if (decimal64Scale < 0) {
throw new AnalysisException("Scale of decimal must not be less than 0."
+ " Scale was set to: " + decimal64Scale + ".");
}
// scale < precision
if (decimal64Scale > decimal64Precision) {
throw new AnalysisException(
"Scale of decimal must be smaller than precision." + " Scale is "
+ decimal64Scale + " and precision is " + decimal64Precision);
}
break;
}
case DECIMAL128: {
int decimal128Precision = scalarType.decimalPrecision();
int decimal128Scale = scalarType.decimalScale();
if (decimal128Precision < 1
|| decimal128Precision > ScalarType.MAX_DECIMAL128_PRECISION) {
throw new AnalysisException("Precision of decimal128 must between 1 and 38."
+ " Precision was set to: " + decimal128Precision + ".");
}
// scale >= 0
if (decimal128Scale < 0) {
throw new AnalysisException("Scale of decimal must not be less than 0."
+ " Scale was set to: " + decimal128Scale + ".");
}
// scale < precision
if (decimal128Scale > decimal128Precision) {
throw new AnalysisException(
"Scale of decimal must be smaller than precision." + " Scale is "
+ decimal128Scale + " and precision is " + decimal128Precision);
}
break;
}
case DECIMAL256: {
if (SessionVariable.getEnableDecimal256()) {
int precision = scalarType.decimalPrecision();
int scale = scalarType.decimalScale();
if (precision < 1 || precision > ScalarType.MAX_DECIMAL256_PRECISION) {
throw new AnalysisException("Precision of decimal256 must between 1 and 76."
+ " Precision was set to: " + precision + ".");
}
// scale >= 0
if (scale < 0) {
throw new AnalysisException("Scale of decimal must not be less than 0."
+ " Scale was set to: " + scale + ".");
}
// scale < precision
if (scale > precision) {
throw new AnalysisException(
"Scale of decimal must be smaller than precision." + " Scale is "
+ scale + " and precision is " + precision);
}
break;
} else {
int precision = scalarType.decimalPrecision();
throw new AnalysisException("Column of type Decimal256 with precision "
+ precision + " in not supported.");
}
}
case TIMEV2:
case DATETIMEV2: {
int precision = scalarType.decimalPrecision();
int scale = scalarType.decimalScale();
// precision: [1, 27]
if (precision != ScalarType.DATETIME_PRECISION) {
throw new AnalysisException(
"Precision of Datetime/Time must be " + ScalarType.DATETIME_PRECISION
+ "." + " Precision was set to: " + precision + ".");
}
// scale: [0, 9]
if (scale < 0 || scale > 6) {
throw new AnalysisException("Scale of Datetime/Time must between 0 and 6."
+ " Scale was set to: " + scale + ".");
}
break;
}
case INVALID_TYPE:
throw new AnalysisException("Invalid type.");
default:
break;
}
}
private void validateNestedType(Type parent, Type child) throws AnalysisException {
if (child.isNull()) {
throw new AnalysisException("Unsupported data type: " + child.toSql());
}
// check whether the sub-type is supported
if (!parent.supportSubType(child)) {
throw new AnalysisException(
parent.getPrimitiveType() + " unsupported sub-type: " + child.toSql());
}
validateDataType(child);
}
/**
* translate to catalog create table stmt
*/

21
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/FixedRangePartition.java
View File

@ -20,10 +20,10 @@ package org.apache.doris.nereids.trees.plans.commands.info;
import org.apache.doris.analysis.PartitionKeyDesc;
import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.analysis.SinglePartitionDesc;
import org.apache.doris.common.FeNameFormat;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import com.google.common.collect.Maps;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -33,12 +33,12 @@ import java.util.stream.Collectors;
* represent fixed range partition
*/
public class FixedRangePartition extends PartitionDefinition {
private final String partitionName;
private List<Expression> lowerBounds;
private List<Expression> upperBounds;
public FixedRangePartition(String partitionName, List<Expression> lowerBounds, List<Expression> upperBounds) {
this.partitionName = partitionName;
public FixedRangePartition(boolean ifNotExists, String partitionName,
List<Expression> lowerBounds, List<Expression> upperBounds) {
super(ifNotExists, partitionName);
this.lowerBounds = lowerBounds;
this.upperBounds = upperBounds;
}
@ -46,6 +46,11 @@ public class FixedRangePartition extends PartitionDefinition {
@Override
public void validate(Map<String, String> properties) {
super.validate(properties);
try {
FeNameFormat.checkPartitionName(partitionName);
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e.getCause());
}
List<Expression> newLowerBounds = new ArrayList<>();
List<Expression> newUpperBounds = new ArrayList<>();
for (int i = 0; i < partitionTypes.size(); ++i) {
@ -72,7 +77,9 @@ public class FixedRangePartition extends PartitionDefinition {
.collect(Collectors.toList());
List<PartitionValue> upperValues = upperBounds.stream().map(this::toLegacyPartitionValueStmt)
.collect(Collectors.toList());
return new SinglePartitionDesc(false, partitionName,
PartitionKeyDesc.createFixed(lowerValues, upperValues), replicaAllocation, Maps.newHashMap());
return new SinglePartitionDesc(ifNotExists, partitionName,
PartitionKeyDesc.createFixed(lowerValues, upperValues), replicaAllocation,
properties, partitionDataProperty, isInMemory, tabletType, versionInfo,
storagePolicy, isMutable);
}
}

19
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/InPartition.java
View File

@ -21,11 +21,10 @@ import org.apache.doris.analysis.AllPartitionDesc;
import org.apache.doris.analysis.PartitionKeyDesc;
import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.analysis.SinglePartitionDesc;
import org.apache.doris.common.FeNameFormat;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import com.google.common.collect.Maps;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -35,17 +34,21 @@ import java.util.stream.Collectors;
* represent in partition
*/
public class InPartition extends PartitionDefinition {
private final String partitionName;
private final List<List<Expression>> values;
public InPartition(String partitionName, List<List<Expression>> values) {
this.partitionName = partitionName;
public InPartition(boolean ifNotExists, String partitionName, List<List<Expression>> values) {
super(ifNotExists, partitionName);
this.values = values;
}
@Override
public void validate(Map<String, String> properties) {
super.validate(properties);
try {
FeNameFormat.checkPartitionName(partitionName);
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e.getCause());
}
if (values.stream().anyMatch(l -> l.stream().anyMatch(MaxValue.class::isInstance))) {
throw new AnalysisException("MAXVALUE cannot be used in 'in partition'");
}
@ -65,7 +68,9 @@ public class InPartition extends PartitionDefinition {
List<List<PartitionValue>> catalogValues = values.stream().map(l -> l.stream()
.map(this::toLegacyPartitionValueStmt)
.collect(Collectors.toList())).collect(Collectors.toList());
return new SinglePartitionDesc(false, partitionName, PartitionKeyDesc.createIn(catalogValues),
replicaAllocation, Maps.newHashMap());
return new SinglePartitionDesc(ifNotExists, partitionName,
PartitionKeyDesc.createIn(catalogValues), replicaAllocation, properties,
partitionDataProperty, isInMemory, tabletType, versionInfo, storagePolicy,
isMutable);
}
}

31
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/LessThanPartition.java
View File

@ -20,10 +20,10 @@ package org.apache.doris.nereids.trees.plans.commands.info;
import org.apache.doris.analysis.PartitionKeyDesc;
import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.analysis.SinglePartitionDesc;
import org.apache.doris.common.FeNameFormat;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import com.google.common.collect.Maps;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@ -32,17 +32,21 @@ import java.util.stream.Collectors;
* represent less than partition
*/
public class LessThanPartition extends PartitionDefinition {
private final String partitionName;
private final List<Expression> values;
public LessThanPartition(String partitionName, List<Expression> values) {
this.partitionName = partitionName;
public LessThanPartition(boolean ifNotExists, String partitionName, List<Expression> values) {
super(ifNotExists, partitionName);
this.values = values;
}
@Override
public void validate(Map<String, String> properties) {
super.validate(properties);
try {
FeNameFormat.checkPartitionName(partitionName);
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e.getCause());
}
}
public String getPartitionName() {
@ -54,13 +58,16 @@ public class LessThanPartition extends PartitionDefinition {
*/
public SinglePartitionDesc translateToCatalogStyle() {
if (values.get(0) instanceof MaxValue) {
return new SinglePartitionDesc(false, partitionName, PartitionKeyDesc.createMaxKeyDesc(),
replicaAllocation, Maps.newHashMap());
return new SinglePartitionDesc(ifNotExists, partitionName,
PartitionKeyDesc.createMaxKeyDesc(), replicaAllocation, properties,
partitionDataProperty, isInMemory, tabletType, versionInfo, storagePolicy,
isMutable);
}
List<PartitionValue> partitionValues = values.stream()
.map(this::toLegacyPartitionValueStmt)
.collect(Collectors.toList());
return new SinglePartitionDesc(false, partitionName,
PartitionKeyDesc.createLessThan(partitionValues), replicaAllocation, Maps.newHashMap());
List<PartitionValue> partitionValues =
values.stream().map(this::toLegacyPartitionValueStmt).collect(Collectors.toList());
return new SinglePartitionDesc(ifNotExists, partitionName,
PartitionKeyDesc.createLessThan(partitionValues), replicaAllocation, properties,
partitionDataProperty, isInMemory, tabletType, versionInfo, storagePolicy,
isMutable);
}
}

88
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.plans.commands.info;
import org.apache.doris.analysis.AllPartitionDesc;
import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.catalog.DataProperty;
import org.apache.doris.catalog.ReplicaAllocation;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.nereids.exceptions.AnalysisException;
@ -27,6 +28,11 @@ import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.shape.LeafExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.thrift.TTabletType;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import java.util.List;
import java.util.Map;
@ -35,9 +41,25 @@ import java.util.Map;
* abstract class for partition definition
*/
public abstract class PartitionDefinition {
protected List<DataType> partitionTypes;
protected boolean ifNotExists;
protected String partitionName;
protected Map<String, String> properties;
protected List<DataType> partitionTypes;
protected DataProperty partitionDataProperty =
new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM);
protected ReplicaAllocation replicaAllocation = ReplicaAllocation.DEFAULT_ALLOCATION;
protected boolean isInMemory = false;
protected TTabletType tabletType = TTabletType.TABLET_TYPE_DISK;
protected Long versionInfo;
protected String storagePolicy = "";
protected boolean isMutable;
public PartitionDefinition(boolean ifNotExists, String partName) {
this.ifNotExists = ifNotExists;
this.partitionName = partName;
}
public PartitionDefinition withProperties(Map<String, String> properties) {
this.properties = properties;
@ -50,12 +72,74 @@ public abstract class PartitionDefinition {
* Validate the properties.
* Derived class can override this method to do more validation.
*/
public void validate(Map<String, String> properties) {
public void validate(Map<String, String> otherProperties) {
try {
if (PropertyAnalyzer.analyzeUniqueKeyMergeOnWrite(otherProperties)) {
String storagePolicy = PropertyAnalyzer.analyzeStoragePolicy(properties);
if (!storagePolicy.isEmpty()) {
throw new AnalysisException(
"Can not create UNIQUE KEY table that enables Merge-On-write"
+ " with storage policy(" + storagePolicy + ")");
}
}
boolean hasStoragePolicy = false;
if (properties != null) {
hasStoragePolicy = properties.keySet().stream().anyMatch(iter -> {
boolean equal = iter
.compareToIgnoreCase(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY) == 0;
// when find has storage policy properties, here will set it in partition
if (equal) {
storagePolicy = properties.get(iter);
}
return equal;
});
}
Map<String, String> mergedMap = Maps.newHashMap();
// Should putAll `otherProperties` before `this.properties`,
// because the priority of partition is higher than table
if (otherProperties != null) {
mergedMap.putAll(otherProperties);
}
if (this.properties != null) {
mergedMap.putAll(this.properties);
}
this.properties = mergedMap;
// analyze data property
partitionDataProperty = PropertyAnalyzer.analyzeDataProperty(properties,
new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
Preconditions.checkNotNull(partitionDataProperty);
replicaAllocation = PropertyAnalyzer.analyzeReplicaAllocation(properties, "");
if (replicaAllocation.isNotSet()) {
replicaAllocation = ReplicaAllocation.DEFAULT_ALLOCATION;
}
// analyze version info
versionInfo = PropertyAnalyzer.analyzeVersionInfo(properties);
// analyze in memory
isInMemory = PropertyAnalyzer.analyzeBooleanProp(properties,
PropertyAnalyzer.PROPERTIES_INMEMORY, false);
if (isInMemory == true) {
throw new AnalysisException("Not support set 'in_memory'='true' now!");
}
// analyze is mutable
isMutable = PropertyAnalyzer.analyzeBooleanProp(properties,
PropertyAnalyzer.PROPERTIES_MUTABLE, true);
tabletType = PropertyAnalyzer.analyzeTabletType(properties);
if (otherProperties == null) {
// check unknown properties
if (properties != null && !properties.isEmpty()) {
if (!hasStoragePolicy) {
Joiner.MapJoiner mapJoiner = Joiner.on(", ").withKeyValueSeparator(" = ");
throw new AnalysisException(
"Unknown properties: " + mapJoiner.join(properties));
}
}
}
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e.getCause());
}

11
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/StepPartition.java
View File

@ -23,8 +23,6 @@ import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import com.google.common.collect.Maps;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@ -38,8 +36,9 @@ public class StepPartition extends PartitionDefinition {
private final long unit;
private final String unitString;
public StepPartition(List<Expression> fromExpression, List<Expression> toExpression, long unit,
String unitString) {
public StepPartition(boolean ifNotExists, String partitionName, List<Expression> fromExpression,
List<Expression> toExpression, long unit, String unitString) {
super(ifNotExists, partitionName);
this.fromExpression = fromExpression;
this.toExpression = toExpression;
this.unit = unit;
@ -68,10 +67,10 @@ public class StepPartition extends PartitionDefinition {
try {
if (unitString == null) {
return new MultiPartitionDesc(PartitionKeyDesc.createMultiFixed(fromValues, toValues, unit),
replicaAllocation, Maps.newHashMap());
replicaAllocation, properties);
}
return new MultiPartitionDesc(PartitionKeyDesc.createMultiFixed(fromValues, toValues, unit, unitString),
replicaAllocation, Maps.newHashMap());
replicaAllocation, properties);
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e.getCause());
}

2
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java
View File

@ -64,7 +64,7 @@ public class MappingTest extends TestWithFeService {
+ ")");
createTable("CREATE TABLE IF NOT EXISTS customer (\n"
+ " C_CUSTKEY INTEGER NOT NULL,\n"
+ " C_NAME VARCHAR(25) NOT NULL,\n"
+ " C_NAME VARCHAR(25) NOT NULL\n"
+ ")\n"
+ "DUPLICATE KEY(C_CUSTKEY, C_NAME)\n"
+ "DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 3\n"

2
regression-test/suites/correctness_p0/test_agg_materialize.groovy
View File

@ -34,7 +34,7 @@ suite("test_agg_materialize") {
sql """drop table if exists c5749_bug_t;"""
sql """CREATE TABLE `c5749_bug_t` (
`org_code` varchar(255) NULL ,
`cash_amt` decimal(27, 9) NULL ,
`cash_amt` decimal(27, 9) NULL
) ENGINE=OLAP
DUPLICATE KEY(`org_code`)

2
regression-test/suites/datatype_p0/date/test_date_key.groovy
View File

@ -20,7 +20,7 @@ suite("test_date_key") {
sql """
create table `test_date_key` (
`k1` datev1, `k2` int,
INDEX idx_k1 (`k1`) USING BITMAP,
INDEX idx_k1 (`k1`) USING BITMAP
) duplicate key(`k1`)
distributed by hash(k2) buckets 3
properties("replication_num" = "1");

2
regression-test/suites/datatype_p0/date/test_datetime_key.groovy
View File

@ -20,7 +20,7 @@ suite("test_datetime_key") {
sql """
create table `test_datetime_key` (
`k1` datetimev1, `k2` int,
INDEX idx_k1 (`k1`) USING BITMAP,
INDEX idx_k1 (`k1`) USING BITMAP
) duplicate key(`k1`)
distributed by hash(k2) buckets 3
properties("replication_num" = "1");

2
regression-test/suites/datatype_p0/datetimev1/test_datetimev1_calc.groovy
View File

@ -29,7 +29,7 @@ suite("test_datetimev1_calc", "nonConcurrent") {
CREATE TABLE IF NOT EXISTS `${table1}` (
`id` int,
`value1` datetimev1 NULL COMMENT "",
`value2` datetimev1 NULL COMMENT "",
`value2` datetimev1 NULL COMMENT ""
) ENGINE=OLAP
DUPLICATE KEY(`id`)
COMMENT "OLAP"

2
regression-test/suites/datatype_p0/datev1/test_datev1_calc.groovy
View File

@ -29,7 +29,7 @@ suite("test_datev1_calc", "nonConcurrent") {
CREATE TABLE IF NOT EXISTS `${table1}` (
`id` int,
`value1` datev1 NULL COMMENT "",
`value2` datev1 NULL COMMENT "",
`value2` datev1 NULL COMMENT ""
) ENGINE=OLAP
DUPLICATE KEY(`id`)
COMMENT "OLAP"

2
regression-test/suites/datatype_p0/decimalv2/test_decimalv2_calc.groovy
View File

@ -31,7 +31,7 @@ suite("test_decimalv2_calc", "nonConcurrent") {
`decimal_key1` decimalv2(8, 5) NULL COMMENT "",
`decimal_key2` decimalv2(16, 5) NULL COMMENT "",
`decimal_value1` decimalv2(8, 5) NULL COMMENT "",
`decimal_value2` decimalv2(16, 5) NULL COMMENT "",
`decimal_value2` decimalv2(16, 5) NULL COMMENT ""
) ENGINE=OLAP
DUPLICATE KEY(`decimal_key1`, `decimal_key2`)
COMMENT "OLAP"

2
regression-test/suites/datatype_p0/decimalv2/test_decimalv2_calc_with_conversion.groovy
View File

@ -31,7 +31,7 @@ suite("test_decimalv2_calc_with_conversion", "nonConcurrent") {
`decimal_key1` decimalv2(8, 5) NULL COMMENT "",
`decimal_key2` decimalv2(16, 5) NULL COMMENT "",
`decimal_value1` decimalv2(8, 5) NULL COMMENT "",
`decimal_value2` decimalv2(16, 5) NULL COMMENT "",
`decimal_value2` decimalv2(16, 5) NULL COMMENT ""
) ENGINE=OLAP
DUPLICATE KEY(`decimal_key1`, `decimal_key2`)
COMMENT "OLAP"

2
regression-test/suites/datatype_p0/decimalv3/test_decimalv3_overflow.groovy
View File

@ -32,7 +32,7 @@ suite("test_decimalv3_overflow") {
def tblName2 = "test_decimalv3_overflow2"
sql "drop table if exists ${tblName2}"
sql """ CREATE TABLE ${tblName2} (
`c2` decimalv3(20, 2),
`c2` decimalv3(20, 2)
) ENGINE=OLAP
UNIQUE KEY(`c2`)
DISTRIBUTED BY HASH(`c2`) BUCKETS 10

2
regression-test/suites/datatype_p0/decimalv3/test_uniq_tab_decimalv2.groovy
View File

@ -26,7 +26,7 @@ suite("test_uniq_tab_decimalv2") {
`decimal_key1` decimalv2(8, 5) NULL COMMENT "",
`decimal_key2` decimalv2(16, 5) NULL COMMENT "",
`decimal_value1` decimalv2(8, 5) NULL COMMENT "",
`decimal_value2` decimalv2(16, 5) NULL COMMENT "",
`decimal_value2` decimalv2(16, 5) NULL COMMENT ""
) ENGINE=OLAP
UNIQUE KEY(`decimal_key1`, `decimal_key2`)
COMMENT "OLAP"

8
regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_unsupport_types.groovy
View File

@ -25,7 +25,7 @@ suite("create_nestedtypes_with_unsupport_types") {
sql """
CREATE TABLE IF NOT EXISTS ${testTablex} (
`k1` INT(11) NULL,
`k2` array<${columnType}> NULL,
`k2` array<${columnType}> NULL
)
DUPLICATE KEY(`k1`)
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
@ -48,7 +48,7 @@ suite("create_nestedtypes_with_unsupport_types") {
sql """
CREATE TABLE IF NOT EXISTS ${testTablex} (
`k1` INT(11) NULL,
`k2` map<${columnType}, String> NULL,
`k2` map<${columnType}, String> NULL
)
DUPLICATE KEY(`k1`)
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
@ -60,7 +60,7 @@ suite("create_nestedtypes_with_unsupport_types") {
sql """
CREATE TABLE IF NOT EXISTS ${testTablex} (
`k1` INT(11) NULL,
`k2` map<String, ${columnType}> NULL,
`k2` map<String, ${columnType}> NULL
)
DUPLICATE KEY(`k1`)
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
@ -84,7 +84,7 @@ suite("create_nestedtypes_with_unsupport_types") {
sql """
CREATE TABLE IF NOT EXISTS ${testTablex} (
`k1` INT(11) NULL,
`k2` STRUCT<f1:${columnType}> NULL,
`k2` STRUCT<f1:${columnType}> NULL
)
DUPLICATE KEY(`k1`)
DISTRIBUTED BY HASH(`k1`) BUCKETS 1

2
regression-test/suites/load_p0/routine_load/ddl/dup_tbl_basic_multi_table_create.sql
View File

@ -29,7 +29,7 @@ CREATE TABLE routine_load_dup_tbl_basic_multi_table
INDEX idx_ngrambf_k116 (`k16`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"),
INDEX idx_ngrambf_k117 (`k17`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"),
INDEX idx_bitmap_k104 (`k02`) USING BITMAP,
INDEX idx_bitmap_k104 (`k02`) USING BITMAP
)
DUPLICATE KEY(k00)

2
regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_or_datetime_computation_negative.groovy
View File

@ -23,7 +23,7 @@ suite("test_date_or_datetime_computation_negative") {
`dateV2` DATEV2 NOT NULL,
`dateV2_null` DATEV2 NULL,
`datetime` DATETIME NOT NULL,
`datetime_null` DATETIME NULL, )
`datetime_null` DATETIME NULL )
DUPLICATE KEY(`row_id`)
DISTRIBUTED BY HASH(`row_id`) BUCKETS 1
PROPERTIES (

4
regression-test/suites/query_p0/session_variable/test_default_limit.groovy
View File

@ -23,7 +23,7 @@ suite('test_default_limit', "arrow_flight_sql") {
create table baseall (
k0 int,
k1 int,
k2 int,
k2 int
)
distributed by hash(k0) buckets 16
properties(
@ -35,7 +35,7 @@ suite('test_default_limit', "arrow_flight_sql") {
create table bigtable (
k0 int,
k1 int,
k2 int,
k2 int
)
distributed by hash(k0) buckets 16
properties(

2
regression-test/suites/query_p0/sql_functions/aggregate_functions/test_count_distinct_with_collection_type.groovy
View File

@ -24,7 +24,7 @@ suite("test_count_distinct_with_collection_type", "p0") {
`id` int(11) NULL,
`a` array<text> NULL,
`m` map<text, int> NULL,
`s` STRUCT<f1:INT, f2:VARCHAR(30), f3: DECIMAL> NULL,
`s` STRUCT<f1:INT, f2:VARCHAR(30), f3: DECIMAL> NULL
) ENGINE=OLAP
DUPLICATE KEY(`id`)
COMMENT 'OLAP'

2
regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions.groovy
View File

@ -314,7 +314,7 @@ suite("test_array_functions") {
`kdcmlv3s1` decimalv3(9, 3) null,
`kdcmlv3s2` decimalv3(15, 5) null,
`kdcmlv3s3` decimalv3(27, 9) null,
`kadcml` array<decimal(27, 9)> null,
`kadcml` array<decimal(27, 9)> null
) engine=olap
DISTRIBUTED BY HASH(`id`) BUCKETS 1
properties("replication_num" = "1")

2
regression-test/suites/query_p0/sql_functions/size_funciton/test_size_function.groovy
View File

@ -37,7 +37,7 @@ suite("test_size_function") {
`id` int(11) NULL,
`c_array1` array<int(11)> NULL,
`c_array2` array<varchar(20)> NULL,
`c_map` map<int(11), string> NULL,
`c_map` map<int(11), string> NULL
) ENGINE=OLAP
DUPLICATE KEY(`id`)
DISTRIBUTED BY HASH(`id`) BUCKETS 1