database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature-wip](array-type) Support array type which doesn't contain null (#9809)

Browse Source
This commit is contained in:
Adonis Ling
2022-06-12 23:35:28 +08:00
committed by GitHub
parent bf984c1e80
commit 415b6b8086
14 changed files with 283 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/cup/sql_parser.cup
View File

@ -276,7 +276,8 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALIAS, KW_ALL, KW_A
KW_UNCOMMITTED, KW_UNBOUNDED, KW_UNION, KW_UNIQUE, KW_UNLOCK, KW_UNSIGNED, KW_USE, KW_USER, KW_USING, KW_UNINSTALL,
KW_VALUE, KW_VALUES, KW_VARCHAR, KW_VARIABLES, KW_VERBOSE, KW_VIEW,
KW_WARNINGS, KW_WEEK, KW_WHEN, KW_WHITELIST, KW_WHERE, KW_WITH, KW_WORK, KW_WRITE,
KW_YEAR;
KW_YEAR,
KW_NOT_NULL;
terminal COMMA, COLON, DOT, DOTDOTDOT, AT, STAR, LPAREN, RPAREN, SEMICOLON, LBRACKET, RBRACKET, DIVIDE, MOD, ADD, SUBTRACT;
terminal BITAND, BITOR, BITXOR, BITNOT;
@ -4628,6 +4629,8 @@ type ::=
{: RESULT = ScalarType.createVarcharType(-1); :}
| KW_ARRAY LESSTHAN type:value_type GREATERTHAN
{: RESULT = new ArrayType(value_type); :}
| KW_ARRAY LESSTHAN KW_NOT_NULL LPAREN type:value_type RPAREN GREATERTHAN
{: RESULT = new ArrayType(value_type, false); :}
| KW_MAP LESSTHAN type:key_type COMMA type:value_type GREATERTHAN
{: RESULT = new MapType(key_type,value_type); :}
| KW_STRUCT LESSTHAN struct_field_list:fields GREATERTHAN

17
fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
View File

@ -35,17 +35,22 @@ import java.util.List;
public class ArrayLiteral extends LiteralExpr {
public ArrayLiteral() {
this.type = new ArrayType(Type.NULL);
type = new ArrayType(Type.NULL, false);
children = new ArrayList<>();
}
public ArrayLiteral(LiteralExpr... v) {
if (v.length < 1) {
this.type = new ArrayType(Type.NULL);
return;
Type itemType = Type.NULL;
boolean containsNull = false;
for (LiteralExpr expr : v) {
if (itemType == Type.NULL || expr.type.getSlotSize() > itemType.getSlotSize()) {
itemType = expr.type;
}
if (expr.isNullable()) {
containsNull = true;
}
}
this.type = new ArrayType(v[0].type);
type = new ArrayType(itemType, containsNull);
children = new ArrayList<>(v.length);
children.addAll(Arrays.asList(v));
}

50
fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
View File

@ -37,22 +37,31 @@ public class ArrayType extends Type {
@SerializedName(value = "itemType")
private Type itemType;
@SerializedName(value = "containsNull")
private boolean containsNull;
public ArrayType() {
this.itemType = NULL;
itemType = NULL;
containsNull = false;
}
public ArrayType(Type itemType) {
this.itemType = itemType;
this(itemType, true);
}
public void setItemType(Type itemType) {
public ArrayType(Type itemType, boolean containsNull) {
this.itemType = itemType;
this.containsNull = containsNull;
}
public Type getItemType() {
return itemType;
}
public boolean getContainsNull() {
return containsNull;
}
@Override
public PrimitiveType getPrimitiveType() {
return PrimitiveType.ARRAY;
@ -69,32 +78,33 @@ public class ArrayType extends Type {
}
// Array(Null) is a virtual Array type, can match any Array(...) type
if (itemType.isNull()) {
if (itemType.isNull() || ((ArrayType) t).getItemType().isNull()) {
return true;
}
if (((ArrayType) t).getItemType().isNull()) {
return true;
}
return itemType.matchesType(((ArrayType) t).itemType);
return itemType.matchesType(((ArrayType) t).itemType)
&& ((ArrayType) t).containsNull == containsNull;
}
public static ArrayType create() {
return new ArrayType();
}
public static ArrayType create(Type type) {
return new ArrayType(type);
public static ArrayType create(Type type, boolean containsNull) {
return new ArrayType(type, containsNull);
}
@Override
public String toSql(int depth) {
return String.format("ARRAY<%s>", itemType.toSql(depth + 1));
if (!containsNull) {
return "ARRAY<NOT_NULL(" + itemType.toSql(depth + 1) + ")>";
} else {
return "ARRAY<" + itemType.toSql(depth + 1) + ">";
}
}
@Override
public int hashCode() {
return Objects.hash(itemType);
return Objects.hash(itemType, containsNull);
}
@Override
@ -103,10 +113,13 @@ public class ArrayType extends Type {
return false;
}
ArrayType otherArrayType = (ArrayType) other;
return otherArrayType.itemType.equals(itemType);
return otherArrayType.itemType.equals(itemType) && otherArrayType.containsNull == containsNull;
}
public static boolean canCastTo(ArrayType type, ArrayType targetType) {
if (!targetType.containsNull && type.containsNull) {
return false;
}
if (targetType.getItemType().isStringType() && type.getItemType().isStringType()) {
return true;
}
@ -119,6 +132,7 @@ public class ArrayType extends Type {
container.types.add(node);
Preconditions.checkNotNull(itemType);
node.setType(TTypeNodeType.ARRAY);
node.setContainsNull(containsNull);
itemType.toThrift(container);
}
@ -130,8 +144,7 @@ public class ArrayType extends Type {
}
// Pass in the padding to make sure nested fields are aligned properly,
// even if we then strip the top-level padding.
String structStr = itemType.prettyPrint(lpad);
structStr = structStr.substring(lpad);
String structStr = itemType.prettyPrint(lpad).substring(lpad);
return String.format("%sARRAY<%s>", leftPadding, structStr);
}
@ -162,10 +175,7 @@ public class ArrayType extends Type {
@Override
public boolean supportsTablePartitioning() {
if (!isSupported() || isComplexType()) {
return false;
}
return true;
return isSupported() && !isComplexType();
}
@Override

5
fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
View File

@ -166,10 +166,7 @@ public class Column implements Writable {
public void createChildrenColumn(Type type, Column column) {
if (type.isArrayType()) {
Column c = new Column(COLUMN_ARRAY_CHILDREN, ((ArrayType) type).getItemType());
// TODO We always set the item type in array nullable.
// We may provide an alternative to configure this property of
// the item type in array in future.
c.setIsAllowNull(true);
c.setIsAllowNull(((ArrayType) type).getContainsNull());
column.addChildrenColumn(c);
}
}

4
fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
View File

@ -134,6 +134,7 @@ public abstract class ColumnType {
ArrayType arrayType = (ArrayType) type;
Text.writeString(out, arrayType.getPrimitiveType().name());
write(out, arrayType.getItemType());
out.writeBoolean(arrayType.getContainsNull());
}
}
@ -141,7 +142,8 @@ public abstract class ColumnType {
PrimitiveType primitiveType = PrimitiveType.valueOf(Text.readString(in));
if (primitiveType == PrimitiveType.ARRAY) {
Type itermType = read(in);
return ArrayType.create(itermType);
boolean containsNull = in.readBoolean();
return ArrayType.create(itermType, containsNull);
} else {
int scale = in.readInt();
int precision = in.readInt();

1
fe/fe-core/src/main/jflex/sql_scanner.flex
View File

@ -432,6 +432,7 @@ import org.apache.doris.qe.SqlModeHelper;
keywordMap.put("year", new Integer(SqlParserSymbols.KW_YEAR));
keywordMap.put("||", new Integer(SqlParserSymbols.KW_PIPE));
keywordMap.put("current_timestamp", new Integer(SqlParserSymbols.KW_CURRENT_TIMESTAMP));
keywordMap.put("not_null", new Integer(SqlParserSymbols.KW_NOT_NULL));
}
// map from token id to token description

2
fe/fe-core/src/test/java/org/apache/doris/analysis/RangePartitionPruneTest.java
View File

@ -128,7 +128,7 @@ public class RangePartitionPruneTest extends PartitionPruneTestBase {
addCase("select * from test.t1 where dt in (20211124, 20211126, 20211122)", "partitions=3/8", "partitions=3/8");
// is null
addCase("select * from test.t1 where dt is null", "partitions=1/8", "partitions=1/8");
addCase("select * from test.not_null where dt is null", "partitions=0/7", "partitions=0/7");
addCase("select * from test.`not_null` where dt is null", "partitions=0/7", "partitions=0/7");
// not equal to
addCase("select * from test.t1 where dt!=20211122", "partitions=8/8", "partitions=8/8");

5
fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
View File

@ -539,5 +539,10 @@ public class CreateTableTest {
createTable("create table test.table2(k1 INT, k2 Array<Array<int>>) duplicate key (k1) "
+ "distributed by hash(k1) buckets 1 properties('replication_num' = '1');");
});
ExceptionChecker.expectThrowsNoException(() -> {
createTable("create table test.table3(k1 INT, k2 Array<not_null(int)>) duplicate key (k1) "
+ "distributed by hash(k1) buckets 1 properties('replication_num' = '1');");
});
}
}