database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature-wip](array-type) Support ArrayLiteral in SQL. (#8089) (#8582)

Browse Source 
Please refer to #8074
This commit is contained in:
Adonis Ling
2022-03-22 15:07:06 +08:00
committed by GitHub
parent cf0a9fd177
commit 38ec3cbbdf
21 changed files with 306 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
fe/fe-core/src/main/cup/sql_parser.cup
View File

@ -379,6 +379,7 @@ nonterminal CaseExpr case_expr;
nonterminal ArrayList<CaseWhenClause> case_when_clause_list;
nonterminal FunctionParams function_params;
nonterminal Expr function_call_expr, array_expr;
nonterminal ArrayLiteral array_literal;
nonterminal StructField struct_field;
nonterminal ArrayList<StructField> struct_field_list;
nonterminal AnalyticWindow opt_window_clause;
@ -4752,6 +4753,17 @@ function_call_expr ::=
:}
;
array_literal ::=
LBRACKET RBRACKET
{:
RESULT = new ArrayLiteral();
:}
| LBRACKET expr_list:list RBRACKET
{:
RESULT = new ArrayLiteral(list.toArray(new LiteralExpr[0]));
:}
;
array_expr ::=
KW_ARRAY LPAREN function_params:params RPAREN
{:
@ -4800,6 +4812,8 @@ non_pred_expr ::=
{: RESULT = l; :}
| array_expr:a
{: RESULT = a; :}
| array_literal:a
{: RESULT = a; :}
| function_call_expr:e
{: RESULT = e; :}
| KW_DATE STRING_LITERAL:l

28
fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
View File

@ -17,6 +17,14 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
import org.apache.commons.lang.StringUtils;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@ -24,12 +32,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Type;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
public class ArrayLiteral extends LiteralExpr {
public ArrayLiteral() {
@ -113,4 +115,18 @@ public class ArrayLiteral extends LiteralExpr {
public Expr clone() {
return new ArrayLiteral(this);
}
@Override
public Expr uncheckedCastTo(Type targetType) throws AnalysisException {
if (!targetType.isArrayType()) {
return super.uncheckedCastTo(targetType);
}
ArrayLiteral literal = new ArrayLiteral(this);
for (int i = 0; i < children.size(); ++ i) {
Expr child = children.get(i);
literal.children.set(i, child.uncheckedCastTo(((ArrayType)targetType).getItemType()));
}
literal.setType(targetType);
return literal;
}
}

10
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
View File

@ -17,10 +17,6 @@
package org.apache.doris.analysis;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.collections.CollectionUtils;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
@ -38,6 +34,12 @@ import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.external.elasticsearch.EsUtil;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

3
fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
View File

@ -1253,7 +1253,8 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
}
public Expr checkTypeCompatibility(Type targetType) throws AnalysisException {
if (targetType.getPrimitiveType().equals(type.getPrimitiveType())) {
if (targetType.getPrimitiveType() != PrimitiveType.ARRAY &&
targetType.getPrimitiveType() == type.getPrimitiveType()) {
return this;
}
// bitmap must match exactly

4
fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
View File

@ -95,6 +95,10 @@ public class ArrayType extends Type {
return otherArrayType.itemType.equals(itemType);
}
public static boolean canCastTo(ArrayType type, ArrayType targetType) {
return Type.canCastTo(type.getItemType(), targetType.getItemType());
}
@Override
public void toThrift(TTypeDesc container) {
TTypeNode node = new TTypeNode();

13
fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
View File

@ -17,9 +17,6 @@
package org.apache.doris.catalog;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.gson.annotations.SerializedName;
import org.apache.doris.alter.SchemaChangeHandler;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.SlotRef;
@ -33,6 +30,11 @@ import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.thrift.TColumn;
import org.apache.doris.thrift.TColumnType;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -158,6 +160,10 @@ public class Column implements Writable {
public void createChildrenColumn(Type type, Column column) {
if (type.isArrayType()) {
Column c = new Column(COLUMN_ARRAY_CHILDREN, ((ArrayType) type).getItemType());
// TODO We always set the item type in array nullable.
// We may provide an alternative to configure this property of
// the item type in array in future.
c.setIsAllowNull(true);
column.addChildrenColumn(c);
}
}
@ -366,6 +372,7 @@ public class Column implements Writable {
childrenTColumnType.setIndexLen(children.getOlapColumnIndexSize());
childrenTColumn.setColumnType(childrenTColumnType);
childrenTColumn.setIsAllowNull(children.isAllowNull());
tColumn.setChildrenColumn(new ArrayList<>());
tColumn.children_column.add(childrenTColumn);

4
fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
View File

@ -384,8 +384,10 @@ public abstract class Type {
public static boolean canCastTo(Type t1, Type t2) {
if (t1.isScalarType() && t2.isScalarType()) {
return ScalarType.canCastTo((ScalarType) t1, (ScalarType) t2);
} else if (t1.isArrayType() && t2.isArrayType()) {
return ArrayType.canCastTo((ArrayType)t1, (ArrayType)t2);
}
return false;
return t1.isNull();
}
/**

18
fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java
View File

@ -19,9 +19,13 @@ package org.apache.doris.analysis;
import org.apache.doris.analysis.ColumnDef.DefaultValue;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.junit.Assert;
import org.junit.Before;
@ -39,7 +43,6 @@ public class ColumnDefTest {
stringCol = new TypeDef(ScalarType.createChar(10));
floatCol = new TypeDef(ScalarType.createType(PrimitiveType.FLOAT));
booleanCol = new TypeDef(ScalarType.createType(PrimitiveType.BOOLEAN));
}
@Test
@ -117,5 +120,16 @@ public class ColumnDefTest {
}
}
@Test
public void testArray() throws AnalysisException {
Config.enable_complex_type_support = true;
TypeDef typeDef = new TypeDef(new ArrayType(Type.INT));
ColumnDef columnDef = new ColumnDef("array", typeDef, false, null, true, DefaultValue.NOT_SET, "");
Column column = columnDef.toColumn();
Assert.assertEquals(1, column.getChildren().size());
Column childColumn = column.getChildren().get(0);
Assert.assertEquals("item", childColumn.getName());
Assert.assertEquals(Type.INT, childColumn.getType());
Assert.assertTrue(childColumn.isAllowNull());
}
}

114
fe/fe-core/src/test/java/org/apache/doris/analysis/InsertArrayStmtTest.java Normal file
View File

@ -0,0 +1,114 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.analysis;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TUniqueId;
import org.apache.doris.utframe.UtFrameUtils;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.StringReader;
import java.util.ArrayList;
public class InsertArrayStmtTest {
private static final String RUNNING_DIR = UtFrameUtils.generateRandomFeRunningDir(InsertArrayStmtTest.class);
private static ConnectContext connectContext;
@BeforeClass
public static void setUp() throws Exception {
Config.enable_complex_type_support = true;
UtFrameUtils.createDorisCluster(RUNNING_DIR);
connectContext = UtFrameUtils.createDefaultCtx();
createDatabase("create database test;");
}
@AfterClass
public static void tearDown() {
UtFrameUtils.cleanDorisFeDir(RUNNING_DIR);
}
private static void createDatabase(String sql) throws Exception {
CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
Catalog.getCurrentCatalog().createDb(createDbStmt);
}
private static void createTable(String sql) throws Exception {
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
Catalog.getCurrentCatalog().createTable(createTableStmt);
}
private static InsertStmt parseAndAnalyze(String sql) throws Exception {
SqlParser parser = new SqlParser(new SqlScanner(
new StringReader(sql), connectContext.getSessionVariable().getSqlMode()
));
InsertStmt insertStmt = (InsertStmt) SqlParserUtils.getFirstStmt(parser);
Analyzer analyzer = new Analyzer(connectContext.getCatalog(), connectContext);
insertStmt.analyze(analyzer);
return insertStmt;
};
@Test
public void testInsertArrayStmt() throws Exception {
ExceptionChecker.expectThrowsNoException(() -> {
createTable("create table test.table1 (k1 INT, v1 Array<int>) duplicate key (k1) " +
" distributed by hash (k1) buckets 1 properties ('replication_num' = '1');");
});
connectContext.setQueryId(new TUniqueId(1, 0));
InsertStmt insertStmt = parseAndAnalyze("insert into test.table1 values (1, [1, 2, 3]);");
ArrayList<Expr> row = ((SelectStmt) insertStmt.getQueryStmt()).getValueList().getFirstRow();
Assert.assertEquals(2, row.size());
Assert.assertTrue(row.get(1) instanceof ArrayLiteral);
ArrayLiteral arrayLiteral = (ArrayLiteral) row.get(1);
Assert.assertEquals(3, arrayLiteral.getChildren().size());
Assert.assertTrue(arrayLiteral.isAnalyzed);
for (int i = 1; i <= 3; ++ i) {
Expr child = arrayLiteral.getChild(i - 1);
Assert.assertTrue(child.isAnalyzed);
Assert.assertSame(PrimitiveType.INT, child.getType().getPrimitiveType());
Assert.assertTrue(child instanceof IntLiteral);
Assert.assertEquals(i, ((IntLiteral) child).getValue());
}
connectContext.setQueryId(new TUniqueId(2, 0));
insertStmt = parseAndAnalyze("insert into test.table1 values (1, []);");
row = ((SelectStmt) insertStmt.getQueryStmt()).getValueList().getFirstRow();
Assert.assertEquals(2, row.size());
Assert.assertTrue(row.get(1) instanceof ArrayLiteral);
arrayLiteral = (ArrayLiteral) row.get(1);
Assert.assertTrue(arrayLiteral.isAnalyzed);
Assert.assertTrue(arrayLiteral.getChildren().isEmpty());
Assert.assertSame(PrimitiveType.INT, ((ArrayType) arrayLiteral.getType()).getItemType().getPrimitiveType());
connectContext.setQueryId(new TUniqueId(3, 0));
ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "type not match", ()-> {
parseAndAnalyze("insert into test.table1 values (1, [[1, 2], [3, 4]]);");
});
}
}

9
fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
View File

@ -61,6 +61,7 @@ import java.nio.channels.SocketChannel;
import java.nio.file.Files;
import java.util.List;
import java.util.Map;
import java.util.UUID;
public class UtFrameUtils {
@ -125,6 +126,14 @@ public class UtFrameUtils {
return statementBases;
}
public static String generateRandomFeRunningDir(Class testSuiteClass) {
return generateRandomFeRunningDir(testSuiteClass.getSimpleName());
}
public static String generateRandomFeRunningDir(String testSuiteName) {
return "fe" + "/mocked/" + testSuiteName + "/" + UUID.randomUUID().toString() + "/";
}
public static int startFEServer(String runningDir) throws EnvVarNotSetException, IOException,
FeStartException, NotInitException, DdlException, InterruptedException {
// get DORIS_HOME