database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature-wip](array-type) Add some regression tests for nested array (#12322)

Browse Source 
#11392 made _input_block in each BetaRowsetReaders sharable. However, for some types (e.g. nested array with more than 1 depth), the _column_vector_batches in RowBlockV2 can be nested which means that there is a ColumnVectorBatch inside another ColumnVectorBatch. In this case, the data of inner ColumnVectorBatch
may be corrupted because the data of _input_block is copied shallowly to the _output_block.
This commit is contained in:
Adonis Ling
2022-09-05 14:05:24 +08:00
committed by GitHub
parent 3b104e334a
commit 8bfb89c100
5 changed files with 173 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
be/src/olap/rowset/beta_rowset_reader.cpp
View File

@ -203,8 +203,18 @@ Status BetaRowsetReader::init(RowsetReaderContext* read_context) {
}
_iterator.reset(final_iterator);
// The data in _input_block will be copied shallowly to _output_block.
// Therefore, for nestable fields, the _input_block can't be shared.
bool has_nestable_fields = false;
for (const auto* field : _input_schema->columns()) {
if (field != nullptr && field->get_sub_field_count() > 0) {
has_nestable_fields = true;
break;
}
}
// init input block
if (can_reuse_schema) {
if (can_reuse_schema && !has_nestable_fields) {
if (read_context->reuse_block == nullptr) {
read_context->reuse_block.reset(
new RowBlockV2(*_input_schema, std::min(1024, read_context->batch_size)));

2
fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
View File

@ -53,7 +53,7 @@ public class ArrayLiteral extends LiteralExpr {
}
}
if (itemType == Type.NULL || itemType == Type.INVALID) {
if (itemType == Type.INVALID) {
throw new AnalysisException("Invalid element type in ARRAY");
}

4
fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
View File

@ -508,6 +508,10 @@ public abstract class Type {
}
return new ArrayType(itemCompatibleType, arrayType1.getContainsNull() || arrayType2.getContainsNull());
} else if (t1.isArrayType() && t2.isNull()) {
return t1;
} else if (t1.isNull() && t2.isArrayType()) {
return t2;
}
return ScalarType.INVALID;

57
regression-test/data/load/insert/test_insert_nested_array.out Normal file
View File

@ -0,0 +1,57 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !select --
1 []
1 []
2 [NULL]
2 [NULL]
3 [[]]
3 [[]]
4 [[1, 2, 3], [4, 5, 6]]
4 [[1, 2, 3], [4, 5, 6]]
5 [[1, 2, 3], NULL, [4, 5, 6]]
5 [[1, 2, 3], NULL, [4, 5, 6]]
6 [[1, 2, NULL], NULL, [4, NULL, 6], NULL, [NULL, 8, 9]]
6 [[1, 2, NULL], NULL, [4, NULL, 6], NULL, [NULL, 8, 9]]
-- !select --
1 []
1 []
2 [NULL]
2 [NULL]
3 [[]]
3 [[]]
4 [[1, 2, 3], [4, 5, 6]]
4 [[1, 2, 3], [4, 5, 6]]
5 [[1, 2, 3], NULL, [4, 5, 6]]
5 [[1, 2, 3], NULL, [4, 5, 6]]
6 [[1, 2, NULL], NULL, [4, NULL, 6], NULL, [NULL, 8, 9]]
6 [[1, 2, NULL], NULL, [4, NULL, 6], NULL, [NULL, 8, 9]]
-- !select --
1 []
1 []
2 [NULL]
2 [NULL]
3 [[]]
3 [[]]
4 [[NULL]]
4 [[NULL]]
5 [[[]]]
5 [[[]]]
6 [[[NULL]], [[1], [2, 3]], [[4, 5, 6], NULL, NULL]]
6 [[[NULL]], [[1], [2, 3]], [[4, 5, 6], NULL, NULL]]
-- !select --
1 []
1 []
2 [NULL]
2 [NULL]
3 [[]]
3 [[]]
4 [[NULL]]
4 [[NULL]]
5 [[[]]]
5 [[[]]]
6 [[[NULL]], [[1], [2, 3]], [[4, 5, 6], NULL, NULL]]
6 [[[NULL]], [[1], [2, 3]], [[4, 5, 6], NULL, NULL]]

100
regression-test/suites/load/insert/test_insert_nested_array.groovy Normal file
View File

@ -0,0 +1,100 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_insert_nested_array", "load") {
def test_nested_array_2_depths = { enable_vectorized ->
sql "ADMIN SET FRONTEND CONFIG ('enable_array_type' = 'true')"
sql "set enable_vectorized_engine = ${enable_vectorized}"
def tableName
if (enable_vectorized) {
tableName = "nested_array_test_2_vectorized"
} else {
tableName = "nested_array_test_2_non_vectorized"
}
sql "DROP TABLE IF EXISTS ${tableName}"
sql """
CREATE TABLE ${tableName} (
`key` INT,
value ARRAY<ARRAY<INT>>
) DUPLICATE KEY (`key`) DISTRIBUTED BY HASH (`key`) BUCKETS 1
PROPERTIES ('replication_num' = '1')
"""
sql "INSERT INTO ${tableName} VALUES (1, [])"
sql "INSERT INTO ${tableName} VALUES (2, [null])"
sql "INSERT INTO ${tableName} VALUES (3, [[]])"
sql "INSERT INTO ${tableName} VALUES (4, [[1, 2, 3], [4, 5, 6]])"
sql "INSERT INTO ${tableName} VALUES (5, [[1, 2, 3], null, [4, 5, 6]])"
sql "INSERT INTO ${tableName} VALUES (6, [[1, 2, null], null, [4, null, 6], null, [null, 8, 9]])"
sql """
INSERT INTO ${tableName} VALUES
(1, []),
(2, [null]),
(3, [[]]),
(4, [[1, 2, 3], [4, 5, 6]]),
(5, [[1, 2, 3], null, [4, 5, 6]]),
(6, [[1, 2, null], null, [4, null, 6], null, [null, 8, 9]])
"""
qt_select "select * from ${tableName} order by `key`"
}
def test_nested_array_3_depths = { enable_vectorized ->
sql "ADMIN SET FRONTEND CONFIG ('enable_array_type' = 'true')"
sql "set enable_vectorized_engine = ${enable_vectorized}"
def tableName
if (enable_vectorized) {
tableName = "nested_array_test_3_vectorized"
} else {
tableName = "nested_array_test_3_non_vectorized"
}
sql "DROP TABLE IF EXISTS ${tableName}"
sql """
CREATE TABLE ${tableName} (
`key` INT,
value ARRAY<ARRAY<ARRAY<INT>>>
) DUPLICATE KEY (`key`) DISTRIBUTED BY HASH (`key`) BUCKETS 1
PROPERTIES ('replication_num' = '1')
"""
sql "INSERT INTO ${tableName} VALUES (1, [])"
sql "INSERT INTO ${tableName} VALUES (2, [null])"
sql "INSERT INTO ${tableName} VALUES (3, [[]])"
sql "INSERT INTO ${tableName} VALUES (4, [[null]])"
sql "INSERT INTO ${tableName} VALUES (5, [[[]]])"
sql "INSERT INTO ${tableName} VALUES (6, [[[null]], [[1], [2, 3]], [[4, 5, 6], null, null]])"
sql """
INSERT INTO ${tableName} VALUES
(1, []),
(2, [null]),
(3, [[]]),
(4, [[null]]),
(5, [[[]]]),
(6, [[[null]], [[1], [2, 3]], [[4, 5, 6], null, null]])
"""
qt_select "select * from ${tableName} order by `key`"
}
test_nested_array_2_depths.call(false)
test_nested_array_2_depths.call(true)
test_nested_array_3_depths.call(false)
test_nested_array_3_depths.call(true)
}