database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Cherry-pick](branch-2.1) Pick "[Featrue](default value) Support bitmap_empty default value (#40364)" (#40487)

Browse Source 
## Proposed changes

Pick #40364 

<!--Describe your changes.-->
This commit is contained in:
abmdocrt
2024-09-09 16:57:38 +08:00
committed by GitHub
parent 2235c1abd3
commit 8f37eccbf2
10 changed files with 209 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
be/src/olap/partial_update_info.cpp
View File

@ -20,6 +20,10 @@
#include <gen_cpp/olap_file.pb.h>
#include "olap/tablet_schema.h"
#include "olap/utils.h"
#include "util/bitmap_value.h"
#include "vec/common/assert_cast.h"
#include "vec/core/block.h"
namespace doris {
@ -141,6 +145,11 @@ void PartialUpdateInfo::_generate_default_values_for_missing_cids(
DateV2Value<DateV2ValueType> dv;
dv.from_unixtime(timestamp_ms / 1000, timezone);
default_value = dv.debug_string();
} else if (UNLIKELY(column.type() == FieldType::OLAP_FIELD_TYPE_OBJECT &&
to_lower(column.default_value()).find(to_lower("BITMAP_EMPTY")) !=
std::string::npos)) {
BitmapValue v = BitmapValue {};
default_value = v.to_string();
} else {
default_value = tablet_schema.column(cur_cid).default_value();
}

1
fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
View File

@ -111,6 +111,7 @@ BINARY: 'BINARY';
BINLOG: 'BINLOG';
BITAND: 'BITAND';
BITMAP: 'BITMAP';
BITMAP_EMPTY: 'BITMAP_EMPTY';
BITMAP_UNION: 'BITMAP_UNION';
BITOR: 'BITOR';
BITXOR: 'BITXOR';

5
fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
View File

@ -589,8 +589,8 @@ columnDef
(aggType=aggTypeDef)?
((NOT)? NULL)?
(AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)?
(DEFAULT (nullValue=NULL | INTEGER_VALUE | stringValue=STRING_LITERAL| CURRENT_DATE
| defaultTimestamp=CURRENT_TIMESTAMP (LEFT_PAREN defaultValuePrecision=number RIGHT_PAREN)?))?
(DEFAULT (nullValue=NULL | INTEGER_VALUE | DECIMAL_VALUE | BITMAP_EMPTY | stringValue=STRING_LITERAL
| CURRENT_DATE | defaultTimestamp=CURRENT_TIMESTAMP (LEFT_PAREN defaultValuePrecision=number RIGHT_PAREN)?))?
(ON UPDATE CURRENT_TIMESTAMP (LEFT_PAREN onUpdateValuePrecision=number RIGHT_PAREN)?)?
(COMMENT comment=STRING_LITERAL)?
;
@ -1038,6 +1038,7 @@ nonReserved
| BIN
| BITAND
| BITMAP
| BITMAP_EMPTY
| BITMAP_UNION
| BITOR
| BITXOR

6
fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
View File

@ -367,8 +367,10 @@ public class ColumnDef {
}
if (type.getPrimitiveType() == PrimitiveType.BITMAP) {
if (defaultValue.isSet && defaultValue != DefaultValue.NULL_DEFAULT_VALUE) {
throw new AnalysisException("Bitmap type column can not set default value");
if (defaultValue.isSet && defaultValue != DefaultValue.NULL_DEFAULT_VALUE
&& !defaultValue.value.equals(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.value)) {
throw new AnalysisException("Bitmap type column default value only support null or "
+ DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.value);
}
defaultValue = DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE;
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -2695,6 +2695,8 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
}
} else if (ctx.CURRENT_DATE() != null) {
defaultValue = Optional.of(DefaultValue.CURRENT_DATE_DEFAULT_VALUE);
} else if (ctx.BITMAP_EMPTY() != null) {
defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE);
}
}
if (ctx.UPDATE() != null) {

8
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
View File

@ -286,13 +286,15 @@ public class ColumnDefinition {
}
defaultValue = Optional.of(DefaultValue.HLL_EMPTY_DEFAULT_VALUE);
} else if (type.isBitmapType()) {
if (defaultValue.isPresent() && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE) {
throw new AnalysisException("Bitmap type column can not set default value");
if (defaultValue.isPresent() && isOlap && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE
&& !defaultValue.get().getValue().equals(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.getValue())) {
throw new AnalysisException("Bitmap type column default value only support "
+ DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE);
}
defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE);
} else if (type.isArrayType() && defaultValue.isPresent() && isOlap
&& defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE && !defaultValue.get()
.getValue().equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.getValue())) {
.getValue().equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.getValue())) {
throw new AnalysisException("Array type column default value only support null or "
+ DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE);
} else if (type.isMapType()) {

4
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java
View File

@ -27,6 +27,8 @@ public class DefaultValue {
public static String CURRENT_DATE = "CURRENT_DATE";
public static String CURRENT_TIMESTAMP = "CURRENT_TIMESTAMP";
public static String NOW = "now";
public static String HLL_EMPTY = "HLL_EMPTY";
public static String BITMAP_EMPTY = "BITMAP_EMPTY";
public static DefaultValue CURRENT_DATE_DEFAULT_VALUE = new DefaultValue(CURRENT_DATE, CURRENT_DATE.toLowerCase());
public static DefaultValue CURRENT_TIMESTAMP_DEFAULT_VALUE = new DefaultValue(CURRENT_TIMESTAMP, NOW);
// default null
@ -36,7 +38,7 @@ public class DefaultValue {
// default "value", "0" means empty hll
public static DefaultValue HLL_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO);
// default "value", "0" means empty bitmap
public static DefaultValue BITMAP_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO);
public static DefaultValue BITMAP_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO, BITMAP_EMPTY);
// default "value", "[]" means empty array
public static DefaultValue ARRAY_EMPTY_DEFAULT_VALUE = new DefaultValue("[]");

43
regression-test/data/correctness_p0/test_default_bitmap_empty.out Normal file
View File

@ -0,0 +1,43 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !insert_into1 --
0
0
0
0
-- !stream_load_csv1 --
0
0
0
0
0
0
-- !select_2 --
0
0
0
0
-- !stream_load_csv2 --
0
0
0
0
0
0
-- !insert_into3 --
0
0
0
0
-- !stream_load_csv3 --
0
0
0
0
0
0

2
regression-test/data/correctness_p0/test_default_bitmap_empty_streamload.csv Normal file
View File

@ -0,0 +1,2 @@
5,5
6,6
1 5 5
2 6 6

137
regression-test/suites/correctness_p0/test_default_bitmap_empty.groovy Normal file
View File

@ -0,0 +1,137 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_default_bitmap_empty") {
sql "SET enable_nereids_planner=true"
sql "SET enable_fallback_to_original_planner=false"
def tableName = "test_default_bitmap_empty"
sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
CREATE TABLE IF NOT EXISTS ${tableName}
(
k TINYINT,
v1 bitmap NOT NULL DEFAULT bitmap_empty,
v2 INT
)
UNIQUE KEY(K)
DISTRIBUTED BY HASH(k)
PROPERTIES("replication_num" = "1");
"""
// test insert into.
sql " insert into ${tableName} (k, v2) values (1, 1); "
sql " insert into ${tableName} (k, v2) values (2, 2); "
sql " insert into ${tableName} (k, v2) values (3, 3); "
sql " insert into ${tableName} (k, v2) values (4, 4); "
sql "sync"
qt_insert_into1 """ select bitmap_count(v1) from ${tableName}; """
// test csv stream load.
streamLoad {
table "${tableName}"
set 'column_separator', ','
set 'columns', 'k, v1=bitmap_empty(), v2'
file 'test_default_bitmap_empty_streamload.csv'
time 10000 // limit inflight 10s
}
sql "sync"
qt_stream_load_csv1 """ select bitmap_count(v1) from ${tableName}; """
// test partial update
sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
CREATE TABLE IF NOT EXISTS ${tableName}
(
k TINYINT,
v1 bitmap NOT NULL DEFAULT bitmap_empty,
v2 INT
)
UNIQUE KEY(K)
DISTRIBUTED BY HASH(k)
PROPERTIES("replication_num" = "1");
"""
sql "set enable_unique_key_partial_update=true;"
sql "set enable_insert_strict=false;"
sql " insert into ${tableName} (k, v2) values (1, 1); "
sql " insert into ${tableName} (k, v2) values (2, 2); "
sql " insert into ${tableName} (k, v2) values (3, 3); "
sql " insert into ${tableName} (k, v2) values (4, 4); "
sql "sync"
qt_select_2 "select bitmap_count(v1) from ${tableName};"
streamLoad {
table "${tableName}"
set 'partial_columns', 'true'
set 'column_separator', ','
set 'columns', 'k, v2'
file 'test_default_bitmap_empty_streamload.csv'
time 10000 // limit inflight 10s
}
sql "sync"
qt_stream_load_csv2 """ select bitmap_count(v1) from ${tableName}; """
sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
CREATE TABLE IF NOT EXISTS ${tableName}
(
k TINYINT,
v1 bitmap BITMAP_UNION default BITMAP_EMPTY,
v2 INT replace_if_not_null
)
aggregate KEY(K)
DISTRIBUTED BY HASH(k)
PROPERTIES("replication_num" = "1");
"""
// test insert into.
sql " insert into ${tableName} (k, v2) values (1, 1); "
sql " insert into ${tableName} (k, v2) values (2, 2); "
sql " insert into ${tableName} (k, v2) values (3, 3); "
sql " insert into ${tableName} (k, v2) values (4, 4); "
sql "sync"
qt_insert_into3 """ select bitmap_count(v1) from ${tableName}; """
// test csv stream load.
streamLoad {
table "${tableName}"
set 'column_separator', ','
set 'columns', 'k, v1=bitmap_empty(), v2'
file 'test_default_bitmap_empty_streamload.csv'
time 10000 // limit inflight 10s
}
sql "sync"
qt_stream_load_csv3 """ select bitmap_count(v1) from ${tableName}; """
}