database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Feature](bitmap/hll)Support return bitmap/hll data in select statement in vectorization (#15224)

Browse Source 
Support return bitmap data in select statement in vectorization mode

In the scenario of using Bitmap to circle people, users need to return the Bitmap results to the upper layer, which is parsing the contents of the Bitmap to deal with high QPS query scenarios
This commit is contained in:
chenlinzhong
2022-12-27 14:49:24 +08:00
committed by GitHub
parent 5a8201320a
commit 524208ab3a
7 changed files with 132 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
be/src/runtime/result_writer.h
View File

@ -45,6 +45,10 @@ public:
virtual bool output_object_data() const { return _output_object_data; }
void set_output_object_data(bool output_object_data) {
_output_object_data = output_object_data;
}
static const std::string NULL_IN_CSV;
virtual void set_header_info(const std::string& header_type, const std::string& header) {
_header_type = header_type;

2
be/src/vec/columns/column.h
View File

@ -527,6 +527,8 @@ public:
virtual bool is_bitmap() const { return false; }
virtual bool is_hll() const { return false; }
// true if column has null element
virtual bool has_null() const { return false; }

1
be/src/vec/columns/column_complex.h
View File

@ -47,6 +47,7 @@ public:
bool is_numeric() const override { return false; }
bool is_bitmap() const override { return std::is_same_v<T, BitmapValue>; }
bool is_hll() const override { return std::is_same_v<T, HyperLogLog>; }
size_t size() const override { return data.size(); }

1
be/src/vec/columns/column_nullable.h
View File

@ -214,6 +214,7 @@ public:
bool is_nullable() const override { return true; }
bool is_bitmap() const override { return get_nested_column().is_bitmap(); }
bool is_hll() const override { return get_nested_column().is_hll(); }
bool is_column_decimal() const override { return get_nested_column().is_column_decimal(); }
bool is_column_string() const override { return get_nested_column().is_column_string(); }
bool is_column_array() const override { return get_nested_column().is_column_array(); }

26
be/src/vec/sink/vmysql_result_writer.cpp
View File

@ -17,11 +17,13 @@
#include "vec/sink/vmysql_result_writer.h"
#include "olap/hll.h"
#include "runtime/buffer_control_block.h"
#include "runtime/jsonb_value.h"
#include "runtime/large_int_value.h"
#include "runtime/runtime_state.h"
#include "vec/columns/column_array.h"
#include "vec/columns/column_complex.h"
#include "vec/columns/column_nullable.h"
#include "vec/columns/column_vector.h"
#include "vec/common/assert_cast.h"
@ -46,7 +48,7 @@ Status VMysqlResultWriter::init(RuntimeState* state) {
if (nullptr == _sinker) {
return Status::InternalError("sinker is NULL pointer.");
}
set_output_object_data(state->return_object_data_as_binary());
return Status::OK();
}
@ -91,7 +93,27 @@ Status VMysqlResultWriter::_add_one_column(const ColumnPtr& column_ptr,
}
if constexpr (type == TYPE_OBJECT) {
buf_ret = _buffer.push_null();
if (column->is_bitmap() && output_object_data()) {
const vectorized::ColumnComplexType<BitmapValue>* pColumnComplexType =
assert_cast<const vectorized::ColumnComplexType<BitmapValue>*>(
column.get());
BitmapValue bitmapValue = pColumnComplexType->get_element(i);
size_t size = bitmapValue.getSizeInBytes();
std::unique_ptr<char[]> buf = std::make_unique<char[]>(size);
bitmapValue.write(buf.get());
buf_ret = _buffer.push_string(buf.get(), size);
} else if (column->is_hll() && output_object_data()) {
const vectorized::ColumnComplexType<HyperLogLog>* pColumnComplexType =
assert_cast<const vectorized::ColumnComplexType<HyperLogLog>*>(
column.get());
HyperLogLog hyperLogLog = pColumnComplexType->get_element(i);
size_t size = hyperLogLog.max_serialized_size();
std::unique_ptr<char[]> buf = std::make_unique<char[]>(size);
hyperLogLog.serialize((uint8*)buf.get());
buf_ret = _buffer.push_string(buf.get(), size);
} else {
buf_ret = _buffer.push_null();
}
}
if constexpr (type == TYPE_VARCHAR) {
const auto string_val = column->get_data_at(i);

50
regression-test/suites/query_p1/return_binaray/test_return_binaray_hll.groovy Normal file
View File

@ -0,0 +1,50 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_return_binary_hll") {
def tableName="test_return_binary_hll"
sql "drop table if exists ${tableName};"
sql """
CREATE TABLE `${tableName}` (
`dt` int(11) NULL,
`page` varchar(10) NULL,
`user_id` hll HLL_UNION NULL
) ENGINE=OLAP
AGGREGATE KEY(`dt`, `page`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`dt`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
"storage_format" = "V2",
"disable_auto_compaction" = "false"
);
"""
sql """
insert into ${tableName} values(1,1,hll_hash(1)),(1,1,hll_hash(2)),(1,1,hll_hash(3)),(1,1,hll_hash(23332));
"""
sql "set enable_vectorized_engine=true;"
sql "set return_object_data_as_binary=false;"
def result1 = sql "select * from ${tableName}"
assertTrue(result1[0][2]==null);
sql "set enable_vectorized_engine=true;"
sql "set return_object_data_as_binary=true;"
def result2 = sql "select * from ${tableName}"
assertTrue(result2[0][2]!=null);
}

50
regression-test/suites/query_p1/return_binaray/test_return_binary_bitmap.groovy Normal file
View File

@ -0,0 +1,50 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_return_binary_bitmap") {
def tableName="test_return_binary_bitmap"
sql "drop table if exists ${tableName};"
sql """
CREATE TABLE `${tableName}` (
`dt` int(11) NULL,
`page` varchar(10) NULL,
`user_id` bitmap BITMAP_UNION NULL
) ENGINE=OLAP
AGGREGATE KEY(`dt`, `page`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`dt`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"in_memory" = "false",
"storage_format" = "V2",
"disable_auto_compaction" = "false"
);
"""
sql """
insert into ${tableName} values(1,1,to_bitmap(1)),(1,1,to_bitmap(2)),(1,1,to_bitmap(3)),(1,1,to_bitmap(23332));
"""
sql "set enable_vectorized_engine=true;"
sql "set return_object_data_as_binary=false;"
def result1 = sql "select * from ${tableName}"
assertTrue(result1[0][2]==null);
sql "set enable_vectorized_engine=true;"
sql "set return_object_data_as_binary=true;"
def result2 = sql "select * from ${tableName}"
assertTrue(result2[0][2]!=null);
}