database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Refact](type system) refact serde for type system and pb serde impl (#18627)

Browse Source
This commit is contained in:
amory
2023-04-18 14:13:56 +08:00
committed by GitHub
parent 18898db09d
commit 564446e52f
50 changed files with 1361 additions and 505 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
be/src/vec/CMakeLists.txt
View File

@ -82,6 +82,19 @@ set(VEC_FILES
core/field.cpp
core/sort_block.cpp
core/materialize_block.cpp
data_types/serde/data_type_serde.cpp
data_types/serde/data_type_map_serde.cpp
data_types/serde/data_type_array_serde.cpp
data_types/serde/data_type_struct_serde.cpp
data_types/serde/data_type_number_serde.cpp
data_types/serde/data_type_string_serde.cpp
data_types/serde/data_type_decimal_serde.cpp
data_types/serde/data_type_object_serde.cpp
data_types/serde/data_type_fixedlengthobject_serde.cpp
data_types/serde/data_type_hll_serde.cpp
data_types/serde/data_type_bitmap_serde.cpp
data_types/serde/data_type_quantilestate_serde.cpp
data_types/serde/data_type_nullable_serde.cpp
data_types/data_type.cpp
data_types/data_type_array.cpp
data_types/data_type_struct.cpp

19
be/src/vec/aggregate_functions/aggregate_function_rpc.h
View File

@ -151,21 +151,10 @@ public:
int end, const DataTypes& argument_types) {
for (int i = 0; i < argument_types.size(); i++) {
PValues* arg = request.add_args();
if (auto* nullable = vectorized::check_and_get_column<const vectorized::ColumnNullable>(
*columns[i])) {
auto data_col = nullable->get_nested_column_ptr();
auto& null_col = nullable->get_null_map_column();
auto data_type = std::reinterpret_pointer_cast<const vectorized::DataTypeNullable>(
argument_types[i]);
data_col->get_data_at(0);
RPCFnImpl::convert_nullable_col_to_pvalue(
data_col->convert_to_full_column_if_const(), data_type->get_nested_type(),
null_col, arg, start, end);
} else {
RPCFnImpl::convert_col_to_pvalue<false>(
columns[i]->convert_to_full_column_if_const(), argument_types[i], arg,
start, end);
auto data_type = argument_types[i];
if (auto st = data_type->get_serde()->write_column_to_pb(*columns[i], *arg, start, end);
st != Status::OK()) {
return st;
}
}
return Status::OK();

5
be/src/vec/common/schema_util.cpp
View File

@ -20,23 +20,18 @@
#include <vec/common/schema_util.h>
#include <vec/core/field.h>
#include <vec/data_types/data_type_array.h>
#include <vec/data_types/data_type_object.h>
#include <vec/functions/simple_function_factory.h>
#include <vec/json/parse2column.h>
#include <vec/data_types/data_type_factory.hpp>
#include <vector>
#include "common/compiler_util.h"
#include "gen_cpp/FrontendService.h"
#include "gen_cpp/HeartbeatService_types.h"
#include "olap/rowset/rowset_writer_context.h"
#include "runtime/client_cache.h"
#include "runtime/descriptors.h"
#include "runtime/exec_env.h"
#include "util/thrift_rpc_helper.h"
#include "vec/columns/column.h"
#include "vec/columns/columns_number.h"
#include "vec/core/types.h"
#include "vec/data_types/data_type.h"
#include "vec/data_types/data_type_nullable.h"

4
be/src/vec/data_types/data_type.h
View File

@ -28,6 +28,7 @@
#include "vec/common/cow.h"
#include "vec/common/string_buffer.hpp"
#include "vec/core/types.h"
#include "vec/data_types/serde/data_type_serde.h"
#include "vec/io/reader_buffer.h"
namespace doris {
@ -75,6 +76,9 @@ public:
virtual std::string to_string(const IColumn& column, size_t row_num) const;
virtual Status from_string(ReadBuffer& rb, IColumn* column) const;
// get specific serializer or deserializer
virtual DataTypeSerDeSPtr get_serde() const = 0;
protected:
virtual String do_get_name() const;

5
be/src/vec/data_types/data_type_array.h
View File

@ -22,6 +22,7 @@
#include <gen_cpp/Types_types.h>
#include "serde/data_type_array_serde.h"
#include "vec/data_types/data_type.h"
namespace doris::vectorized {
@ -89,6 +90,10 @@ public:
std::string to_string(const IColumn& column, size_t row_num) const override;
void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
Status from_string(ReadBuffer& rb, IColumn* column) const override;
DataTypeSerDeSPtr get_serde() const override {
return std::make_shared<DataTypeArraySerDe>(nested->get_serde());
};
};
} // namespace doris::vectorized

5
be/src/vec/data_types/data_type_bitmap.h
View File

@ -18,6 +18,7 @@
#pragma once
#include <gen_cpp/Types_types.h>
#include "serde/data_type_bitmap_serde.h"
#include "util/bitmap_value.h"
#include "vec/columns/column.h"
#include "vec/columns/column_complex.h"
@ -87,6 +88,10 @@ public:
static void serialize_as_stream(const BitmapValue& value, BufferWritable& buf);
static void deserialize_as_stream(BitmapValue& value, BufferReadable& buf);
DataTypeSerDeSPtr get_serde() const override {
return std::make_shared<DataTypeBitMapSerDe>();
};
};
} // namespace doris::vectorized

4
be/src/vec/data_types/data_type_decimal.h
View File

@ -24,6 +24,7 @@
#include "common/config.h"
#include "runtime/define_primitive_type.h"
#include "serde/data_type_decimal_serde.h"
#include "vec/columns/column_decimal.h"
#include "vec/common/arithmetic_overflow.h"
#include "vec/common/typeid_cast.h"
@ -198,6 +199,9 @@ public:
std::string to_string(const IColumn& column, size_t row_num) const override;
void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
Status from_string(ReadBuffer& rb, IColumn* column) const override;
DataTypeSerDeSPtr get_serde() const override {
return std::make_shared<DataTypeDecimalSerDe<T>>();
};
/// Decimal specific

4
be/src/vec/data_types/data_type_fixed_length_object.h
View File

@ -17,6 +17,7 @@
#pragma once
#include "serde/data_type_fixedlengthobject_serde.h"
#include "vec/columns/column_fixed_length_object.h"
#include "vec/core/types.h"
#include "vec/data_types/data_type.h"
@ -61,6 +62,9 @@ public:
bool is_categorial() const override { return is_value_represented_by_integer(); }
bool can_be_inside_low_cardinality() const override { return false; }
DataTypeSerDeSPtr get_serde() const override {
return std::make_shared<DataTypeFixedLengthObjectSerDe>();
};
};
} // namespace doris::vectorized

3
be/src/vec/data_types/data_type_hll.h
View File

@ -17,6 +17,7 @@
#pragma once
#include "olap/hll.h"
#include "serde/data_type_hll_serde.h"
#include "vec/columns/column.h"
#include "vec/columns/column_complex.h"
#include "vec/core/types.h"
@ -82,6 +83,8 @@ public:
static void serialize_as_stream(const HyperLogLog& value, BufferWritable& buf);
static void deserialize_as_stream(HyperLogLog& value, BufferReadable& buf);
DataTypeSerDeSPtr get_serde() const override { return std::make_shared<DataTypeHLLSerDe>(); };
};
} // namespace doris::vectorized

3
be/src/vec/data_types/data_type_jsonb.h
View File

@ -66,6 +66,9 @@ public:
std::string to_string(const IColumn& column, size_t row_num) const override;
void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
Status from_string(ReadBuffer& rb, IColumn* column) const override;
DataTypeSerDeSPtr get_serde() const override {
return std::make_shared<DataTypeStringSerDe>();
};
private:
DataTypeString data_type_string;

4
be/src/vec/data_types/data_type_map.h
View File

@ -21,6 +21,7 @@
#pragma once
#include "gen_cpp/data.pb.h"
#include "serde/data_type_map_serde.h"
#include "util/stack_util.h"
#include "vec/columns/column_array.h"
#include "vec/columns/column_map.h"
@ -79,6 +80,9 @@ public:
std::string to_string(const IColumn& column, size_t row_num) const override;
void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
Status from_string(ReadBuffer& rb, IColumn* column) const override;
DataTypeSerDeSPtr get_serde() const override {
return std::make_shared<DataTypeMapSerDe>(key_type->get_serde(), value_type->get_serde());
};
};
} // namespace doris::vectorized

3
be/src/vec/data_types/data_type_nothing.h
View File

@ -68,6 +68,9 @@ public:
bool have_subtypes() const override { return false; }
bool cannot_be_stored_in_tables() const override { return true; }
DataTypeSerDeSPtr get_serde() const override {
LOG(FATAL) << get_name() << " not support serde";
};
};
} // namespace doris::vectorized

5
be/src/vec/data_types/data_type_nullable.h
View File

@ -21,6 +21,7 @@
#pragma once
#include "vec/data_types/data_type.h"
#include "vec/data_types/serde/data_type_nullable_serde.h"
namespace doris::vectorized {
@ -98,6 +99,10 @@ public:
const DataTypePtr& get_nested_type() const { return nested_data_type; }
bool is_null_literal() const override { return nested_data_type->is_null_literal(); }
DataTypeSerDeSPtr get_serde() const override {
return std::make_shared<DataTypeNullableSerDe>(nested_data_type->get_serde());
}
private:
DataTypePtr nested_data_type;
};

4
be/src/vec/data_types/data_type_number_base.h
View File

@ -22,6 +22,7 @@
#include <cstdint>
#include "serde/data_type_number_serde.h"
#include "vec/columns/column_vector.h"
#include "vec/core/types.h"
#include "vec/data_types/data_type.h"
@ -119,6 +120,9 @@ public:
Status from_string(ReadBuffer& rb, IColumn* column) const override;
bool is_null_literal() const override { return _is_null_literal; }
void set_null_literal(bool flag) { _is_null_literal = flag; }
DataTypeSerDeSPtr get_serde() const override {
return std::make_shared<DataTypeNumberSerDe<T>>();
};
private:
bool _is_null_literal = false;

6
be/src/vec/data_types/data_type_object.h
View File

@ -22,6 +22,9 @@
#include <vec/columns/column_object.h>
#include <vec/core/field.h>
#include <vec/data_types/data_type.h>
#include "serde/data_type_object_serde.h"
namespace doris::vectorized {
class DataTypeObject : public IDataType {
private:
@ -54,5 +57,8 @@ public:
[[noreturn]] Field get_default() const override {
LOG(FATAL) << "Method getDefault() is not implemented for data type " << get_name();
}
DataTypeSerDeSPtr get_serde() const override {
return std::make_shared<DataTypeObjectSerDe>();
};
};
} // namespace doris::vectorized

6
be/src/vec/data_types/data_type_quantilestate.h
View File

@ -16,11 +16,14 @@
// under the License.
#pragma once
#include "serde/data_type_quantilestate_serde.h"
#include "util/quantile_state.h"
#include "vec/columns/column.h"
#include "vec/columns/column_complex.h"
#include "vec/core/types.h"
#include "vec/data_types/data_type.h"
namespace doris::vectorized {
template <typename T>
class DataTypeQuantileState : public IDataType {
@ -82,6 +85,9 @@ public:
static void serialize_as_stream(const QuantileState<T>& value, BufferWritable& buf);
static void deserialize_as_stream(QuantileState<T>& value, BufferReadable& buf);
DataTypeSerDeSPtr get_serde() const override {
return std::make_shared<DataTypeQuantileStateSerDe<T>>();
};
};
using DataTypeQuantileStateDouble = DataTypeQuantileState<double>;

4
be/src/vec/data_types/data_type_string.h
View File

@ -22,6 +22,7 @@
#include <ostream>
#include "serde/data_type_string_serde.h"
#include "vec/data_types/data_type.h"
namespace doris::vectorized {
@ -65,6 +66,9 @@ public:
std::string to_string(const IColumn& column, size_t row_num) const override;
void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
Status from_string(ReadBuffer& rb, IColumn* column) const override;
DataTypeSerDeSPtr get_serde() const override {
return std::make_shared<DataTypeStringSerDe>();
};
};
} // namespace doris::vectorized

8
be/src/vec/data_types/data_type_struct.h
View File

@ -24,6 +24,7 @@
#include <optional>
#include "gen_cpp/data.pb.h"
#include "serde/data_type_struct_serde.h"
#include "util/stack_util.h"
#include "vec/columns/column_array.h"
#include "vec/columns/column_nullable.h"
@ -99,6 +100,13 @@ public:
std::string to_string(const IColumn& column, size_t row_num) const override;
void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
bool get_have_explicit_names() const { return have_explicit_names; }
DataTypeSerDeSPtr get_serde() const override {
DataTypeSerDeSPtrs ptrs;
for (auto iter = elems.begin(); iter < elems.end(); ++iter) {
ptrs.push_back((*iter)->get_serde());
}
return std::make_shared<DataTypeStructSerDe>(ptrs);
};
};
} // namespace doris::vectorized

3
be/src/vec/data_types/data_type_time.h
View File

@ -52,6 +52,9 @@ public:
using PromotedType = DataTypeNumber<NearestFieldType<Float64>>;
return std::make_shared<PromotedType>();
}
DataTypeSerDeSPtr get_serde() const override {
return std::make_shared<DataTypeNumberSerDe<Float64>>();
};
};
} // namespace doris::vectorized

3
be/src/vec/data_types/data_type_time_v2.h
View File

@ -85,6 +85,9 @@ public:
std::string to_string(const IColumn& column, size_t row_num) const override;
void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
Status from_string(ReadBuffer& rb, IColumn* column) const override;
DataTypeSerDeSPtr get_serde() const override {
return std::make_shared<DataTypeNumberSerDe<UInt64>>();
};
MutableColumnPtr create_column() const override;

24
be/src/vec/data_types/serde/data_type_array_serde.cpp Normal file
View File

@ -0,0 +1,24 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_array_serde.h"
#include "vec/columns/column_array.cpp"
namespace doris {
namespace vectorized {}
} // namespace doris

39
be/src/vec/data_types/serde/data_type_array_serde.h Normal file
View File

@ -0,0 +1,39 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_serde.h"
namespace doris {
namespace vectorized {
class DataTypeArraySerDe : public DataTypeSerDe {
public:
DataTypeArraySerDe(const DataTypeSerDeSPtr& _nested_serde) : nested_serde(_nested_serde) {}
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override {
LOG(FATAL) << "Not support write array column to pb";
}
Status read_column_from_pb(IColumn& column, const PValues& arg) const override {
LOG(FATAL) << "Not support read from pb to array";
}
private:
DataTypeSerDeSPtr nested_serde;
};
} // namespace vectorized
} // namespace doris

52
be/src/vec/data_types/serde/data_type_bitmap_serde.cpp Normal file
View File

@ -0,0 +1,52 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_bitmap_serde.h"
#include "vec/columns/column_complex.h"
#include "vec/data_types/data_type.h"
namespace doris {
namespace vectorized {
Status DataTypeBitMapSerDe::write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const {
auto ptype = result.mutable_type();
ptype->set_id(PGenericType::BITMAP);
auto& data_column = assert_cast<const ColumnBitmap&>(column);
int row_count = end - start;
result.mutable_bytes_value()->Reserve(row_count);
for (int row = start; row < end; ++row) {
auto& value = const_cast<BitmapValue&>(data_column.get_element(row));
std::string memory_buffer;
int bytesize = value.getSizeInBytes();
memory_buffer.resize(bytesize);
value.write_to(const_cast<char*>(memory_buffer.data()));
result.add_bytes_value(memory_buffer);
}
return Status::OK();
}
Status DataTypeBitMapSerDe::read_column_from_pb(IColumn& column, const PValues& arg) const {
auto& col = reinterpret_cast<ColumnBitmap&>(column);
for (int i = 0; i < arg.bytes_value_size(); ++i) {
BitmapValue value(arg.bytes_value(i).data());
col.insert_value(value);
}
return Status::OK();
}
} // namespace vectorized
} // namespace doris

30
be/src/vec/data_types/serde/data_type_bitmap_serde.h Normal file
View File

@ -0,0 +1,30 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_serde.h"
namespace doris {
namespace vectorized {
class DataTypeBitMapSerDe : public DataTypeSerDe {
public:
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override;
Status read_column_from_pb(IColumn& column, const PValues& arg) const override;
};
} // namespace vectorized
} // namespace doris

22
be/src/vec/data_types/serde/data_type_decimal_serde.cpp Normal file
View File

@ -0,0 +1,22 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_decimal_serde.h"
namespace doris {
namespace vectorized {} // namespace vectorized
} // namespace doris

75
be/src/vec/data_types/serde/data_type_decimal_serde.h Normal file
View File

@ -0,0 +1,75 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_serde.h"
#include "vec/columns/column_decimal.h"
namespace doris {
namespace vectorized {
template <typename T>
class DataTypeDecimalSerDe : public DataTypeSerDe {
static_assert(IsDecimalNumber<T>);
public:
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override;
Status read_column_from_pb(IColumn& column, const PValues& arg) const override;
};
template <typename T>
Status DataTypeDecimalSerDe<T>::write_column_to_pb(const IColumn& column, PValues& result,
int start, int end) const {
int row_count = end - start;
const auto* col = check_and_get_column<ColumnDecimal<T>>(column);
auto ptype = result.mutable_type();
if constexpr (std::is_same_v<T, Decimal<Int128>>) {
ptype->set_id(PGenericType::DECIMAL128);
} else if constexpr (std::is_same_v<T, Decimal<Int128I>>) {
ptype->set_id(PGenericType::DECIMAL128I);
} else if constexpr (std::is_same_v<T, Decimal<Int32>>) {
ptype->set_id(PGenericType::INT32);
} else if constexpr (std::is_same_v<T, Decimal<Int64>>) {
ptype->set_id(PGenericType::INT64);
} else {
return Status::NotSupported("unknown ColumnType for writing to pb");
}
result.mutable_bytes_value()->Reserve(row_count);
for (size_t row_num = start; row_num < end; ++row_num) {
StringRef single_data = col->get_data_at(row_num);
result.add_bytes_value(single_data.data, single_data.size);
}
return Status::OK();
}
template <typename T>
Status DataTypeDecimalSerDe<T>::read_column_from_pb(IColumn& column, const PValues& arg) const {
if constexpr (std::is_same_v<T, Decimal<Int128>> || std::is_same_v<T, Decimal<Int128I>> ||
std::is_same_v<T, Decimal<Int16>> || std::is_same_v<T, Decimal<Int32>>) {
column.resize(arg.bytes_value_size());
auto& data = reinterpret_cast<ColumnDecimal<T>&>(column).get_data();
for (int i = 0; i < arg.bytes_value_size(); ++i) {
data[i] = *(int128_t*)(arg.bytes_value(i).c_str());
}
return Status::OK();
}
return Status::NotSupported("unknown ColumnType for reading from pb");
}
} // namespace vectorized
} // namespace doris

22
be/src/vec/data_types/serde/data_type_fixedlengthobject_serde.cpp Normal file
View File

@ -0,0 +1,22 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_fixedlengthobject_serde.h"
namespace doris {
namespace vectorized {}
} // namespace doris

34
be/src/vec/data_types/serde/data_type_fixedlengthobject_serde.h Normal file
View File

@ -0,0 +1,34 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_serde.h"
namespace doris {
namespace vectorized {
class DataTypeFixedLengthObjectSerDe : public DataTypeSerDe {
public:
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override {
LOG(FATAL) << "Not support write FixedLengthObject column to pb";
}
Status read_column_from_pb(IColumn& column, const PValues& arg) const override {
LOG(FATAL) << "Not support read from pb to FixedLengthObject";
};
};
} // namespace vectorized
} // namespace doris

52
be/src/vec/data_types/serde/data_type_hll_serde.cpp Normal file
View File

@ -0,0 +1,52 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_hll_serde.h"
#include "vec/columns/column_complex.h"
namespace doris {
namespace vectorized {
Status DataTypeHLLSerDe::write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const {
auto ptype = result.mutable_type();
ptype->set_id(PGenericType::HLL);
auto& data_column = assert_cast<const ColumnHLL&>(column);
int row_count = end - start;
result.mutable_bytes_value()->Reserve(row_count);
for (size_t row_num = start; row_num < end; ++row_num) {
auto& value = const_cast<HyperLogLog&>(data_column.get_element(row_num));
std::string memory_buffer(value.max_serialized_size(), '0');
value.serialize((uint8_t*)memory_buffer.data());
result.add_bytes_value(memory_buffer.data(), memory_buffer.size());
}
return Status::OK();
}
Status DataTypeHLLSerDe::read_column_from_pb(IColumn& column, const PValues& arg) const {
auto& col = reinterpret_cast<ColumnHLL&>(column);
for (int i = 0; i < arg.bytes_value_size(); ++i) {
HyperLogLog value;
value.deserialize(Slice(arg.bytes_value(i)));
col.insert_value(value);
}
return Status::OK();
}
} // namespace vectorized
} // namespace doris

30
be/src/vec/data_types/serde/data_type_hll_serde.h Normal file
View File

@ -0,0 +1,30 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_serde.h"
namespace doris {
namespace vectorized {
class DataTypeHLLSerDe : public DataTypeSerDe {
public:
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override;
Status read_column_from_pb(IColumn& column, const PValues& arg) const override;
};
} // namespace vectorized
} // namespace doris

22
be/src/vec/data_types/serde/data_type_map_serde.cpp Normal file
View File

@ -0,0 +1,22 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_map_serde.h"
namespace doris {
namespace vectorized {}
} // namespace doris

41
be/src/vec/data_types/serde/data_type_map_serde.h Normal file
View File

@ -0,0 +1,41 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_serde.h"
namespace doris {
namespace vectorized {
class DataTypeMapSerDe : public DataTypeSerDe {
public:
DataTypeMapSerDe(const DataTypeSerDeSPtr& _key_serde, const DataTypeSerDeSPtr& _value_serde)
: key_serde(_key_serde), value_serde(_value_serde) {}
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override {
LOG(FATAL) << "Not support write map column to pb";
}
Status read_column_from_pb(IColumn& column, const PValues& arg) const override {
LOG(FATAL) << "Not support read from pb to map";
}
private:
DataTypeSerDeSPtr key_serde;
DataTypeSerDeSPtr value_serde;
};
} // namespace vectorized
} // namespace doris

63
be/src/vec/data_types/serde/data_type_nullable_serde.cpp Normal file
View File

@ -0,0 +1,63 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_nullable_serde.h"
#include "vec/columns/column_nullable.h"
namespace doris {
namespace vectorized {
Status DataTypeNullableSerDe::write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const {
int row_count = end - start;
auto& nullable_col = assert_cast<const ColumnNullable&>(column);
auto& null_col = nullable_col.get_null_map_column();
if (nullable_col.has_null(row_count)) {
result.set_has_null(true);
auto* null_map = result.mutable_null_map();
null_map->Reserve(row_count);
const auto* col = check_and_get_column<ColumnUInt8>(null_col);
auto& data = col->get_data();
null_map->Add(data.begin() + start, data.begin() + end);
}
return nested_serde->write_column_to_pb(nullable_col.get_nested_column(), result, start, end);
}
// read from PValues to column
Status DataTypeNullableSerDe::read_column_from_pb(IColumn& column, const PValues& arg) const {
auto& col = reinterpret_cast<ColumnNullable&>(column);
auto& null_map_data = col.get_null_map_data();
auto& nested = col.get_nested_column();
if (Status st = nested_serde->read_column_from_pb(nested, arg); st != Status::OK()) {
return st;
}
null_map_data.resize(nested.size());
if (arg.has_null()) {
for (int i = 0; i < arg.null_map_size(); ++i) {
null_map_data[i] = arg.null_map(i);
}
} else {
for (int i = 0; i < nested.size(); ++i) {
null_map_data[i] = false;
}
}
return Status::OK();
}
} // namespace vectorized
} // namespace doris

35
be/src/vec/data_types/serde/data_type_nullable_serde.h Normal file
View File

@ -0,0 +1,35 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_serde.h"
namespace doris {
namespace vectorized {
class DataTypeNullableSerDe : public DataTypeSerDe {
public:
DataTypeNullableSerDe(const DataTypeSerDeSPtr& _nested_serde) : nested_serde(_nested_serde) {}
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override;
Status read_column_from_pb(IColumn& column, const PValues& arg) const override;
private:
DataTypeSerDeSPtr nested_serde;
};
} // namespace vectorized
} // namespace doris

22
be/src/vec/data_types/serde/data_type_number_serde.cpp Normal file
View File

@ -0,0 +1,22 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_number_serde.h"
namespace doris {
namespace vectorized {} // namespace vectorized
} // namespace doris

161
be/src/vec/data_types/serde/data_type_number_serde.h Normal file
View File

@ -0,0 +1,161 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_serde.h"
#include "vec/columns/column_vector.h"
namespace doris {
namespace vectorized {
template <typename T>
class DataTypeNumberSerDe : public DataTypeSerDe {
static_assert(IsNumber<T>);
public:
using ColumnType = ColumnVector<T>;
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override;
Status read_column_from_pb(IColumn& column, const PValues& arg) const override;
};
template <typename T>
Status DataTypeNumberSerDe<T>::read_column_from_pb(IColumn& column, const PValues& arg) const {
if constexpr (std::is_same_v<T, UInt8> || std::is_same_v<T, UInt16> ||
std::is_same_v<T, UInt32>) {
column.resize(arg.uint32_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.uint32_value_size(); ++i) {
data[i] = arg.uint32_value(i);
}
} else if constexpr (std::is_same_v<T, Int8> || std::is_same_v<T, Int16> ||
std::is_same_v<T, Int32>) {
column.resize(arg.int32_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.int32_value_size(); ++i) {
data[i] = arg.int32_value(i);
}
} else if constexpr (std::is_same_v<T, UInt64>) {
column.resize(arg.uint64_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.uint64_value_size(); ++i) {
data[i] = arg.uint64_value(i);
}
} else if constexpr (std::is_same_v<T, Int64>) {
column.resize(arg.int64_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.int64_value_size(); ++i) {
data[i] = arg.int64_value(i);
}
} else if constexpr (std::is_same_v<T, float>) {
column.resize(arg.float_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.float_value_size(); ++i) {
data[i] = arg.float_value(i);
}
} else if constexpr (std::is_same_v<T, double>) {
column.resize(arg.double_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.float_value_size(); ++i) {
data[i] = arg.double_value(i);
}
} else if constexpr (std::is_same_v<T, Int128>) {
column.resize(arg.bytes_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
for (int i = 0; i < arg.bytes_value_size(); ++i) {
data[i] = *(int128_t*)(arg.bytes_value(i).c_str());
}
} else {
return Status::NotSupported("unknown ColumnType for reading from pb");
}
return Status::OK();
}
template <typename T>
Status DataTypeNumberSerDe<T>::write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const {
int row_count = end - start;
auto ptype = result.mutable_type();
const auto* col = check_and_get_column<ColumnVector<T>>(column);
if constexpr (std::is_same_v<T, Int128>) {
ptype->set_id(PGenericType::INT128);
result.mutable_bytes_value()->Reserve(row_count);
for (size_t row_num = start; row_num < end; ++row_num) {
StringRef single_data = col->get_data_at(row_num);
result.add_bytes_value(single_data.data, single_data.size);
}
return Status::OK();
}
auto& data = col->get_data();
if constexpr (std::is_same_v<T, UInt8>) {
ptype->set_id(PGenericType::UINT8);
auto* values = result.mutable_uint32_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else if constexpr (std::is_same_v<T, UInt16>) {
ptype->set_id(PGenericType::UINT16);
auto* values = result.mutable_uint32_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else if constexpr (std::is_same_v<T, UInt32>) {
ptype->set_id(PGenericType::UINT32);
auto* values = result.mutable_uint32_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else if constexpr (std::is_same_v<T, UInt64>) {
ptype->set_id(PGenericType::UINT64);
auto* values = result.mutable_uint64_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else if constexpr (std::is_same_v<T, Int8>) {
ptype->set_id(PGenericType::INT8);
auto* values = result.mutable_int32_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else if constexpr (std::is_same_v<T, Int16>) {
ptype->set_id(PGenericType::INT16);
auto* values = result.mutable_int32_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else if constexpr (std::is_same_v<T, Int32>) {
ptype->set_id(PGenericType::INT32);
auto* values = result.mutable_int32_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else if constexpr (std::is_same_v<T, Int64>) {
ptype->set_id(PGenericType::INT64);
auto* values = result.mutable_int64_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else if constexpr (std::is_same_v<T, float>) {
ptype->set_id(PGenericType::FLOAT);
auto* values = result.mutable_float_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else if constexpr (std::is_same_v<T, double>) {
ptype->set_id(PGenericType::DOUBLE);
auto* values = result.mutable_double_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
} else {
return Status::NotSupported("unknown ColumnType for writing to pb");
}
return Status::OK();
}
} // namespace vectorized
} // namespace doris

22
be/src/vec/data_types/serde/data_type_object_serde.cpp Normal file
View File

@ -0,0 +1,22 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_object_serde.h"
namespace doris {
namespace vectorized {}
} // namespace doris

34
be/src/vec/data_types/serde/data_type_object_serde.h Normal file
View File

@ -0,0 +1,34 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_serde.h"
namespace doris {
namespace vectorized {
class DataTypeObjectSerDe : public DataTypeSerDe {
public:
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override {
LOG(FATAL) << "Not support write object column to pb";
}
Status read_column_from_pb(IColumn& column, const PValues& arg) const override {
LOG(FATAL) << "Not support read from pb to object";
}
};
} // namespace vectorized
} // namespace doris

22
be/src/vec/data_types/serde/data_type_quantilestate_serde.cpp Normal file
View File

@ -0,0 +1,22 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_quantilestate_serde.h"
namespace doris {
namespace vectorized {} // namespace vectorized
} // namespace doris

52
be/src/vec/data_types/serde/data_type_quantilestate_serde.h Normal file
View File

@ -0,0 +1,52 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_serde.h"
namespace doris {
namespace vectorized {
template <typename T>
class DataTypeQuantileStateSerDe : public DataTypeSerDe {
public:
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override;
Status read_column_from_pb(IColumn& column, const PValues& arg) const override;
};
template <typename T>
Status DataTypeQuantileStateSerDe<T>::write_column_to_pb(const IColumn& column, PValues& result,
int start, int end) const {
result.mutable_bytes_value()->Reserve(end - start);
for (size_t row_num = start; row_num < end; ++row_num) {
StringRef data = column.get_data_at(row_num);
result.add_bytes_value(data.to_string());
}
return Status::OK();
}
template <typename T>
Status DataTypeQuantileStateSerDe<T>::read_column_from_pb(IColumn& column,
const PValues& arg) const {
column.reserve(arg.bytes_value_size());
for (int i = 0; i < arg.bytes_value_size(); ++i) {
column.insert_data(arg.bytes_value(i).c_str(), arg.bytes_value(i).size());
}
return Status::OK();
}
} // namespace vectorized
} // namespace doris

24
be/src/vec/data_types/serde/data_type_serde.cpp Normal file
View File

@ -0,0 +1,24 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_serde.h"
namespace doris {
namespace vectorized {
DataTypeSerDe::DataTypeSerDe() = default;
DataTypeSerDe::~DataTypeSerDe() = default;
} // namespace vectorized
} // namespace doris

68
be/src/vec/data_types/serde/data_type_serde.h Normal file
View File

@ -0,0 +1,68 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include <string>
#include "common/status.h"
#include "vec/columns/column.h"
namespace doris {
namespace vectorized {
// Deserialize means read from different file format or memory format,
// for example read from arrow, read from parquet.
// Serialize means write the column cell or the total column into another
// memory format or file format.
// Every data type should implement this interface.
// In the past, there are many switch cases in the code and we do not know
// how many cases or files we has to modify when we add a new type. And also
// it is very difficult to add a new read file format or write file format because
// the developer does not know how many datatypes has to deal.
class DataTypeSerDe {
public:
DataTypeSerDe();
virtual ~DataTypeSerDe();
// Protobuf serializer and deserializer
virtual Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const = 0;
virtual Status read_column_from_pb(IColumn& column, const PValues& arg) const = 0;
// JSONB serializer and deserializer
// MySQL serializer and deserializer
// Thrift serializer and deserializer
// ORC serializer and deserializer
// CSV serializer and deserializer
// JSON serializer and deserializer
// Arrow serializer and deserializer
};
using DataTypeSerDeSPtr = std::shared_ptr<DataTypeSerDe>;
using DataTypeSerDeSPtrs = std::vector<DataTypeSerDeSPtr>;
} // namespace vectorized
} // namespace doris

43
be/src/vec/data_types/serde/data_type_string_serde.cpp Normal file
View File

@ -0,0 +1,43 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_string_serde.h"
#include "vec/columns/column_string.h"
namespace doris {
namespace vectorized {
Status DataTypeStringSerDe::write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const {
result.mutable_bytes_value()->Reserve(end - start);
for (size_t row_num = start; row_num < end; ++row_num) {
StringRef data = column.get_data_at(row_num);
result.add_string_value(data.to_string());
}
return Status::OK();
}
Status DataTypeStringSerDe::read_column_from_pb(IColumn& column, const PValues& arg) const {
auto& col = reinterpret_cast<ColumnString&>(column);
col.reserve(arg.string_value_size());
for (int i = 0; i < arg.string_value_size(); ++i) {
column.insert_data(arg.string_value(i).c_str(), arg.string_value(i).size());
}
return Status::OK();
}
} // namespace vectorized
} // namespace doris

30
be/src/vec/data_types/serde/data_type_string_serde.h Normal file
View File

@ -0,0 +1,30 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_serde.h"
namespace doris {
namespace vectorized {
class DataTypeStringSerDe : public DataTypeSerDe {
public:
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override;
Status read_column_from_pb(IColumn& column, const PValues& arg) const override;
};
} // namespace vectorized
} // namespace doris

22
be/src/vec/data_types/serde/data_type_struct_serde.cpp Normal file
View File

@ -0,0 +1,22 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_struct_serde.h"
namespace doris {
namespace vectorized {}
} // namespace doris

40
be/src/vec/data_types/serde/data_type_struct_serde.h Normal file
View File

@ -0,0 +1,40 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "data_type_serde.h"
namespace doris {
namespace vectorized {
class DataTypeStructSerDe : public DataTypeSerDe {
public:
DataTypeStructSerDe(const DataTypeSerDeSPtrs& _elemSerDeSPtrs)
: elemSerDeSPtrs(_elemSerDeSPtrs) {}
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override {
LOG(FATAL) << "Not support write struct column to pb";
}
Status read_column_from_pb(IColumn& column, const PValues& arg) const override {
LOG(FATAL) << "Not support read from pb to strut";
}
private:
DataTypeSerDeSPtrs elemSerDeSPtrs;
};
} // namespace vectorized
} // namespace doris

479
be/src/vec/functions/function_rpc.cpp
View File

@ -36,446 +36,6 @@ RPCFnImpl::RPCFnImpl(const TFunction& fn) : _fn(fn) {
_fn.scalar_fn.symbol);
}
void RPCFnImpl::convert_nullable_col_to_pvalue(const ColumnPtr& column,
const DataTypePtr& data_type,
const ColumnUInt8& null_col, PValues* arg, int start,
int end) {
int row_count = end - start;
if (column->has_null(row_count)) {
auto* null_map = arg->mutable_null_map();
null_map->Reserve(row_count);
const auto* col = check_and_get_column<ColumnUInt8>(null_col);
auto& data = col->get_data();
null_map->Add(data.begin() + start, data.begin() + end);
RPCFnImpl::convert_col_to_pvalue<true>(column, data_type, arg, start, end);
} else {
RPCFnImpl::convert_col_to_pvalue<false>(column, data_type, arg, start, end);
}
}
template <bool nullable>
void RPCFnImpl::convert_col_to_pvalue(const ColumnPtr& column, const DataTypePtr& data_type,
PValues* arg, int start, int end) {
int row_count = end - start;
PGenericType* ptype = arg->mutable_type();
switch (data_type->get_type_id()) {
case TypeIndex::UInt8: {
ptype->set_id(PGenericType::UINT8);
auto* values = arg->mutable_bool_value();
values->Reserve(row_count);
const auto* col = check_and_get_column<ColumnUInt8>(column);
auto& data = col->get_data();
values->Add(data.begin() + start, data.begin() + end);
break;
}
case TypeIndex::UInt16: {
ptype->set_id(PGenericType::UINT16);
auto* values = arg->mutable_uint32_value();
values->Reserve(row_count);
const auto* col = check_and_get_column<ColumnUInt16>(column);
auto& data = col->get_data();
values->Add(data.begin() + start, data.begin() + end);
break;
}
case TypeIndex::UInt32: {
ptype->set_id(PGenericType::UINT32);
auto* values = arg->mutable_uint32_value();
values->Reserve(row_count);
const auto* col = check_and_get_column<ColumnUInt32>(column);
auto& data = col->get_data();
values->Add(data.begin() + start, data.begin() + end);
break;
}
case TypeIndex::UInt64: {
ptype->set_id(PGenericType::UINT64);
auto* values = arg->mutable_uint64_value();
values->Reserve(row_count);
const auto* col = check_and_get_column<ColumnUInt64>(column);
auto& data = col->get_data();
values->Add(data.begin() + start, data.begin() + end);
break;
}
case TypeIndex::UInt128: {
ptype->set_id(PGenericType::UINT128);
arg->mutable_bytes_value()->Reserve(row_count);
for (size_t row_num = start; row_num < end; ++row_num) {
if constexpr (nullable) {
if (column->is_null_at(row_num)) {
arg->add_bytes_value(nullptr);
} else {
StringRef data = column->get_data_at(row_num);
arg->add_bytes_value(data.data, data.size);
}
} else {
StringRef data = column->get_data_at(row_num);
arg->add_bytes_value(data.data, data.size);
}
}
break;
}
case TypeIndex::Int8: {
ptype->set_id(PGenericType::INT8);
auto* values = arg->mutable_int32_value();
values->Reserve(row_count);
const auto* col = check_and_get_column<ColumnInt8>(column);
auto& data = col->get_data();
values->Add(data.begin() + start, data.begin() + end);
break;
}
case TypeIndex::Int16: {
ptype->set_id(PGenericType::INT16);
auto* values = arg->mutable_int32_value();
values->Reserve(row_count);
const auto* col = check_and_get_column<ColumnInt16>(column);
auto& data = col->get_data();
values->Add(data.begin() + start, data.begin() + end);
break;
}
case TypeIndex::Int32: {
ptype->set_id(PGenericType::INT32);
auto* values = arg->mutable_int32_value();
values->Reserve(row_count);
const auto* col = check_and_get_column<ColumnInt32>(column);
auto& data = col->get_data();
values->Add(data.begin() + start, data.begin() + end);
break;
}
case TypeIndex::Int64: {
ptype->set_id(PGenericType::INT64);
auto* values = arg->mutable_int64_value();
values->Reserve(row_count);
const auto* col = check_and_get_column<ColumnInt64>(column);
auto& data = col->get_data();
values->Add(data.begin() + start, data.begin() + end);
break;
}
case TypeIndex::Int128: {
ptype->set_id(PGenericType::INT128);
arg->mutable_bytes_value()->Reserve(row_count);
for (size_t row_num = start; row_num < end; ++row_num) {
if constexpr (nullable) {
if (column->is_null_at(row_num)) {
arg->add_bytes_value(nullptr);
} else {
StringRef data = column->get_data_at(row_num);
arg->add_bytes_value(data.data, data.size);
}
} else {
StringRef data = column->get_data_at(row_num);
arg->add_bytes_value(data.data, data.size);
}
}
break;
}
case TypeIndex::Float32: {
ptype->set_id(PGenericType::FLOAT);
auto* values = arg->mutable_float_value();
values->Reserve(row_count);
const auto* col = check_and_get_column<ColumnFloat32>(column);
auto& data = col->get_data();
values->Add(data.begin() + start, data.begin() + end);
break;
}
case TypeIndex::Float64: {
ptype->set_id(PGenericType::DOUBLE);
auto* values = arg->mutable_double_value();
values->Reserve(row_count);
const auto* col = check_and_get_column<ColumnFloat64>(column);
auto& data = col->get_data();
values->Add(data.begin() + start, data.begin() + end);
break;
}
case TypeIndex::String: {
ptype->set_id(PGenericType::STRING);
arg->mutable_bytes_value()->Reserve(row_count);
for (size_t row_num = start; row_num < end; ++row_num) {
if constexpr (nullable) {
if (column->is_null_at(row_num)) {
arg->add_string_value(nullptr);
} else {
StringRef data = column->get_data_at(row_num);
arg->add_string_value(data.to_string());
}
} else {
StringRef data = column->get_data_at(row_num);
arg->add_string_value(data.to_string());
}
}
break;
}
case TypeIndex::Date: {
ptype->set_id(PGenericType::DATE);
arg->mutable_datetime_value()->Reserve(row_count);
for (size_t row_num = start; row_num < end; ++row_num) {
PDateTime* date_time = arg->add_datetime_value();
if constexpr (nullable) {
if (!column->is_null_at(row_num)) {
VecDateTimeValue v =
VecDateTimeValue::create_from_olap_date(column->get_int(row_num));
date_time->set_day(v.day());
date_time->set_month(v.month());
date_time->set_year(v.year());
}
} else {
VecDateTimeValue v =
VecDateTimeValue::create_from_olap_date(column->get_int(row_num));
date_time->set_day(v.day());
date_time->set_month(v.month());
date_time->set_year(v.year());
}
}
break;
}
case TypeIndex::DateTime: {
ptype->set_id(PGenericType::DATETIME);
arg->mutable_datetime_value()->Reserve(row_count);
for (size_t row_num = start; row_num < end; ++row_num) {
PDateTime* date_time = arg->add_datetime_value();
if constexpr (nullable) {
if (!column->is_null_at(row_num)) {
VecDateTimeValue v =
VecDateTimeValue::create_from_olap_datetime(column->get_int(row_num));
date_time->set_day(v.day());
date_time->set_month(v.month());
date_time->set_year(v.year());
date_time->set_hour(v.hour());
date_time->set_minute(v.minute());
date_time->set_second(v.second());
}
} else {
VecDateTimeValue v =
VecDateTimeValue::create_from_olap_datetime(column->get_int(row_num));
date_time->set_day(v.day());
date_time->set_month(v.month());
date_time->set_year(v.year());
date_time->set_hour(v.hour());
date_time->set_minute(v.minute());
date_time->set_second(v.second());
}
}
break;
}
case TypeIndex::BitMap: {
ptype->set_id(PGenericType::BITMAP);
arg->mutable_bytes_value()->Reserve(row_count);
for (size_t row_num = start; row_num < end; ++row_num) {
if constexpr (nullable) {
if (column->is_null_at(row_num)) {
arg->add_bytes_value(nullptr);
} else {
StringRef data = column->get_data_at(row_num);
arg->add_bytes_value(data.data, data.size);
}
} else {
StringRef data = column->get_data_at(row_num);
arg->add_bytes_value(data.data, data.size);
}
}
break;
}
case TypeIndex::HLL: {
ptype->set_id(PGenericType::HLL);
arg->mutable_bytes_value()->Reserve(row_count);
for (size_t row_num = start; row_num < end; ++row_num) {
if constexpr (nullable) {
if (column->is_null_at(row_num)) {
arg->add_bytes_value(nullptr);
} else {
StringRef data = column->get_data_at(row_num);
arg->add_bytes_value(data.data, data.size);
}
} else {
StringRef data = column->get_data_at(row_num);
arg->add_bytes_value(data.data, data.size);
}
}
break;
}
case TypeIndex::QuantileState: {
ptype->set_id(PGenericType::QUANTILE_STATE);
arg->mutable_bytes_value()->Reserve(row_count);
for (size_t row_num = start; row_num < end; ++row_num) {
if constexpr (nullable) {
if (column->is_null_at(row_num)) {
arg->add_bytes_value(nullptr);
} else {
StringRef data = column->get_data_at(row_num);
arg->add_bytes_value(data.data, data.size);
}
} else {
StringRef data = column->get_data_at(row_num);
arg->add_bytes_value(data.data, data.size);
}
}
break;
}
default:
LOG(INFO) << "unknown type: " << data_type->get_name();
ptype->set_id(PGenericType::UNKNOWN);
break;
}
}
template <bool nullable>
void RPCFnImpl::_convert_to_column(MutableColumnPtr& column, const PValues& result) {
switch (result.type().id()) {
case PGenericType::UINT8: {
column->reserve(result.uint32_value_size());
column->resize(result.uint32_value_size());
auto& data = reinterpret_cast<ColumnUInt8*>(column.get())->get_data();
for (int i = 0; i < result.uint32_value_size(); ++i) {
data[i] = result.uint32_value(i);
}
break;
}
case PGenericType::UINT16: {
column->reserve(result.uint32_value_size());
column->resize(result.uint32_value_size());
auto& data = reinterpret_cast<ColumnUInt16*>(column.get())->get_data();
for (int i = 0; i < result.uint32_value_size(); ++i) {
data[i] = result.uint32_value(i);
}
break;
}
case PGenericType::UINT32: {
column->reserve(result.uint32_value_size());
column->resize(result.uint32_value_size());
auto& data = reinterpret_cast<ColumnUInt32*>(column.get())->get_data();
for (int i = 0; i < result.uint32_value_size(); ++i) {
data[i] = result.uint32_value(i);
}
break;
}
case PGenericType::UINT64: {
column->reserve(result.uint64_value_size());
column->resize(result.uint64_value_size());
auto& data = reinterpret_cast<ColumnUInt64*>(column.get())->get_data();
for (int i = 0; i < result.uint64_value_size(); ++i) {
data[i] = result.uint64_value(i);
}
break;
}
case PGenericType::INT8: {
column->reserve(result.int32_value_size());
column->resize(result.int32_value_size());
auto& data = reinterpret_cast<ColumnInt16*>(column.get())->get_data();
for (int i = 0; i < result.int32_value_size(); ++i) {
data[i] = result.int32_value(i);
}
break;
}
case PGenericType::INT16: {
column->reserve(result.int32_value_size());
column->resize(result.int32_value_size());
auto& data = reinterpret_cast<ColumnInt16*>(column.get())->get_data();
for (int i = 0; i < result.int32_value_size(); ++i) {
data[i] = result.int32_value(i);
}
break;
}
case PGenericType::INT32: {
column->reserve(result.int32_value_size());
column->resize(result.int32_value_size());
auto& data = reinterpret_cast<ColumnInt32*>(column.get())->get_data();
for (int i = 0; i < result.int32_value_size(); ++i) {
data[i] = result.int32_value(i);
}
break;
}
case PGenericType::INT64: {
column->reserve(result.int64_value_size());
column->resize(result.int64_value_size());
auto& data = reinterpret_cast<ColumnInt64*>(column.get())->get_data();
for (int i = 0; i < result.int64_value_size(); ++i) {
data[i] = result.int64_value(i);
}
break;
}
case PGenericType::DATE:
case PGenericType::DATETIME: {
column->reserve(result.datetime_value_size());
column->resize(result.datetime_value_size());
auto& data = reinterpret_cast<ColumnInt64*>(column.get())->get_data();
for (int i = 0; i < result.datetime_value_size(); ++i) {
VecDateTimeValue v;
PDateTime pv = result.datetime_value(i);
v.set_time(pv.year(), pv.month(), pv.day(), pv.hour(), pv.minute(), pv.minute());
data[i] = binary_cast<VecDateTimeValue, Int64>(v);
}
break;
}
case PGenericType::FLOAT: {
column->reserve(result.float_value_size());
column->resize(result.float_value_size());
auto& data = reinterpret_cast<ColumnFloat32*>(column.get())->get_data();
for (int i = 0; i < result.float_value_size(); ++i) {
data[i] = result.float_value(i);
}
break;
}
case PGenericType::DOUBLE: {
column->reserve(result.double_value_size());
column->resize(result.double_value_size());
auto& data = reinterpret_cast<ColumnFloat64*>(column.get())->get_data();
for (int i = 0; i < result.double_value_size(); ++i) {
data[i] = result.double_value(i);
}
break;
}
case PGenericType::INT128: {
column->reserve(result.bytes_value_size());
column->resize(result.bytes_value_size());
auto& data = reinterpret_cast<ColumnInt128*>(column.get())->get_data();
for (int i = 0; i < result.bytes_value_size(); ++i) {
data[i] = *(int128_t*)(result.bytes_value(i).c_str());
}
break;
}
case PGenericType::STRING: {
column->reserve(result.string_value_size());
for (int i = 0; i < result.string_value_size(); ++i) {
column->insert_data(result.string_value(i).c_str(), result.string_value(i).size());
}
break;
}
case PGenericType::DECIMAL128: {
column->reserve(result.bytes_value_size());
column->resize(result.bytes_value_size());
auto& data = reinterpret_cast<ColumnDecimal128*>(column.get())->get_data();
for (int i = 0; i < result.bytes_value_size(); ++i) {
data[i] = *(int128_t*)(result.bytes_value(i).c_str());
}
break;
}
case PGenericType::BITMAP: {
column->reserve(result.bytes_value_size());
for (int i = 0; i < result.bytes_value_size(); ++i) {
column->insert_data(result.bytes_value(i).c_str(), result.bytes_value(i).size());
}
break;
}
case PGenericType::HLL: {
column->reserve(result.bytes_value_size());
for (int i = 0; i < result.bytes_value_size(); ++i) {
column->insert_data(result.bytes_value(i).c_str(), result.bytes_value(i).size());
}
break;
}
case PGenericType::QUANTILE_STATE: {
column->reserve(result.bytes_value_size());
for (int i = 0; i < result.bytes_value_size(); ++i) {
column->insert_data(result.bytes_value(i).c_str(), result.bytes_value(i).size());
}
break;
}
default: {
LOG(WARNING) << "unknown PGenericType: " << result.type().DebugString();
break;
}
}
}
Status RPCFnImpl::vec_call(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
size_t result, size_t input_rows_count) {
PFunctionCallRequest request;
@ -508,45 +68,16 @@ void RPCFnImpl::_convert_block_to_proto(Block& block, const ColumnNumbers& argum
ColumnWithTypeAndName& column = block.get_by_position(col_idx);
arg->set_has_null(column.column->has_null(row_count));
auto col = column.column->convert_to_full_column_if_const();
if (auto* nullable = check_and_get_column<const ColumnNullable>(*col)) {
auto data_col = nullable->get_nested_column_ptr();
auto& null_col = nullable->get_null_map_column();
auto data_type = std::reinterpret_pointer_cast<const DataTypeNullable>(column.type);
convert_nullable_col_to_pvalue(data_col->convert_to_full_column_if_const(),
data_type->get_nested_type(), null_col, arg, 0,
row_count);
} else {
convert_col_to_pvalue<false>(col, column.type, arg, 0, row_count);
}
column.type->get_serde()->write_column_to_pb(*col, *arg, 0, row_count);
}
}
void RPCFnImpl::_convert_to_block(Block& block, const PValues& result, size_t pos) {
auto data_type = block.get_data_type(pos);
if (data_type->is_nullable()) {
auto null_type = std::reinterpret_pointer_cast<const DataTypeNullable>(data_type);
auto data_col = null_type->get_nested_type()->create_column();
_convert_to_column<true>(data_col, result);
auto null_col = ColumnUInt8::create(data_col->size(), 0);
auto& null_map_data = null_col->get_data();
null_col->reserve(data_col->size());
null_col->resize(data_col->size());
if (result.has_null()) {
for (int i = 0; i < data_col->size(); ++i) {
null_map_data[i] = result.null_map(i);
}
} else {
for (int i = 0; i < data_col->size(); ++i) {
null_map_data[i] = false;
}
}
block.replace_by_position(pos,
ColumnNullable::create(std::move(data_col), std::move(null_col)));
} else {
auto column = data_type->create_column();
_convert_to_column<false>(column, result);
block.replace_by_position(pos, std::move(column));
}
auto col = data_type->create_column();
auto serde = data_type->get_serde();
serde->read_column_from_pb(*col, result);
block.replace_by_position(pos, std::move(col));
}
FunctionRPC::FunctionRPC(const TFunction& fn, const DataTypes& argument_types,

11
be/src/vec/functions/function_rpc.h
View File

@ -31,22 +31,11 @@ public:
const std::vector<size_t>& arguments, size_t result, size_t input_rows_count);
bool available() { return _client != nullptr; }
static void convert_nullable_col_to_pvalue(const vectorized::ColumnPtr& column,
const vectorized::DataTypePtr& data_type,
const vectorized::ColumnUInt8& null_col,
PValues* arg, int start, int end);
template <bool nullable>
static void convert_col_to_pvalue(const vectorized::ColumnPtr& column,
const vectorized::DataTypePtr& data_type, PValues* arg,
int start, int end);
private:
void _convert_block_to_proto(vectorized::Block& block,
const vectorized::ColumnNumbers& arguments,
size_t input_rows_count, PFunctionCallRequest* request);
void _convert_to_block(vectorized::Block& block, const PValues& result, size_t pos);
template <bool nullable>
void _convert_to_column(vectorized::MutableColumnPtr& column, const PValues& result);
std::shared_ptr<PFunctionService_Stub> _client;
std::string _function_name;

1
be/test/CMakeLists.txt
View File

@ -228,6 +228,7 @@ set(VEC_TEST_FILES
vec/columns/column_decimal_test.cpp
vec/columns/column_fixed_length_object_test.cpp
vec/data_types/complex_type_test.cpp
vec/data_types/serde/data_type_serde_test.cpp
vec/core/block_test.cpp
vec/core/block_spill_test.cpp
vec/core/column_array_test.cpp

183
be/test/vec/data_types/serde/data_type_serde_test.cpp Normal file
View File

@ -0,0 +1,183 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <memory>
#include <string>
#include "vec/columns/column_complex.h"
#include "vec/columns/column_decimal.h"
#include "vec/columns/column_nullable.h"
#include "vec/data_types/data_type_bitmap.h"
#include "vec/data_types/data_type_decimal.h"
#include "vec/data_types/data_type_hll.h"
#include "vec/data_types/data_type_nullable.h"
#include "vec/data_types/data_type_number.h"
#include "vec/data_types/data_type_quantilestate.h"
#include "vec/data_types/data_type_string.h"
namespace doris::vectorized {
void column_to_pb(const DataTypePtr data_type, const IColumn& col, PValues* result) {
const DataTypeSerDeSPtr serde = data_type->get_serde();
serde->write_column_to_pb(col, *result, 0, col.size());
}
void pb_to_column(const DataTypePtr data_type, PValues& result, IColumn& col) {
auto serde = data_type->get_serde();
serde->read_column_from_pb(col, result);
}
void check_pb_col(const DataTypePtr data_type, const IColumn& col) {
PValues pv = PValues();
column_to_pb(data_type, col, &pv);
std::string s1 = pv.DebugString();
auto col1 = data_type->create_column();
pb_to_column(data_type, pv, *col1);
PValues as_pv = PValues();
column_to_pb(data_type, *col1, &as_pv);
std::string s2 = as_pv.DebugString();
EXPECT_EQ(s1, s2);
}
void serialize_and_deserialize_pb_test() {
// int
{
auto vec = vectorized::ColumnVector<Int32>::create();
auto& data = vec->get_data();
for (int i = 0; i < 1024; ++i) {
data.push_back(i);
}
vectorized::DataTypePtr data_type(std::make_shared<vectorized::DataTypeInt32>());
check_pb_col(data_type, *vec.get());
}
// string
{
auto strcol = vectorized::ColumnString::create();
for (int i = 0; i < 1024; ++i) {
std::string is = std::to_string(i);
strcol->insert_data(is.c_str(), is.size());
}
vectorized::DataTypePtr data_type(std::make_shared<vectorized::DataTypeString>());
check_pb_col(data_type, *strcol.get());
}
// decimal
{
vectorized::DataTypePtr decimal_data_type(doris::vectorized::create_decimal(27, 9, true));
auto decimal_column = decimal_data_type->create_column();
auto& data = ((vectorized::ColumnDecimal<vectorized::Decimal<vectorized::Int128>>*)
decimal_column.get())
->get_data();
for (int i = 0; i < 1024; ++i) {
__int128_t value = i * pow(10, 9) + i * pow(10, 8);
data.push_back(value);
}
check_pb_col(decimal_data_type, *decimal_column.get());
}
// bitmap
{
vectorized::DataTypePtr bitmap_data_type(std::make_shared<vectorized::DataTypeBitMap>());
auto bitmap_column = bitmap_data_type->create_column();
std::vector<BitmapValue>& container =
((vectorized::ColumnBitmap*)bitmap_column.get())->get_data();
for (int i = 0; i < 4; ++i) {
BitmapValue bv;
for (int j = 0; j <= i; ++j) {
bv.add(j);
}
container.push_back(bv);
}
check_pb_col(bitmap_data_type, *bitmap_column.get());
}
// hll
{
vectorized::DataTypePtr hll_data_type(std::make_shared<vectorized::DataTypeHLL>());
auto hll_column = hll_data_type->create_column();
std::vector<HyperLogLog>& container =
((vectorized::ColumnHLL*)hll_column.get())->get_data();
for (int i = 0; i < 4; ++i) {
HyperLogLog hll;
hll.update(i);
container.push_back(hll);
}
check_pb_col(hll_data_type, *hll_column.get());
}
// quantilestate
{
vectorized::DataTypePtr quantile_data_type(
std::make_shared<vectorized::DataTypeQuantileStateDouble>());
auto quantile_column = quantile_data_type->create_column();
std::vector<QuantileStateDouble>& container =
((vectorized::ColumnQuantileStateDouble*)quantile_column.get())->get_data();
const long max_rand = 1000000L;
double lower_bound = 0;
double upper_bound = 100;
srandom(time(nullptr));
for (int i = 0; i < 1024; ++i) {
QuantileStateDouble q;
double random_double =
lower_bound + (upper_bound - lower_bound) * (random() % max_rand) / max_rand;
q.add_value(random_double);
container.push_back(q);
}
check_pb_col(quantile_data_type, *quantile_column.get());
}
// nullable string
{
vectorized::DataTypePtr string_data_type(std::make_shared<vectorized::DataTypeString>());
vectorized::DataTypePtr nullable_data_type(
std::make_shared<vectorized::DataTypeNullable>(string_data_type));
auto nullable_column = nullable_data_type->create_column();
((vectorized::ColumnNullable*)nullable_column.get())->insert_null_elements(1024);
check_pb_col(nullable_data_type, *nullable_column.get());
}
// nullable decimal
{
vectorized::DataTypePtr decimal_data_type(doris::vectorized::create_decimal(27, 9, true));
vectorized::DataTypePtr nullable_data_type(
std::make_shared<vectorized::DataTypeNullable>(decimal_data_type));
auto nullable_column = nullable_data_type->create_column();
((vectorized::ColumnNullable*)nullable_column.get())->insert_null_elements(1024);
check_pb_col(nullable_data_type, *nullable_column.get());
}
// int with 1024 batch size
{
auto vec = vectorized::ColumnVector<Int32>::create();
auto& data = vec->get_data();
for (int i = 0; i < 1024; ++i) {
data.push_back(i);
}
std::cout << vec->size() << std::endl;
vectorized::DataTypePtr data_type(std::make_shared<vectorized::DataTypeInt32>());
vectorized::DataTypePtr nullable_data_type(
std::make_shared<vectorized::DataTypeNullable>(data_type));
auto nullable_column = nullable_data_type->create_column();
((vectorized::ColumnNullable*)nullable_column.get())
->insert_range_from_not_nullable(*vec, 0, 1024);
check_pb_col(nullable_data_type, *nullable_column.get());
}
}
TEST(DataTypeSerDeTest, DataTypeScalaSerDeTest) {
serialize_and_deserialize_pb_test();
}
} // namespace doris::vectorized