database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[FIX](mysql_writer) fix mysql output binary object works (#20154)

Browse Source 
* fix struct_export out data

* fix mysql writer output with binary true
This commit is contained in:
amory
2023-05-29 16:53:33 +08:00
committed by GitHub
parent cc47ee480c
commit 91dae8a5b6
29 changed files with 276 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
be/src/vec/data_types/serde/data_type_array_serde.cpp
View File

@ -86,8 +86,9 @@ void DataTypeArraySerDe::read_column_from_arrow(IColumn& column, const arrow::Ar
template <bool is_binary_format>
Status DataTypeArraySerDe::_write_column_to_mysql(
const IColumn& column, std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx,
int start, int end, bool col_const) const {
const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx, int start, int end,
bool col_const) const {
int buf_ret = 0;
auto& column_array = assert_cast<const ColumnArray&>(column);
auto& offsets = column_array.get_offsets();
@ -109,12 +110,14 @@ Status DataTypeArraySerDe::_write_column_to_mysql(
} else {
if (is_nested_string) {
buf_ret = result[row_idx].push_string("\"", 1);
RETURN_IF_ERROR(nested_serde->write_column_to_mysql(data, result, row_idx, j,
j + 1, col_const));
RETURN_IF_ERROR(nested_serde->write_column_to_mysql(
data, return_object_data_as_binary, result, row_idx, j, j + 1,
col_const));
buf_ret = result[row_idx].push_string("\"", 1);
} else {
RETURN_IF_ERROR(nested_serde->write_column_to_mysql(data, result, row_idx, j,
j + 1, col_const));
RETURN_IF_ERROR(nested_serde->write_column_to_mysql(
data, return_object_data_as_binary, result, row_idx, j, j + 1,
col_const));
}
}
}

18
be/src/vec/data_types/serde/data_type_array_serde.h
View File

@ -56,19 +56,23 @@ public:
int end) const override;
void read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int start,
int end, const cctz::time_zone& ctz) const override;
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
private:
template <bool is_binary_format>
Status _write_column_to_mysql(const IColumn& column,
Status _write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result,
int row_idx, int start, int end, bool col_const) const;

17
be/src/vec/data_types/serde/data_type_bitmap_serde.cpp
View File

@ -84,14 +84,25 @@ void DataTypeBitMapSerDe::read_one_cell_from_jsonb(IColumn& column, const JsonbV
template <bool is_binary_format>
Status DataTypeBitMapSerDe::_write_column_to_mysql(
const IColumn& column, std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx,
int start, int end, bool col_const) const {
const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx, int start, int end,
bool col_const) const {
auto& data_column = assert_cast<const ColumnBitmap&>(column);
int buf_ret = 0;
for (ssize_t i = start; i < end; ++i) {
if (0 != buf_ret) {
return Status::InternalError("pack mysql buffer failed.");
}
buf_ret = result[row_idx].push_null();
if (return_object_data_as_binary) {
const auto col_index = index_check_const(i, col_const);
BitmapValue bitmapValue = data_column.get_element(col_index);
size_t size = bitmapValue.getSizeInBytes();
std::unique_ptr<char[]> buf = std::make_unique<char[]>(size);
bitmapValue.write_to(buf.get());
buf_ret = result[row_idx].push_string(buf.get(), size);
} else {
buf_ret = result[row_idx].push_null();
}
++row_idx;
}
return Status::OK();

18
be/src/vec/data_types/serde/data_type_bitmap_serde.h
View File

@ -50,20 +50,24 @@ public:
int end, const cctz::time_zone& ctz) const override {
LOG(FATAL) << "Not support read bitmap column from arrow";
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
private:
// Bitmap is binary data which is not shown by mysql.
template <bool is_binary_format>
Status _write_column_to_mysql(const IColumn& column,
Status _write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result,
int row_idx, int start, int end, bool col_const) const;
};

5
be/src/vec/data_types/serde/data_type_date64_serde.cpp
View File

@ -111,8 +111,9 @@ void DataTypeDate64SerDe::read_column_from_arrow(IColumn& column, const arrow::A
template <bool is_binary_format>
Status DataTypeDate64SerDe::_write_column_to_mysql(
const IColumn& column, std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx,
int start, int end, bool col_const) const {
const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx, int start, int end,
bool col_const) const {
auto& data = assert_cast<const ColumnVector<Int64>&>(column).get_data();
int buf_ret = 0;
for (ssize_t i = start; i < end; ++i) {

18
be/src/vec/data_types/serde/data_type_date64_serde.h
View File

@ -47,19 +47,23 @@ class DataTypeDate64SerDe : public DataTypeNumberSerDe<Int64> {
int end) const override;
void read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int start,
int end, const cctz::time_zone& ctz) const override;
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
private:
template <bool is_binary_format>
Status _write_column_to_mysql(const IColumn& column,
Status _write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result,
int row_idx, int start, int end, bool col_const) const;
};

5
be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp
View File

@ -48,8 +48,9 @@ void DataTypeDateTimeV2SerDe::write_column_to_arrow(const IColumn& column, const
}
template <bool is_binary_format>
Status DataTypeDateTimeV2SerDe::_write_column_to_mysql(
const IColumn& column, std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx,
int start, int end, bool col_const) const {
const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx, int start, int end,
bool col_const) const {
auto& data = assert_cast<const ColumnVector<UInt64>&>(column).get_data();
int buf_ret = 0;
for (ssize_t i = start; i < end; ++i) {

18
be/src/vec/data_types/serde/data_type_datetimev2_serde.h
View File

@ -51,19 +51,23 @@ public:
int end, const cctz::time_zone& ctz) const override {
LOG(FATAL) << "not support read arrow array to uint64 column";
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
private:
template <bool is_binary_format>
Status _write_column_to_mysql(const IColumn& column,
Status _write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result,
int row_idx, int start, int end, bool col_const) const;
int scale;

5
be/src/vec/data_types/serde/data_type_datev2_serde.cpp
View File

@ -67,8 +67,9 @@ void DataTypeDateV2SerDe::read_column_from_arrow(IColumn& column, const arrow::A
}
template <bool is_binary_format>
Status DataTypeDateV2SerDe::_write_column_to_mysql(
const IColumn& column, std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx,
int start, int end, bool col_const) const {
const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx, int start, int end,
bool col_const) const {
auto& data = assert_cast<const ColumnVector<UInt32>&>(column).get_data();
int buf_ret = 0;
for (ssize_t i = start; i < end; ++i) {

18
be/src/vec/data_types/serde/data_type_datev2_serde.h
View File

@ -47,19 +47,23 @@ class DataTypeDateV2SerDe : public DataTypeNumberSerDe<UInt32> {
int end) const override;
void read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int start,
int end, const cctz::time_zone& ctz) const override;
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
private:
template <bool is_binary_format>
Status _write_column_to_mysql(const IColumn& column,
Status _write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result,
int row_idx, int start, int end, bool col_const) const;
};

23
be/src/vec/data_types/serde/data_type_decimal_serde.h
View File

@ -63,19 +63,23 @@ public:
int end) const override;
void read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int start,
int end, const cctz::time_zone& ctz) const override;
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
private:
template <bool is_binary_format>
Status _write_column_to_mysql(const IColumn& column,
Status _write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result,
int row_idx, int start, int end, bool col_const) const;
@ -85,8 +89,9 @@ private:
template <typename T>
template <bool is_binary_format>
Status DataTypeDecimalSerDe<T>::_write_column_to_mysql(
const IColumn& column, std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx,
int start, int end, bool col_const) const {
const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx, int start, int end,
bool col_const) const {
int buf_ret = 0;
auto& data = assert_cast<const ColumnDecimal<T>&>(column).get_data();
for (int i = start; i < end; ++i) {

10
be/src/vec/data_types/serde/data_type_fixedlengthobject_serde.h
View File

@ -60,12 +60,14 @@ public:
int end, const cctz::time_zone& ctz) const override {
LOG(FATAL) << "Not support read FixedLengthObject column from arrow";
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
LOG(FATAL) << "Not support write object column to mysql";
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
LOG(FATAL) << "Not support write object column to mysql";
}
};

17
be/src/vec/data_types/serde/data_type_hll_serde.cpp
View File

@ -103,14 +103,25 @@ void DataTypeHLLSerDe::write_column_to_arrow(const IColumn& column, const UInt8*
template <bool is_binary_format>
Status DataTypeHLLSerDe::_write_column_to_mysql(
const IColumn& column, std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx,
int start, int end, bool col_const) const {
const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx, int start, int end,
bool col_const) const {
auto& data_column = assert_cast<const ColumnHLL&>(column);
int buf_ret = 0;
for (ssize_t i = start; i < end; ++i) {
if (0 != buf_ret) {
return Status::InternalError("pack mysql buffer failed.");
}
buf_ret = result[row_idx].push_null();
if (return_object_data_as_binary) {
const auto col_index = index_check_const(i, col_const);
HyperLogLog hyperLogLog = data_column.get_element(col_index);
size_t size = hyperLogLog.max_serialized_size();
std::unique_ptr<char[]> buf = std::make_unique<char[]>(size);
hyperLogLog.serialize((uint8*)buf.get());
buf_ret = result[row_idx].push_string(buf.get(), size);
} else {
buf_ret = result[row_idx].push_null();
}
++row_idx;
}
return Status::OK();

18
be/src/vec/data_types/serde/data_type_hll_serde.h
View File

@ -48,20 +48,24 @@ public:
int end, const cctz::time_zone& ctz) const override {
LOG(FATAL) << "Not support read hll column from arrow";
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
private:
// Hll is binary data which is not shown by mysql.
template <bool is_binary_format>
Status _write_column_to_mysql(const IColumn& column,
Status _write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result,
int row_idx, int start, int end, bool col_const) const;
};

18
be/src/vec/data_types/serde/data_type_jsonb_serde.h
View File

@ -34,18 +34,22 @@ namespace vectorized {
class Arena;
class DataTypeJsonbSerDe : public DataTypeStringSerDe {
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_jsonb_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_jsonb_column_to_mysql(column, return_object_data_as_binary, result, row_idx,
start, end, col_const);
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_jsonb_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_jsonb_column_to_mysql(column, return_object_data_as_binary, result, row_idx,
start, end, col_const);
}
private:
template <bool is_binary_format>
Status _write_jsonb_column_to_mysql(const IColumn& column,
Status _write_jsonb_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result,
int row_idx, int start, int end, bool col_const) const {
int buf_ret = 0;

21
be/src/vec/data_types/serde/data_type_map_serde.cpp
View File

@ -56,8 +56,9 @@ void DataTypeMapSerDe::read_column_from_arrow(IColumn& column, const arrow::Arra
}
template <bool is_binary_format>
Status DataTypeMapSerDe::_write_column_to_mysql(
const IColumn& column, std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx,
int start, int end, bool col_const) const {
const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx, int start, int end,
bool col_const) const {
int buf_ret = 0;
auto& map_column = assert_cast<const ColumnMap&>(column);
const IColumn& nested_keys_column = map_column.get_keys();
@ -82,12 +83,14 @@ Status DataTypeMapSerDe::_write_column_to_mysql(
} else {
if (is_key_string) {
buf_ret = result[row_idx].push_string("\"", 1);
RETURN_IF_ERROR(key_serde->write_column_to_mysql(nested_keys_column, result,
row_idx, j, j + 1, col_const));
RETURN_IF_ERROR(key_serde->write_column_to_mysql(
nested_keys_column, return_object_data_as_binary, result, row_idx, j,
j + 1, col_const));
buf_ret = result[row_idx].push_string("\"", 1);
} else {
RETURN_IF_ERROR(key_serde->write_column_to_mysql(nested_keys_column, result,
row_idx, j, j + 1, col_const));
RETURN_IF_ERROR(key_serde->write_column_to_mysql(
nested_keys_column, return_object_data_as_binary, result, row_idx, j,
j + 1, col_const));
}
}
buf_ret = result[row_idx].push_string(":", 1);
@ -97,11 +100,13 @@ Status DataTypeMapSerDe::_write_column_to_mysql(
if (is_val_string) {
buf_ret = result[row_idx].push_string("\"", 1);
RETURN_IF_ERROR(value_serde->write_column_to_mysql(
nested_values_column, result, row_idx, j, j + 1, col_const));
nested_values_column, return_object_data_as_binary, result, row_idx, j,
j + 1, col_const));
buf_ret = result[row_idx].push_string("\"", 1);
} else {
RETURN_IF_ERROR(value_serde->write_column_to_mysql(
nested_values_column, result, row_idx, j, j + 1, col_const));
nested_values_column, return_object_data_as_binary, result, row_idx, j,
j + 1, col_const));
}
}
}

18
be/src/vec/data_types/serde/data_type_map_serde.h
View File

@ -55,19 +55,23 @@ public:
int end) const override;
void read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int start,
int end, const cctz::time_zone& ctz) const override;
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
private:
template <bool is_binary_format>
Status _write_column_to_mysql(const IColumn& column,
Status _write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result,
int row_idx, int start, int end, bool col_const) const;

8
be/src/vec/data_types/serde/data_type_nullable_serde.cpp
View File

@ -134,8 +134,9 @@ void DataTypeNullableSerDe::read_column_from_arrow(IColumn& column, const arrow:
template <bool is_binary_format>
Status DataTypeNullableSerDe::_write_column_to_mysql(
const IColumn& column, std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx,
int start, int end, bool col_const) const {
const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx, int start, int end,
bool col_const) const {
int buf_ret = 0;
auto& col = static_cast<const ColumnNullable&>(column);
auto& nested_col = col.get_nested_column();
@ -148,7 +149,8 @@ Status DataTypeNullableSerDe::_write_column_to_mysql(
buf_ret = result[row_idx].push_null();
} else {
RETURN_IF_ERROR(nested_serde->write_column_to_mysql(
nested_col, result, row_idx, col_index, col_index + 1, col_const));
nested_col, return_object_data_as_binary, result, row_idx, col_index,
col_index + 1, col_const));
}
++row_idx;
}

18
be/src/vec/data_types/serde/data_type_nullable_serde.h
View File

@ -48,19 +48,23 @@ public:
int end) const override;
void read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int start,
int end, const cctz::time_zone& ctz) const override;
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
private:
template <bool is_binary_format>
Status _write_column_to_mysql(const IColumn& column,
Status _write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result,
int row_idx, int start, int end, bool col_const) const;

23
be/src/vec/data_types/serde/data_type_number_serde.h
View File

@ -68,19 +68,23 @@ public:
int end) const override;
void read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int start,
int end, const cctz::time_zone& ctz) const override;
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
private:
template <bool is_binary_format>
Status _write_column_to_mysql(const IColumn& column,
Status _write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result,
int row_idx, int start, int end, bool col_const) const;
};
@ -88,8 +92,9 @@ private:
template <typename T>
template <bool is_binary_format>
Status DataTypeNumberSerDe<T>::_write_column_to_mysql(
const IColumn& column, std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx,
int start, int end, bool col_const) const {
const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx, int start, int end,
bool col_const) const {
int buf_ret = 0;
auto& data = assert_cast<const ColumnType&>(column).get_data();
for (auto i = start; i < end; ++i) {

10
be/src/vec/data_types/serde/data_type_object_serde.h
View File

@ -61,12 +61,14 @@ public:
int end, const cctz::time_zone& ctz) const override {
LOG(FATAL) << "Not support read object column from arrow";
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
LOG(FATAL) << "Not support write object column to mysql";
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
LOG(FATAL) << "Not support write object column to mysql";
}
};

36
be/src/vec/data_types/serde/data_type_quantilestate_serde.h
View File

@ -57,19 +57,23 @@ public:
int end, const cctz::time_zone& ctz) const override {
LOG(FATAL) << "Not support read " << column.get_name() << " from arrow";
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
private:
template <bool is_binary_format>
Status _write_column_to_mysql(const IColumn& column,
Status _write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result,
int row_idx, int start, int end, bool col_const) const;
};
@ -123,14 +127,26 @@ void DataTypeQuantileStateSerDe<T>::read_one_cell_from_jsonb(IColumn& column,
template <typename T>
template <bool is_binary_format>
Status DataTypeQuantileStateSerDe<T>::_write_column_to_mysql(
const IColumn& column, std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx,
int start, int end, bool col_const) const {
const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx, int start, int end,
bool col_const) const {
auto& data_column = reinterpret_cast<const ColumnQuantileState<T>&>(column);
int buf_ret = 0;
for (ssize_t i = start; i < end; ++i) {
if (0 != buf_ret) {
return Status::InternalError("pack mysql buffer failed.");
}
buf_ret = result[row_idx].push_null();
if (return_object_data_as_binary) {
const auto col_index = index_check_const(i, col_const);
auto& quantile_value =
const_cast<QuantileState<T>&>(data_column.get_element(col_index));
size_t size = quantile_value.get_serialized_size();
std::unique_ptr<char[]> buf = std::make_unique<char[]>(size);
quantile_value.serialize((uint8_t*)buf.get());
buf_ret = result[row_idx].push_string(buf.get(), size);
} else {
buf_ret = result[row_idx].push_null();
}
++row_idx;
}
return Status::OK();

4
be/src/vec/data_types/serde/data_type_serde.h
View File

@ -74,11 +74,11 @@ public:
virtual void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const = 0;
// MySQL serializer and deserializer
virtual Status write_column_to_mysql(const IColumn& column,
virtual Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx,
int start, int end, bool col_const) const = 0;
virtual Status write_column_to_mysql(const IColumn& column,
virtual Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int start,
int row_idx, int end, bool col_const) const = 0;
// Thrift serializer and deserializer

5
be/src/vec/data_types/serde/data_type_string_serde.cpp
View File

@ -125,8 +125,9 @@ void DataTypeStringSerDe::read_column_from_arrow(IColumn& column, const arrow::A
}
template <bool is_binary_format>
Status DataTypeStringSerDe::_write_column_to_mysql(
const IColumn& column, std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx,
int start, int end, bool col_const) const {
const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx, int start, int end,
bool col_const) const {
int buf_ret = 0;
auto& col = assert_cast<const ColumnString&>(column);
for (ssize_t i = start; i < end; ++i) {

18
be/src/vec/data_types/serde/data_type_string_serde.h
View File

@ -46,19 +46,23 @@ public:
int end) const override;
void read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int start,
int end, const cctz::time_zone& ctz) const override;
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
private:
template <bool is_binary_format>
Status _write_column_to_mysql(const IColumn& column,
Status _write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result,
int row_idx, int start, int end, bool col_const) const;
};

13
be/src/vec/data_types/serde/data_type_struct_serde.cpp
View File

@ -58,8 +58,9 @@ void DataTypeStructSerDe::read_column_from_arrow(IColumn& column, const arrow::A
}
template <bool is_binary_format>
Status DataTypeStructSerDe::_write_column_to_mysql(
const IColumn& column, std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx,
int start, int end, bool col_const) const {
const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result, int row_idx, int start, int end,
bool col_const) const {
int buf_ret = 0;
auto& col = assert_cast<const ColumnStruct&>(column);
for (ssize_t i = start; i < end; ++i) {
@ -81,13 +82,13 @@ Status DataTypeStructSerDe::_write_column_to_mysql(
if (remove_nullable(col.get_column_ptr(j))->is_column_string()) {
buf_ret = result[row_idx].push_string("\"", 1);
RETURN_IF_ERROR(elemSerDeSPtrs[j]->write_column_to_mysql(
col.get_column(j), result, row_idx, col_index, col_index + 1,
col_const));
col.get_column(j), return_object_data_as_binary, result, row_idx,
col_index, col_index + 1, col_const));
buf_ret = result[row_idx].push_string("\"", 1);
} else {
RETURN_IF_ERROR(elemSerDeSPtrs[j]->write_column_to_mysql(
col.get_column(j), result, row_idx, col_index, col_index + 1,
col_const));
col.get_column(j), return_object_data_as_binary, result, row_idx,
col_index, col_index + 1, col_const));
}
}
begin = false;

18
be/src/vec/data_types/serde/data_type_struct_serde.h
View File

@ -56,19 +56,23 @@ public:
int end) const override;
void read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int start,
int end, const cctz::time_zone& ctz) const override;
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_column_to_mysql(column, return_object_data_as_binary, result, row_idx, start,
end, col_const);
}
private:
template <bool is_binary_format>
Status _write_column_to_mysql(const IColumn& column,
Status _write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result,
int row_idx, int start, int end, bool col_const) const;

17
be/src/vec/data_types/serde/data_type_time_serde.h
View File

@ -31,18 +31,23 @@ namespace vectorized {
class Arena;
class DataTypeTimeSerDe : public DataTypeNumberSerDe<Float64> {
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<false>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_date_time_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<false>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_date_time_column_to_mysql(column, return_object_data_as_binary, result,
row_idx, start, end, col_const);
}
Status write_column_to_mysql(const IColumn& column, std::vector<MysqlRowBuffer<true>>& result,
int row_idx, int start, int end, bool col_const) const override {
return _write_date_time_column_to_mysql(column, result, row_idx, start, end, col_const);
Status write_column_to_mysql(const IColumn& column, bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<true>>& result, int row_idx, int start,
int end, bool col_const) const override {
return _write_date_time_column_to_mysql(column, return_object_data_as_binary, result,
row_idx, start, end, col_const);
}
private:
template <bool is_binary_format>
Status _write_date_time_column_to_mysql(const IColumn& column,
bool return_object_data_as_binary,
std::vector<MysqlRowBuffer<is_binary_format>>& result,
int row_idx, int start, int end, bool col_const) const {
int buf_ret = 0;

4
be/src/vec/sink/vmysql_result_writer.cpp
View File

@ -624,8 +624,8 @@ Status VMysqlResultWriter<is_binary_format>::append_block(Block& input_block) {
<< fmt::format("block's rows({}) != column{}'s size({})", num_rows, i,
block.get_by_position(i).column->size());
RETURN_IF_ERROR(type_ptr->get_serde()->write_column_to_mysql(*column_ptr, rows_buffer, 0, 0,
num_rows, col_const));
RETURN_IF_ERROR(type_ptr->get_serde()->write_column_to_mysql(
*column_ptr, output_object_data(), rows_buffer, 0, 0, num_rows, col_const));
if (!status) {
LOG(WARNING) << "convert row to mysql result failed. block_struct="