database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bug](jdbc) support get_default on complex type (#23325)

Browse Source 
support get_default on complex type
This commit is contained in:
Pxl
2023-08-25 14:08:24 +08:00
committed by GitHub
parent 0ccb7262a7
commit b96b8f4370
8 changed files with 111 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
be/CMakeLists.txt
View File

@ -225,15 +225,18 @@ function(TRY_TO_CHANGE_LINKER LINKER_COMMAND LINKER_NAME)
endif()
endfunction()
# In terms of performance, mold> lld> gold> ld
set(CUSTUM_LINKER_COMMAND "ld")
TRY_TO_CHANGE_LINKER("mold" "mold")
TRY_TO_CHANGE_LINKER("lld" "LLD")
TRY_TO_CHANGE_LINKER("gold" "GNU gold")
if (NOT CUSTUM_LINKER_COMMAND STREQUAL "ld")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${CUSTUM_LINKER_COMMAND}")
if (NOT OS_MACOSX) # MACOSX's lld will core dump
# In terms of performance, mold> lld> gold> ld
set(CUSTUM_LINKER_COMMAND "ld")
TRY_TO_CHANGE_LINKER("mold" "mold")
TRY_TO_CHANGE_LINKER("lld" "LLD")
TRY_TO_CHANGE_LINKER("gold" "GNU gold")
if (NOT CUSTUM_LINKER_COMMAND STREQUAL "ld")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${CUSTUM_LINKER_COMMAND}")
endif()
endif()
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_C_STANDARD 17)

58
be/src/vec/common/field_visitors.h
View File

@ -63,64 +63,6 @@ typename std::decay_t<Visitor>::ResultType apply_visitor(Visitor&& visitor, F&&
return visitor(field.template get<DecimalField<Decimal128>>());
case Field::Types::Decimal128I:
return visitor(field.template get<DecimalField<Decimal128I>>());
case Field::Types::AggregateFunctionState:
return visitor(field.template get<AggregateFunctionStateData>());
default:
LOG(FATAL) << "Bad type of Field";
return {};
}
}
template <typename Visitor, typename F1, typename F2>
typename std::decay_t<Visitor>::ResultType apply_visitor(Visitor&& visitor, F1&& field1,
F2&& field2) {
switch (field1.getType()) {
case Field::Types::Null:
return apply_binary_visitor_impl(std::forward<Visitor>(visitor),
field1.template get<Null>(), std::forward<F2>(field2));
case Field::Types::UInt64:
return apply_binary_visitor_impl(std::forward<Visitor>(visitor),
field1.template get<UInt64>(), std::forward<F2>(field2));
case Field::Types::UInt128:
return apply_binary_visitor_impl(std::forward<Visitor>(visitor),
field1.template get<UInt128>(), std::forward<F2>(field2));
case Field::Types::Int64:
return apply_binary_visitor_impl(std::forward<Visitor>(visitor),
field1.template get<Int64>(), std::forward<F2>(field2));
case Field::Types::Float64:
return apply_binary_visitor_impl(std::forward<Visitor>(visitor),
field1.template get<Float64>(), std::forward<F2>(field2));
case Field::Types::String:
return apply_binary_visitor_impl(std::forward<Visitor>(visitor),
field1.template get<String>(), std::forward<F2>(field2));
case Field::Types::Array:
return apply_binary_visitor_impl(std::forward<Visitor>(visitor),
field1.template get<Array>(), std::forward<F2>(field2));
case Field::Types::Tuple:
return apply_binary_visitor_impl(std::forward<Visitor>(visitor),
field1.template get<Tuple>(), std::forward<F2>(field2));
case Field::Types::Decimal32:
return apply_binary_visitor_impl(std::forward<Visitor>(visitor),
field1.template get<DecimalField<Decimal32>>(),
std::forward<F2>(field2));
case Field::Types::Decimal64:
return apply_binary_visitor_impl(std::forward<Visitor>(visitor),
field1.template get<DecimalField<Decimal64>>(),
std::forward<F2>(field2));
case Field::Types::Decimal128:
return apply_binary_visitor_impl(std::forward<Visitor>(visitor),
field1.template get<DecimalField<Decimal128>>(),
std::forward<F2>(field2));
case Field::Types::Decimal128I:
return apply_binary_visitor_impl(std::forward<Visitor>(visitor),
field1.template get<DecimalField<Decimal128I>>(),
std::forward<F2>(field2));
case Field::Types::AggregateFunctionState:
return apply_binary_visitor_impl(std::forward<Visitor>(visitor),
field1.template get<AggregateFunctionStateData>(),
std::forward<F2>(field2));
default:
LOG(FATAL) << "Bad type of Field";
return {};

12
be/src/vec/core/field.cpp
View File

@ -83,13 +83,6 @@ void read_binary(Array& x, BufferReadable& buf) {
x.push_back(value);
break;
}
case Field::Types::AggregateFunctionState: {
AggregateFunctionStateData value;
doris::vectorized::read_string_binary(value.name, buf);
doris::vectorized::read_string_binary(value.data, buf);
x.push_back(value);
break;
}
}
}
}
@ -129,11 +122,6 @@ void write_binary(const Array& x, BufferWritable& buf) {
doris::vectorized::write_json_binary(get<JsonbField>(*it), buf);
break;
}
case Field::Types::AggregateFunctionState: {
doris::vectorized::write_string_binary(it->get<AggregateFunctionStateData>().name, buf);
doris::vectorized::write_string_binary(it->get<AggregateFunctionStateData>().data, buf);
break;
}
}
};
}

359
be/src/vec/core/field.h
View File

@ -39,6 +39,9 @@
// IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
#include "common/compiler_util.h" // IWYU pragma: keep
#include "olap/hll.h"
#include "util/bitmap_value.h"
#include "util/quantile_state.h"
#include "vec/common/uint128.h"
#include "vec/core/types.h"
@ -55,7 +58,9 @@ struct PackedInt128;
namespace doris::vectorized {
template <typename T>
struct NearestFieldTypeImpl;
struct NearestFieldTypeImpl {
using Type = T;
};
template <typename T>
using NearestFieldType = typename NearestFieldTypeImpl<T>::Type;
@ -117,37 +122,6 @@ using FieldMap = std::map<String, Field, std::less<String>>;
DEFINE_FIELD_MAP(VariantMap);
#undef DEFINE_FIELD_MAP
struct AggregateFunctionStateData {
String name; /// Name with arguments.
String data;
bool operator<(const AggregateFunctionStateData&) const {
LOG(FATAL) << "Operator < is not implemented for AggregateFunctionStateData.";
}
bool operator<=(const AggregateFunctionStateData&) const {
LOG(FATAL) << "Operator <= is not implemented for AggregateFunctionStateData.";
}
bool operator>(const AggregateFunctionStateData&) const {
LOG(FATAL) << "Operator <= is not implemented for AggregateFunctionStateData.";
}
bool operator>=(const AggregateFunctionStateData&) const {
LOG(FATAL) << "Operator >= is not implemented for AggregateFunctionStateData.";
}
bool operator==(const AggregateFunctionStateData& rhs) const {
if (name != rhs.name) {
LOG(FATAL) << fmt::format(
"Comparing aggregate functions with different types: {} and {}", name,
rhs.name);
}
return data == rhs.data;
}
};
class JsonbField {
public:
JsonbField() = default;
@ -342,6 +316,9 @@ public:
Decimal128I = 24,
Map = 25,
VariantMap = 26,
Bitmap = 27,
HyperLogLog = 28,
QuantileState = 29,
};
static const int MIN_NON_POD = 16;
@ -378,16 +355,20 @@ public:
return "Decimal128";
case Decimal128I:
return "Decimal128I";
case AggregateFunctionState:
return "AggregateFunctionState";
case FixedLengthObject:
return "FixedLengthObject";
case VariantMap:
return "VariantMap";
case Bitmap:
return "Bitmap";
case HyperLogLog:
return "HyperLogLog";
case QuantileState:
return "QuantileState";
default:
LOG(FATAL) << "type not supported, type=" << Types::to_string(which);
break;
}
LOG(FATAL) << "Bad type of Field";
return nullptr;
}
};
@ -523,153 +504,65 @@ public:
return get<T>();
}
bool operator<(const Field& rhs) const {
if (which < rhs.which) return true;
if (which > rhs.which) return false;
switch (which) {
case Types::Null:
return false;
case Types::UInt64:
return get<UInt64>() < rhs.get<UInt64>();
case Types::UInt128:
return get<UInt128>() < rhs.get<UInt128>();
case Types::Int64:
return get<Int64>() < rhs.get<Int64>();
case Types::Int128:
return get<Int128>() < rhs.get<Int128>();
case Types::Float64:
return get<Float64>() < rhs.get<Float64>();
case Types::String:
return get<String>() < rhs.get<String>();
case Types::JSONB:
return get<JsonbField>() < rhs.get<JsonbField>();
case Types::Array:
return get<Array>() < rhs.get<Array>();
case Types::Tuple:
return get<Tuple>() < rhs.get<Tuple>();
case Types::Map:
return get<Map>() < rhs.get<Map>();
case Types::Decimal32:
return get<DecimalField<Decimal32>>() < rhs.get<DecimalField<Decimal32>>();
case Types::Decimal64:
return get<DecimalField<Decimal64>>() < rhs.get<DecimalField<Decimal64>>();
case Types::Decimal128:
return get<DecimalField<Decimal128>>() < rhs.get<DecimalField<Decimal128>>();
case Types::Decimal128I:
return get<DecimalField<Decimal128I>>() < rhs.get<DecimalField<Decimal128I>>();
case Types::AggregateFunctionState:
return get<AggregateFunctionStateData>() < rhs.get<AggregateFunctionStateData>();
case Types::FixedLengthObject:
break;
case Types::VariantMap:
return get<VariantMap>() < rhs.get<VariantMap>();
}
LOG(FATAL) << "Bad type of Field";
return {};
}
bool operator>(const Field& rhs) const { return rhs < *this; }
bool operator<=(const Field& rhs) const {
if (which < rhs.which) return true;
if (which > rhs.which) return false;
switch (which) {
case Types::Null:
return true;
case Types::UInt64:
return get<UInt64>() <= rhs.get<UInt64>();
case Types::UInt128:
return get<UInt128>() <= rhs.get<UInt128>();
case Types::Int64:
return get<Int64>() <= rhs.get<Int64>();
case Types::Int128:
return get<Int128>() <= rhs.get<Int128>();
case Types::Float64:
return get<Float64>() <= rhs.get<Float64>();
case Types::String:
return get<String>() <= rhs.get<String>();
case Types::JSONB:
return get<JsonbField>() <= rhs.get<JsonbField>();
case Types::Array:
return get<Array>() <= rhs.get<Array>();
case Types::Tuple:
return get<Tuple>() <= rhs.get<Tuple>();
case Types::Map:
return get<Map>() < rhs.get<Map>();
case Types::Decimal32:
return get<DecimalField<Decimal32>>() <= rhs.get<DecimalField<Decimal32>>();
case Types::Decimal64:
return get<DecimalField<Decimal64>>() <= rhs.get<DecimalField<Decimal64>>();
case Types::Decimal128:
return get<DecimalField<Decimal128>>() <= rhs.get<DecimalField<Decimal128>>();
case Types::Decimal128I:
return get<DecimalField<Decimal128I>>() <= rhs.get<DecimalField<Decimal128I>>();
case Types::AggregateFunctionState:
return get<AggregateFunctionStateData>() <= rhs.get<AggregateFunctionStateData>();
case Types::FixedLengthObject:
break;
case Types::VariantMap:
return get<VariantMap>() <= rhs.get<VariantMap>();
}
LOG(FATAL) << "Bad type of Field";
return {};
}
bool operator>=(const Field& rhs) const { return rhs <= *this; }
bool operator==(const Field& rhs) const {
if (which != rhs.which) return false;
switch (which) {
case Types::Null:
return true;
case Types::UInt64:
case Types::Int64:
case Types::Float64:
return get<UInt64>() == rhs.get<UInt64>();
case Types::String:
return get<String>() == rhs.get<String>();
case Types::JSONB:
return get<JsonbField>() == rhs.get<JsonbField>();
case Types::Array:
return get<Array>() == rhs.get<Array>();
case Types::Tuple:
return get<Tuple>() == rhs.get<Tuple>();
case Types::Map:
return get<Map>() < rhs.get<Map>();
case Types::UInt128:
return get<UInt128>() == rhs.get<UInt128>();
case Types::Int128:
return get<Int128>() == rhs.get<Int128>();
case Types::Decimal32:
return get<DecimalField<Decimal32>>() == rhs.get<DecimalField<Decimal32>>();
case Types::Decimal64:
return get<DecimalField<Decimal64>>() == rhs.get<DecimalField<Decimal64>>();
case Types::Decimal128:
return get<DecimalField<Decimal128>>() == rhs.get<DecimalField<Decimal128>>();
case Types::Decimal128I:
return get<DecimalField<Decimal128I>>() == rhs.get<DecimalField<Decimal128I>>();
case Types::AggregateFunctionState:
return get<AggregateFunctionStateData>() == rhs.get<AggregateFunctionStateData>();
case Types::FixedLengthObject:
break;
case Types::VariantMap:
return get<VariantMap>() == rhs.get<VariantMap>();
}
CHECK(false) << "Bad type of Field";
return operator<=>(rhs) == std::strong_ordering::equal;
}
bool operator!=(const Field& rhs) const { return !(*this == rhs); }
std::strong_ordering operator<=>(const Field& rhs) const {
if (which == Types::Null || rhs == Types::Null) {
return std::strong_ordering::equal;
}
if (which != rhs.which) {
LOG(FATAL) << "lhs type not equal with rhs, lhs=" << Types::to_string(which)
<< ", rhs=" << Types::to_string(rhs.which);
}
switch (which) {
case Types::Bitmap:
case Types::HyperLogLog:
case Types::QuantileState:
case Types::FixedLengthObject:
case Types::JSONB:
case Types::Null:
case Types::Array:
case Types::Tuple:
case Types::Map:
case Types::VariantMap:
return std::strong_ordering::equal;
case Types::UInt64:
return get<UInt64>() <=> rhs.get<UInt64>();
case Types::UInt128:
return get<UInt128>() <=> rhs.get<UInt128>();
case Types::Int64:
return get<Int64>() <=> rhs.get<Int64>();
case Types::Int128:
return get<Int128>() <=> rhs.get<Int128>();
case Types::Float64:
return get<Float64>() < rhs.get<Float64>() ? std::strong_ordering::less
: get<Float64>() == rhs.get<Float64>() ? std::strong_ordering::equal
: std::strong_ordering::greater;
case Types::String:
return get<String>() <=> rhs.get<String>();
case Types::Decimal32:
return get<Decimal32>() <=> rhs.get<Decimal32>();
case Types::Decimal64:
return get<Decimal64>() <=> rhs.get<Decimal64>();
case Types::Decimal128:
return get<Decimal128>() <=> rhs.get<Decimal128>();
case Types::Decimal128I:
return get<Decimal128I>() <=> rhs.get<Decimal128I>();
default:
LOG(FATAL) << "lhs type not equal with rhs, lhs=" << Types::to_string(which)
<< ", rhs=" << Types::to_string(rhs.which);
break;
}
}
private:
std::aligned_union_t<DBMS_MIN_FIELD_SIZE - sizeof(Types::Which), Null, UInt64, UInt128, Int64,
Int128, Float64, String, JsonbField, Array, Tuple, Map, VariantMap,
DecimalField<Decimal32>, DecimalField<Decimal64>, DecimalField<Decimal128>,
DecimalField<Decimal128I>, AggregateFunctionStateData>
DecimalField<Decimal128I>, BitmapValue, HyperLogLog, QuantileState<double>>
storage;
Types::Which which;
@ -747,15 +640,21 @@ private:
case Types::Decimal128I:
f(field.template get<DecimalField<Decimal128I>>());
return;
case Types::AggregateFunctionState:
f(field.template get<AggregateFunctionStateData>());
return;
case Types::FixedLengthObject:
LOG(FATAL) << "FixedLengthObject not supported";
break;
case Types::VariantMap:
f(field.template get<VariantMap>());
return;
case Types::Bitmap:
f(field.template get<BitmapValue>());
return;
case Types::HyperLogLog:
f(field.template get<HyperLogLog>());
return;
case Types::QuantileState:
f(field.template get<QuantileState<double>>());
return;
default:
LOG(FATAL) << "type not supported, type=" << Types::to_string(field.which);
break;
}
}
@ -813,9 +712,6 @@ private:
case Types::Map:
destroy<Map>();
break;
case Types::AggregateFunctionState:
destroy<AggregateFunctionStateData>();
break;
case Types::VariantMap:
destroy<VariantMap>();
break;
@ -836,10 +732,6 @@ private:
#undef DBMS_MIN_FIELD_SIZE
template <>
struct TypeId<AggregateFunctionStateData> {
static constexpr const TypeIndex value = TypeIndex::AggregateFunction;
};
template <>
struct TypeId<Tuple> {
static constexpr const TypeIndex value = TypeIndex::Tuple;
@ -921,14 +813,25 @@ struct Field::TypeToEnum<DecimalField<Decimal128I>> {
static constexpr Types::Which value = Types::Decimal128I;
};
template <>
struct Field::TypeToEnum<AggregateFunctionStateData> {
static constexpr Types::Which value = Types::AggregateFunctionState;
};
template <>
struct Field::TypeToEnum<VariantMap> {
static constexpr Types::Which value = Types::VariantMap;
};
template <>
struct Field::TypeToEnum<BitmapValue> {
static constexpr Types::Which value = Types::Bitmap;
};
template <>
struct Field::TypeToEnum<HyperLogLog> {
static constexpr Types::Which value = Types::HyperLogLog;
};
template <>
struct Field::TypeToEnum<QuantileState<double>> {
static constexpr Types::Which value = Types::QuantileState;
};
template <>
struct Field::EnumToType<Field::Types::Null> {
using Type = Null;
@ -990,10 +893,6 @@ struct Field::EnumToType<Field::Types::Decimal128I> {
using Type = DecimalField<Decimal128I>;
};
template <>
struct Field::EnumToType<Field::Types::AggregateFunctionState> {
using Type = DecimalField<AggregateFunctionStateData>;
};
template <>
struct Field::EnumToType<Field::Types::VariantMap> {
using Type = VariantMap;
};
@ -1035,10 +934,6 @@ template <>
struct TypeName<Map> {
static std::string get() { return "Map"; }
};
template <>
struct TypeName<AggregateFunctionStateData> {
static std::string get() { return "AggregateFunctionState"; }
};
/// char may be signed or unsigned, and behave identically to signed char or unsigned char,
/// but they are always three different types.
@ -1065,11 +960,6 @@ struct NearestFieldTypeImpl<UInt32> {
using Type = UInt64;
};
template <>
struct NearestFieldTypeImpl<UInt128> {
using Type = UInt128;
};
//template <> struct NearestFieldTypeImpl<UUID> { using Type = UInt128; };
template <>
struct NearestFieldTypeImpl<Int16> {
using Type = Int64;
@ -1079,34 +969,13 @@ struct NearestFieldTypeImpl<Int32> {
using Type = Int64;
};
template <>
struct NearestFieldTypeImpl<VariantMap> {
using Type = VariantMap;
};
/// long and long long are always different types that may behave identically or not.
/// This is different on Linux and Mac.
template <>
struct NearestFieldTypeImpl<long> {
using Type = Int64;
};
template <>
struct NearestFieldTypeImpl<long long> {
using Type = Int64;
};
template <>
struct NearestFieldTypeImpl<unsigned long> {
using Type = UInt64;
};
template <>
struct NearestFieldTypeImpl<unsigned long long> {
using Type = UInt64;
};
template <>
struct NearestFieldTypeImpl<Int128> {
using Type = Int128;
};
template <>
struct NearestFieldTypeImpl<Decimal32> {
using Type = DecimalField<Decimal32>;
@ -1144,52 +1013,19 @@ struct NearestFieldTypeImpl<Float32> {
using Type = Float64;
};
template <>
struct NearestFieldTypeImpl<Float64> {
using Type = Float64;
};
template <>
struct NearestFieldTypeImpl<const char*> {
using Type = String;
};
template <>
struct NearestFieldTypeImpl<String> {
using Type = String;
};
template <>
struct NearestFieldTypeImpl<JsonbField> {
using Type = JsonbField;
};
template <>
struct NearestFieldTypeImpl<Array> {
using Type = Array;
};
template <>
struct NearestFieldTypeImpl<Tuple> {
using Type = Tuple;
};
template <>
struct NearestFieldTypeImpl<Map> {
using Type = Map;
};
template <>
struct NearestFieldTypeImpl<bool> {
using Type = UInt64;
};
template <>
struct NearestFieldTypeImpl<Null> {
using Type = Null;
};
template <>
struct NearestFieldTypeImpl<std::string_view> {
using Type = String;
};
template <>
struct NearestFieldTypeImpl<AggregateFunctionStateData> {
using Type = AggregateFunctionStateData;
};
template <>
struct Field::TypeToEnum<PackedInt128> {
static const Types::Which value = Types::Int128;
@ -1234,8 +1070,9 @@ Field& Field::operator=(T&& rhs) {
if (which != TypeToEnum<std::decay_t<U>>::value) {
destroy();
create_concrete(std::forward<U>(val));
} else
} else {
assign_concrete(std::forward<U>(val));
}
return *this;
}

3
be/src/vec/data_types/convert_field_to_type.cpp
View File

@ -82,9 +82,6 @@ public:
[[noreturn]] String operator()(const DecimalField<Decimal128I>& x) const {
LOG(FATAL) << "not implemeted";
}
[[noreturn]] String operator()(const AggregateFunctionStateData& x) const {
LOG(FATAL) << "not implemeted";
}
};
namespace {

5
be/src/vec/data_types/data_type_bitmap.h
View File

@ -94,10 +94,7 @@ public:
}
void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
[[noreturn]] virtual Field get_default() const override {
LOG(FATAL) << "Method get_default() is not implemented for data type " << get_name();
__builtin_unreachable();
}
Field get_default() const override { return BitmapValue(); }
[[noreturn]] Field get_field(const TExprNode& node) const override {
LOG(FATAL) << "Unimplemented get_field for BitMap";

6
be/src/vec/data_types/data_type_hll.h
View File

@ -91,11 +91,7 @@ public:
void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
Status from_string(ReadBuffer& rb, IColumn* column) const override;
Field get_default() const override {
LOG(FATAL) << "Method get_default() is not implemented for data type " << get_name();
// unreachable
return String();
}
Field get_default() const override { return HyperLogLog::empty(); }
[[noreturn]] Field get_field(const TExprNode& node) const override {
LOG(FATAL) << "Unimplemented get_field for HLL";

5
be/src/vec/data_types/data_type_quantilestate.h
View File

@ -93,10 +93,7 @@ public:
}
void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
[[noreturn]] virtual Field get_default() const override {
LOG(FATAL) << "Method get_default() is not implemented for data type " << get_name();
__builtin_unreachable();
}
Field get_default() const override { return QuantileState<T>(); }
[[noreturn]] Field get_field(const TExprNode& node) const override {
LOG(FATAL) << "Unimplemented get_field for quantilestate";