Add `JSON` datatype, following features are implemented by this PR: 1. `CREATE` tables with `JSON` type columns 2. `INSERT` values containing `JSON` type value stored in `String`, which is represented as binary format(AKA `JSONB`) at BE 3. `SELECT` JSON columns Detail design refers [DSIP-016: Support JSON type](https://cwiki.apache.org/confluence/display/DORIS/DSIP-016%3A+Support+JSON+type) * add JSONB data storage format type * fix JsonLiteral resolve bug * add DataTypeJson case in data_type_factory * add JSON syntax check in FE * add operators for jsonb_document, currently not support comparison between any JSON type value * add ColumnJson and DataTypeJson * add JsonField to store JsonValue * add JsonValue to convert String JSON to BINARY JSON and JsonLiteral case for vliteral * add push_json for MysqlResultWriter * JSON column need no zone_map_index * Revert "JSON column need no zone_map_index" This reverts commit f71d1ce1ded9dbae44a5d58abcec338816b70d79. * add JSON writer and reader, ignore zone-map for JSON column * add json_to_string for DataTypeJson * add olap_data_convertor for JSON type * add some enum * add OLAP_FIELD_TYPE_JSON type, FieldTypeTraits for it and corresponding cases or functions * fix column_json offsets overflow bug, format code * remove useless TODOs, add CmpType cases for JSON type * add license header * format license * format be codes * resolve rebase master conflicts * fix bugs for CREATE and meta related code * refactor JsonValue constructors, add fe JSON cases and fix some bugs, reformat codes * modification be codes along code review advice * fix rebase conflicts with master * add unit test for json_value and column_json * fix rebase error * rename json to jsonb * fix some data convert bugs, set Mysql type to JSON
203 lines
6.5 KiB
C++
203 lines
6.5 KiB
C++
// Licensed to the Apache Software Foundation (ASF) under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
// This file is copied from
|
|
// https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Field.cpp
|
|
// and modified by Doris
|
|
|
|
#include "vec/core/field.h"
|
|
|
|
#include "vec/core/decimal_comparison.h"
|
|
#include "vec/io/io_helper.h"
|
|
|
|
namespace doris::vectorized {
|
|
|
|
void read_binary(Array& x, BufferReadable& buf) {
|
|
size_t size;
|
|
UInt8 type;
|
|
doris::vectorized::read_binary(type, buf);
|
|
doris::vectorized::read_binary(size, buf);
|
|
|
|
for (size_t index = 0; index < size; ++index) {
|
|
switch (type) {
|
|
case Field::Types::Null: {
|
|
x.push_back(doris::vectorized::Field());
|
|
break;
|
|
}
|
|
case Field::Types::UInt64: {
|
|
UInt64 value;
|
|
doris::vectorized::read_var_uint(value, buf);
|
|
x.push_back(value);
|
|
break;
|
|
}
|
|
case Field::Types::UInt128: {
|
|
UInt128 value;
|
|
doris::vectorized::read_binary(value, buf);
|
|
x.push_back(value);
|
|
break;
|
|
}
|
|
case Field::Types::Int64: {
|
|
Int64 value;
|
|
doris::vectorized::read_var_int(value, buf);
|
|
x.push_back(value);
|
|
break;
|
|
}
|
|
case Field::Types::Float64: {
|
|
Float64 value;
|
|
doris::vectorized::read_float_binary(value, buf);
|
|
x.push_back(value);
|
|
break;
|
|
}
|
|
case Field::Types::String: {
|
|
std::string value;
|
|
doris::vectorized::read_string_binary(value, buf);
|
|
x.push_back(value);
|
|
break;
|
|
}
|
|
case Field::Types::JSONB: {
|
|
JsonbField value;
|
|
doris::vectorized::read_json_binary(value, buf);
|
|
x.push_back(value);
|
|
break;
|
|
}
|
|
case Field::Types::AggregateFunctionState: {
|
|
AggregateFunctionStateData value;
|
|
doris::vectorized::read_string_binary(value.name, buf);
|
|
doris::vectorized::read_string_binary(value.data, buf);
|
|
x.push_back(value);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void write_binary(const Array& x, BufferWritable& buf) {
|
|
UInt8 type = Field::Types::Null;
|
|
size_t size = x.size();
|
|
if (size) type = x.front().get_type();
|
|
doris::vectorized::write_binary(type, buf);
|
|
doris::vectorized::write_binary(size, buf);
|
|
|
|
for (Array::const_iterator it = x.begin(); it != x.end(); ++it) {
|
|
switch (type) {
|
|
case Field::Types::Null:
|
|
break;
|
|
case Field::Types::UInt64: {
|
|
doris::vectorized::write_var_uint(get<UInt64>(*it), buf);
|
|
break;
|
|
}
|
|
case Field::Types::UInt128: {
|
|
doris::vectorized::write_binary(get<UInt128>(*it), buf);
|
|
break;
|
|
}
|
|
case Field::Types::Int64: {
|
|
doris::vectorized::write_var_int(get<Int64>(*it), buf);
|
|
break;
|
|
}
|
|
case Field::Types::Float64: {
|
|
doris::vectorized::write_float_binary(get<Float64>(*it), buf);
|
|
break;
|
|
}
|
|
case Field::Types::String: {
|
|
doris::vectorized::write_string_binary(get<std::string>(*it), buf);
|
|
break;
|
|
}
|
|
case Field::Types::JSONB: {
|
|
doris::vectorized::write_json_binary(get<JsonbField>(*it), buf);
|
|
break;
|
|
}
|
|
case Field::Types::AggregateFunctionState: {
|
|
doris::vectorized::write_string_binary(it->get<AggregateFunctionStateData>().name, buf);
|
|
doris::vectorized::write_string_binary(it->get<AggregateFunctionStateData>().data, buf);
|
|
break;
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
template <>
|
|
Decimal32 DecimalField<Decimal32>::get_scale_multiplier() const {
|
|
return DataTypeDecimal<Decimal32>::get_scale_multiplier(scale);
|
|
}
|
|
|
|
template <>
|
|
Decimal64 DecimalField<Decimal64>::get_scale_multiplier() const {
|
|
return DataTypeDecimal<Decimal64>::get_scale_multiplier(scale);
|
|
}
|
|
|
|
template <>
|
|
Decimal128 DecimalField<Decimal128>::get_scale_multiplier() const {
|
|
return DataTypeDecimal<Decimal128>::get_scale_multiplier(scale);
|
|
}
|
|
|
|
template <typename T>
|
|
static bool dec_equal(T x, T y, UInt32 x_scale, UInt32 y_scale) {
|
|
using Comparator = DecimalComparison<T, T, EqualsOp>;
|
|
return Comparator::compare(x, y, x_scale, y_scale);
|
|
}
|
|
|
|
template <typename T>
|
|
static bool dec_less(T x, T y, UInt32 x_scale, UInt32 y_scale) {
|
|
using Comparator = DecimalComparison<T, T, LessOp>;
|
|
return Comparator::compare(x, y, x_scale, y_scale);
|
|
}
|
|
|
|
template <typename T>
|
|
static bool dec_less_or_equal(T x, T y, UInt32 x_scale, UInt32 y_scale) {
|
|
using Comparator = DecimalComparison<T, T, LessOrEqualsOp>;
|
|
return Comparator::compare(x, y, x_scale, y_scale);
|
|
}
|
|
|
|
template <>
|
|
bool decimal_equal(Decimal32 x, Decimal32 y, UInt32 xs, UInt32 ys) {
|
|
return dec_equal(x, y, xs, ys);
|
|
}
|
|
template <>
|
|
bool decimal_less(Decimal32 x, Decimal32 y, UInt32 xs, UInt32 ys) {
|
|
return dec_less(x, y, xs, ys);
|
|
}
|
|
template <>
|
|
bool decimal_less_or_equal(Decimal32 x, Decimal32 y, UInt32 xs, UInt32 ys) {
|
|
return dec_less_or_equal(x, y, xs, ys);
|
|
}
|
|
|
|
template <>
|
|
bool decimal_equal(Decimal64 x, Decimal64 y, UInt32 xs, UInt32 ys) {
|
|
return dec_equal(x, y, xs, ys);
|
|
}
|
|
template <>
|
|
bool decimal_less(Decimal64 x, Decimal64 y, UInt32 xs, UInt32 ys) {
|
|
return dec_less(x, y, xs, ys);
|
|
}
|
|
template <>
|
|
bool decimal_less_or_equal(Decimal64 x, Decimal64 y, UInt32 xs, UInt32 ys) {
|
|
return dec_less_or_equal(x, y, xs, ys);
|
|
}
|
|
|
|
template <>
|
|
bool decimal_equal(Decimal128 x, Decimal128 y, UInt32 xs, UInt32 ys) {
|
|
return dec_equal(x, y, xs, ys);
|
|
}
|
|
template <>
|
|
bool decimal_less(Decimal128 x, Decimal128 y, UInt32 xs, UInt32 ys) {
|
|
return dec_less(x, y, xs, ys);
|
|
}
|
|
template <>
|
|
bool decimal_less_or_equal(Decimal128 x, Decimal128 y, UInt32 xs, UInt32 ys) {
|
|
return dec_less_or_equal(x, y, xs, ys);
|
|
}
|
|
} // namespace doris::vectorized
|