Files
doris/be/src/vec/core/field.cpp
Shane 59699a4321 [feature](JSON datatype)Support JSON datatype (#10322)
Add `JSON` datatype, following features are implemented by this PR:
1. `CREATE` tables with `JSON` type columns
2. `INSERT` values containing `JSON` type value stored in `String`, which is represented as binary format(AKA `JSONB`) at BE 
3. `SELECT` JSON columns

Detail design refers [DSIP-016: Support JSON type](https://cwiki.apache.org/confluence/display/DORIS/DSIP-016%3A+Support+JSON+type)

* add JSONB data storage format type

* fix JsonLiteral resolve bug

* add DataTypeJson case in data_type_factory

* add JSON syntax check in FE

* add operators for jsonb_document, currently not support comparison between any JSON type value

* add ColumnJson and DataTypeJson

* add JsonField to store JsonValue

* add JsonValue to convert String JSON to BINARY JSON and JsonLiteral case for vliteral

* add push_json for MysqlResultWriter

* JSON column need no zone_map_index

* Revert "JSON column need no zone_map_index"

This reverts commit f71d1ce1ded9dbae44a5d58abcec338816b70d79.

* add JSON writer and reader, ignore zone-map for JSON column

* add json_to_string for DataTypeJson

* add olap_data_convertor for JSON type

* add some enum

* add OLAP_FIELD_TYPE_JSON type, FieldTypeTraits for it and corresponding cases or functions

* fix column_json offsets overflow bug, format code

* remove useless TODOs, add CmpType cases for JSON type

* add license header

* format license

* format be codes

* resolve rebase master conflicts

* fix bugs for CREATE and meta related code

* refactor JsonValue constructors, add fe JSON cases and fix some bugs, reformat codes

* modification be codes along code review advice

* fix rebase conflicts with master

* add unit test for json_value and column_json

* fix rebase error

* rename json to jsonb

* fix some data convert bugs, set Mysql type to JSON
2022-09-25 14:06:49 +08:00

203 lines
6.5 KiB
C++

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
// This file is copied from
// https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Field.cpp
// and modified by Doris
#include "vec/core/field.h"
#include "vec/core/decimal_comparison.h"
#include "vec/io/io_helper.h"
namespace doris::vectorized {
void read_binary(Array& x, BufferReadable& buf) {
size_t size;
UInt8 type;
doris::vectorized::read_binary(type, buf);
doris::vectorized::read_binary(size, buf);
for (size_t index = 0; index < size; ++index) {
switch (type) {
case Field::Types::Null: {
x.push_back(doris::vectorized::Field());
break;
}
case Field::Types::UInt64: {
UInt64 value;
doris::vectorized::read_var_uint(value, buf);
x.push_back(value);
break;
}
case Field::Types::UInt128: {
UInt128 value;
doris::vectorized::read_binary(value, buf);
x.push_back(value);
break;
}
case Field::Types::Int64: {
Int64 value;
doris::vectorized::read_var_int(value, buf);
x.push_back(value);
break;
}
case Field::Types::Float64: {
Float64 value;
doris::vectorized::read_float_binary(value, buf);
x.push_back(value);
break;
}
case Field::Types::String: {
std::string value;
doris::vectorized::read_string_binary(value, buf);
x.push_back(value);
break;
}
case Field::Types::JSONB: {
JsonbField value;
doris::vectorized::read_json_binary(value, buf);
x.push_back(value);
break;
}
case Field::Types::AggregateFunctionState: {
AggregateFunctionStateData value;
doris::vectorized::read_string_binary(value.name, buf);
doris::vectorized::read_string_binary(value.data, buf);
x.push_back(value);
break;
}
}
}
}
void write_binary(const Array& x, BufferWritable& buf) {
UInt8 type = Field::Types::Null;
size_t size = x.size();
if (size) type = x.front().get_type();
doris::vectorized::write_binary(type, buf);
doris::vectorized::write_binary(size, buf);
for (Array::const_iterator it = x.begin(); it != x.end(); ++it) {
switch (type) {
case Field::Types::Null:
break;
case Field::Types::UInt64: {
doris::vectorized::write_var_uint(get<UInt64>(*it), buf);
break;
}
case Field::Types::UInt128: {
doris::vectorized::write_binary(get<UInt128>(*it), buf);
break;
}
case Field::Types::Int64: {
doris::vectorized::write_var_int(get<Int64>(*it), buf);
break;
}
case Field::Types::Float64: {
doris::vectorized::write_float_binary(get<Float64>(*it), buf);
break;
}
case Field::Types::String: {
doris::vectorized::write_string_binary(get<std::string>(*it), buf);
break;
}
case Field::Types::JSONB: {
doris::vectorized::write_json_binary(get<JsonbField>(*it), buf);
break;
}
case Field::Types::AggregateFunctionState: {
doris::vectorized::write_string_binary(it->get<AggregateFunctionStateData>().name, buf);
doris::vectorized::write_string_binary(it->get<AggregateFunctionStateData>().data, buf);
break;
}
}
};
}
template <>
Decimal32 DecimalField<Decimal32>::get_scale_multiplier() const {
return DataTypeDecimal<Decimal32>::get_scale_multiplier(scale);
}
template <>
Decimal64 DecimalField<Decimal64>::get_scale_multiplier() const {
return DataTypeDecimal<Decimal64>::get_scale_multiplier(scale);
}
template <>
Decimal128 DecimalField<Decimal128>::get_scale_multiplier() const {
return DataTypeDecimal<Decimal128>::get_scale_multiplier(scale);
}
template <typename T>
static bool dec_equal(T x, T y, UInt32 x_scale, UInt32 y_scale) {
using Comparator = DecimalComparison<T, T, EqualsOp>;
return Comparator::compare(x, y, x_scale, y_scale);
}
template <typename T>
static bool dec_less(T x, T y, UInt32 x_scale, UInt32 y_scale) {
using Comparator = DecimalComparison<T, T, LessOp>;
return Comparator::compare(x, y, x_scale, y_scale);
}
template <typename T>
static bool dec_less_or_equal(T x, T y, UInt32 x_scale, UInt32 y_scale) {
using Comparator = DecimalComparison<T, T, LessOrEqualsOp>;
return Comparator::compare(x, y, x_scale, y_scale);
}
template <>
bool decimal_equal(Decimal32 x, Decimal32 y, UInt32 xs, UInt32 ys) {
return dec_equal(x, y, xs, ys);
}
template <>
bool decimal_less(Decimal32 x, Decimal32 y, UInt32 xs, UInt32 ys) {
return dec_less(x, y, xs, ys);
}
template <>
bool decimal_less_or_equal(Decimal32 x, Decimal32 y, UInt32 xs, UInt32 ys) {
return dec_less_or_equal(x, y, xs, ys);
}
template <>
bool decimal_equal(Decimal64 x, Decimal64 y, UInt32 xs, UInt32 ys) {
return dec_equal(x, y, xs, ys);
}
template <>
bool decimal_less(Decimal64 x, Decimal64 y, UInt32 xs, UInt32 ys) {
return dec_less(x, y, xs, ys);
}
template <>
bool decimal_less_or_equal(Decimal64 x, Decimal64 y, UInt32 xs, UInt32 ys) {
return dec_less_or_equal(x, y, xs, ys);
}
template <>
bool decimal_equal(Decimal128 x, Decimal128 y, UInt32 xs, UInt32 ys) {
return dec_equal(x, y, xs, ys);
}
template <>
bool decimal_less(Decimal128 x, Decimal128 y, UInt32 xs, UInt32 ys) {
return dec_less(x, y, xs, ys);
}
template <>
bool decimal_less_or_equal(Decimal128 x, Decimal128 y, UInt32 xs, UInt32 ys) {
return dec_less_or_equal(x, y, xs, ys);
}
} // namespace doris::vectorized