From 9f088f6e901ef6468078bd32d94ce5f241a0f5e8 Mon Sep 17 00:00:00 2001 From: xueweizhang Date: Thu, 2 Mar 2023 14:08:52 +0800 Subject: [PATCH] [feature](json) add json_valid function (#17247) add json_valid function Signed-off-by: nextdreamblue --- be/src/vec/functions/function_json.cpp | 65 ++++++++++++++++ .../json-functions/json_valid.md | 75 +++++++++++++++++++ docs/sidebars.json | 3 +- .../json-functions/json_valid.md | 75 +++++++++++++++++++ gensrc/script/doris_builtins_functions.py | 1 + .../jsonb_p0/test_jsonb_load_and_function.out | 31 ++++++++ .../test_jsonb_load_and_function.groovy | 4 + 7 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 docs/en/docs/sql-manual/sql-functions/json-functions/json_valid.md create mode 100644 docs/zh-CN/docs/sql-manual/sql-functions/json-functions/json_valid.md diff --git a/be/src/vec/functions/function_json.cpp b/be/src/vec/functions/function_json.cpp index 4014b65610..9a22fc94f0 100644 --- a/be/src/vec/functions/function_json.cpp +++ b/be/src/vec/functions/function_json.cpp @@ -655,6 +655,69 @@ using FunctionGetJsonDouble = FunctionBinaryStringOperateToNullType; using FunctionGetJsonString = FunctionBinaryStringOperateToNullType; +class FunctionJsonValid : public IFunction { +public: + static constexpr auto name = "json_valid"; + static FunctionPtr create() { return std::make_shared(); } + + String get_name() const override { return name; } + + size_t get_number_of_arguments() const override { return 1; } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + return make_nullable(std::make_shared()); + } + + bool use_default_implementation_for_nulls() const override { return false; } + + bool use_default_implementation_for_constants() const override { return true; } + + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + size_t result, size_t input_rows_count) override { + const IColumn& col_from = *(block.get_by_position(arguments[0]).column); + + auto null_map = ColumnUInt8::create(input_rows_count, 0); + + const ColumnString* col_from_string = check_and_get_column(col_from); + if (auto* nullable = check_and_get_column(col_from)) { + col_from_string = + check_and_get_column(*nullable->get_nested_column_ptr()); + } + + if (!col_from_string) { + return Status::RuntimeError("Illegal column {} should be ColumnString", + col_from.get_name()); + } + + auto col_to = ColumnVector::create(); + auto& vec_to = col_to->get_data(); + size_t size = col_from.size(); + vec_to.resize(size); + + // parser can be reused for performance + JsonbParserSIMD parser; + for (size_t i = 0; i < input_rows_count; ++i) { + if (col_from.is_null_at(i)) { + null_map->get_data()[i] = 1; + vec_to[i] = 0; + continue; + } + + const auto& val = col_from_string->get_data_at(i); + if (parser.parse(val.data, val.size)) { + vec_to[i] = 1; + } else { + vec_to[i] = 0; + } + } + + block.replace_by_position(result, + ColumnNullable::create(std::move(col_to), std::move(null_map))); + + return Status::OK(); + } +}; + void register_function_json(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); @@ -663,6 +726,8 @@ void register_function_json(SimpleFunctionFactory& factory) { factory.register_function>(); factory.register_function>(); factory.register_function>(); + + factory.register_function(); } } // namespace doris::vectorized diff --git a/docs/en/docs/sql-manual/sql-functions/json-functions/json_valid.md b/docs/en/docs/sql-manual/sql-functions/json-functions/json_valid.md new file mode 100644 index 0000000000..844fd5289c --- /dev/null +++ b/docs/en/docs/sql-manual/sql-functions/json-functions/json_valid.md @@ -0,0 +1,75 @@ +--- +{ + "title": "json_valid", + "language": "en" +} +--- + + + +## json_valid +### description + +json_valid functions returns 0 or 1 to indicate whether a value is valid JSON and Returns NULL if the argument is NULL. + +#### Syntax + +`JSONB json_valid(VARCHAR json_str)` + +### example + +1. parse valid JSON string + +``` +MySQL > SELECT json_valid('{"k1":"v31","k2":300}'); ++-------------------------------------+ +| json_valid('{"k1":"v31","k2":300}') | ++-------------------------------------+ +| 1 | ++-------------------------------------+ +1 row in set (0.02 sec) +``` + +2. parse invalid JSON string + +``` +MySQL > SELECT json_valid('invalid json'); ++----------------------------+ +| json_valid('invalid json') | ++----------------------------+ +| 0 | ++----------------------------+ +1 row in set (0.02 sec) +``` + +3. parse NULL + +``` +MySQL > select json_valid(NULL); ++------------------+ +| json_valid(NULL) | ++------------------+ +| NULL | ++------------------+ +1 row in set (0.02 sec) +``` + +### keywords +JSON, VALID, JSON_VALID diff --git a/docs/sidebars.json b/docs/sidebars.json index 1c3f2db74f..363a7da352 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -576,7 +576,8 @@ "sql-manual/sql-functions/json-functions/get_json_string", "sql-manual/sql-functions/json-functions/json_array", "sql-manual/sql-functions/json-functions/json_object", - "sql-manual/sql-functions/json-functions/json_quote" + "sql-manual/sql-functions/json-functions/json_quote", + "sql-manual/sql-functions/json-functions/json_valid" ] }, { diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/json-functions/json_valid.md b/docs/zh-CN/docs/sql-manual/sql-functions/json-functions/json_valid.md new file mode 100644 index 0000000000..5dc70e373e --- /dev/null +++ b/docs/zh-CN/docs/sql-manual/sql-functions/json-functions/json_valid.md @@ -0,0 +1,75 @@ +--- +{ + "title": "json_valid", + "language": "zh-CN" +} +--- + + + +## json_valid +### description + +json_valid 函数返回0或1以表明是否为有效的JSON, 如果参数是NULL则返回NULL。 + +#### Syntax + +`JSONB json_valid(VARCHAR json_str)` + +### example + +1. 正常JSON字符串 + +``` +MySQL > SELECT json_valid('{"k1":"v31","k2":300}'); ++-------------------------------------+ +| json_valid('{"k1":"v31","k2":300}') | ++-------------------------------------+ +| 1 | ++-------------------------------------+ +1 row in set (0.02 sec) +``` + +2. 无效的JSON字符串 + +``` +MySQL > SELECT json_valid('invalid json'); ++----------------------------+ +| json_valid('invalid json') | ++----------------------------+ +| 0 | ++----------------------------+ +1 row in set (0.02 sec) +``` + +3. NULL参数 + +``` +MySQL > select json_valid(NULL); ++------------------+ +| json_valid(NULL) | ++------------------+ +| NULL | ++------------------+ +1 row in set (0.02 sec) +``` + +### keywords +JSON, VALID, JSON_VALID diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index b9eb605aa2..f650e0a28a 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -1385,6 +1385,7 @@ visible_functions = [ [['json_array'], 'VARCHAR', ['VARCHAR', '...'], 'ALWAYS_NOT_NULLABLE'], [['json_object'], 'VARCHAR', ['VARCHAR', '...'], 'ALWAYS_NOT_NULLABLE'], [['json_quote'], 'VARCHAR', ['VARCHAR'], ''], + [['json_valid'], 'INT', ['VARCHAR'], 'ALWAYS_NULLABLE'], #hll function [['hll_cardinality'], 'BIGINT', ['HLL'], 'ALWAYS_NOT_NULLABLE'], diff --git a/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out b/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out index d834c24dcd..8172b0f011 100644 --- a/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out +++ b/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out @@ -3944,3 +3944,34 @@ false -- !select -- \N +-- !select -- +1 \N +2 1 +3 1 +4 1 +5 1 +6 1 +7 1 +8 1 +9 1 +10 1 +11 1 +12 1 +13 1 +14 1 +15 1 +16 1 +17 1 +18 1 +26 \N +27 1 + +-- !select -- +1 + +-- !select -- +0 + +-- !select -- +\N + diff --git a/regression-test/suites/jsonb_p0/test_jsonb_load_and_function.groovy b/regression-test/suites/jsonb_p0/test_jsonb_load_and_function.groovy index e52210b37d..707c225ce1 100644 --- a/regression-test/suites/jsonb_p0/test_jsonb_load_and_function.groovy +++ b/regression-test/suites/jsonb_p0/test_jsonb_load_and_function.groovy @@ -416,4 +416,8 @@ suite("test_jsonb_load_and_function", "p0") { qt_select """SELECT CAST('{x' AS JSONB)""" qt_select """SELECT CAST('[123, abc]' AS JSONB)""" + qt_select """SELECT id, JSON_VALID(j) FROM ${testTable} ORDER BY id""" + qt_select """SELECT JSON_VALID('{"k1":"v31","k2":300}')""" + qt_select """SELECT JSON_VALID('invalid json')""" + qt_select """SELECT JSON_VALID(NULL)""" }