[bugfix](jsonb) fix jsonb parser crash on noavx2 host (#18977)

support avx2 and noavx2 for jsonb parser using __AVX2__ macro.
This commit is contained in:
Kang
2023-04-26 15:10:12 +08:00
committed by GitHub
parent 45874bbf62
commit 1dfc5ea34c
5 changed files with 20 additions and 5 deletions

View File

@ -26,7 +26,11 @@
#include "common/status.h"
#include "util/hash_util.hpp"
#ifdef __AVX2__
#include "util/jsonb_parser_simd.h"
#else
#include "util/jsonb_parser.h"
#endif
namespace doris {
@ -36,7 +40,7 @@ struct JsonBinaryValue {
// default nullprt and size 0 for invalid or NULL value
const char* ptr = nullptr;
size_t len = 0;
JsonbParserSIMD parser;
JsonbParser parser;
JsonBinaryValue() : ptr(nullptr), len(0) {}
JsonBinaryValue(char* ptr, int len) { from_json_string(const_cast<const char*>(ptr), len); }

View File

@ -665,12 +665,15 @@ private:
case 4:
*--out = ((uc | 0x80) & 0xBF);
uc >>= 6;
[[fallthrough]];
case 3:
*--out = ((uc | 0x80) & 0xBF);
uc >>= 6;
[[fallthrough]];
case 2:
*--out = ((uc | 0x80) & 0xBF);
uc >>= 6;
[[fallthrough]];
case 1:
// Mask the first byte according to the standard.
*--out = (uc | firstByteMark[len - 1]);

View File

@ -343,7 +343,7 @@ private:
JsonbErrType err_;
};
using JsonbParserSIMD = JsonbParserTSIMD<JsonbOutStream>;
using JsonbParser = JsonbParserTSIMD<JsonbOutStream>;
} // namespace doris

View File

@ -41,7 +41,11 @@
#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/status.h"
#include "exprs/json_functions.h"
#ifdef __AVX2__
#include "util/jsonb_parser_simd.h"
#else
#include "util/jsonb_parser.h"
#endif
#include "util/string_parser.hpp"
#include "util/string_util.h"
#include "vec/aggregate_functions/aggregate_function.h"
@ -910,7 +914,7 @@ public:
vec_to.resize(size);
// parser can be reused for performance
JsonbParserSIMD parser;
JsonbParser parser;
for (size_t i = 0; i < input_rows_count; ++i) {
if (col_from.is_null_at(i)) {
null_map->get_data()[i] = 1;

View File

@ -33,7 +33,11 @@
#include "udf/udf.h"
#include "util/jsonb_document.h"
#include "util/jsonb_error.h"
#ifdef __AVX2__
#include "util/jsonb_parser_simd.h"
#else
#include "util/jsonb_parser.h"
#endif
#include "util/jsonb_stream.h"
#include "util/jsonb_utils.h"
#include "util/jsonb_writer.h"
@ -74,7 +78,7 @@ enum class JsonbParseErrorMode { FAIL = 0, RETURN_NULL, RETURN_VALUE, RETURN_INV
template <NullalbeMode nullable_mode, JsonbParseErrorMode parse_error_handle_mode>
class FunctionJsonbParseBase : public IFunction {
private:
JsonbParserSIMD default_value_parser;
JsonbParser default_value_parser;
bool has_const_default_value = false;
public:
@ -221,7 +225,7 @@ public:
col_to->reserve(size);
// parser can be reused for performance
JsonbParserSIMD parser;
JsonbParser parser;
JsonbErrType error = JsonbErrType::E_NONE;
for (size_t i = 0; i < input_rows_count; ++i) {