database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[chore](macOS) Fix BE UT (#14307)

Browse Source 
#13195 left some unresolved issues. One of them is that some BE unit tests fail.
This PR fixes this issue. Now, we can run the command ./run-be-ut.sh --run successfully on macOS.
This commit is contained in:
Adonis Ling
2022-11-18 10:13:38 +08:00
committed by GitHub
parent da0b09caea
commit 2b6f85ab96
12 changed files with 160 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
be/src/exprs/encryption_functions.h
View File

@ -28,7 +28,7 @@ namespace doris {
class Expr;
struct ExprValue;
class TupleRow;
static StringCaseUnorderedMap<EncryptionMode> aes_mode_map {
inline StringCaseUnorderedMap<EncryptionMode> aes_mode_map {
{"AES_128_ECB", EncryptionMode::AES_128_ECB},
{"AES_192_ECB", EncryptionMode::AES_192_ECB},
{"AES_256_ECB", EncryptionMode::AES_256_ECB},
@ -53,7 +53,7 @@ static StringCaseUnorderedMap<EncryptionMode> aes_mode_map {
{"AES_128_OFB", EncryptionMode::AES_128_OFB},
{"AES_192_OFB", EncryptionMode::AES_192_OFB},
{"AES_256_OFB", EncryptionMode::AES_256_OFB}};
static StringCaseUnorderedMap<EncryptionMode> sm4_mode_map {
inline StringCaseUnorderedMap<EncryptionMode> sm4_mode_map {
{"SM4_128_ECB", EncryptionMode::SM4_128_ECB},
{"SM4_128_CBC", EncryptionMode::SM4_128_CBC},
{"SM4_128_CFB128", EncryptionMode::SM4_128_CFB128},

2
be/src/util/string_util.h
View File

@ -104,7 +104,7 @@ public:
if (lhs.size() != rhs.size()) {
return false;
}
return strncasecmp(lhs.c_str(), rhs.c_str(), 0) == 0;
return strncasecmp(lhs.c_str(), rhs.c_str(), lhs.size()) == 0;
}
};

5
be/test/CMakeLists.txt
View File

@ -324,6 +324,10 @@ set(UTIL_TEST_FILES
util/interval_tree_test.cpp
util/key_util_test.cpp
)
if (OS_MACOSX)
list(REMOVE_ITEM UTIL_TEST_FILES util/system_metrics_test.cpp)
endif()
set(VEC_TEST_FILES
vec/aggregate_functions/agg_collect_test.cpp
vec/aggregate_functions/agg_test.cpp
@ -372,7 +376,6 @@ set(VEC_TEST_FILES
vec/utils/arrow_column_to_doris_column_test.cpp
vec/olap/char_type_padding_test.cpp
)
add_executable(doris_be_test
${AGENT_TEST_FILES}
${COMMON_TEST_FILES}

4
be/test/env/env_posix_test.cpp vendored
View File

@ -55,7 +55,11 @@ TEST_F(EnvPosixTest, random_access) {
auto st = env->new_writable_file(fname, &wfile);
EXPECT_TRUE(st.ok());
st = wfile->pre_allocate(1024);
#ifndef __APPLE__
EXPECT_TRUE(st.ok());
#else
EXPECT_FALSE(st.ok());
#endif
// write data
Slice field1("123456789");
st = wfile->append(field1);

2
be/test/runtime/memory_scratch_sink_test.cpp
View File

@ -71,7 +71,7 @@ public:
EXPECT_EQ(system("mkdir -p ./test_run/output/"), 0);
EXPECT_EQ(system("pwd"), 0);
EXPECT_EQ(system("cp -r ./be/test/runtime/test_data/ ./test_run/."), 0);
EXPECT_EQ(system("cp -r ./be/test/runtime/test_data ./test_run/."), 0);
init();
}

86
be/test/testutil/any_type.h Normal file
View File

@ -0,0 +1,86 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <any>
#include <typeinfo>
#include <variant>
namespace doris {
// Typeinfo for __int128 is missing on macOS, we can't use std::any directly.
struct AnyType : public std::variant<std::any, __int128_t> {
AnyType() = default;
template <typename T>
AnyType(T value) {
this->emplace<std::any>(std::move(value));
}
AnyType(__int128_t value) { this->emplace<__int128_t>(value); }
const std::type_info* type() const {
if (std::holds_alternative<std::any>(*this)) {
const auto& type_info = std::get<std::any>(*this).type();
return &type_info;
} else {
#ifdef __APPLE__
return nullptr;
#else
static const auto& type_info = typeid(__int128_t);
return &type_info;
#endif
}
}
template <class ValueType>
friend ValueType any_cast(const AnyType& operand);
template <class ValueType>
friend ValueType any_cast(AnyType& operand);
template <class ValueType>
friend ValueType any_cast(AnyType&& operand);
};
template <class ValueType>
inline ValueType any_cast(const AnyType& operand) {
return std::any_cast<ValueType>(std::get<std::any>(operand));
}
template <>
inline __int128_t any_cast(const AnyType& operand) {
return std::get<__int128_t>(operand);
}
template <class ValueType>
inline ValueType any_cast(AnyType& operand) {
return std::any_cast<ValueType>(std::get<std::any>(operand));
}
template <>
inline __int128_t any_cast(AnyType& operand) {
return std::get<__int128_t>(operand);
}
template <class ValueType>
inline ValueType any_cast(AnyType&& operand) {
return std::any_cast<ValueType>(std::get<std::any>(operand));
}
template <>
inline __int128_t any_cast(AnyType&& operand) {
return std::get<__int128_t>(operand);
}
} // namespace doris

10
be/test/testutil/test_util.cpp
View File

@ -19,6 +19,8 @@
#ifndef __APPLE__
#include <linux/limits.h>
#else
#include <mach-o/dyld.h>
#endif
#include <common/configbase.h>
@ -56,9 +58,17 @@ std::string GetCurrentRunningDir() {
char exe[PATH_MAX];
ssize_t r;
#ifdef __APPLE__
uint32_t size = PATH_MAX;
if (_NSGetExecutablePath(exe, &size) < 0) {
return std::string();
}
r = strlen(exe) + 1;
#else
if ((r = readlink("/proc/self/exe", exe, PATH_MAX)) < 0) {
return std::string();
}
#endif
if (r == PATH_MAX) {
r -= 1;

2
be/test/util/arrow/arrow_work_flow_test.cpp
View File

@ -53,7 +53,7 @@ protected:
EXPECT_EQ(system("mkdir -p ./test_run/output/"), 0);
EXPECT_EQ(system("pwd"), 0);
EXPECT_EQ(system("cp -r ./be/test/util/test_data/ ./test_run/."), 0);
EXPECT_EQ(system("cp -r ./be/test/util/test_data ./test_run/."), 0);
init();
}

4
be/test/util/path_util_test.cpp
View File

@ -58,7 +58,11 @@ TEST(TestPathUtil, DirNameTest) {
EXPECT_EQ(".", path_util::dir_name("."));
EXPECT_EQ(".", path_util::dir_name(".."));
EXPECT_EQ("/", path_util::dir_name("/"));
#ifndef __APPLE__
EXPECT_EQ("//", path_util::dir_name("//"));
#else
EXPECT_EQ("/", path_util::dir_name("//"));
#endif
EXPECT_EQ(".", path_util::dir_name("a"));
EXPECT_EQ(".", path_util::dir_name("ab"));
EXPECT_EQ(".", path_util::dir_name("ab/"));

6
be/test/util/threadpool_test.cpp
View File

@ -327,9 +327,15 @@ TEST_F(ThreadPoolTest, TestZeroQueueSize) {
#ifndef THREAD_SANITIZER
TEST_F(ThreadPoolTest, TestDeadlocks) {
::testing::FLAGS_gtest_death_test_style = "threadsafe";
#ifdef __APPLE__
const char* death_msg =
"_ZNSt3__1L8__invokeIRNS_6__bindIMN5doris10ThreadPoolEFvvEJPS3_EEEJEEEDTclscT_fp_"
"spscT0_fp0_EEOS9_DpOSA_";
#else
const char* death_msg =
"_ZNSt5_BindIFMN5doris10ThreadPoolEFvvEPS1_EE6__callIvJEJLm0EEEET_OSt5tupleIJDpT0_"
"EESt12_Index_tupleIJXspT1_EEE";
#endif
EXPECT_DEATH(
{
EXPECT_TRUE(rebuild_pool_with_min_max(1, 1).ok());

60
be/test/vec/function/function_test_util.cpp
View File

@ -48,7 +48,7 @@ uint64_t str_to_datetime_v2(std::string datetime_str, std::string datetime_forma
return binary_cast<DateV2Value<DateTimeV2ValueType>, UInt64>(v);
}
size_t type_index_to_data_type(const std::vector<std::any>& input_types, size_t index,
size_t type_index_to_data_type(const std::vector<AnyType>& input_types, size_t index,
ut_type::UTDataTypeDesc& ut_desc, DataTypePtr& type) {
doris_udf::FunctionContext::TypeDesc& desc = ut_desc.type_desc;
if (index < 0 || index >= input_types.size()) {
@ -56,19 +56,19 @@ size_t type_index_to_data_type(const std::vector<std::any>& input_types, size_t
}
TypeIndex tp;
if (input_types[index].type() == typeid(Consted)) {
tp = std::any_cast<Consted>(input_types[index]).tp;
} else if (input_types[index].type() == typeid(ConstedNotnull)) {
tp = std::any_cast<ConstedNotnull>(input_types[index]).tp;
if (input_types[index].type() == &typeid(Consted)) {
tp = any_cast<Consted>(input_types[index]).tp;
} else if (input_types[index].type() == &typeid(ConstedNotnull)) {
tp = any_cast<ConstedNotnull>(input_types[index]).tp;
ut_desc.is_nullable = false;
} else if (input_types[index].type() == typeid(Nullable)) {
tp = std::any_cast<Nullable>(input_types[index]).tp;
} else if (input_types[index].type() == &typeid(Nullable)) {
tp = any_cast<Nullable>(input_types[index]).tp;
ut_desc.is_nullable = true;
} else if (input_types[index].type() == typeid(Notnull)) {
tp = std::any_cast<Notnull>(input_types[index]).tp;
} else if (input_types[index].type() == &typeid(Notnull)) {
tp = any_cast<Notnull>(input_types[index]).tp;
ut_desc.is_nullable = false;
} else {
tp = std::any_cast<TypeIndex>(input_types[index]);
tp = any_cast<TypeIndex>(input_types[index]);
}
switch (tp) {
@ -164,13 +164,13 @@ size_t type_index_to_data_type(const std::vector<std::any>& input_types, size_t
return 0;
}
}
bool parse_ut_data_type(const std::vector<std::any>& input_types, ut_type::UTDataTypeDescs& descs) {
bool parse_ut_data_type(const std::vector<AnyType>& input_types, ut_type::UTDataTypeDescs& descs) {
descs.clear();
descs.reserve(input_types.size());
for (size_t i = 0; i < input_types.size();) {
ut_type::UTDataTypeDesc desc;
if (input_types[i].type() == typeid(Consted) ||
input_types[i].type() == typeid(ConstedNotnull)) {
if (input_types[i].type() == &typeid(Consted) ||
input_types[i].type() == &typeid(ConstedNotnull)) {
desc.is_const = true;
}
size_t res = type_index_to_data_type(input_types, i, desc, desc.data_type);
@ -188,8 +188,8 @@ bool parse_ut_data_type(const std::vector<std::any>& input_types, ut_type::UTDat
}
template <typename Date, TypeIndex type_index = TypeIndex::Nothing>
bool insert_date_cell(MutableColumnPtr& column, const std::string& format, const std::any& cell) {
auto datetime_str = std::any_cast<std::string>(cell);
bool insert_date_cell(MutableColumnPtr& column, const std::string& format, const AnyType& cell) {
auto datetime_str = any_cast<std::string>(cell);
Date v;
auto result = v.from_date_format_str(format.c_str(), format.size(), datetime_str.c_str(),
datetime_str.size());
@ -208,8 +208,8 @@ bool insert_date_cell(MutableColumnPtr& column, const std::string& format, const
return true;
}
bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const std::any& cell) {
if (cell.type() == typeid(Null)) {
bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const AnyType& cell) {
if (cell.type() == &typeid(Null)) {
column->insert_data(nullptr, 0);
return true;
}
@ -219,41 +219,41 @@ bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const std::any&
WhichDataType type(type_ptr);
if (type.is_string()) {
auto str = std::any_cast<ut_type::STRING>(cell);
auto str = any_cast<ut_type::STRING>(cell);
column->insert_data(str.c_str(), str.size());
} else if (type.is_json()) {
auto str = std::any_cast<ut_type::STRING>(cell);
auto str = any_cast<ut_type::STRING>(cell);
JsonBinaryValue jsonb_val(str.c_str(), str.size());
column->insert_data(jsonb_val.value(), jsonb_val.size());
} else if (type.idx == TypeIndex::BitMap) {
BitmapValue* bitmap = std::any_cast<BitmapValue*>(cell);
BitmapValue* bitmap = any_cast<BitmapValue*>(cell);
column->insert_data((char*)bitmap, sizeof(BitmapValue));
} else if (type.is_uint8()) {
auto value = std::any_cast<ut_type::BOOLEAN>(cell);
auto value = any_cast<ut_type::BOOLEAN>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_int8()) {
auto value = std::any_cast<ut_type::TINYINT>(cell);
auto value = any_cast<ut_type::TINYINT>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_int16()) {
auto value = std::any_cast<ut_type::SMALLINT>(cell);
auto value = any_cast<ut_type::SMALLINT>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_int32()) {
auto value = std::any_cast<ut_type::INT>(cell);
auto value = any_cast<ut_type::INT>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_int64()) {
auto value = std::any_cast<ut_type::BIGINT>(cell);
auto value = any_cast<ut_type::BIGINT>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_int128()) {
auto value = std::any_cast<ut_type::LARGEINT>(cell);
auto value = any_cast<ut_type::LARGEINT>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_float32()) {
auto value = std::any_cast<ut_type::FLOAT>(cell);
auto value = any_cast<ut_type::FLOAT>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_float64()) {
auto value = std::any_cast<ut_type::DOUBLE>(cell);
auto value = any_cast<ut_type::DOUBLE>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_decimal128()) {
auto value = std::any_cast<Decimal<Int128>>(cell);
auto value = any_cast<Decimal<Int128>>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_date_time()) {
static std::string date_time_format("%Y-%m-%d %H:%i:%s");
@ -272,7 +272,7 @@ bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const std::any&
RETURN_IF_FALSE((insert_date_cell<DateV2Value<DateTimeV2ValueType>>(
column, date_time_format, cell)));
} else if (type.is_array()) {
auto v = std::any_cast<Array>(cell);
auto v = any_cast<Array>(cell);
column->insert(v);
} else {
LOG(WARNING) << "dataset not supported for TypeIndex:" << (int)type.idx;

21
be/test/vec/function/function_test_util.h
View File

@ -23,6 +23,7 @@
#include <string>
#include "exprs/table_function/table_function.h"
#include "testutil/any_type.h"
#include "testutil/function_utils.h"
#include "udf/udf.h"
#include "udf/udf_internal.h"
@ -38,12 +39,12 @@
namespace doris::vectorized {
using InputDataSet = std::vector<std::vector<std::any>>; // without result
using CellSet = std::vector<std::any>;
using Expect = std::any;
using InputDataSet = std::vector<std::vector<AnyType>>; // without result
using CellSet = std::vector<AnyType>;
using Expect = AnyType;
using Row = std::pair<CellSet, Expect>;
using DataSet = std::vector<Row>;
using InputTypeSet = std::vector<std::any>;
using InputTypeSet = std::vector<AnyType>;
int64_t str_to_date_time(std::string datetime_str, bool data_time = true);
uint32_t str_to_date_v2(std::string datetime_str, std::string datetime_format);
@ -157,11 +158,11 @@ using UTDataTypeDescs = std::vector<UTDataTypeDesc>;
} // namespace ut_type
size_t type_index_to_data_type(const std::vector<std::any>& input_types, size_t index,
size_t type_index_to_data_type(const std::vector<AnyType>& input_types, size_t index,
ut_type::UTDataTypeDesc& ut_desc, DataTypePtr& type);
bool parse_ut_data_type(const std::vector<std::any>& input_types, ut_type::UTDataTypeDescs& descs);
bool parse_ut_data_type(const std::vector<AnyType>& input_types, ut_type::UTDataTypeDescs& descs);
bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const std::any& cell);
bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const AnyType& cell);
Block* create_block_from_inputset(const InputTypeSet& input_types, const InputDataSet& input_set);
@ -271,7 +272,7 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty
for (int i = 0; i < row_size; ++i) {
auto check_column_data = [&]() {
if constexpr (std::is_same_v<ReturnType, DataTypeJsonb>) {
const auto& expect_data = std::any_cast<String>(data_set[i].second);
const auto& expect_data = any_cast<String>(data_set[i].second);
auto s = column->get_data_at(i);
if (expect_data.size() == 0) {
// zero size result means invalid
@ -286,7 +287,7 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty
column->get(i, field);
const auto& expect_data =
std::any_cast<typename ReturnType::FieldType>(data_set[i].second);
any_cast<typename ReturnType::FieldType>(data_set[i].second);
if constexpr (std::is_same_v<ReturnType, DataTypeDecimal<Decimal128>>) {
const auto& column_data = field.get<DecimalField<Decimal128>>().get_value();
@ -303,7 +304,7 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty
};
if constexpr (nullable) {
bool is_null = data_set[i].second.type() == typeid(Null);
bool is_null = data_set[i].second.type() == &typeid(Null);
EXPECT_EQ(is_null, column->is_null_at(i)) << " at row " << i;
if (!is_null) check_column_data();
} else {