[chore](macOS) Support LLVM Clang 15 (#16991)

Remove the deprecated classes std::codecvt_utf8_utf16<char16_t> and std::wstring_convert.
Use libiconv to convert UTF-8 strings to UTF-16LE ones.
This commit is contained in:
Adonis Ling
2023-02-22 15:04:48 +08:00
committed by GitHub
parent 3636d0a561
commit 0b3e18d060
2 changed files with 37 additions and 5 deletions

View File

@ -17,9 +17,10 @@
#include "exec/table_connector.h"
#include <fmt/core.h>
#include <gen_cpp/Types_types.h>
#include <codecvt>
#include <glog/logging.h>
#include <iconv.h>
#include "runtime/define_primitive_type.h"
#include "runtime/primitive_type.h"
@ -46,8 +47,39 @@ void TableConnector::init_profile(doris::RuntimeProfile* profile) {
}
std::u16string TableConnector::utf8_to_u16string(const char* first, const char* last) {
std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> utf8_utf16_cvt;
return utf8_utf16_cvt.from_bytes(first, last);
auto deleter = [](auto convertor) {
if (convertor == reinterpret_cast<decltype(convertor)>(-1)) {
return;
}
iconv_close(convertor);
};
std::unique_ptr<std::remove_pointer_t<iconv_t>, decltype(deleter)> convertor(
iconv_open("UTF-16LE", "UTF-8"), deleter);
char* in = const_cast<char*>(first);
size_t inbytesleft = last - first;
char16_t buffer[1024];
char* out = reinterpret_cast<char*>(&buffer[0]);
size_t outbytesleft = sizeof(buffer);
std::u16string result;
while (inbytesleft > 0) {
if (iconv(convertor.get(), &in, &inbytesleft, &out, &outbytesleft)) {
if (errno == E2BIG) {
result += std::u16string_view(buffer,
(sizeof(buffer) - outbytesleft) / sizeof(char16_t));
out = reinterpret_cast<char*>(&buffer[0]);
outbytesleft = sizeof(buffer);
} else {
LOG(WARNING) << fmt::format("Failed to convert the UTF-8 string {} to UTF-16LE",
std::string(first, last));
return result;
}
}
}
result += std::u16string_view(buffer, (sizeof(buffer) - outbytesleft) / sizeof(char16_t));
return result;
}
Status TableConnector::append(const std::string& table_name, vectorized::Block* block,

View File

@ -72,7 +72,7 @@ ${BASH_SOURCE[0]} start|stop|clean [OPTIONS ...]
stop Stop the FE cluster.
clean Stop the data (rm -rf "\$(pwd)"/fe*).
clean Clean the data (rm -rf "\$(pwd)"/fe*).
EOF
exit 1
}