From 852046de2946d31d28a0011fc68c32388b01feed Mon Sep 17 00:00:00 2001 From: yangzhg <780531911@qq.com> Date: Tue, 7 Jan 2020 19:16:10 +0800 Subject: [PATCH] Fix incompatibility with arm architecture in olap #2645 (#2682) --- be/src/common/atomic.h | 4 + be/src/exec/aggregation_node.cpp | 1 - be/src/exec/broker_scanner.cpp | 4 +- be/src/olap/atomic.h | 231 ------------------ be/src/olap/bhp_lib.h | 22 -- be/src/olap/olap_index.h | 1 - be/src/olap/rowset/segment_group.cpp | 4 +- be/src/olap/rowset/segment_group.h | 3 +- .../rowset/segment_v2/bitshuffle_wrapper.cpp | 6 + be/src/olap/storage_engine.h | 4 - be/src/olap/txn_manager.h | 1 - be/src/olap/utils.cpp | 7 +- be/src/olap/utils.h | 2 + 13 files changed, 22 insertions(+), 268 deletions(-) delete mode 100644 be/src/olap/atomic.h diff --git a/be/src/common/atomic.h b/be/src/common/atomic.h index d1bb581568..766c496f40 100644 --- a/be/src/common/atomic.h +++ b/be/src/common/atomic.h @@ -35,7 +35,11 @@ public: // should be: // while (1) CpuWait(); static ALWAYS_INLINE void cpu_wait() { +#if (defined(__i386) || defined(__x86_64__)) asm volatile("pause\n": : :"memory"); +#elif defined(__aarch64__) + asm volatile("yield\n" ::: "memory"); +#endif } /// Provides "barrier" semantics (see below) without a memory access. diff --git a/be/src/exec/aggregation_node.cpp b/be/src/exec/aggregation_node.cpp index 5f9beeb2ab..1b3ff19147 100644 --- a/be/src/exec/aggregation_node.cpp +++ b/be/src/exec/aggregation_node.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include #include "codegen/codegen_anyval.h" diff --git a/be/src/exec/broker_scanner.cpp b/be/src/exec/broker_scanner.cpp index 1155db9927..1478d37ebb 100644 --- a/be/src/exec/broker_scanner.cpp +++ b/be/src/exec/broker_scanner.cpp @@ -34,7 +34,7 @@ #include "exec/local_file_reader.h" #include "exec/broker_reader.h" #include "exec/decompressor.h" -#include "util/simdutf8check.h" +#include "util/utf8_check.h" namespace doris { @@ -400,7 +400,7 @@ bool BrokerScanner::convert_one_row( // Convert one row to this tuple bool BrokerScanner::line_to_src_tuple(const Slice& line) { - if (!validate_utf8_fast(line.data, line.size)) { + if (!validate_utf8(line.data, line.size)) { std::stringstream error_msg; error_msg << "data is not encoded by UTF-8"; _state->append_error_msg_to_file(std::string(line.data, line.size), diff --git a/be/src/olap/atomic.h b/be/src/olap/atomic.h deleted file mode 100644 index bd6eb2d015..0000000000 --- a/be/src/olap/atomic.h +++ /dev/null @@ -1,231 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_OLAP_ATOMIC_H -#define DORIS_BE_SRC_OLAP_ATOMIC_H - -namespace doris { - -#define LOCK_PREFIX "lock " - -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) - -typedef volatile long atomic64; -typedef atomic64 atomic_t; - -#define atomic_xchg atomic64_xchg -#define atomic_cmpxchg atomic64_cmpxchg -#define atomic_add atomic64_add -#define atomic_sub atomic64_sub -#define atomic_sub_and_test atomic64_sub_and_test -#define atomic_inc atomic64_inc -#define atomic_dec atomic64_dec -#define atomic_dec_and_test atomic64_dec_and_test -#define atomic_inc_and_test atomic64_inc_and_test -#define atomic_add_negative atomic64_add_negative -#define atomic_add_return atomic64_add_return -#define atomic_sub_return atomic64_sub_return -#define atomic_add_unless atomic64_add_unless - -#define atomic64_inc_return(v) (atomic64_add_return(1, (v))) -#define atomic64_dec_return(v) (atomic64_sub_return(1, (v))) - -#define atomic_inc_return atomic64_inc_return -#define atomic_dec_return atomic64_dec_return - -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) - -static inline long -atomic64_xchg(atomic64 x, atomic64* ptr) { - __asm__ __volatile__( - LOCK_PREFIX "xchgq %0,%1" - : "=r"(x) - : "m"(*ptr), "0"(x) - : "memory" - ); - - return x; -} - -static inline long -atomic64_cmpxchg(atomic64* ptr, atomic64 old_value, atomic64 new_value) { - atomic64 prev; - - __asm__ __volatile__( - LOCK_PREFIX "cmpxchgq %1,%2" - : "=a"(prev) - : "r"(new_value), "m"(*ptr), "0"(old_value) - : "memory"); - - return prev; -} - -static inline void -atomic64_add(long i, atomic64* v) { - __asm__ __volatile__( - LOCK_PREFIX "addq %1,%0" - : "=m"(*v) - : "ir"(i), "m"(*v) - ); -} - -static inline void -atomic64_sub(long i, atomic64* v) { - __asm__ __volatile__( - LOCK_PREFIX "subq %1,%0" - : "=m"(*v) - : "er"(i), "m"(*v) - ); -} - -static inline int -atomic64_sub_and_test(long i, atomic64* v) { - unsigned char c; - - __asm__ __volatile__( - LOCK_PREFIX "subq %2,%0; sete %1" - : "=m"(*v), "=qm"(c) - : "er"(i), "m"(*v) - : "memory" - ); - - return c; -} - -static inline void -atomic64_inc(atomic64* v) { - __asm__ __volatile__( - LOCK_PREFIX "incq %0" - : "+m"(*v) - ); -} - -static inline void -atomic64_dec(atomic64* v) { - __asm__ __volatile__( - LOCK_PREFIX "decq %0" - : "+m"(*v) - ); -} - -static inline int -atomic64_dec_and_test(atomic64* v) { - unsigned char c; - - __asm__ __volatile__( - LOCK_PREFIX "decq %0; sete %1" - : "=m"(*v), "=qm"(c) - : "m"(*v) - : "memory" - ); - - return c != 0; -} - -static inline int -atomic64_inc_and_test(atomic64* v) { - unsigned char c; - - __asm__ __volatile__( - LOCK_PREFIX "incq %0; sete %1" - : "=m"(*v), "=qm"(c) - : "m"(*v) - : "memory" - ); - - return c != 0; -} - -static inline int -atomic64_add_negative(long i, atomic64* v) { - unsigned char c; - - __asm__ __volatile__( - LOCK_PREFIX "addq %2,%0; sets %1" - : "=m"(*v), "=qm"(c) - : "er"(i), "m"(*v) - : "memory" - ); - - return c; -} - -static inline long -atomic64_add_return(long i, atomic64* v) { - long __i = i; - - __asm__ __volatile__( - LOCK_PREFIX "xaddq %0, %1;" - : "+r"(i), "+m"(*v) - : - : "memory" - ); - - return i + __i; -} - -static inline long -atomic64_sub_return(long i, atomic64* v) { - return atomic64_add_return(-i, v); -} - -static inline int -atomic64_add_unless(atomic64* v, long a, long u) { - long c, old; - - c = *v; - - for (;;) { - if (unlikely(c == (u))) { - break; - } - - old = atomic64_cmpxchg((v), c, c + (a)); - - if (likely(old == c)) { - break; - } - - c = old; - } - - return c != (u); -} - -static inline void -atomic_or_long(unsigned long* v1, unsigned long v2) { - __asm__ __volatile__( - LOCK_PREFIX "orq %1, %0" - : "+m"(*v1) - : "r"(v2) - ); -} - -static inline short int -atomic_inc_short(short int* v) { - __asm__ __volatile__( - LOCK_PREFIX "addw $1, %0" - : "+m"(*v) - ); - - return *v; -} - -} // namespace doris - -#endif // DORIS_BE_SRC_OLAP_ATOMIC_H diff --git a/be/src/olap/bhp_lib.h b/be/src/olap/bhp_lib.h index 0e237497c7..81a37c182c 100644 --- a/be/src/olap/bhp_lib.h +++ b/be/src/olap/bhp_lib.h @@ -768,28 +768,6 @@ inline int baidu_crc32_qw(char const* src, int crc, unsigned int length) { return crc; } -inline int check_sse4_2() { - int ret; - __asm__ __volatile__( - "mov $1, %%rax;" - "cpuid;" - "test $0x000100000, %%ecx;" - "jnz 1f;" - "xor %%eax, %%eax;" - "jmp 2f;" - - "1:" - "mov $1, %%eax;" - - "2:" - "mov %%eax, %0;" - :"=r"(ret) - : - :"%rax", "%rcx", "rdx", "%rbx", "memory" - ); - return ret; -} - } // namespace doris #endif // DORIS_BE_SRC_OLAP_BHP_LIB_H diff --git a/be/src/olap/olap_index.h b/be/src/olap/olap_index.h index 947f7850c1..0db0e97ec5 100644 --- a/be/src/olap/olap_index.h +++ b/be/src/olap/olap_index.h @@ -28,7 +28,6 @@ #include "gen_cpp/olap_file.pb.h" #include "gen_cpp/column_data_file.pb.h" -#include "olap/atomic.h" #include "olap/field.h" #include "olap/file_helper.h" #include "olap/olap_common.h" diff --git a/be/src/olap/rowset/segment_group.cpp b/be/src/olap/rowset/segment_group.cpp index 41fcab51e8..5d3a6007bc 100644 --- a/be/src/olap/rowset/segment_group.cpp +++ b/be/src/olap/rowset/segment_group.cpp @@ -190,7 +190,7 @@ std::string SegmentGroup::construct_data_file_path(int32_t segment_id) const { } void SegmentGroup::acquire() { - atomic_inc(&_ref_count); + ++_ref_count; } int64_t SegmentGroup::ref_count() { @@ -198,7 +198,7 @@ int64_t SegmentGroup::ref_count() { } void SegmentGroup::release() { - atomic_dec(&_ref_count); + --_ref_count; } bool SegmentGroup::is_in_use() { diff --git a/be/src/olap/rowset/segment_group.h b/be/src/olap/rowset/segment_group.h index d9b4bb70b7..09597be17c 100644 --- a/be/src/olap/rowset/segment_group.h +++ b/be/src/olap/rowset/segment_group.h @@ -28,7 +28,6 @@ #include "gen_cpp/olap_file.pb.h" #include "gen_cpp/column_data_file.pb.h" -#include "olap/atomic.h" #include "olap/field.h" #include "olap/file_helper.h" #include "olap/olap_common.h" @@ -291,7 +290,7 @@ private: PUniqueId _load_id; // load id for segmentgroup int32_t _num_segments; // number of segments in this segmentgroup bool _index_loaded; // whether the segmentgroup has been read - atomic_t _ref_count; // reference count + std::atomic _ref_count; // reference count MemIndex _index; bool _is_pending; TPartitionId _partition_id; diff --git a/be/src/olap/rowset/segment_v2/bitshuffle_wrapper.cpp b/be/src/olap/rowset/segment_v2/bitshuffle_wrapper.cpp index 22c280ae1a..5e3c1632e4 100644 --- a/be/src/olap/rowset/segment_v2/bitshuffle_wrapper.cpp +++ b/be/src/olap/rowset/segment_v2/bitshuffle_wrapper.cpp @@ -54,6 +54,7 @@ decltype(&bshuf_decompress_lz4) g_bshuf_decompress_lz4; // the cost of a 'std::once' call. __attribute__((constructor)) void SelectBitshuffleFunctions() { +#if (defined(__i386) || defined(__x86_64__)) if (CPU().has_avx2()) { g_bshuf_compress_lz4_bound = bshuf_compress_lz4_bound_avx2; g_bshuf_compress_lz4 = bshuf_compress_lz4_avx2; @@ -63,6 +64,11 @@ void SelectBitshuffleFunctions() { g_bshuf_compress_lz4 = bshuf_compress_lz4; g_bshuf_decompress_lz4 = bshuf_decompress_lz4; } +#else + g_bshuf_compress_lz4_bound = bshuf_compress_lz4_bound; + g_bshuf_compress_lz4 = bshuf_compress_lz4; + g_bshuf_decompress_lz4 = bshuf_decompress_lz4; +#endif } int64_t compress_lz4(void* in, void* out, size_t size, diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 542f5fda0c..9cb9294134 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -36,8 +36,6 @@ #include "gen_cpp/AgentService_types.h" #include "gen_cpp/BackendService_types.h" #include "gen_cpp/MasterService_types.h" -#include "olap/atomic.h" -#include "olap/lru_cache.h" #include "olap/olap_common.h" #include "olap/olap_define.h" #include "olap/tablet.h" @@ -344,8 +342,6 @@ private: // thread to run tablet checkpoint std::vector _tablet_checkpoint_threads; - static atomic_t _s_request_number; - // for tablet and disk report std::mutex _report_mtx; std::condition_variable _report_cv; diff --git a/be/src/olap/txn_manager.h b/be/src/olap/txn_manager.h index 4b22266fc1..c51e253e0b 100755 --- a/be/src/olap/txn_manager.h +++ b/be/src/olap/txn_manager.h @@ -38,7 +38,6 @@ #include "gen_cpp/AgentService_types.h" #include "gen_cpp/BackendService_types.h" #include "gen_cpp/MasterService_types.h" -#include "olap/atomic.h" #include "olap/lru_cache.h" #include "olap/olap_common.h" #include "olap/olap_define.h" diff --git a/be/src/olap/utils.cpp b/be/src/olap/utils.cpp index 7b0784f02f..f4b09e95fe 100644 --- a/be/src/olap/utils.cpp +++ b/be/src/olap/utils.cpp @@ -934,12 +934,15 @@ unsigned int crc32c_lut(char const * b, unsigned int off, unsigned int len, unsi } uint32_t olap_crc32(uint32_t crc32, const char* buf, size_t len) { - static int sse4_flag = check_sse4_2(); - if (OLAP_LIKELY(1 == sse4_flag)) { +#if defined(__i386) || defined(__x86_64__) + if (OLAP_LIKELY(CpuInfo::is_supported(CpuInfo::SSE4_2))) { return baidu_crc32_qw(buf, crc32, len); } else { return crc32c_lut(buf, 0, len, crc32); } +#else + return crc32c_lut(buf, 0, len, crc32); +#endif } OLAPStatus gen_timestamp_string(string* out_string) { diff --git a/be/src/olap/utils.h b/be/src/olap/utils.h index 8c58ad49f3..cb59582421 100644 --- a/be/src/olap/utils.h +++ b/be/src/olap/utils.h @@ -37,7 +37,9 @@ #include #include "common/logging.h" +#if defined(__i386) || defined(__x86_64__) #include "olap/bhp_lib.h" +#endif #include "olap/olap_common.h" #include "olap/olap_define.h"