database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Chore](build) remove some unused code and remove some wno (#20326)

Browse Source 
remove some unused code about spinlock
remove some wno and fix warning
remove varadic macro usage
This commit is contained in:
Pxl
2023-06-05 10:48:07 +08:00
committed by GitHub
parent 59a0f80233
commit 7dc7ed97eb
39 changed files with 75 additions and 7305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
LICENSE.txt
View File

@ -251,46 +251,6 @@ be/src/gutil/utf/*: licensed under the following terms:
--------------------------------------------------------------------------------
be/src/gutil/valgrind.h: licensed under the following terms:
This file is part of Valgrind, a dynamic binary instrumentation
framework.
Copyright (C) 2000-2008 Julian Seward. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. The origin of this software must not be misrepresented; you must
not claim that you wrote the original software. If you use this
software in a product, an acknowledgment in the product
documentation would be appreciated but is not required.
3. Altered source versions must be plainly marked as such, and must
not be misrepresented as being the original software.
4. The name of the author may not be used to endorse or promote
products derived from this software without specific prior written
permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--------------------------------------------------------------------------------
be/src/gutil: 3-clause BSD
Except above files in this module, other files of this module are derived from code in

13
be/CMakeLists.txt
View File

@ -509,8 +509,7 @@ add_compile_options(-g
$<$<COMPILE_LANGUAGE:CXX>:-Wnon-virtual-dtor>)
add_compile_options(-Wno-unused-parameter
-Wno-sign-compare
-Wno-array-bounds)
-Wno-sign-compare)
if (COMPILER_GCC)
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11.1")
@ -530,14 +529,14 @@ if (COMPILER_CLANG)
add_compile_options(-fcolor-diagnostics
-Wpedantic
-Wunused
-Wunused-command-line-argument
-Wunused-exception-parameter
-Wunused-volatile-lvalue
-Wunused-template
-Wunused-private-field
-Wunused-property-ivar
-Wunused-member-function
-Wunused-macros)
add_compile_options(-Wno-variadic-macros
-Wno-gnu-zero-variadic-macro-arguments
-Wno-vla-extension)
add_compile_options(-Wno-vla-extension)
if (USE_LIBCPP)
add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>)
if (NOT OS_MACOSX)

5
be/src/agent/task_worker_pool.cpp
View File

@ -297,7 +297,6 @@ void TaskWorkerPool::_remove_task_info(const TTaskType::type task_type, int64_t
VLOG_NOTICE << "remove task info. type=" << task_type << ", signature=" << signature
<< ", queue_size=" << queue_size;
TRACE("remove task info");
}
void TaskWorkerPool::_finish_task(const TFinishTaskRequest& finish_task_request) {
@ -322,7 +321,6 @@ void TaskWorkerPool::_finish_task(const TFinishTaskRequest& finish_task_request)
}
sleep(config::sleep_one_second);
}
TRACE("finish task");
}
void TaskWorkerPool::_alter_inverted_index_worker_thread_callback() {
@ -1209,7 +1207,8 @@ void CreateTableTaskPool::_create_tablet_worker_thread_callback() {
});
ADOPT_TRACE(trace.get());
DorisMetrics::instance()->create_tablet_requests_total->increment(1);
TRACE("start to create tablet $0", create_tablet_req.tablet_id);
VLOG_NOTICE << "start to create tablet " << create_tablet_req.tablet_id;
std::vector<TTabletInfo> finish_tablet_infos;
VLOG_NOTICE << "create tablet: " << create_tablet_req;
Status status = _env->storage_engine()->create_tablet(create_tablet_req);

5
be/src/agent/user_resource_listener.cpp
View File

@ -86,10 +86,11 @@ void UserResourceListener::update_users_resource(int64_t new_version) {
if (!master_status.ok()) {
LOG(WARNING) << "Reopen to get frontend client failed, with address:"
<< _master_info.network_address.hostname << ":"
<< _master_info.network_address.port;
<< _master_info.network_address.port << ", reason=" << e.what();
return;
}
LOG(WARNING) << "fetchResource from frontend failed, retry!";
LOG(WARNING) << "fetchResource from frontend failed"
<< ", reason=" << e.what();
client->fetchResource(new_fetched_resource);
}
} catch (TException& e) {

38
be/src/common/logging.h
View File

@ -17,6 +17,8 @@
#pragma once
#include <memory>
// GLOG defines this based on the system but doesn't check if it's already
// been defined. undef it first to avoid warnings.
// glog MUST be included before gflags. Instead of including them,
@ -70,43 +72,45 @@ bool init_glog(const char* basename);
// flushed. May only be called once.
void shutdown_logging();
/// Wrap a glog stream and tag on the log. usage:
/// LOG_INFO("here is an info for a {} query", query_type).tag("query_id", queryId);
#define LOG_INFO(...) doris::TaggableLogger(LOG(INFO), ##__VA_ARGS__)
#define LOG_WARNING(...) doris::TaggableLogger(LOG(WARNING), ##__VA_ARGS__)
#define LOG_ERROR(...) doris::TaggableLogger(LOG(ERROR), ##__VA_ARGS__)
#define LOG_FATAL(...) doris::TaggableLogger(LOG(FATAL), ##__VA_ARGS__)
class TaggableLogger {
public:
TaggableLogger(const char* file, int line, google::LogSeverity severity)
: _msg(file, line, severity) {}
template <typename... Args>
TaggableLogger(std::ostream& stream, std::string_view fmt, Args&&... args) : _stream(stream) {
TaggableLogger& operator()(const std::string_view& fmt, Args&&... args) {
if constexpr (sizeof...(args) == 0) {
_stream << fmt;
_msg.stream() << fmt;
} else {
_stream << fmt::format(fmt, std::forward<Args>(args)...);
_msg.stream() << fmt::format(fmt, std::forward<Args&&>(args)...);
}
return *this;
}
template <typename V>
TaggableLogger& tag(std::string_view key, const V& value) {
_stream << '|' << key << '=';
TaggableLogger& tag(std::string_view key, V&& value) {
_msg.stream() << '|' << key << '=';
if constexpr (std::is_same_v<V, TUniqueId> || std::is_same_v<V, PUniqueId>) {
_stream << print_id(value);
_msg.stream() << print_id(value);
} else {
_stream << value;
_msg.stream() << value;
}
return *this;
}
template <typename E>
TaggableLogger& error(const E& error) {
_stream << "|error=" << error;
TaggableLogger& error(E&& error) {
_msg.stream() << "|error=" << error;
return *this;
}
private:
std::ostream& _stream;
google::LogMessage _msg;
};
#define LOG_INFO TaggableLogger(__FILE__, __LINE__, google::GLOG_INFO)
#define LOG_WARNING TaggableLogger(__FILE__, __LINE__, google::GLOG_WARNING)
#define LOG_ERROR TaggableLogger(__FILE__, __LINE__, google::GLOG_ERROR)
#define LOG_FATAL TaggableLogger(__FILE__, __LINE__, google::GLOG_FATAL)
} // namespace doris

1
be/src/gutil/CMakeLists.txt
View File

@ -31,7 +31,6 @@ SET(SOURCE_FILES
hash/jenkins.cc
int128.cc
ref_counted.cc
spinlock_internal.cc
stringprintf.cc
strings/ascii_ctype.cc
strings/charset.cc

44
be/src/gutil/dynamic_annotations.c
View File

@ -40,11 +40,6 @@
#include "gutil/dynamic_annotations.h"
#ifdef __GNUC__
/* valgrind.h uses gcc extensions so it won't build with other compilers */
#include "gutil/valgrind.h"
#endif
/* Compiler-based ThreadSanitizer defines
DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL = 1
and provides its own definitions of the functions. */
@ -131,43 +126,4 @@ void AnnotateFlushState(const char *file, int line){}
#if DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL == 0
static int GetRunningOnValgrind(void) {
#ifdef RUNNING_ON_VALGRIND
if (RUNNING_ON_VALGRIND) return 1;
#endif
char *running_on_valgrind_str = getenv("RUNNING_ON_VALGRIND");
if (running_on_valgrind_str) {
return strcmp(running_on_valgrind_str, "0") != 0;
}
return 0;
}
/* See the comments in dynamic_annotations.h */
int RunningOnValgrind(void) {
static volatile int running_on_valgrind = -1;
int local_running_on_valgrind = running_on_valgrind;
/* C doesn't have thread-safe initialization of statics, and we
don't want to depend on pthread_once here, so hack it. */
ANNOTATE_BENIGN_RACE(&running_on_valgrind, "safe hack");
if (local_running_on_valgrind == -1)
running_on_valgrind = local_running_on_valgrind = GetRunningOnValgrind();
return local_running_on_valgrind;
}
/* See the comments in dynamic_annotations.h */
double ValgrindSlowdown(void) {
/* Same initialization hack as in RunningOnValgrind(). */
static volatile double slowdown = 0.0;
double local_slowdown = slowdown;
ANNOTATE_BENIGN_RACE(&slowdown, "safe hack");
if (RunningOnValgrind() == 0) {
return 1.0;
}
if (local_slowdown == 0.0) {
char *env = getenv("VALGRIND_SLOWDOWN");
slowdown = local_slowdown = env ? atof(env) : 50.0;
}
return local_slowdown;
}
#endif /* DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL == 0 */

30
be/src/gutil/dynamic_annotations.h
View File

@ -482,36 +482,6 @@ void AnnotateEnableRaceDetection(const char* file, int line, int enable);
void AnnotateNoOp(const char* file, int line, const volatile void* arg);
void AnnotateFlushState(const char* file, int line);
/* Return non-zero value if running under valgrind.
If "valgrind.h" is included into dynamic_annotations.c,
the regular valgrind mechanism will be used.
See http://valgrind.org/docs/manual/manual-core-adv.html about
RUNNING_ON_VALGRIND and other valgrind "client requests".
The file "valgrind.h" may be obtained by doing
svn co svn://svn.valgrind.org/valgrind/trunk/include
If for some reason you can't use "valgrind.h" or want to fake valgrind,
there are two ways to make this function return non-zero:
- Use environment variable: export RUNNING_ON_VALGRIND=1
- Make your tool intercept the function RunningOnValgrind() and
change its return value.
*/
int RunningOnValgrind(void);
/* ValgrindSlowdown returns:
* 1.0, if (RunningOnValgrind() == 0)
* 50.0, if (RunningOnValgrind() != 0 && getenv("VALGRIND_SLOWDOWN") == NULL)
* atof(getenv("VALGRIND_SLOWDOWN")) otherwise
This function can be used to scale timeout values:
EXAMPLE:
for (;;) {
DoExpensiveBackgroundTask();
SleepForSeconds(5 * ValgrindSlowdown());
}
*/
double ValgrindSlowdown(void);
/* AddressSanitizer annotations from LLVM asan_interface.h */
#if defined(__SANITIZE_ADDRESS__) || defined(ADDRESS_SANITIZER)

2777
be/src/gutil/linux_syscall_support.h
View File

File diff suppressed because it is too large Load Diff

125
be/src/gutil/spinlock_internal.cc
View File

@ -1,125 +0,0 @@
// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
/* Copyright (c) 2010, Google Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Google Inc. nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
// The OS-specific header included below must provide two calls:
// base::internal::SpinLockDelay() and base::internal::SpinLockWake().
// See spinlock_internal.h for the spec of SpinLockWake().
// void SpinLockDelay(volatile Atomic32 *w, int32 value, int loop)
// SpinLockDelay() generates an appropriate spin delay on iteration "loop" of a
// spin loop on location *w, whose previously observed value was "value".
// SpinLockDelay() may do nothing, may yield the CPU, may sleep a clock tick,
// or may wait for a delay that can be truncated by a call to SpinlockWake(w).
// In all cases, it must return in bounded time even if SpinlockWake() is not
// called.
#include "gutil/spinlock_internal.h"
// forward declaration for use by spinlock_*-inl.h
namespace base {
namespace internal {
static int SuggestedDelayNS(int loop);
}
} // namespace base
#if defined(_WIN32)
#include "gutil/spinlock_win32-inl.h"
#elif defined(__linux__)
#include "gutil/spinlock_linux-inl.h"
#else
#include "gutil/spinlock_posix-inl.h"
#endif
namespace base {
namespace internal {
// See spinlock_internal.h for spec.
int32 SpinLockWait(volatile Atomic32* w, int n, const SpinLockWaitTransition trans[]) {
int32 v;
bool done = false;
for (int loop = 0; !done; loop++) {
v = base::subtle::Acquire_Load(w);
int i;
for (i = 0; i != n && v != trans[i].from; i++) {
}
if (i == n) {
SpinLockDelay(w, v, loop); // no matching transition
} else if (trans[i].to == v || // null transition
base::subtle::Acquire_CompareAndSwap(w, v, trans[i].to) == v) {
done = trans[i].done;
}
}
return v;
}
// Return a suggested delay in nanoseconds for iteration number "loop"
static int SuggestedDelayNS(int loop) {
// Weak pseudo-random number generator to get some spread between threads
// when many are spinning.
#ifdef BASE_HAS_ATOMIC64
static base::subtle::Atomic64 rand;
uint64 r = base::subtle::NoBarrier_Load(&rand);
r = 0x5deece66dLL * r + 0xb; // numbers from nrand48()
base::subtle::NoBarrier_Store(&rand, r);
r <<= 16; // 48-bit random number now in top 48-bits.
if (loop < 0 || loop > 32) { // limit loop to 0..32
loop = 32;
}
// loop>>3 cannot exceed 4 because loop cannot exceed 32.
// Select top 20..24 bits of lower 48 bits,
// giving approximately 0ms to 16ms.
// Mean is exponential in loop for first 32 iterations, then 8ms.
// The futex path multiplies this by 16, since we expect explicit wakeups
// almost always on that path.
return r >> (44 - (loop >> 3));
#else
static Atomic32 rand;
uint32 r = base::subtle::NoBarrier_Load(&rand);
r = 0x343fd * r + 0x269ec3; // numbers from MSVC++
base::subtle::NoBarrier_Store(&rand, r);
r <<= 1; // 31-bit random number now in top 31-bits.
if (loop < 0 || loop > 32) { // limit loop to 0..32
loop = 32;
}
// loop>>3 cannot exceed 4 because loop cannot exceed 32.
// Select top 20..24 bits of lower 31 bits,
// giving approximately 0ms to 16ms.
// Mean is exponential in loop for first 32 iterations, then 8ms.
// The futex path multiplies this by 16, since we expect explicit wakeups
// almost always on that path.
return r >> (12 - (loop >> 3));
#endif
}
} // namespace internal
} // namespace base

61
be/src/gutil/spinlock_internal.h
View File

@ -1,61 +0,0 @@
// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
/* Copyright (c) 2010, Google Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Google Inc. nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ---
* This file is an internal part spinlock.cc and once.cc
* It may not be used directly by code outside of //base.
*/
#pragma once
#include "gutil/atomicops.h"
#include "gutil/integral_types.h"
namespace base {
namespace internal {
// SpinLockWait() waits until it can perform one of several transitions from
// "from" to "to". It returns when it performs a transition where done==true.
struct SpinLockWaitTransition {
int32 from;
int32 to;
bool done;
};
// Wait until *w can transition from trans[i].from to trans[i].to for some i
// satisfying 0<=i<n && trans[i].done, atomically make the transition,
// then return the old value of *w. Make any other atomic transitions
// where !trans[i].done, but continue waiting.
int32 SpinLockWait(volatile Atomic32* w, int n, const SpinLockWaitTransition trans[]);
void SpinLockWake(volatile Atomic32* w, bool all);
void SpinLockDelay(volatile Atomic32* w, int32 value, int loop);
} // namespace internal
} // namespace base

98
be/src/gutil/spinlock_linux-inl.h
View File

@ -1,98 +0,0 @@
// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
/* Copyright (c) 2009, Google Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Google Inc. nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ---
* This file is a Linux-specific part of spinlock_internal.cc
*/
#include <errno.h>
#include <limits.h>
#include <sched.h>
#include <time.h>
#include "gutil/linux_syscall_support.h"
#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
#define FUTEX_PRIVATE_FLAG 128
static bool have_futex;
static int futex_private_flag = FUTEX_PRIVATE_FLAG;
namespace {
static struct InitModule {
InitModule() {
int x = 0;
// futexes are ints, so we can use them only when
// that's the same size as the lockword_ in SpinLock.
have_futex = (sizeof(Atomic32) == sizeof(int) &&
sys_futex(&x, FUTEX_WAKE, 1, NULL, NULL, 0) >= 0);
if (have_futex && sys_futex(&x, FUTEX_WAKE | futex_private_flag, 1, NULL, NULL, 0) < 0) {
futex_private_flag = 0;
}
}
} init_module;
} // anonymous namespace
namespace base {
namespace internal {
void SpinLockDelay(volatile Atomic32* w, int32 value, int loop) {
if (loop != 0) {
int save_errno = errno;
struct timespec tm;
tm.tv_sec = 0;
if (have_futex) {
tm.tv_nsec = base::internal::SuggestedDelayNS(loop);
} else {
tm.tv_nsec = 2000001; // above 2ms so linux 2.4 doesn't spin
}
if (have_futex) {
tm.tv_nsec *= 16; // increase the delay; we expect explicit wakeups
sys_futex(reinterpret_cast<int*>(const_cast<Atomic32*>(w)),
FUTEX_WAIT | futex_private_flag, value,
reinterpret_cast<struct kernel_timespec*>(&tm), NULL, 0);
} else {
nanosleep(&tm, NULL);
}
errno = save_errno;
}
}
void SpinLockWake(volatile Atomic32* w, bool all) {
if (have_futex) {
sys_futex(reinterpret_cast<int*>(const_cast<Atomic32*>(w)), FUTEX_WAKE | futex_private_flag,
all ? INT_MAX : 1, NULL, NULL, 0);
}
}
} // namespace internal
} // namespace base

61
be/src/gutil/spinlock_posix-inl.h
View File

@ -1,61 +0,0 @@
// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
/* Copyright (c) 2009, Google Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Google Inc. nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ---
* This file is a Posix-specific part of spinlock_internal.cc
*/
#include <errno.h>
#if defined(HAVE_SCHED_H) || defined(__APPLE__)
#include <sched.h> /* For sched_yield() */
#endif
#include <time.h> /* For nanosleep() */
namespace base {
namespace internal {
void SpinLockDelay(volatile Atomic32* w, int32 value, int loop) {
int save_errno = errno;
if (loop == 0) {
} else if (loop == 1) {
sched_yield();
} else {
struct timespec tm;
tm.tv_sec = 0;
tm.tv_nsec = base::internal::SuggestedDelayNS(loop);
nanosleep(&tm, NULL);
}
errno = save_errno;
}
void SpinLockWake(volatile Atomic32* w, bool all) {}
} // namespace internal
} // namespace base

52
be/src/gutil/spinlock_win32-inl.h
View File

@ -1,52 +0,0 @@
// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
/* Copyright (c) 2009, Google Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Google Inc. nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ---
* This file is a Win32-specific part of spinlock_internal.cc
*/
#include <windows.h>
namespace base {
namespace internal {
void SpinLockDelay(volatile Atomic32* w, int32 value, int loop) {
if (loop == 0) {
} else if (loop == 1) {
Sleep(0);
} else {
Sleep(base::internal::SuggestedDelayNS(loop) / 1000000);
}
}
void SpinLockWake(volatile Atomic32* w, bool all) {}
} // namespace internal
} // namespace base

3716
be/src/gutil/valgrind.h
View File

File diff suppressed because it is too large Load Diff

4
be/src/http/action/check_rpc_channel_action.cpp
View File

@ -64,8 +64,8 @@ void CheckRPCChannelAction::handle(HttpRequest* req) {
return;
}
} catch (const std::exception& e) {
std::string err = fmt::format("invalid argument. port:{0}, payload_size: {1}", req_port,
req_payload_size);
std::string err = fmt::format("invalid argument. port: {0}, payload_size: {1}, reason: {}",
req_port, req_payload_size, e.what());
LOG(WARNING) << err;
HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, err);
return;

6
be/src/http/action/stream_load.cpp
View File

@ -334,7 +334,7 @@ Status StreamLoadAction::_on_header(HttpRequest* http_req, std::shared_ptr<Strea
try {
ctx->timeout_second = std::stoi(http_req->header(HTTP_TIMEOUT));
} catch (const std::invalid_argument& e) {
return Status::InvalidArgument("Invalid timeout format");
return Status::InvalidArgument("Invalid timeout format, {}", e.what());
}
}
if (!http_req->header(HTTP_COMMENT).empty()) {
@ -470,7 +470,7 @@ Status StreamLoadAction::_process_put(HttpRequest* http_req,
try {
request.__set_execMemLimit(std::stoll(http_req->header(HTTP_EXEC_MEM_LIMIT)));
} catch (const std::invalid_argument& e) {
return Status::InvalidArgument("Invalid mem limit format");
return Status::InvalidArgument("Invalid mem limit format, {}", e.what());
}
}
if (!http_req->header(HTTP_JSONPATHS).empty()) {
@ -527,7 +527,7 @@ Status StreamLoadAction::_process_put(HttpRequest* http_req,
request.__set_send_batch_parallelism(
std::stoi(http_req->header(HTTP_SEND_BATCH_PARALLELISM)));
} catch (const std::invalid_argument& e) {
return Status::InvalidArgument("Invalid send_batch_parallelism format");
return Status::InvalidArgument("Invalid send_batch_parallelism format, {}", e.what());
}
}

3
be/src/http/action/stream_load_2pc.cpp
View File

@ -53,7 +53,8 @@ void StreamLoad2PCAction::handle(HttpRequest* req) {
try {
ctx->txn_id = std::stoull(req_txn_id);
} catch (const std::exception& e) {
status = Status::InternalError("convert txn_id [{}] failed", req_txn_id);
status = Status::InternalError("convert txn_id [{}] failed, reason={}", req_txn_id,
e.what());
status_result = status.to_json();
HttpChannel::send_reply(req, HttpStatus::OK, status_result);
return;

4
be/src/http/action/tablets_distribution_action.cpp
View File

@ -59,8 +59,8 @@ void TabletsDistributionAction::handle(HttpRequest* req) {
try {
partition_id = std::stoull(req_partition_id);
} catch (const std::exception& e) {
LOG(WARNING) << "invalid argument. partition_id:" << req_partition_id;
Status status = Status::InternalError("invalid argument: {}", req_partition_id);
Status status = Status::InternalError("invalid argument: {}, reason:{}",
req_partition_id, e.what());
std::string status_result = status.to_json();
HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, status_result);
return;

2
be/src/io/fs/broker_file_reader.cpp
View File

@ -67,7 +67,7 @@ Status BrokerFileReader::close() {
try {
try {
(*_client)->closeReader(response, request);
} catch (apache::thrift::transport::TTransportException& e) {
} catch (apache::thrift::transport::TTransportException&) {
std::this_thread::sleep_for(std::chrono::seconds(1));
RETURN_IF_ERROR((*_client).reopen());
(*_client)->closeReader(response, request);

12
be/src/io/fs/broker_file_system.cpp
View File

@ -121,7 +121,7 @@ Status BrokerFileSystem::open_file_internal(const Path& file, int64_t file_size,
Status status;
try {
(*_client)->openReader(*response, request);
} catch (apache::thrift::transport::TTransportException& e) {
} catch (apache::thrift::transport::TTransportException&) {
std::this_thread::sleep_for(std::chrono::seconds(1));
RETURN_IF_ERROR((*_client).reopen());
(*_client)->openReader(*response, request);
@ -156,7 +156,7 @@ Status BrokerFileSystem::delete_file_impl(const Path& file) {
try {
(*_client)->deletePath(del_rep, del_req);
} catch (apache::thrift::transport::TTransportException& e) {
} catch (apache::thrift::transport::TTransportException&) {
RETURN_IF_ERROR((*_client).reopen());
(*_client)->deletePath(del_rep, del_req);
}
@ -196,7 +196,7 @@ Status BrokerFileSystem::exists_impl(const Path& path, bool* res) const {
try {
(*_client)->checkPathExist(check_rep, check_req);
} catch (apache::thrift::transport::TTransportException& e) {
} catch (apache::thrift::transport::TTransportException&) {
RETURN_IF_ERROR((*_client).reopen());
(*_client)->checkPathExist(check_rep, check_req);
}
@ -228,7 +228,7 @@ Status BrokerFileSystem::file_size_impl(const Path& path, int64_t* file_size) co
TBrokerFileSizeResponse resp;
try {
(*_client)->fileSize(resp, req);
} catch (apache::thrift::transport::TTransportException& e) {
} catch (apache::thrift::transport::TTransportException&) {
RETURN_IF_ERROR((*_client).reopen());
(*_client)->fileSize(resp, req);
}
@ -269,7 +269,7 @@ Status BrokerFileSystem::list_impl(const Path& dir, bool only_file, std::vector<
try {
(*_client)->listPath(list_rep, list_req);
} catch (apache::thrift::transport::TTransportException& e) {
} catch (apache::thrift::transport::TTransportException&) {
RETURN_IF_ERROR((*_client).reopen());
(*_client)->listPath(list_rep, list_req);
}
@ -319,7 +319,7 @@ Status BrokerFileSystem::rename_impl(const Path& orig_name, const Path& new_name
try {
(*_client)->renamePath(op_status, rename_req);
} catch (apache::thrift::transport::TTransportException& e) {
} catch (apache::thrift::transport::TTransportException&) {
RETURN_IF_ERROR((*_client).reopen());
(*_client)->renamePath(op_status, rename_req);
}

4
be/src/io/fs/broker_file_writer.cpp
View File

@ -183,7 +183,7 @@ Status BrokerFileWriter::_open() {
try {
client->openWriter(response, request);
} catch (apache::thrift::transport::TTransportException& e) {
} catch (apache::thrift::transport::TTransportException&) {
RETURN_IF_ERROR(client.reopen());
client->openWriter(response, request);
}
@ -237,7 +237,7 @@ Status BrokerFileWriter::_write(const uint8_t* buf, size_t buf_len, size_t* writ
try {
client->pwrite(response, request);
} catch (apache::thrift::transport::TTransportException& e) {
} catch (apache::thrift::transport::TTransportException&) {
RETURN_IF_ERROR(client.reopen());
// broker server will check write offset, so it is safe to re-try
client->pwrite(response, request);

5
be/src/olap/base_compaction.cpp
View File

@ -52,11 +52,9 @@ Status BaseCompaction::prepare_compact() {
LOG(WARNING) << "another base compaction is running. tablet=" << _tablet->full_name();
return Status::Error<TRY_LOCK_FAILED>();
}
TRACE("got base compaction lock");
// 1. pick rowsets to compact
RETURN_IF_ERROR(pick_rowsets_to_compact());
TRACE("rowsets picked");
TRACE_COUNTER_INCREMENT("input_rowsets_count", _input_rowsets.size());
_tablet->set_clone_occurred(false);
@ -74,7 +72,6 @@ Status BaseCompaction::execute_compact_impl() {
LOG(WARNING) << "another base compaction is running. tablet=" << _tablet->full_name();
return Status::Error<TRY_LOCK_FAILED>();
}
TRACE("got base compaction lock");
// Clone task may happen after compaction task is submitted to thread pool, and rowsets picked
// for compaction may change. In this case, current compaction task should not be executed.
@ -88,7 +85,6 @@ Status BaseCompaction::execute_compact_impl() {
// 2. do base compaction, merge rowsets
int64_t permits = get_compaction_permits();
RETURN_IF_ERROR(do_compaction(permits));
TRACE("compaction finished");
// 3. set state to success
_state = CompactionState::SUCCESS;
@ -96,7 +92,6 @@ Status BaseCompaction::execute_compact_impl() {
// 4. add metric to base compaction
DorisMetrics::instance()->base_compaction_deltas_total->increment(_input_rowsets.size());
DorisMetrics::instance()->base_compaction_bytes_total->increment(_input_rowsets_size);
TRACE("save base compaction metrics");
return Status::OK();
}

7
be/src/olap/compaction.cpp
View File

@ -86,7 +86,6 @@ Status Compaction::execute_compact() {
}
Status Compaction::do_compaction(int64_t permits) {
TRACE("start to do compaction");
uint32_t checksum_before;
uint32_t checksum_after;
if (config::enable_compaction_checksum) {
@ -270,7 +269,6 @@ Status Compaction::do_compaction_impl(int64_t permits) {
if (handle_ordered_data_compaction()) {
RETURN_IF_ERROR(modify_rowsets());
TRACE("modify rowsets finished");
int64_t now = UnixMillis();
if (compaction_type() == ReaderType::READER_CUMULATIVE_COMPACTION) {
@ -300,7 +298,6 @@ Status Compaction::do_compaction_impl(int64_t permits) {
if (compaction_type() == ReaderType::READER_COLD_DATA_COMPACTION) {
Tablet::add_pending_remote_rowset(_output_rs_writer->rowset_id().to_string());
}
TRACE("prepare finished");
// 2. write merged rows to output rowset
// The test results show that merger is low-memory-footprint, there is no need to tracker its mem pool
@ -328,7 +325,6 @@ Status Compaction::do_compaction_impl(int64_t permits) {
<< ", output_version=" << _output_version;
return res;
}
TRACE("merge rowsets finished");
TRACE_COUNTER_INCREMENT("merged_rows", stats.merged_rows);
TRACE_COUNTER_INCREMENT("filtered_rows", stats.filtered_rows);
@ -342,11 +338,9 @@ Status Compaction::do_compaction_impl(int64_t permits) {
TRACE_COUNTER_INCREMENT("output_rowset_data_size", _output_rowset->data_disk_size());
TRACE_COUNTER_INCREMENT("output_row_num", _output_rowset->num_rows());
TRACE_COUNTER_INCREMENT("output_segments_num", _output_rowset->num_segments());
TRACE("output rowset built");
// 3. check correctness
RETURN_IF_ERROR(check_correctness(stats));
TRACE("check correctness finished");
if (_input_row_num > 0 && stats.rowid_conversion && config::inverted_index_compaction_enable) {
OlapStopWatch inverted_watch;
@ -414,7 +408,6 @@ Status Compaction::do_compaction_impl(int64_t permits) {
// 4. modify rowsets in memory
RETURN_IF_ERROR(modify_rowsets(&stats));
TRACE("modify rowsets finished");
// 5. update last success compaction time
int64_t now = UnixMillis();

6
be/src/olap/cumulative_compaction.cpp
View File

@ -52,17 +52,14 @@ Status CumulativeCompaction::prepare_compact() {
LOG(INFO) << "The tablet is under cumulative compaction. tablet=" << _tablet->full_name();
return Status::Error<TRY_LOCK_FAILED>();
}
TRACE("got cumulative compaction lock");
// 1. calculate cumulative point
_tablet->calculate_cumulative_point();
TRACE("calculated cumulative point");
VLOG_CRITICAL << "after calculate, current cumulative point is "
<< _tablet->cumulative_layer_point() << ", tablet=" << _tablet->full_name();
// 2. pick rowsets to compact
RETURN_IF_ERROR(pick_rowsets_to_compact());
TRACE("rowsets picked");
TRACE_COUNTER_INCREMENT("input_rowsets_count", _input_rowsets.size());
_tablet->set_clone_occurred(false);
@ -75,7 +72,6 @@ Status CumulativeCompaction::execute_compact_impl() {
LOG(INFO) << "The tablet is under cumulative compaction. tablet=" << _tablet->full_name();
return Status::Error<TRY_LOCK_FAILED>();
}
TRACE("got cumulative compaction lock");
// Clone task may happen after compaction task is submitted to thread pool, and rowsets picked
// for compaction may change. In this case, current compaction task should not be executed.
@ -89,7 +85,6 @@ Status CumulativeCompaction::execute_compact_impl() {
// 3. do cumulative compaction, merge rowsets
int64_t permits = get_compaction_permits();
RETURN_IF_ERROR(do_compaction(permits));
TRACE("compaction finished");
// 4. set state to success
_state = CompactionState::SUCCESS;
@ -103,7 +98,6 @@ Status CumulativeCompaction::execute_compact_impl() {
// 6. add metric to cumulative compaction
DorisMetrics::instance()->cumulative_compaction_deltas_total->increment(_input_rowsets.size());
DorisMetrics::instance()->cumulative_compaction_bytes_total->increment(_input_rowsets_size);
TRACE("save cumulative compaction metrics");
return Status::OK();
}

9
be/src/olap/single_replica_compaction.cpp
View File

@ -62,17 +62,14 @@ Status SingleReplicaCompaction::prepare_compact() {
LOG(INFO) << "The tablet is under cumulative compaction. tablet=" << _tablet->full_name();
return Status::Error<TRY_LOCK_FAILED>();
}
TRACE("got cumulative compaction lock");
std::unique_lock<std::mutex> lock_base(_tablet->get_base_compaction_lock(), std::try_to_lock);
if (!lock_base.owns_lock()) {
LOG(WARNING) << "another base compaction is running. tablet=" << _tablet->full_name();
return Status::Error<TRY_LOCK_FAILED>();
}
TRACE("got base compaction lock");
// 1. pick rowsets to compact
RETURN_IF_ERROR(pick_rowsets_to_compact());
TRACE("rowsets picked");
_tablet->set_clone_occurred(false);
if (_input_rowsets.size() == 1) {
return Status::Error<CUMULATIVE_NO_SUITABLE_VERSION>();
@ -101,14 +98,12 @@ Status SingleReplicaCompaction::execute_compact_impl() {
LOG(INFO) << "The tablet is under cumulative compaction. tablet=" << _tablet->full_name();
return Status::Error<TRY_LOCK_FAILED>();
}
TRACE("got cumulative compaction lock");
std::unique_lock<std::mutex> lock_base(_tablet->get_base_compaction_lock(), std::try_to_lock);
if (!lock_base.owns_lock()) {
LOG(WARNING) << "another base compaction is running. tablet=" << _tablet->full_name();
return Status::Error<TRY_LOCK_FAILED>();
}
TRACE("got base compaction lock");
// Clone task may happen after compaction task is submitted to thread pool, and rowsets picked
// for compaction may change. In this case, current compaction task should not be executed.
@ -121,7 +116,6 @@ Status SingleReplicaCompaction::execute_compact_impl() {
// 2. do single replica compaction
RETURN_IF_ERROR(_do_single_replica_compaction());
TRACE("single replica compaction finished");
// 3. set state to success
_state = CompactionState::SUCCESS;
@ -130,8 +124,6 @@ Status SingleReplicaCompaction::execute_compact_impl() {
}
Status SingleReplicaCompaction::_do_single_replica_compaction() {
TRACE("start to do single replica compaction");
_tablet->data_dir()->disks_compaction_num_increment(1);
Status st = _do_single_replica_compaction_impl();
_tablet->data_dir()->disks_compaction_num_increment(-1);
@ -163,7 +155,6 @@ Status SingleReplicaCompaction::_do_single_replica_compaction_impl() {
RETURN_IF_ERROR(_fetch_rowset(addr, token, proper_version));
// 5. modify rowsets in memory
RETURN_IF_ERROR(modify_rowsets());
TRACE("modify rowsets finished");
// 6. update last success compaction time
if (compaction_type() == ReaderType::READER_CUMULATIVE_COMPACTION) {

1
be/src/olap/storage_engine.cpp
View File

@ -1055,7 +1055,6 @@ Status StorageEngine::create_tablet(const TCreateTabletReq& request) {
LOG(WARNING) << "there is no available disk that can be used to create tablet.";
return Status::Error<CE_CMD_PARAMS_ERROR>();
}
TRACE("got data directory for create tablet");
return _tablet_manager->create_tablet(request, stores);
}

6
be/src/olap/tablet.cpp
View File

@ -1660,7 +1660,6 @@ Status Tablet::prepare_compaction_and_calculate_permits(CompactionType compactio
});
ADOPT_TRACE(trace.get());
TRACE("create cumulative compaction");
StorageEngine::instance()->create_cumulative_compaction(tablet, _cumulative_compaction);
DorisMetrics::instance()->cumulative_compaction_request_total->increment(1);
Status res = _cumulative_compaction->prepare_compact();
@ -1689,7 +1688,6 @@ Status Tablet::prepare_compaction_and_calculate_permits(CompactionType compactio
});
ADOPT_TRACE(trace.get());
TRACE("create base compaction");
StorageEngine::instance()->create_base_compaction(tablet, _base_compaction);
DorisMetrics::instance()->base_compaction_request_total->increment(1);
Status res = _base_compaction->prepare_compact();
@ -1718,7 +1716,6 @@ Status Tablet::prepare_single_replica_compaction(TabletSharedPtr tablet,
CompactionType compaction_type) {
scoped_refptr<Trace> trace(new Trace);
ADOPT_TRACE(trace.get());
TRACE("create single replica compaction");
StorageEngine::instance()->create_single_replica_compaction(tablet, _single_replica_compaction,
compaction_type);
@ -1734,7 +1731,6 @@ Status Tablet::prepare_single_replica_compaction(TabletSharedPtr tablet,
void Tablet::execute_single_replica_compaction(CompactionType compaction_type) {
scoped_refptr<Trace> trace(new Trace);
ADOPT_TRACE(trace.get());
TRACE("execute single replica compaction");
Status res = _single_replica_compaction->execute_compact();
if (!res.ok()) {
if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) {
@ -1783,7 +1779,6 @@ void Tablet::execute_compaction(CompactionType compaction_type) {
});
ADOPT_TRACE(trace.get());
TRACE("execute cumulative compaction");
Status res = _cumulative_compaction->execute_compact();
if (!res.ok()) {
set_last_cumu_compaction_failure_time(UnixMillis());
@ -1806,7 +1801,6 @@ void Tablet::execute_compaction(CompactionType compaction_type) {
});
ADOPT_TRACE(trace.get());
TRACE("create base compaction");
Status res = _base_compaction->execute_compact();
if (!res.ok()) {
set_last_base_compaction_failure_time(UnixMillis());

3
be/src/olap/tablet_manager.cpp
View File

@ -247,7 +247,6 @@ Status TabletManager::create_tablet(const TCreateTabletReq& request, std::vector
LOG(INFO) << "begin to create tablet. tablet_id=" << tablet_id;
std::lock_guard<std::shared_mutex> wrlock(_get_tablets_shard_lock(tablet_id));
TRACE("got tablets shard lock");
// Make create_tablet operation to be idempotent:
// 1. Return true if tablet with same tablet_id and schema_hash exist;
// false if tablet with same tablet_id but different schema_hash exist.
@ -290,7 +289,6 @@ Status TabletManager::create_tablet(const TCreateTabletReq& request, std::vector
DorisMetrics::instance()->create_tablet_requests_failed->increment(1);
return Status::Error<CE_CMD_PARAMS_ERROR>();
}
TRACE("succeed to create tablet");
LOG(INFO) << "success to create tablet. tablet_id=" << tablet_id;
return Status::OK();
@ -364,7 +362,6 @@ TabletSharedPtr TabletManager::_internal_create_tablet_unlocked(
LOG(WARNING) << "fail to get tablet. res=" << res;
break;
}
TRACE("add tablet to StorageEngine");
} while (0);
if (res.ok()) {

2
be/src/runtime/broker_mgr.cpp
View File

@ -95,7 +95,7 @@ void BrokerMgr::ping(const TNetworkAddress& addr) {
status = client.reopen();
if (!status.ok()) {
LOG(WARNING) << "Create broker client failed. broker=" << addr
<< ", status=" << status;
<< ", status=" << status << ", reason=" << e.what();
return;
}
client->ping(response, request);

7
be/src/runtime/thread_context.h
View File

@ -62,8 +62,10 @@
// And count the memory during thread execution (is actually also the code segment that executes the function)
// to specify MemTrackerLimiter, and expect to handle when the memory exceeds the limit, for example cancel query.
// Usage is similar to SCOPED_CONSUME_MEM_TRACKER.
#define SCOPED_ATTACH_TASK(arg1, ...) \
auto VARNAME_LINENUM(attach_task) = AttachTask(arg1, ##__VA_ARGS__)
#define SCOPED_ATTACH_TASK(arg1) auto VARNAME_LINENUM(attach_task) = AttachTask(arg1)
#define SCOPED_ATTACH_TASK_WITH_ID(arg1, arg2, arg3) \
auto VARNAME_LINENUM(attach_task) = AttachTask(arg1, arg2, arg3)
// Switch MemTrackerLimiter for count memory during thread execution.
// Usually used after SCOPED_ATTACH_TASK, in order to count the memory of the specified code segment into another
@ -72,6 +74,7 @@
auto VARNAME_LINENUM(switch_mem_tracker) = SwitchThreadMemTrackerLimiter(mem_tracker_limiter)
#else
#define SCOPED_ATTACH_TASK(arg1, ...) (void)0
#define SCOPED_ATTACH_TASK_WITH_ID(arg1, arg2, arg3) (void)0
#define SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(mem_tracker_limiter) (void)0
#endif

2
be/src/util/cidr.cpp
View File

@ -59,7 +59,7 @@ bool CIDR::reset(const std::string& cidr_str) {
try {
len = std::stoi(suffix, &pos);
} catch (const std::exception& e) {
LOG(WARNING) << "Wrong CIDR format. network=" << cidr_str;
LOG(WARNING) << "Wrong CIDR format. network=" << cidr_str << ", reason=" << e.what();
return false;
}

7
be/src/util/hash_util.hpp
View File

@ -330,6 +330,10 @@ public:
seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
}
#if defined(__clang__)
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wused-but-marked-unused"
#endif
// xxHash function for a byte array. For convenience, a 64-bit seed is also
// hashed into the result. The mapping may change from time to time.
static xxh_u64 xxHash64WithSeed(const char* s, size_t len, xxh_u64 seed) {
@ -341,6 +345,9 @@ public:
static const int INT_VALUE = 0;
return XXH3_64bits_withSeed(reinterpret_cast<const char*>(&INT_VALUE), sizeof(int), seed);
}
#if defined(__clang__)
#pragma clang diagnostic pop
#endif
};
} // namespace doris

31
be/src/util/trace.h
View File

@ -47,22 +47,6 @@ class Trace;
// 't' should be a Trace* pointer.
#define ADOPT_TRACE(t) doris::ScopedAdoptTrace _adopt_trace(t);
// Issue a trace message, if tracing is enabled in the current thread.
// See Trace::SubstituteAndTrace for arguments.
// Example:
// TRACE("Acquired timestamp $0", timestamp);
#define TRACE(format, substitutions...) \
do { \
doris::Trace* _trace = doris::Trace::CurrentTrace(); \
if (_trace) { \
_trace->SubstituteAndTrace(__FILE__, __LINE__, (format), ##substitutions); \
} \
} while (0)
// Like the above, but takes the trace pointer as an explicit argument.
#define TRACE_TO(trace, format, substitutions...) \
(trace)->SubstituteAndTrace(__FILE__, __LINE__, (format), ##substitutions)
// Increment a counter associated with the current trace.
//
// Each trace contains a map of counters which can be used to keep
@ -101,21 +85,6 @@ class Trace;
#define TRACE_COUNTER_SCOPE_LATENCY_US(counter_name) \
::doris::ScopedTraceLatencyCounter _scoped_latency(counter_name)
// Construct a constant C string counter name which acts as a sort of
// coarse-grained histogram for trace metrics.
#define BUCKETED_COUNTER_NAME(prefix, duration_us) \
[]() { \
if (duration_us >= 100 * 1000) { \
return prefix "_gt_100_ms"; \
} else if (duration_us >= 10 * 1000) { \
return prefix "_10-100_ms"; \
} else if (duration_us >= 1000) { \
return prefix "_1-10_ms"; \
} else { \
return prefix "_lt_1ms"; \
} \
}()
// If this scope times out, make a simple trace.
// It will log the cost time only.
// Timeout is chrono duration struct, eg: 5ms, 100 * 1s.

20
be/src/vec/aggregate_functions/aggregate_function_java_udaf.h
View File

@ -270,8 +270,8 @@ public:
} catch (std::bad_alloc const& e) { \
throw doris::Exception( \
ErrorCode::INTERNAL_ERROR, \
"memory allocate failed in column string, buffer:{},size:{}", \
increase_buffer_size, buffer_size); \
"memory allocate failed in column string, buffer:{},size:{},reason:{}", \
increase_buffer_size, buffer_size, e.what()); \
} \
*output_value_buffer = reinterpret_cast<int64_t>(chars.data()); \
*output_intermediate_state_ptr = chars.size(); \
@ -317,10 +317,10 @@ public:
chars.resize(buffer_size); \
offsets.resize(buffer_size); \
} catch (std::bad_alloc const& e) { \
throw doris::Exception( \
ErrorCode::INTERNAL_ERROR, \
"memory allocate failed in array column string, buffer:{},size:{}", \
increase_buffer_size, buffer_size); \
throw doris::Exception(ErrorCode::INTERNAL_ERROR, \
"memory allocate failed in array column string, " \
"buffer:{},size:{},reason:{}", \
increase_buffer_size, buffer_size, e.what()); \
} \
*output_array_null_ptr = reinterpret_cast<int64_t>(null_map_data.data()); \
*output_value_buffer = reinterpret_cast<int64_t>(chars.data()); \
@ -342,10 +342,10 @@ public:
null_map_data.resize(buffer_size); \
data_column->resize(buffer_size); \
} catch (std::bad_alloc const& e) { \
throw doris::Exception( \
ErrorCode::INTERNAL_ERROR, \
"memory allocate failed in array number column, buffer:{},size:{}", \
increase_buffer_size, buffer_size); \
throw doris::Exception(ErrorCode::INTERNAL_ERROR, \
"memory allocate failed in array number column, " \
"buffer:{},size:{},reason:{}", \
increase_buffer_size, buffer_size, e.what()); \
} \
*output_array_null_ptr = reinterpret_cast<int64_t>(null_map_data.data()); \
*output_value_buffer = \

2
be/src/vec/runtime/vdata_stream_recvr.cpp
View File

@ -360,7 +360,7 @@ Status VDataStreamRecvr::create_merger(const VExprContextSPtrs& ordering_expr,
void VDataStreamRecvr::add_block(const PBlock& pblock, int sender_id, int be_number,
int64_t packet_seq, ::google::protobuf::Closure** done) {
SCOPED_ATTACH_TASK(_query_mem_tracker, _query_id, _fragment_instance_id);
SCOPED_ATTACH_TASK_WITH_ID(_query_mem_tracker, _query_id, _fragment_instance_id);
int use_sender_id = _is_merging ? sender_id : 0;
_sender_queues[use_sender_id]->add_block(pblock, be_number, packet_seq, done);
}

1
be/test/CMakeLists.txt
View File

@ -202,7 +202,6 @@ set(UTIL_TEST_FILES
util/thread_test.cpp
util/threadpool_test.cpp
util/mysql_row_buffer_test.cpp
util/trace_test.cpp
util/easy_json-test.cpp
util/http_channel_test.cpp
util/histogram_test.cpp

130
be/test/util/trace_test.cpp
View File

@ -1,130 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "util/trace.h"
#include <gtest/gtest-message.h>
#include <gtest/gtest-test-part.h>
#include <rapidjson/document.h>
#include <cctype>
// IWYU pragma: no_include <bits/chrono.h>
#include <chrono> // IWYU pragma: keep
#include <map>
#include <string>
#include <thread>
#include <vector>
#include "gtest/gtest_pred_impl.h"
#include "gutil/ref_counted.h"
#include "util/trace_metrics.h"
using rapidjson::Document;
using rapidjson::Value;
using std::string;
using std::thread;
using std::vector;
namespace doris {
class TraceTest : public ::testing::Test {};
// Replace all digits in 's' with the character 'X'.
static std::string XOutDigits(const string& s) {
std::string ret;
ret.reserve(s.size());
for (char c : s) {
if (isdigit(c)) {
ret.push_back('X');
} else {
ret.push_back(c);
}
}
return ret;
}
TEST_F(TraceTest, TestBasic) {
scoped_refptr<Trace> t(new Trace);
TRACE_TO(t, "hello $0, $1", "world", 12345);
TRACE_TO(t, "goodbye $0, $1", "cruel world", 54321);
std::string result = XOutDigits(t->DumpToString(Trace::NO_FLAGS));
EXPECT_EQ(
"XXXX XX:XX:XX.XXXXXX trace_test.cpp:XX] hello world, XXXXX\n"
"XXXX XX:XX:XX.XXXXXX trace_test.cpp:XX] goodbye cruel world, XXXXX\n",
result);
}
TEST_F(TraceTest, TestAttach) {
scoped_refptr<Trace> traceA(new Trace);
scoped_refptr<Trace> traceB(new Trace);
{
ADOPT_TRACE(traceA.get());
EXPECT_EQ(traceA.get(), Trace::CurrentTrace());
{
ADOPT_TRACE(traceB.get());
EXPECT_EQ(traceB.get(), Trace::CurrentTrace());
TRACE("hello from traceB");
}
EXPECT_EQ(traceA.get(), Trace::CurrentTrace());
TRACE("hello from traceA");
}
EXPECT_TRUE(Trace::CurrentTrace() == nullptr);
TRACE("this goes nowhere");
EXPECT_EQ("XXXX XX:XX:XX.XXXXXX trace_test.cpp:XX] hello from traceA\n",
XOutDigits(traceA->DumpToString(Trace::NO_FLAGS)));
EXPECT_EQ("XXXX XX:XX:XX.XXXXXX trace_test.cpp:XX] hello from traceB\n",
XOutDigits(traceB->DumpToString(Trace::NO_FLAGS)));
}
TEST_F(TraceTest, TestChildTrace) {
scoped_refptr<Trace> traceA(new Trace);
scoped_refptr<Trace> traceB(new Trace);
ADOPT_TRACE(traceA.get());
traceA->AddChildTrace("child", traceB.get());
TRACE("hello from traceA");
{
ADOPT_TRACE(traceB.get());
TRACE("hello from traceB");
}
EXPECT_EQ(
"XXXX XX:XX:XX.XXXXXX trace_test.cpp:XXX] hello from traceA\n"
"Related trace 'child':\n"
"XXXX XX:XX:XX.XXXXXX trace_test.cpp:XXX] hello from traceB\n",
XOutDigits(traceA->DumpToString(Trace::NO_FLAGS)));
}
TEST_F(TraceTest, TestTraceMetrics) {
scoped_refptr<Trace> trace(new Trace);
trace->metrics()->Increment("foo", 10);
trace->metrics()->Increment("bar", 10);
for (int i = 0; i < 1000; i++) {
trace->metrics()->Increment("baz", i);
}
EXPECT_EQ("{\"bar\":10,\"baz\":499500,\"foo\":10}", trace->MetricsAsJSON());
{
ADOPT_TRACE(trace.get());
TRACE_COUNTER_SCOPE_LATENCY_US("test_scope_us");
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
auto m = trace->metrics()->Get();
EXPECT_GE(m["test_scope_us"], 80 * 1000);
}
} // namespace doris

40
dist/LICENSE-dist.txt vendored
View File

@ -252,46 +252,6 @@ be/src/gutil/utf/*: licensed under the following terms:
--------------------------------------------------------------------------------
be/src/gutil/valgrind.h: licensed under the following terms:
This file is part of Valgrind, a dynamic binary instrumentation
framework.
Copyright (C) 2000-2008 Julian Seward. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. The origin of this software must not be misrepresented; you must
not claim that you wrote the original software. If you use this
software in a product, an acknowledgment in the product
documentation would be appreciated but is not required.
3. Altered source versions must be plainly marked as such, and must
not be misrepresented as being the original software.
4. The name of the author may not be used to endorse or promote
products derived from this software without specific prior written
permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--------------------------------------------------------------------------------
be/src/gutil: 3-clause BSD
Except above files in this module, other files of this module are derived from code in