hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE]MiniCore

Browse Source
This commit is contained in:
nroskill
2023-06-05 02:12:43 +00:00
committed by ob-robot
parent 05822a3c70
commit d627936f7d
7 changed files with 145 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
deps/oblib/src/lib/oblog/ob_async_log_struct.cpp vendored
View File

@ -70,7 +70,7 @@ int ObPLogFileStruct::reopen(const bool redirect_flag)
if (OB_UNLIKELY(strlen(filename_) <= 0)) {
LOG_STDERR("invalid argument log_file = %p\n", filename_);
ret = OB_INVALID_ARGUMENT;
} else if (OB_UNLIKELY((tmp_fd = ::open(filename_, O_WRONLY | O_CREAT | O_APPEND , LOG_FILE_MODE)) < 0)) {
} else if (OB_UNLIKELY((tmp_fd = ::open(filename_, O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, LOG_FILE_MODE)) < 0)) {
LOG_STDERR("open file = %s errno = %d error = %m\n", filename_, errno);
ret = OB_ERR_UNEXPECTED;
} else if (OB_UNLIKELY(0 != fstat(tmp_fd, &stat_))) {
@ -112,7 +112,7 @@ int ObPLogFileStruct::reopen_wf()
} else {
char tmp_file_name[MAX_LOG_FILE_NAME_SIZE];
(void)snprintf(tmp_file_name, sizeof(tmp_file_name), "%s.wf", filename_);
if (OB_UNLIKELY((tmp_fd = ::open(tmp_file_name, O_WRONLY | O_CREAT | O_APPEND , LOG_FILE_MODE)) < 0)) {
if (OB_UNLIKELY((tmp_fd = ::open(tmp_file_name, O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, LOG_FILE_MODE)) < 0)) {
LOG_STDERR("open file = %s errno = %d error = %m\n", tmp_file_name, errno);
ret = OB_ERR_UNEXPECTED;
} else if (OB_UNLIKELY(0 != fstat(tmp_fd, &wf_stat_))) {

6
deps/oblib/src/lib/oblog/ob_log.cpp vendored
View File

@ -742,7 +742,7 @@ void ObLogger::rotate_log(const char *filename,
tm.tm_hour, tm.tm_min, tm.tm_sec, static_cast<int>(t.tv_usec/1000));
ret = rename(filename, old_log_file); //If failed, TODO
int tmp_fd = open(filename, O_WRONLY | O_CREAT | O_APPEND, ObPLogFileStruct::LOG_FILE_MODE);
int tmp_fd = open(filename, O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, ObPLogFileStruct::LOG_FILE_MODE);
if (tmp_fd > 0) {
if (fd > STDERR_FILENO) {
(void)dup2(tmp_fd, fd);
@ -786,7 +786,7 @@ void ObLogger::rotate_log(const char *filename,
}
ret = rename(filename, old_log_file); //If failed, TODO
int tmp_fd = open(filename, O_WRONLY | O_CREAT | O_APPEND, ObPLogFileStruct::LOG_FILE_MODE);
int tmp_fd = open(filename, O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, ObPLogFileStruct::LOG_FILE_MODE);
if (tmp_fd > 0) {
if (redirect_flag) {
(void)dup2(tmp_fd, STDERR_FILENO);
@ -820,7 +820,7 @@ void ObLogger::rotate_log(const char *filename,
}
}
ret = rename(wf_filename, old_wf_log_file); //If failed, TODO
tmp_fd = open(wf_filename, O_WRONLY | O_CREAT | O_APPEND, ObPLogFileStruct::LOG_FILE_MODE);
tmp_fd = open(wf_filename, O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, ObPLogFileStruct::LOG_FILE_MODE);
if (tmp_fd > 0) {
if (wf_fd > STDERR_FILENO) {
(void)dup2(tmp_fd, wf_fd);

3
deps/oblib/src/lib/oblog/ob_log.h vendored
View File

@ -42,6 +42,7 @@
#include "lib/oblog/ob_async_log_struct.h"
#include "lib/utility/ob_defer.h"
#include "lib/oblog/ob_syslog_rate_limiter.h"
#include "lib/signal/ob_signal_handlers.h"
#define OB_LOG_MAX_PAR_MOD_SIZE 32
#define OB_LOG_MAX_SUB_MOD_SIZE 32
@ -1112,7 +1113,7 @@ inline void ObLogger::check_probe(
break;
}
case ProbeAction::PROBE_STACK: {
IGNORE_RETURN raise(60);
IGNORE_RETURN faststack();
break;
}
default: {

90
deps/oblib/src/lib/signal/ob_signal_handlers.cpp vendored
View File

@ -15,6 +15,9 @@
#include "lib/signal/ob_signal_handlers.h"
#include <sys/prctl.h>
#include <dirent.h>
#include <unistd.h>
#include <fstream>
#include <sys/wait.h>
#include "lib/profile/ob_trace_id.h"
#include "lib/utility/utility.h"
#include "lib/signal/ob_libunwind.h"
@ -29,6 +32,17 @@ namespace oceanbase
namespace common
{
static const int SIG_SET[] = {SIGABRT, SIGBUS, SIGFPE, SIGSEGV, SIGURG};
static constexpr char MINICORE_SHELL_PATH[] = "tools/minicore.sh";
static constexpr char FASTSTACK_SHELL_PATH[] = "tools/callstack.sh";
static constexpr char MINICORE_SCRIPT[] = "if [ -e bin/minicore.py ]; then\n"
" python bin/minicore.py `cat $(pwd)/run/observer.pid` -c -o core.`cat $(pwd)/run/observer.pid`.mini\n"
"fi\n"
"[ $(ls -1 core.*.mini 2>/dev/null | wc -l) -gt 5 ] && ls -1 core.*.mini | sort | head -n 1 | xargs rm -f";
static constexpr char FASTSTACK_SCRIPT[] = "if [ -x \"$(command -v obstack)\" ]; then\n"
" obstack `cat $(pwd)/run/observer.pid` > stack.`cat $(pwd)/run/observer.pid`.`date +%Y%m%d%H%M%S`\n"
"fi\n"
"[ $(ls -1 stack.* 2>/dev/null | wc -l) -gt 100 ] && ls -1 stack.* | sort | head -n 1 | xargs rm -f";
static inline void handler(int sig, siginfo_t *s, void *p)
{
@ -138,15 +152,13 @@ void close_socket_fd()
void coredump_cb(int sig, siginfo_t *si, void *context)
{
int ret = OB_SUCCESS;
if (g_coredump_num++ < 1) {
pid_t pid;
close_socket_fd();
ret = minicoredump(sig, GETTID(), pid);
send_request_and_wait(VERB_LEVEL_2,
syscall(SYS_gettid)/*exclude_id*/);
#define MINICORE 0
#if MINICORE
int pid = 0;
if ((pid = fork()) != 0) {
#endif
// parent or fork failed
timespec time = {0, 0};
clock_gettime(CLOCK_REALTIME, &time);
@ -208,23 +220,69 @@ void coredump_cb(int sig, siginfo_t *si, void *context)
iov[3].iov_base = end;
iov[3].iov_len = strlen(end);
writev(STDERR_FILENO, iov, sizeof(iov) / sizeof(iov[0]));
#if MINICORE
} else {
// child
prctl(PR_SET_NAME, "minicoredump");
int64_t total_size = 0;
if (lib::g_mem_cutter != nullptr) {
lib::g_mem_cutter->cut(total_size);
if (OB_SUCC(ret)) {
int status = 0;
waitpid(pid, &status, __WALL);
}
DLOG(INFO, "[MINICORE], TOTAL FREED: %ld", total_size);
}
#endif
}
// Reset back to the default handler
signal(sig, SIG_DFL);
raise(sig);
}
int minicoredump(int sig, int64_t tid, pid_t& pid)
{
static constexpr int64_t MIN_INTERVAL = 5 * 60 * 1000 * 1000; // 5min
static int64_t last_ts = 0;
int64_t now = ObTimeUtility::fast_current_time();
int64_t last = ATOMIC_LOAD(&last_ts);
int ret = OB_SUCCESS;
UNUSED(sig);
UNUSED(tid);
if (now - last < MIN_INTERVAL) {
ret = OB_EAGAIN;
} else if (!ATOMIC_BCAS(&last_ts, last, now)) {
ret = OB_EAGAIN;
} else if (-1 == access("bin/minicore.py", R_OK)) {
ret = OB_FILE_NOT_EXIST;
} else if (-1 == access(MINICORE_SHELL_PATH, R_OK)) {
if (0 == (pid = syscall(__NR_clone, CLONE_VFORK, nullptr, nullptr, nullptr, nullptr))) {
IGNORE_RETURN execlp("sh", "sh", "-c", MINICORE_SCRIPT, nullptr);
_exit(EXIT_FAILURE);
}
} else if (-1 != access(MINICORE_SHELL_PATH, X_OK)) {
if (0 == (pid = syscall(__NR_clone, CLONE_VFORK, nullptr, nullptr, nullptr, nullptr))) {
IGNORE_RETURN execlp("sh", "sh", MINICORE_SHELL_PATH, nullptr);
_exit(EXIT_FAILURE);
}
}
return ret;
}
int faststack()
{
static constexpr int64_t MIN_INTERVAL = 1 * 60 * 1000 * 1000; // 1min
static int64_t last_ts = 0;
int64_t now = ObTimeUtility::fast_current_time();
int64_t last = ATOMIC_LOAD(&last_ts);
int ret = OB_SUCCESS;
if (now - last < MIN_INTERVAL) {
ret = OB_EAGAIN;
} else if (!ATOMIC_BCAS(&last_ts, last, now)) {
ret = OB_EAGAIN;
} else if (-1 == access(FASTSTACK_SHELL_PATH, R_OK)) {
if (0 == syscall(__NR_clone, CLONE_VFORK | CLONE_PARENT, nullptr, nullptr, nullptr, nullptr)) {
IGNORE_RETURN execlp("sh", "sh", "-c", FASTSTACK_SCRIPT, nullptr);
_exit(EXIT_FAILURE);
}
} else if (-1 != access(FASTSTACK_SHELL_PATH, X_OK)) {
if (0 == syscall(__NR_clone, CLONE_VFORK | CLONE_PARENT, nullptr, nullptr, nullptr, nullptr)) {
IGNORE_RETURN execlp("sh", "sh", FASTSTACK_SHELL_PATH, nullptr);
_exit(EXIT_FAILURE);
}
}
return ret;
}
} // namespace common
} // namespace oceanbase

2
deps/oblib/src/lib/signal/ob_signal_handlers.h vendored
View File

@ -21,6 +21,8 @@ namespace oceanbase
{
namespace common
{
extern int minicoredump(int sig, int64_t tid, pid_t& pid);
extern int faststack();
} // namespace common
} // namespace oceanbase

2
src/observer/omt/ob_tenant.cpp
View File

@ -1264,6 +1264,8 @@ int ObTenant::recv_request(ObRequest &req)
if (OB_SUCC(ret)) {
ObTenantStatEstGuard guard(id_);
EVENT_INC(REQUEST_ENQUEUE_COUNT);
} else if (OB_SIZE_OVERFLOW == ret) {
IGNORE_RETURN faststack();
}
return ret;

1
src/observer/omt/ob_tenant_node_balancer.cpp
View File

@ -77,6 +77,7 @@ void ObTenantNodeBalancer::run1()
{
int ret = OB_SUCCESS;
lib::set_thread_name("OmtNodeBalancer");
while (!has_set_stop()) {
TenantUnits units;
int64_t sys_unit_cnt = 0;