[FEAT MERGE]MiniCore

deps/oblib/src/lib/oblog/ob_async_log_struct.cpp

@@ -70,7 +70,7 @@ int ObPLogFileStruct::reopen(const bool redirect_flag)
   if (OB_UNLIKELY(strlen(filename_) <= 0)) {
     LOG_STDERR("invalid argument log_file = %p\n", filename_);
     ret = OB_INVALID_ARGUMENT;
-  } else if (OB_UNLIKELY((tmp_fd = ::open(filename_, O_WRONLY | O_CREAT | O_APPEND , LOG_FILE_MODE)) < 0)) {
+  } else if (OB_UNLIKELY((tmp_fd = ::open(filename_, O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, LOG_FILE_MODE)) < 0)) {
     LOG_STDERR("open file = %s errno = %d error = %m\n", filename_, errno);
     ret = OB_ERR_UNEXPECTED;
   } else if (OB_UNLIKELY(0 != fstat(tmp_fd, &stat_))) {
@@ -112,7 +112,7 @@ int ObPLogFileStruct::reopen_wf()
   } else {
     char tmp_file_name[MAX_LOG_FILE_NAME_SIZE];
     (void)snprintf(tmp_file_name, sizeof(tmp_file_name), "%s.wf", filename_);
-    if (OB_UNLIKELY((tmp_fd = ::open(tmp_file_name, O_WRONLY | O_CREAT | O_APPEND , LOG_FILE_MODE)) < 0)) {
+    if (OB_UNLIKELY((tmp_fd = ::open(tmp_file_name, O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, LOG_FILE_MODE)) < 0)) {
       LOG_STDERR("open file = %s errno = %d error = %m\n", tmp_file_name, errno);
       ret = OB_ERR_UNEXPECTED;
     } else if (OB_UNLIKELY(0 != fstat(tmp_fd, &wf_stat_))) {

deps/oblib/src/lib/oblog/ob_log.cpp

@@ -742,7 +742,7 @@ void ObLogger::rotate_log(const char *filename,
                 tm.tm_hour, tm.tm_min, tm.tm_sec, static_cast<int>(t.tv_usec/1000));
 
         ret = rename(filename, old_log_file); //If failed, TODO
-        int tmp_fd = open(filename, O_WRONLY | O_CREAT | O_APPEND, ObPLogFileStruct::LOG_FILE_MODE);
+        int tmp_fd = open(filename, O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, ObPLogFileStruct::LOG_FILE_MODE);
         if (tmp_fd > 0) {
           if (fd > STDERR_FILENO) {
             (void)dup2(tmp_fd, fd);
@@ -786,7 +786,7 @@ void ObLogger::rotate_log(const char *filename,
         }
 
         ret = rename(filename, old_log_file); //If failed, TODO
-        int tmp_fd = open(filename, O_WRONLY | O_CREAT | O_APPEND, ObPLogFileStruct::LOG_FILE_MODE);
+        int tmp_fd = open(filename, O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, ObPLogFileStruct::LOG_FILE_MODE);
         if (tmp_fd > 0) {
           if (redirect_flag) {
             (void)dup2(tmp_fd, STDERR_FILENO);
@@ -820,7 +820,7 @@ void ObLogger::rotate_log(const char *filename,
             }
           }
           ret = rename(wf_filename, old_wf_log_file); //If failed, TODO
-          tmp_fd = open(wf_filename, O_WRONLY | O_CREAT | O_APPEND, ObPLogFileStruct::LOG_FILE_MODE);
+          tmp_fd = open(wf_filename, O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, ObPLogFileStruct::LOG_FILE_MODE);
           if (tmp_fd > 0) {
             if (wf_fd > STDERR_FILENO) {
               (void)dup2(tmp_fd, wf_fd);

deps/oblib/src/lib/oblog/ob_log.h

@@ -42,6 +42,7 @@
 #include "lib/oblog/ob_async_log_struct.h"
 #include "lib/utility/ob_defer.h"
 #include "lib/oblog/ob_syslog_rate_limiter.h"
+#include "lib/signal/ob_signal_handlers.h"
 
 #define OB_LOG_MAX_PAR_MOD_SIZE 32
 #define OB_LOG_MAX_SUB_MOD_SIZE 32
@@ -1112,7 +1113,7 @@ inline void ObLogger::check_probe(
             break;
           }
           case ProbeAction::PROBE_STACK: {
-            IGNORE_RETURN raise(60);
+            IGNORE_RETURN faststack();
             break;
           }
           default: {

deps/oblib/src/lib/signal/ob_signal_handlers.cpp

@@ -15,6 +15,9 @@
 #include "lib/signal/ob_signal_handlers.h"
 #include <sys/prctl.h>
 #include <dirent.h>
+#include <unistd.h>
+#include <fstream>
+#include <sys/wait.h>
 #include "lib/profile/ob_trace_id.h"
 #include "lib/utility/utility.h"
 #include "lib/signal/ob_libunwind.h"
@@ -29,6 +32,17 @@ namespace oceanbase
 namespace common
 {
 static const int SIG_SET[] = {SIGABRT, SIGBUS, SIGFPE, SIGSEGV, SIGURG};
+static constexpr char MINICORE_SHELL_PATH[] = "tools/minicore.sh";
+static constexpr char FASTSTACK_SHELL_PATH[] = "tools/callstack.sh";
+static constexpr char MINICORE_SCRIPT[] = "if [ -e bin/minicore.py ]; then\n"
+"  python bin/minicore.py `cat $(pwd)/run/observer.pid` -c -o core.`cat $(pwd)/run/observer.pid`.mini\n"
+"fi\n"
+"[ $(ls -1 core.*.mini 2>/dev/null | wc -l) -gt 5 ] && ls -1 core.*.mini | sort | head -n 1 | xargs rm -f";
+
+static constexpr char FASTSTACK_SCRIPT[] = "if [ -x \"$(command -v obstack)\" ]; then\n"
+"  obstack `cat $(pwd)/run/observer.pid` > stack.`cat $(pwd)/run/observer.pid`.`date +%Y%m%d%H%M%S`\n"
+"fi\n"
+"[ $(ls -1 stack.* 2>/dev/null | wc -l) -gt 100 ] && ls -1 stack.* | sort | head -n 1 | xargs rm -f";
 
 static inline void handler(int sig, siginfo_t *s, void *p)
 {
@@ -138,93 +152,137 @@ void close_socket_fd()
 
 void coredump_cb(int sig, siginfo_t *si, void *context)
 {
+  int ret = OB_SUCCESS;
   if (g_coredump_num++ < 1) {
+    pid_t pid;
     close_socket_fd();
+    ret = minicoredump(sig, GETTID(), pid);
     send_request_and_wait(VERB_LEVEL_2,
                           syscall(SYS_gettid)/*exclude_id*/);
-    #define MINICORE 0
+    // parent or fork failed
-    #if MINICORE
+    timespec time = {0, 0};
-    int pid = 0;
+    clock_gettime(CLOCK_REALTIME, &time);
-    if ((pid = fork()) != 0) {
+    int64_t ts = time.tv_sec * 1000000 + time.tv_nsec / 1000;
-    #endif
+    // thread_name
-      // parent or fork failed
+    char tname[16];
-      timespec time = {0, 0};
+    prctl(PR_GET_NAME, tname);
-      clock_gettime(CLOCK_REALTIME, &time);
+    // backtrace
-      int64_t ts = time.tv_sec * 1000000 + time.tv_nsec / 1000;
+    char bt[256];
-      // thread_name
+    int64_t len = 0;
-      char tname[16];
-      prctl(PR_GET_NAME, tname);
-      // backtrace
-      char bt[256];
-      int64_t len = 0;
 #ifdef __x86_64__
-      safe_backtrace(bt, sizeof(bt) - 1, &len);
+    safe_backtrace(bt, sizeof(bt) - 1, &len);
 #endif
-      bt[len++] = '\0';
+    bt[len++] = '\0';
-      // extra
+    // extra
-      const ObFatalErrExtraInfoGuard *extra_info = nullptr; // TODO: May deadlock, ObFatalErrExtraInfoGuard::get_thd_local_val_ptr();
+    const ObFatalErrExtraInfoGuard *extra_info = nullptr; // TODO: May deadlock, ObFatalErrExtraInfoGuard::get_thd_local_val_ptr();
-      uint64_t uval[4] = {0};
+    uint64_t uval[4] = {0};
-      auto *trace_id = ObCurTraceId::get_trace_id();
+    auto *trace_id = ObCurTraceId::get_trace_id();
-      if (trace_id != nullptr) {
+    if (trace_id != nullptr) {
-        trace_id->get_uval(uval);
+      trace_id->get_uval(uval);
-      }
+    }
-      char print_buf[512];
+    char print_buf[512];
-      const ucontext_t *con = (ucontext_t *)context;
+    const ucontext_t *con = (ucontext_t *)context;
-#if defined(__x86_64__)
+#if defined(__x86_64__)
-      int64_t ip = con->uc_mcontext.gregs[REG_RIP];
+    int64_t ip = con->uc_mcontext.gregs[REG_RIP];
-      int64_t bp = con->uc_mcontext.gregs[REG_RBP]; // stack base
+    int64_t bp = con->uc_mcontext.gregs[REG_RBP]; // stack base
-#else
+#else
-      // TODO: ARM
+    // TODO: ARM
-      int64_t ip = -1;
+    int64_t ip = -1;
-      int64_t bp = -1;
+    int64_t bp = -1;
-#endif
+#endif
-      char rlimit_core[32] = "unlimited";
+    char rlimit_core[32] = "unlimited";
-      if (UINT64_MAX != g_rlimit_core) {
+    if (UINT64_MAX != g_rlimit_core) {
-        safe_snprintf(rlimit_core, sizeof(rlimit_core), "%lu", g_rlimit_core);
+      safe_snprintf(rlimit_core, sizeof(rlimit_core), "%lu", g_rlimit_core);
-      }
+    }
-      char crash_info[128] = "CRASH ERROR!!!";
+    char crash_info[128] = "CRASH ERROR!!!";
-      int64_t fatal_error_thread_id = get_fatal_error_thread_id();
+    int64_t fatal_error_thread_id = get_fatal_error_thread_id();
-      if (-1 != fatal_error_thread_id) {
+    if (-1 != fatal_error_thread_id) {
-        safe_snprintf(crash_info, sizeof(crash_info),
+      safe_snprintf(crash_info, sizeof(crash_info),
-                      "Right to Die or Duty to Live's Thread Existed before CRASH ERROR!!!"
+                    "Right to Die or Duty to Live's Thread Existed before CRASH ERROR!!!"
-                      "ThreadId=%ld,", fatal_error_thread_id);
+                    "ThreadId=%ld,", fatal_error_thread_id);
-      }
+    }
-      ssize_t print_len = safe_snprintf(print_buf, sizeof(print_buf),
+    ssize_t print_len = safe_snprintf(print_buf, sizeof(print_buf),
-                                       "%s IP=%lx, RBP=%lx, sig=%d, sig_code=%d, sig_addr=%p, RLIMIT_CORE=%s, "COMMON_FMT", ",
+                                     "%s IP=%lx, RBP=%lx, sig=%d, sig_code=%d, sig_addr=%p, RLIMIT_CORE=%s, "COMMON_FMT", ",
-                                       crash_info, ip, bp, sig, si->si_code, si->si_addr, rlimit_core,
+                                     crash_info, ip, bp, sig, si->si_code, si->si_addr, rlimit_core,
-                                       ts, GETTID(), tname, uval[0], uval[1], uval[2], uval[3],
+                                     ts, GETTID(), tname, uval[0], uval[1], uval[2], uval[3],
-                                       (NULL == extra_info) ? NULL : to_cstring(*extra_info), bt);
+                                     (NULL == extra_info) ? NULL : to_cstring(*extra_info), bt);
-      const auto &si_guard = ObSqlInfoGuard::get_cur_guard();
+    const auto &si_guard = ObSqlInfoGuard::get_cur_guard();
-      char sql[] = "SQL=";
+    char sql[] = "SQL=";
-      char end[] = "\n";
+    char end[] = "\n";
-      struct iovec iov[4];
+    struct iovec iov[4];
-      memset(iov, 0, sizeof(iov));
+    memset(iov, 0, sizeof(iov));
-      iov[0].iov_base = print_buf;
+    iov[0].iov_base = print_buf;
-      iov[0].iov_len = print_len;
+    iov[0].iov_len = print_len;
-      iov[1].iov_base = sql;
+    iov[1].iov_base = sql;
-      iov[1].iov_len = strlen(sql);
+    iov[1].iov_len = strlen(sql);
-      iov[2].iov_base = NULL != si_guard ? si_guard->sql_.ptr() : NULL;
+    iov[2].iov_base = NULL != si_guard ? si_guard->sql_.ptr() : NULL;
-      iov[2].iov_len = NULL != si_guard ? si_guard->sql_.length() : 0;
+    iov[2].iov_len = NULL != si_guard ? si_guard->sql_.length() : 0;
-      iov[3].iov_base = end;
+    iov[3].iov_base = end;
-      iov[3].iov_len = strlen(end);
+    iov[3].iov_len = strlen(end);
-      writev(STDERR_FILENO, iov, sizeof(iov) / sizeof(iov[0]));
+    writev(STDERR_FILENO, iov, sizeof(iov) / sizeof(iov[0]));
-
+    if (OB_SUCC(ret)) {
-    #if MINICORE
+      int status = 0;
-    } else {
+      waitpid(pid, &status, __WALL);
-      // child
-      prctl(PR_SET_NAME, "minicoredump");
-      int64_t total_size = 0;
-      if (lib::g_mem_cutter != nullptr) {
-        lib::g_mem_cutter->cut(total_size);
-      }
-      DLOG(INFO, "[MINICORE], TOTAL FREED: %ld", total_size);
     }
-    #endif
   }
   // Reset back to the default handler
   signal(sig, SIG_DFL);
   raise(sig);
 }
 
+int minicoredump(int sig, int64_t tid, pid_t& pid)
+{
+  static constexpr int64_t MIN_INTERVAL = 5 * 60 * 1000 * 1000; // 5min
+  static int64_t last_ts = 0;
+  int64_t now = ObTimeUtility::fast_current_time();
+  int64_t last = ATOMIC_LOAD(&last_ts);
+  int ret = OB_SUCCESS;
+  UNUSED(sig);
+  UNUSED(tid);
+  if (now - last < MIN_INTERVAL) {
+    ret = OB_EAGAIN;
+  } else if (!ATOMIC_BCAS(&last_ts, last, now)) {
+    ret = OB_EAGAIN;
+  } else if (-1 == access("bin/minicore.py", R_OK)) {
+    ret = OB_FILE_NOT_EXIST;
+  } else if (-1 == access(MINICORE_SHELL_PATH, R_OK)) {
+    if (0 == (pid = syscall(__NR_clone, CLONE_VFORK, nullptr, nullptr, nullptr, nullptr))) {
+      IGNORE_RETURN execlp("sh", "sh", "-c", MINICORE_SCRIPT, nullptr);
+      _exit(EXIT_FAILURE);
+    }
+  } else if (-1 != access(MINICORE_SHELL_PATH, X_OK)) {
+    if (0 == (pid = syscall(__NR_clone, CLONE_VFORK, nullptr, nullptr, nullptr, nullptr))) {
+      IGNORE_RETURN execlp("sh", "sh", MINICORE_SHELL_PATH, nullptr);
+      _exit(EXIT_FAILURE);
+    }
+  }
+  return ret;
+}
+
+int faststack()
+{
+  static constexpr int64_t MIN_INTERVAL = 1 * 60 * 1000 * 1000; // 1min
+  static int64_t last_ts = 0;
+  int64_t now = ObTimeUtility::fast_current_time();
+  int64_t last = ATOMIC_LOAD(&last_ts);
+  int ret = OB_SUCCESS;
+  if (now - last < MIN_INTERVAL) {
+    ret = OB_EAGAIN;
+  } else if (!ATOMIC_BCAS(&last_ts, last, now)) {
+    ret = OB_EAGAIN;
+  } else if (-1 == access(FASTSTACK_SHELL_PATH, R_OK)) {
+    if (0 == syscall(__NR_clone, CLONE_VFORK | CLONE_PARENT, nullptr, nullptr, nullptr, nullptr)) {
+      IGNORE_RETURN execlp("sh", "sh", "-c", FASTSTACK_SCRIPT, nullptr);
+      _exit(EXIT_FAILURE);
+    }
+  } else if (-1 != access(FASTSTACK_SHELL_PATH, X_OK)) {
+    if (0 == syscall(__NR_clone, CLONE_VFORK | CLONE_PARENT, nullptr, nullptr, nullptr, nullptr)) {
+      IGNORE_RETURN execlp("sh", "sh", FASTSTACK_SHELL_PATH, nullptr);
+      _exit(EXIT_FAILURE);
+    }
+  }
+  return ret;
+}
+
 } // namespace common
 } // namespace oceanbase

deps/oblib/src/lib/signal/ob_signal_handlers.h

@@ -21,6 +21,8 @@ namespace oceanbase
 {
 namespace common
 {
+extern int minicoredump(int sig, int64_t tid, pid_t& pid);
+extern int faststack();
 } // namespace common
 } // namespace oceanbase
 

src/observer/omt/ob_tenant.cpp

@@ -1264,6 +1264,8 @@ int ObTenant::recv_request(ObRequest &req)
   if (OB_SUCC(ret)) {
     ObTenantStatEstGuard guard(id_);
     EVENT_INC(REQUEST_ENQUEUE_COUNT);
+  } else if (OB_SIZE_OVERFLOW == ret) {
+    IGNORE_RETURN faststack();
   }
 
   return ret;

src/observer/omt/ob_tenant_node_balancer.cpp

@@ -77,6 +77,7 @@ void ObTenantNodeBalancer::run1()
 {
   int ret = OB_SUCCESS;
   lib::set_thread_name("OmtNodeBalancer");
+
   while (!has_set_stop()) {
     TenantUnits units;
     int64_t sys_unit_cnt = 0;