Add file names and line numbers to stacktraces

The GLIBC backtrace functionality doesn't generate file names and line
numbers in the generated stacktrace. This can to be done manually by
executing a set of system commands.

Conceptually doing non-signal-safe operations in a signal handler is very
wrong but as stacktraces are only printed when something has gone horribly
wrong, there is no real need to worry about making things worse.

As a safeguard for fatal errors while the stacktrace is being generated,
it is first dumped into the standard error output of the process. This
will function even if malloc is corrupted.

server/core/gateway.cc

@@ -374,6 +374,96 @@ volatile sig_atomic_t fatal_handling = 0;
 
 static int signal_set(int sig, void (*handler)(int));
 
+void get_command_output(char* output, size_t size, const char* format, ...)
+{
+    va_list valist;
+    va_start(valist, format);
+    int cmd_len = vsnprintf(NULL, 0, format, valist);
+    va_end(valist);
+
+    va_start(valist, format);
+    char cmd[cmd_len + 1];
+    vsnprintf(cmd, cmd_len + 1, format, valist);
+    va_end(valist);
+
+    *output = '\0';
+    FILE* file = popen(cmd, "r");
+
+    if (file)
+    {
+        size_t nread = fread(output, 1, size, file);
+        nread = nread < size ? nread : size - 1;
+        output[nread--] = '\0';
+
+        // Trim trailing newlines
+        while (output + nread > output && output[nread] == '\n')
+        {
+            output[nread--] = '\0';
+        }
+
+        pclose(file);
+    }
+}
+
+void extract_file_and_line(const char* symbols, char* cmd, size_t size)
+{
+    const char* filename_end = strchr(symbols, '(');
+    const char* symname_end = strchr(symbols, ')');
+
+    if (filename_end && symname_end)
+    {
+        // This appears to be a symbol in a library
+        char filename[PATH_MAX + 1];
+        char symname[512];
+        char offset[512];
+        snprintf(filename, sizeof(filename), "%.*s", (int)(filename_end - symbols), symbols);
+
+        const char* symname_start = filename_end + 1;
+
+        if (*symname_start != '+')
+        {
+            // We have a string form symbol name and an offset, we need to
+            // extract the symbol address
+
+            const char* addr_offset = symname_start;
+
+            while (addr_offset < symname_end && *addr_offset != '+')
+            {
+                addr_offset++;
+            }
+
+            snprintf(symname, sizeof(symname), "%.*s", (int)(addr_offset - symname_start), symname_start);
+
+            if (addr_offset < symname_end && *addr_offset == '+')
+            {
+                addr_offset++;
+            }
+
+            snprintf(offset, sizeof(offset), "%.*s", (int)(symname_end - addr_offset), addr_offset);
+
+            // Get the hexadecimal address of the symbol
+            get_command_output(cmd, size,
+                               "nm %s |grep ' %s$'|sed -e 's/ .*//' -e 's/^/0x/'",
+                               filename, symname);
+            long long symaddr = strtoll(cmd, NULL, 16);
+            long long offsetaddr = strtoll(offset, NULL, 16);
+
+            // Calculate the file and line now that we have the raw offset into
+            // the library
+            get_command_output(cmd, size,
+                               "addr2line -e %s 0x%x",
+                               filename, symaddr + offsetaddr);
+        }
+        else
+        {
+            // Raw offset into library
+            symname_start++;
+            snprintf(symname, sizeof(symname), "%.*s", (int)(symname_end - symname_start), symname_start);
+            get_command_output(cmd, size, "addr2line -e %s %s", filename, symname);
+        }
+    }
+}
+
 static void
 sigfatal_handler(int i)
 {
@@ -408,7 +498,9 @@ sigfatal_handler(int i)
     {
         for (int n = 0; n < count; n++)
         {
-            MXS_ALERT("  %s\n", symbols[n]);
+            char cmd[PATH_MAX + 1024] = "<not found>";
+            extract_file_and_line(symbols[n], cmd, sizeof(cmd));
+            MXS_ALERT("  %s: %s", symbols[n], cmd);
         }
         MXS_FREE(symbols);
     }