Ensure exit is possible

If shutdown has been initiated (via maxadmin or by sending a
SIGINT or SIGTERM) and a SIGTERM is received, the process
is terminated.

If shutdown has been initiated (via maxadmin or by sending a
SIGINT or SIGTERM) and a SIGINT (Ctrl-C) is received, a warning
is printed that shutdown is in progess. Then, if an additional
SIGINT is received, the process is terminated.

So, in practice:
- If MaxScale is running as a daemon, the first SIGTERM initiates
  shutdown and a second one unconditionally terminates the process.
- If MaxScale is running in the console, the first Ctrl-C initiates
  shutdown, the second prints a warning and the third terminates
  the process.

This is to ensure that MaxScale can be forced to exit, in
case some thread is hung for whatever reason, and is thus preventing
the controlled shutdown.

include/maxscale/maxscale.h

@@ -44,4 +44,14 @@ void maxscale_reset_starttime(void);
 time_t maxscale_started(void);
 int maxscale_uptime(void);
 
+/**
+ * Initiate shutdown of MaxScale.
+ *
+ * This functions informs all threads that they should stop the
+ * processing and exit.
+ *
+ * @return How many times maxscale_shutdown() has been called.
+ */
+int maxscale_shutdown(void);
+
 MXS_END_DECLS

server/core/gateway.c

@@ -181,7 +181,6 @@ static int set_user(const char* user);
 bool pid_file_exists();
 void write_child_exit_code(int fd, int code);
 static bool change_cwd();
-void shutdown_server();
 static void log_exit_status();
 static bool daemonize();
 static bool sniff_configuration(const char* filepath);
@@ -288,20 +287,45 @@ static void sigusr1_handler (int i)
 }
 
 static const char shutdown_msg[] = "\n\nShutting down MaxScale\n\n";
+static const char patience_msg[] =
+    "\n"
+    "Patience is a virtue...\n"
+    "Shutdown in progress, but one more Ctrl-C or SIGTERM and MaxScale goes down,\n"
+    "no questions asked.\n";
 
 static void sigterm_handler(int i)
 {
     last_signal = i;
-    shutdown_server();
-    write(STDERR_FILENO, shutdown_msg, sizeof(shutdown_msg) - 1);
+    int n_shutdowns = maxscale_shutdown();
+
+    if (n_shutdowns == 1)
+    {
+        write(STDERR_FILENO, shutdown_msg, sizeof(shutdown_msg) - 1);
+    }
+    else
+    {
+        exit(EXIT_FAILURE);
+    }
 }
 
 static void
 sigint_handler(int i)
 {
     last_signal = i;
-    shutdown_server();
-    write(STDERR_FILENO, shutdown_msg, sizeof(shutdown_msg) - 1);
+    int n_shutdowns = maxscale_shutdown();
+
+    if (n_shutdowns == 1)
+    {
+        write(STDERR_FILENO, shutdown_msg, sizeof(shutdown_msg) - 1);
+    }
+    else if (n_shutdowns == 2)
+    {
+        write(STDERR_FILENO, patience_msg, sizeof(patience_msg) - 1);
+    }
+    else
+    {
+        exit(EXIT_FAILURE);
+    }
 }
 
 static void
@@ -2041,14 +2065,22 @@ return_main:
 /*<
  * Shutdown MaxScale server
  */
-void
-shutdown_server()
+int maxscale_shutdown()
 {
-    service_shutdown();
-    poll_shutdown();
-    hkshutdown();
-    memlog_flush_all();
-    log_flush_shutdown();
+    static int n_shutdowns = 0;
+
+    int n = atomic_add(&n_shutdowns, 1);
+
+    if (n == 0)
+    {
+        service_shutdown();
+        poll_shutdown();
+        hkshutdown();
+        memlog_flush_all();
+        log_flush_shutdown();
+    }
+
+    return n + 1;
 }
 
 static void log_flush_shutdown(void)

server/modules/routing/debugcli/debugcmd.c

@@ -70,6 +70,7 @@
 #include <debugcli.h>
 #include <maxscale/housekeeper.h>
 #include <maxscale/listmanager.h>
+#include <maxscale/maxscale.h>
 
 #include <maxscale/log_manager.h>
 #include <sys/syslog.h>
@@ -276,7 +277,11 @@ struct subcommand listoptions[] = {
       {0, 0, 0} }
 };
 
-extern void shutdown_server();
+static void shutdown_server()
+{
+    maxscale_shutdown();
+}
+
 static void shutdown_service(DCB *dcb, SERVICE *service);
 static void shutdown_monitor(DCB *dcb, MONITOR *monitor);