Wait until housekeeper has finished

When calling hkshutdown(), only a shutdown flag will be set.
In main() we will then actually wait for the housekeeper thread
to finish before exiting.

include/maxscale/housekeeper.h

@@ -51,11 +51,34 @@ typedef struct hktask
     struct hktask *next;      /*< Next task in the list */
 } HKTASK;
 
-extern void hkinit();
+/**
+ * Initialises the housekeeper mechanism.
+ *
+ * A call to any of the other housekeeper functions can be made only if
+ * this function returns successfully.
+ *
+ * @return True if the housekeeper mechanism was initialized, false otherwise.
+ */
+extern bool hkinit();
+
+/**
+ * Shuts down the housekeeper mechanism.
+ *
+ * Should be called @b only if @c hkinit() returned successfully.
+ *
+ * @see hkinit hkfinish
+ */
+extern void hkshutdown();
+
+/**
+ * Waits for the housekeeper thread to finish. Should be called only after
+ * hkshutdown() has been called.
+ */
+extern void hkfinish();
+
 extern int  hktask_add(const char *name, void (*task)(void *), void *data, int frequency);
 extern int  hktask_oneshot(const char *name, void (*task)(void *), void *data, int when);
 extern int  hktask_remove(const char *name);
-extern void hkshutdown();
 extern void hkshow_tasks(DCB *pdcb);
 
 MXS_END_DECLS

include/maxscale/semaphore.h

@@ -0,0 +1,22 @@
+#pragma once
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl.
+ *
+ * Change Date: 2019-07-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+/**
+ * @file semaphore.h  Semaphores used by MaxScale.
+ */
+
+// As a minimal preparation for other environments than Linux, components
+// include <maxscale/semaphore.h>,  instead of including <semaphore.h>
+// directly.
+#include <semaphore.h>

server/core/gateway.c

@@ -1936,7 +1936,13 @@ int main(int argc, char **argv)
     /*
      * Start the housekeeper thread
      */
-    hkinit();
+    if (!hkinit())
+    {
+        char* logerr = "Failed to start housekeeper thread.";
+        print_log_n_stderr(true, true, logerr, logerr, 0);
+        rc = MAXSCALE_INTERNALERROR;
+        goto return_main;
+    }
 
     /*<
      * Start the polling threads, note this is one less than is
@@ -1974,6 +1980,11 @@ int main(int argc, char **argv)
      */
     poll_waitevents((void *)0);
 
+    /*<
+     * Wait for the housekeeper to finish.
+     */
+    hkfinish();
+
     /*<
      * Wait server threads' completion.
      */

server/core/housekeeper.c

@@ -10,13 +10,14 @@
  * of this software will be governed by version 2 or later of the General
  * Public License.
  */
+#include <maxscale/housekeeper.h>
 #include <stdlib.h>
 #include <string.h>
 #include <maxscale/alloc.h>
-#include <maxscale/housekeeper.h>
+#include <maxscale/atomic.h>
-#include <maxscale/thread.h>
+#include <maxscale/semaphore.h>
 #include <maxscale/spinlock.h>
-#include <maxscale/log_manager.h>
+#include <maxscale/thread.h>
 
 /**
  * @file housekeeper.c  Provide a mechanism to run periodic tasks
@@ -49,22 +50,28 @@ static HKTASK *tasks = NULL;
  */
 static SPINLOCK tasklock = SPINLOCK_INIT;
 
-static int do_shutdown = 0;
+static bool do_shutdown = 0;
+
 long hkheartbeat = 0; /*< One heartbeat is 100 milliseconds */
 static THREAD hk_thr_handle;
 
 static void hkthread(void *);
 
-/**
+bool
- * Initialise the housekeeper thread
- */
-void
 hkinit()
 {
-    if (thread_start(&hk_thr_handle, hkthread, NULL) == NULL)
+    bool inited = false;
+
+    if (thread_start(&hk_thr_handle, hkthread, NULL) != NULL)
     {
-        MXS_ERROR("Failed to start housekeeper thread.");
+        inited = true;
     }
+    else
+    {
+        MXS_ALERT("Failed to start housekeeper thread.");
+    }
+
+    return inited;
 }
 
 /**
@@ -255,21 +262,17 @@ hkthread(void *data)
     void *taskdata;
     int i;
 
-    for (;;)
+    while (!do_shutdown)
     {
         for (i = 0; i < 10; i++)
         {
-            if (do_shutdown)
-            {
-                return;
-            }
             thread_millisleep(100);
             hkheartbeat++;
         }
         now = time(0);
         spinlock_acquire(&tasklock);
         ptr = tasks;
-        while (ptr)
+        while (!do_shutdown && ptr)
         {
             if (ptr->nextdue <= now)
             {
@@ -297,16 +300,25 @@ hkthread(void *data)
         }
         spinlock_release(&tasklock);
     }
+
+    MXS_NOTICE("Housekeeper shutting down.");
 }
 
-/**
- * Called to shutdown the housekeeper
- *
- */
 void
 hkshutdown()
 {
-    do_shutdown = 1;
+    do_shutdown = true;
+    atomic_synchronize();
+}
+
+void hkfinish()
+{
+    ss_dassert(do_shutdown);
+
+    MXS_NOTICE("Waiting for housekeeper to shut down.");
+    thread_wait(hk_thr_handle);
+    do_shutdown = false;
+    MXS_NOTICE("Housekeeper has shut down.");
 }
 
 /**