MXS-2196: Start listeners after worker threads

If the startup of the listeners requires communication with all of the
workers, the workers must be up and running for that to happen.

Due to the fact that the main thread is still a worker thread, the
initialization code is not extra straightforward. By queuing an event to
the main worker, the startup of all listeners is done at a fully
operational state with all workers fully functional.

The service initialization code was also flawed in the sense that it would
cause a deadlock if any of the threads would have to check for the user
permissions. This is mainly a problem with the authenticator modules but
the benefits of the per service pre-loading of users is most likely
superficial. In theory startup will be faster as each thread now queries
the users in parallel.

server/core/gateway.cc

@@ -1413,6 +1413,33 @@ int main(int argc, char** argv)
     const char* specified_user = NULL;
     char export_cnf[PATH_MAX + 1] = "";
 
+    /**
+     * The following lambda function is executed as the first event on the main worker. This is what starts
+     * up the listeners for all services.
+     *
+     * Due to the fact that the main thread runs a worker thread we have to queue the starting
+     * of the listeners to happen after all workers have started. This allows worker messages to be used
+     * when listeners are being started.
+     *
+     * Once the main worker is dedicated to doing work other than handling traffic the code could be executed
+     * immediately after the worker thread have been started. This would make the startup logic clearer as
+     * the order of the events would be the way they appear to be.
+     */
+    auto do_startup = [&]() {
+            if (!service_launch_all())
+            {
+                const char* logerr = "Failed to start all MaxScale services. Exiting.";
+                print_log_n_stderr(true, true, logerr, logerr, 0);
+                rc = MAXSCALE_NOSERVICES;
+                RoutingWorker::shutdown_all();
+            }
+            else if (daemon_mode)
+            {
+                // Successful start, notify the parent process that it can exit.
+                write_child_exit_code(daemon_pipe[1], rc);
+            }
+        };
+
     config_init();
     config_set_global_defaults();
     mxb_assert(cnf);
@@ -2177,17 +2204,6 @@ int main(int argc, char** argv)
     /** Start all monitors */
     monitor_start_all();
 
-    /** Start the services that were created above */
-    n_services = service_launch_all();
-
-    if (n_services == -1)
-    {
-        const char* logerr = "Failed to start all MaxScale services. Exiting.";
-        print_log_n_stderr(true, true, logerr, logerr, 0);
-        rc = MAXSCALE_NOSERVICES;
-        goto return_main;
-    }
-
     if (cnf->config_check)
     {
         MXS_NOTICE("Configuration was successfully verified.");
@@ -2241,20 +2257,20 @@ int main(int argc, char** argv)
                config_threadcount(),
                config_thread_stack_size());
 
-    /**
-     * Successful start, notify the parent process that it can exit.
-     */
-    mxb_assert(rc == MAXSCALE_SHUTDOWN);
-    if (daemon_mode)
+    worker = RoutingWorker::get(RoutingWorker::MAIN);
+    mxb_assert(worker);
+
+    if (!worker->execute(do_startup, RoutingWorker::EXECUTE_QUEUED))
     {
-        write_child_exit_code(daemon_pipe[1], rc);
+        const char* logerr = "Failed to queue startup task.";
+        print_log_n_stderr(true, true, logerr, logerr, 0);
+        rc = MAXSCALE_INTERNALERROR;
+        goto return_main;
     }
 
     /*<
      * Run worker 0 in the main thread.
      */
-    worker = RoutingWorker::get(RoutingWorker::MAIN);
-    mxb_assert(worker);
     worker->run();
 
     /** Stop administrative interface */

server/core/internal/service.hh

@@ -195,18 +195,9 @@ void service_destroy_instances(void);
  * Initialize and start all services. This should only be called once by the
  * main initialization code.
  *
- * @return Number of successfully started services or -1 on error
+ * @return False if a fatal error occurred
  */
-int service_launch_all(void);
-
-/**
- * Perform thread-specific initialization
- *
- * Currently this function only pre-loads users for all threads.
- *
- * @return True on success, false on error (currently always returns true).
- */
-bool service_thread_init();
+bool service_launch_all(void);
 
 /**
  * @brief Remove a listener from use

server/core/routingworker.cc

@@ -558,7 +558,7 @@ bool RoutingWorker::pre_run()
 {
     this_thread.current_worker_id = m_id;
 
-    bool rv = modules_thread_init() && service_thread_init() && qc_thread_init(QC_INIT_SELF);
+    bool rv = modules_thread_init() && qc_thread_init(QC_INIT_SELF);
 
     if (!rv)
     {

server/core/service.cc

@@ -422,10 +422,10 @@ bool serviceStartListener(SERVICE* svc, const char* name)
     return listener && listener->service() == svc && listener->start();
 }
 
-int service_launch_all()
+bool service_launch_all()
 {
     int n = 0, i;
-    bool error = false;
+    bool ok = true;
     int num_svc = this_unit.services.size();
 
     MXS_NOTICE("Starting a total of %d services...", num_svc);
@@ -439,7 +439,7 @@ int service_launch_all()
         if (i == 0)
         {
             MXS_ERROR("Failed to start service '%s'.", service->name);
-            error = true;
+            ok = false;
         }
 
         if (maxscale_is_shutting_down())
@@ -448,7 +448,7 @@ int service_launch_all()
         }
     }
 
-    return error ? -1 : n;
+    return ok;
 }
 
 bool serviceStop(SERVICE* service)
@@ -1911,21 +1911,6 @@ uint64_t service_get_version(const SERVICE* svc, service_version_which_t which)
     return version;
 }
 
-bool service_thread_init()
-{
-    LockGuard guard(this_unit.lock);
-
-    for (Service* service : this_unit.services)
-    {
-        if (service->capabilities & ACAP_TYPE_ASYNC)
-        {
-            service_refresh_users(service);
-        }
-    }
-
-    return true;
-}
-
 bool Service::is_basic_parameter(const std::string& name)
 {
     static const std::set<std::string> names =