MXS-1948: Do round-robin worker assignment

Due to the skewed accept distribution without SO_REUSEPORT, we use
round-robin assignment of workers for new client connections. This
provides better performance as work is more likely to be evenly
distributed across all threads.

Using a least-busy-worker algorithm would provide a more stable result but
this is not trivially simple to implement. For this reason, the
round-robin based approach was chosen for 2.2.

server/core/dcb.cc

@@ -3532,23 +3532,30 @@ int poll_add_dcb(DCB *dcb)
         new_state = DCB_STATE_LISTENING;
         worker_id = MXS_WORKER_ALL;
     }
-    else if (dcb->dcb_role == DCB_ROLE_CLIENT_HANDLER &&
+    else if (dcb->dcb_role == DCB_ROLE_CLIENT_HANDLER)
-             (strcasecmp(dcb->service->routerModule, "cli") == 0 ||
-              strcasecmp(dcb->service->routerModule, "maxinfo") == 0))
     {
-        // If the DCB refers to an accepted maxadmin/maxinfo socket, we force it
+        if (strcasecmp(dcb->service->routerModule, "cli") == 0 ||
-        // to the main thread. That's done in order to prevent a deadlock
+            strcasecmp(dcb->service->routerModule, "maxinfo") == 0)
-        // that may happen if there are multiple concurrent administrative calls,
+        {
-        // handled by different worker threads.
+            // If the DCB refers to an accepted maxadmin/maxinfo socket, we force it
-        // See: https://jira.mariadb.org/browse/MXS-1805 and https://jira.mariadb.org/browse/MXS-1833
+            // to the main thread. That's done in order to prevent a deadlock
+            // that may happen if there are multiple concurrent administrative calls,
+            // handled by different worker threads.
+            // See: https://jira.mariadb.org/browse/MXS-1805 and https://jira.mariadb.org/browse/MXS-1833
+            dcb->poll.thread.id = 0;
+        }
+        else
+        {
+            // Round-robin the client connection worker assignment
+            dcb->poll.thread.id = Worker::pick_worker_id();
+        }
+
         new_state = DCB_STATE_POLLING;
-        dcb->poll.thread.id = 0;
         worker_id = dcb->poll.thread.id;
     }
     else
     {
-        ss_dassert(dcb->dcb_role == DCB_ROLE_CLIENT_HANDLER ||
+        ss_dassert(dcb->dcb_role == DCB_ROLE_BACKEND_HANDLER);
-                   dcb->dcb_role == DCB_ROLE_BACKEND_HANDLER);
         ss_dassert(Worker::get_current_id() != -1);
         ss_dassert(Worker::get_current_id() == dcb->poll.thread.id);
 

server/core/internal/worker.hh

@@ -495,6 +495,13 @@ public:
      */
     static void set_maxwait(unsigned int maxwait);
 
+    /**
+     * Get next Worker's ID
+     *
+     * @return The ID of the worker where work should be assigned
+     */
+    static int pick_worker_id();
+
 private:
     Worker(int id,
            int epoll_fd);

server/core/worker.cc

@@ -582,6 +582,13 @@ void Worker::set_maxwait(unsigned int maxwait)
     this_unit.max_poll_sleep = maxwait;
 }
 
+// static
+int Worker::pick_worker_id()
+{
+    static int id_generator = 0;
+    return atomic_add(&id_generator, 1) % this_unit.n_workers;
+}
+
 bool Worker::post(Task* pTask, Semaphore* pSem, enum execute_mode_t mode)
 {
     // No logging here, function must be signal safe.