MXS-1921: Explain why session was closed

When a client connection is closed by MaxScale before the client initiates
a controlled closing of the connection, an error message is sent. This
error message now also explains why the connection was closed to make
problem resolution easier.

include/maxscale/session.h

@@ -135,6 +135,17 @@ typedef struct mxs_upstream
     int32_t (*error)(void *instance, void *session, void *);
 } MXS_UPSTREAM;
 
+/* Specific reasons why a session was closed */
+typedef enum
+{
+    SESSION_CLOSE_NONE = 0,             // No special reason
+    SESSION_CLOSE_TIMEOUT,              // Connection timed out
+    SESSION_CLOSE_HANDLEERROR_FAILED,   // Router returned an error from handleError
+    SESSION_CLOSE_ROUTING_FAILED,       // Router closed DCB
+    SESSION_CLOSE_KILLED,               // Killed by another connection
+    SESSION_CLOSE_TOO_MANY_CONNECTIONS, // Too many connections
+} session_close_t;
+
 /**
  * The session status block
  *
@@ -169,6 +180,7 @@ typedef struct session
     } stmt;  /**< Current statement being executed */
     bool qualifies_for_pooling; /**< Whether this session qualifies for the connection pool */
     SessionStmtQueue*      last_statements;  /*< The N last statements by the client */
+    session_close_t        close_reason; /**< Reason why the session was closed */
     skygw_chk_t     ses_chk_tail;
 } MXS_SESSION;
 
@@ -534,4 +546,15 @@ void session_set_dump_statements(session_dump_statements_t value);
  */
 session_dump_statements_t session_get_dump_statements();
 
+
+/**
+ * Get the reason why a session was closed
+ *
+ * @param session Session to inspect
+ *
+ * @return String representation of the reason why the session was closed. If
+ *         the session was closed normally, an empty string is returned.
+ */
+const char* session_get_close_reason(const MXS_SESSION* session);
+
 MXS_END_DECLS

server/core/dcb.cc

@@ -2473,6 +2473,7 @@ dcb_accept(DCB *dcb)
                 {
                     client_dcb->func.connlimit(client_dcb, client_dcb->service->max_connections);
                 }
+                dcb->session->close_reason = SESSION_CLOSE_TOO_MANY_CONNECTIONS;
                 dcb_close(client_dcb);
                 client_dcb = NULL;
             }
@@ -2881,6 +2882,7 @@ void dcb_process_idle_sessions(int thr)
                                     dcb->user ? dcb->user : "<unknown>",
                                     dcb->remote ? dcb->remote : "<unknown>",
                                     (float)idle / 10.f);
+                        dcb->session->close_reason = SESSION_CLOSE_TIMEOUT;
                         poll_fake_hangup_event(dcb);
                     }
                 }

server/core/session.cc

@@ -129,6 +129,7 @@ static MXS_SESSION* session_alloc_body(SERVICE* service, DCB* client_dcb,
     session->stmt.buffer = NULL;
     session->stmt.target = NULL;
     session->qualifies_for_pooling = false;
+    session->close_reason = SESSION_CLOSE_NONE;
 
     MXS_CONFIG *config = config_get_global_options();
     // If MaxScale is running in Oracle mode, then autocommit needs to
@@ -1211,3 +1212,31 @@ void session_dump_statements(MXS_SESSION* pSession)
         }
     }
 }
+
+const char* session_get_close_reason(const MXS_SESSION* session)
+{
+    switch (session->close_reason)
+    {
+        case SESSION_CLOSE_NONE:
+            return "";
+
+        case SESSION_CLOSE_TIMEOUT:
+            return "Timed out by MaxScale";
+
+        case SESSION_CLOSE_HANDLEERROR_FAILED:
+            return "Router could not recover from connection errors";
+
+        case SESSION_CLOSE_ROUTING_FAILED:
+            return "Router could not route query";
+
+        case SESSION_CLOSE_KILLED:
+            return "Killed by another connection";
+
+        case SESSION_CLOSE_TOO_MANY_CONNECTIONS:
+            return "Too many connections";
+
+        default:
+            ss_dassert(!true);
+            return "Internal error";
+    }
+}

server/modules/protocol/MySQL/mariadbbackend/mysql_backend.c

@@ -583,6 +583,7 @@ static void do_handle_error(DCB *dcb, mxs_error_action_t action, const char *err
      */
     if (!succp)
     {
+        session->close_reason = SESSION_CLOSE_HANDLEERROR_FAILED;
         poll_fake_hangup_event(session->client_dcb);
     }
 }

server/modules/protocol/MySQL/mariadbclient/mysql_client.cc

@@ -1136,12 +1136,14 @@ gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint64_t capabilities)
     if (return_code != 0)
     {
         /** Routing failed, close the client connection */
+        dcb->session->close_reason = SESSION_CLOSE_ROUTING_FAILED;
         dcb_close(dcb);
         MXS_ERROR("Routing the query failed. Session will be closed.");
     }
     else if (proto->current_command == MXS_COM_QUIT)
     {
         /** Close router session which causes closing of backends */
+        ss_info_dassert(session_valid_for_pool(dcb->session), "Session should qualify for pooling");
         dcb_close(dcb);
     }
 
@@ -1471,7 +1473,15 @@ static int gw_client_hangup_event(DCB *dcb)
             }
 
             // The client did not send a COM_QUIT packet
-            modutil_send_mysql_err_packet(dcb, 0, 0, 1927, "08S01", "Connection killed by MaxScale");
+            std::string errmsg{"Connection killed by MaxScale"};
+            std::string extra{session_get_close_reason(dcb->session)};
+
+            if (!extra.empty())
+            {
+                errmsg += ": " + extra;
+            }
+
+            modutil_send_mysql_err_packet(dcb, 0, 0, 1927, "08S01", errmsg.c_str());
         }
         dcb_close(dcb);
     }

server/modules/protocol/MySQL/mysql_common.cc

@@ -1704,6 +1704,7 @@ static bool kill_func(DCB *dcb, void *data)
         else
         {
             // DCB is not yet connected, send a hangup to forcibly close it
+            dcb->session->close_reason = SESSION_CLOSE_KILLED;
             poll_fake_hangup_event(dcb);
         }
     }