MXS-2544 Use separate sqlite tables for user authentication data

The data is now split into three tables similar to the server.

server/modules/authenticator/PAM/PAMAuth/pam_auth.cc

@@ -22,12 +22,24 @@
 #include "../pam_auth_common.hh"
 
 using std::string;
+using SSQLite = SQLite::SSQLite;
+
+/** Table and column names. The names mostly match the ones in the server. */
+const string TABLE_USER = "user";
+const string TABLE_DB = "db";
+const string TABLE_ROLES_MAPPING = "roles_mapping";
+
 const string FIELD_USER = "user";
 const string FIELD_HOST = "host";
-const string FIELD_DB = "db";
-const string FIELD_ANYDB = "anydb";
 const string FIELD_AUTHSTR = "authentication_string";
-const string FIELD_PROXY = "proxy_grant";
+const string FIELD_DEF_ROLE = "default_role";
+const string FIELD_ANYDB = "anydb";
+const string FIELD_IS_ROLE = "is_role";
+const string FIELD_HAS_PROXY = "proxy_grant";
+
+const string FIELD_DB = "db";
+const string FIELD_ROLE = "role";
+
 const int NUM_FIELDS = 6;
 
 const char* SQLITE_OPEN_FAIL = "Failed to open SQLite3 handle for file '%s': '%s'";

server/modules/authenticator/PAM/PAMAuth/pam_auth.hh

@@ -32,12 +32,19 @@ extern const string FIELD_HOST;
 extern const string FIELD_DB;
 extern const string FIELD_ANYDB;
 extern const string FIELD_AUTHSTR;
-extern const string FIELD_PROXY;
+extern const string FIELD_DEF_ROLE;
+extern const string FIELD_HAS_PROXY;
+extern const string FIELD_IS_ROLE;
+extern const string FIELD_ROLE;
 extern const int NUM_FIELDS;
 
 extern const char* SQLITE_OPEN_FAIL;
 extern const char* SQLITE_OPEN_OOM;
 
+extern const string TABLE_USER;
+extern const string TABLE_DB;
+extern const string TABLE_ROLES_MAPPING;
+
 struct sqlite3;
 
 class SQLite;

server/modules/authenticator/PAM/PAMAuth/pam_client_session.cc

@@ -15,6 +15,7 @@
 
 #include <sstream>
 #include <maxbase/pam_utils.hh>
+#include <maxbase/format.hh>
 #include <maxscale/event.hh>
 
 using maxscale::Buffer;
@@ -76,6 +77,39 @@ int user_services_cb(PamClientSession::StringVector* data, int columns, char** c
     }
     return 0;
 }
+
+struct UserData
+{
+    string host;
+    string authentication_string;
+    string default_role;
+    bool anydb {false};
+};
+
+using UserDataArr = std::vector<UserData>;
+
+/**
+ * Helper callback for PamClientSession::get_pam_user_services(). See SQLite3
+ * documentation for more information.
+ *
+ * @param data Application data
+ * @param columns Number of columns, must be 1
+ * @param column_vals Column values
+ * @param column_names Column names
+ * @return Always 0
+ */
+int user_data_cb(UserDataArr* data, int columns, char** column_vals, char** column_names)
+{
+    mxb_assert(columns == 4);
+    UserData new_row;
+    new_row.host = column_vals[0];
+    new_row.authentication_string = column_vals[1];
+    new_row.default_role = column_vals[2];
+    new_row.anydb = (column_vals[3][0] == '1');
+    data->push_back(new_row);
+    return 0;
+}
+
 }
 
 PamClientSession::PamClientSession(const PamInstance& instance, SSQLite sqlite)
@@ -113,37 +147,52 @@ PamClientSession* PamClientSession::create(const PamInstance& inst)
 void PamClientSession::get_pam_user_services(const DCB* dcb, const MYSQL_session* session,
                                              StringVector* services_out)
 {
-    string services_query = string("SELECT authentication_string FROM ") + m_instance.m_tablename + " WHERE "
+    const char* user = session->user;
-        + FIELD_USER + " = '" + session->user + "'"
+    const char* host = dcb->remote;
-        + " AND '" + dcb->remote + "' LIKE " + FIELD_HOST
+    const char* db = session->db;
-        + " AND (" + FIELD_ANYDB + " = '1' OR '" + session->db + "' IN ('information_schema', '') OR '"
+    // First search for a normal matching user.
-        + session->db + "' LIKE " + FIELD_DB + ")"
+    const string columns = "host, authentication_string, default_role, anydb";
-        + " AND " + FIELD_PROXY + " = '0' ORDER BY authentication_string;";
+    const string filter = "('%s' LIKE " + FIELD_HOST + ") AND (" + FIELD_IS_ROLE + " = 0)";
-    MXS_DEBUG("PAM services search sql: '%s'.", services_query.c_str());
+    const string users_filter = "(" + FIELD_USER + " = '%s') AND " + filter;
 
-    if (!m_sqlite->exec(services_query, user_services_cb, services_out))
+    const string users_query_fmt = "SELECT " + columns + " FROM " + TABLE_USER + " WHERE "
+            + users_filter + ";";
+    string users_query = mxb::string_printf(users_query_fmt.c_str(), user, host);
+    UserDataArr matching_users;
+    m_sqlite->exec(users_query, user_data_cb, &matching_users);
+
+    // If any of the rows returned has a global priv we have a valid service name.
+    for (auto entry : matching_users)
     {
-        MXS_ERROR("Failed to execute query: '%s'", m_sqlite->error());
+        // TODO: Order entries according to https://mariadb.com/kb/en/library/create-user/
+        // -> User Name Component and only return one service.
+        if (entry.anydb)
+        {
+            services_out->push_back(entry.authentication_string);
+        }
+        // TODO: add support for roles
     }
-
     auto word_entry = [](size_t num) -> const char* {
-            return (num == 1) ? "entry" : "entries";
+           return (num == 1) ? "entry" : "entries";
-        };
+    };
 
-    if (!services_out->empty())
+    // TODO: Check database grants.
+
+    if (!matching_users.empty())
     {
-        auto num_services = services_out->size();
+        auto num_services = matching_users.size();
         MXS_INFO("Found %lu valid PAM user %s for '%s'@'%s'.",
-                 num_services, word_entry(num_services), session->user, dcb->remote);
+                 num_services, word_entry(num_services), user, host);
     }
     else
     {
         // No service found for user with correct username & host.
         // Check if a matching anonymous user exists.
-        const string anon_query = string("SELECT authentication_string FROM ") + m_instance.m_tablename
+        const string anon_filter = "(" + FIELD_USER + " = '') AND " + filter + " AND ("
-            + " WHERE " + FIELD_USER + " = ''"
+                + FIELD_HAS_PROXY + " = '0')";
-            + " AND '" + dcb->remote + "' LIKE " + FIELD_HOST
+        const string anon_query_fmt = string("SELECT authentication_string FROM ") + TABLE_USER
-            + " AND " + FIELD_PROXY + " = '1' ORDER BY authentication_string;";
+            + " WHERE " + anon_filter + ";";
+        string anon_query = mxb::string_printf(anon_query_fmt.c_str(), host);
         MXS_DEBUG("PAM proxy user services search sql: '%s'.", anon_query.c_str());
 
         if (m_sqlite->exec(anon_query, user_services_cb, services_out))

server/modules/authenticator/PAM/PAMAuth/pam_instance.cc

@@ -15,12 +15,14 @@
 
 #include <string>
 #include <string.h>
+#include <maxbase/format.hh>
 #include <maxscale/jansson.hh>
 #include <maxscale/paths.h>
 #include <maxscale/secrets.h>
 #include <maxscale/mysql_utils.hh>
 
 using std::string;
+using mxq::QueryResult;
 
 /**
  * Create an instance.
@@ -34,18 +36,6 @@ PamInstance* PamInstance::create(char** options)
     // TODO: Once Centos6 is no longer needed and Sqlite version 3.7+ can be assumed,
     // use a memory-only db with a URI filename (e.g. file:pam.db?mode=memory&cache=shared)
     const string pam_db_fname = string(get_cachedir()) + "/pam_db.sqlite3";
-    // The table name where we store the users
-    const string pam_table_name = "pam_users";
-    /** Deletion statement for the in-memory table */
-    const string drop_sql = string("DROP TABLE IF EXISTS ") + pam_table_name + ";";
-    /** CREATE TABLE statement for the in-memory table */
-    const string create_sql = string("CREATE TABLE ") + pam_table_name
-        + " (" + FIELD_USER + " varchar(255), "
-        + FIELD_HOST + " varchar(255), "
-        + FIELD_DB + " varchar(255), "
-        + FIELD_ANYDB + " boolean, "
-        + FIELD_AUTHSTR + " text, "
-        + FIELD_PROXY + " boolean);";
 
     if (sqlite3_threadsafe() == 0)
     {
@@ -85,7 +75,6 @@ PamInstance* PamInstance::create(char** options)
  */
 PamInstance::PamInstance(SQLite::SSQLite dbhandle, const string& dbname)
     : m_dbname(dbname)
-    , m_tablename("pam_users")
     , m_sqlite(std::move(dbhandle))
 {
 }
@@ -132,29 +121,49 @@ bool PamInstance::prepare_tables()
         return rval;
     };
 
-    /** Table names and columns. The tables mostly match the ones in the server, but have only
+    auto drop_recreate_table = [gen_drop_sql, gen_create_sql](SQLite* db, const string& tblname,
-     *  a subset of columns. */
+                                                              const ColDefArray& coldefs) {
+        bool rval = false;
+        string drop_query = gen_drop_sql(tblname);
+        string create_query = gen_create_sql(tblname, coldefs);
+        if (!db->exec(drop_query))
+        {
+            MXB_ERROR("Failed to delete sqlite3 table: %s", db->error());
+        }
+        else if (!db->exec(create_query))
+        {
+            MXB_ERROR("Failed to create sqlite3 table: %s", db->error());
+        }
+        else
+        {
+            rval = true;
+        }
+        return rval;
+    };
 
-    const string users = m_tablename;
-    const string drop_users_sql = gen_drop_sql(users);
     // Sqlite3 doesn't require datatypes in the create-statement but it's good to have for clarity.
     const ColDefArray users_coldef = {{FIELD_USER, Type::TEXT},
                                       {FIELD_HOST, Type::TEXT},
-                                      {FIELD_DB, Type::TEXT},
-                                      {FIELD_ANYDB, Type::BOOL},
                                       {FIELD_AUTHSTR, Type::TEXT},
-                                      {FIELD_PROXY, Type::BOOL}};
+                                      {FIELD_DEF_ROLE, Type::TEXT},
-    const string create_users_sql = gen_create_sql(users, users_coldef);
+                                      {FIELD_ANYDB, Type::BOOL},
+                                      {FIELD_IS_ROLE, Type::BOOL},
+                                      {FIELD_HAS_PROXY, Type::BOOL}};
+    const ColDefArray dbs_coldef = {{FIELD_USER, Type::TEXT},
+                                    {FIELD_HOST, Type::TEXT},
+                                    {FIELD_DB, Type::TEXT}};
+    const ColDefArray roles_coldef = {{FIELD_USER, Type::TEXT},
+                                      {FIELD_HOST, Type::TEXT},
+                                      {FIELD_ROLE, Type::TEXT}};
 
     bool rval = false;
-    if (m_sqlite->exec(drop_users_sql) && m_sqlite->exec(create_users_sql))
+    auto sqlite = m_sqlite.get();
+    if (drop_recreate_table(sqlite, TABLE_USER, users_coldef)
+        && drop_recreate_table(sqlite, TABLE_DB, dbs_coldef)
+        && drop_recreate_table(sqlite, TABLE_ROLES_MAPPING, roles_coldef))
     {
         rval = true;
     }
-    else
-    {
-        MXB_ERROR("Failed to create sqlite3 table: %s", m_sqlite->error());
-    }
     return rval;
 }
 
@@ -178,7 +187,7 @@ void PamInstance::add_pam_user(const char* user, const char* host, const char* d
      * no quotes around them. The quotes for strings are added in this function.
      */
     const string insert_sql_template =
-        "INSERT INTO " + m_tablename + " VALUES ('%s', '%s', %s, '%s', %s, '%s')";
+        "INSERT INTO " + TABLE_USER + " VALUES ('%s', '%s', %s, '%s', %s, '%s')";
 
     /** Used for NULL value creation in the INSERT query */
     const char NULL_TOKEN[] = "NULL";
@@ -238,7 +247,7 @@ void PamInstance::add_pam_user(const char* user, const char* host, const char* d
 void PamInstance::delete_old_users()
 {
     /** Delete query used to clean up the database before loading new users */
-    const string delete_query = "DELETE FROM " + m_tablename;
+    const string delete_query = "DELETE FROM " + TABLE_USER + ";";
     if (!m_sqlite->exec(delete_query))
     {
         MXB_ERROR("Failed to delete old users: %s", m_sqlite->error());
@@ -255,73 +264,202 @@ void PamInstance::delete_old_users()
 int PamInstance::load_users(SERVICE* service)
 {
     /** Query that gets all users that authenticate via the pam plugin */
-    const char PAM_USERS_QUERY[] =
+
-        "SELECT u.user, u.host, d.db, u.select_priv, u.authentication_string FROM "
+    string users_query, db_query, role_query;
-        "mysql.user AS u LEFT JOIN mysql.db AS d ON (u.user = d.user AND u.host = d.host) WHERE "
+    auto prepare_queries = [&users_query, &db_query, &role_query](bool using_roles) {
-        "(u.plugin = 'pam' AND (d.db IS NOT NULL OR u.select_priv = 'Y')) "
+        string user_cols = "user, host, select_priv, insert_priv, update_priv, delete_priv, "
-        "UNION "
+                           "authentication_string";
-        "SELECT u.user, u.host, t.db, u.select_priv, u.authentication_string FROM "
+        string filter = "plugin = 'pam'";
-        "mysql.user AS u LEFT JOIN mysql.tables_priv AS t ON (u.user = t.user AND u.host = t.host) WHERE "
+        if (using_roles)
-        "(u.plugin = 'pam' AND t.db IS NOT NULL AND u.select_priv = 'N') "
+        {
-        "ORDER BY user";
+            user_cols += ", default_role, is_role";
-#if defined (SS_DEBUG)
+            filter += " OR is_role = 'Y'";     // If using roles, accept them as well.
-    const unsigned int PAM_USERS_QUERY_NUM_FIELDS = 5;
+        }
-#endif
+        else
+        {
+            user_cols += ", '' AS default_role, 'N' AS is_role"; // keeps the number of columns constant
+        }
+        users_query = mxb::string_printf("SELECT %s FROM mysql.user WHERE %s;",
+                                         user_cols.c_str(), filter.c_str());
+
+        string join_filter = "b.plugin = 'pam'";
+        if (using_roles)
+        {
+            // Roles do not have plugins, yet may affect authentication.
+            join_filter += " OR b.is_role = 'Y'";
+        }
+        const string inner_join = "INNER JOIN mysql.user AS b ON (a.user = b.user AND a.host = b.host "
+                                  "AND (" + join_filter + "))";
+
+        // Read database grants for pam users and roles. This is combined with table grants.
+        db_query = "SELECT DISTINCT * FROM ("
+                   // Select users/roles with general db-level privs ...
+                   "(SELECT a.user, a.host, a.db FROM mysql.db AS a " + inner_join + ") "
+                   "UNION "
+                   // and combine with table privs counting as db-level privs.
+                   "(SELECT a.user, a.host, a.db FROM mysql.tables_priv AS a " + inner_join + ")) AS c;";
+
+        if (using_roles)
+        {
+            role_query = "SELECT a.user, a.host, a.role FROM mysql.roles_mapping AS a "
+                    + inner_join + ";";
+        }
+
+    };
 
     const char* user;
-    const char* password;
+    const char* pw_crypt;
-    serviceGetUser(service, &user, &password);
+    serviceGetUser(service, &user, &pw_crypt);
     int rval = MXS_AUTH_LOADUSERS_ERROR;
-    char* pw;
 
-    if ((pw = decrypt_password(password)))
+    char* pw_clear = decrypt_password(pw_crypt);
+    if (pw_clear)
     {
-        for (SERVER_REF* servers = service->dbref; servers; servers = servers->next)
+        bool found_valid_server = false;
+        bool got_data = false;
+        for (auto sref = service->dbref; sref && !got_data; sref = sref->next)
         {
-            MYSQL* mysql = mysql_init(NULL);
+            SERVER* srv = sref->server;
-            if (mxs_mysql_real_connect(mysql, servers->server, user, pw))
+            if (srv->is_active && srv->is_usable())
             {
-                if (mysql_query(mysql, PAM_USERS_QUERY))
+                bool using_roles = false;
+                auto version = srv->version();
+                // Default roles are in server version 10.1.1.
+                if (version.major > 10 || (version.major == 10 && (version.minor > 1
+                    || (version.minor == 1 && version.patch == 1))))
                 {
-                    MXS_ERROR("Failed to query server '%s' for PAM users: '%s'.",
+                    using_roles = true;
-                              servers->server->name(), mysql_error(mysql));
                 }
-                else
+                prepare_queries(using_roles);
+
+                found_valid_server = true;
+                MYSQL* mysql = mysql_init(NULL);
+                if (mxs_mysql_real_connect(mysql, srv, user, pw_clear))
                 {
-                    MYSQL_RES* res = mysql_store_result(mysql);
+                    string error_msg;
-                    delete_old_users();
+                    QResult users_res, dbs_res, roles_res;
-                    if (res)
+                    // Perform the queries. All must succeed on the same backend.
+                    // TODO: Think if it would be faster to do these queries concurrently.
+                    if (((users_res = mxs::execute_query(mysql, users_query, &error_msg)) != nullptr)
+                        && ((dbs_res = mxs::execute_query(mysql, db_query, &error_msg)) != nullptr))
                     {
-                        mxb_assert(mysql_num_fields(res) == PAM_USERS_QUERY_NUM_FIELDS);
+                        if (using_roles)
-                        MYSQL_ROW row;
-                        while ((row = mysql_fetch_row(res)))
                         {
-                            add_pam_user(row[0], row[1],                                // user, host
+                            if ((roles_res = mxs::execute_query(mysql, role_query, &error_msg)) != nullptr)
-                                         row[2], row[3] && strcasecmp(row[3], "Y") == 0,// db, anydb
+                            {
-                                         row[4],                                        // pam service
+                                got_data = true;
-                                         false);                                        // not a proxy
+                            }
+                        }
+                        else
+                        {
+                            got_data = true;
                         }
-                        mysql_free_result(res);
                     }
 
-                    if (fetch_anon_proxy_users(servers->server, mysql))
+                    if (got_data)
                     {
+                        fill_user_arrays(std::move(users_res), std::move(dbs_res), std::move(roles_res));
                         rval = MXS_AUTH_LOADUSERS_OK;
                     }
+                    else
+                    {
+                        MXB_ERROR("Failed to query server '%s' for PAM users: '%s'.",
+                                  srv->name(), error_msg.c_str());
+                    }
                 }
                 mysql_close(mysql);
-
-                if (rval == MXS_AUTH_LOADUSERS_OK)
-                {
-                    break;
-                }
             }
         }
-        MXS_FREE(pw);
+
+        if (!found_valid_server)
+        {
+            MXB_ERROR("Service '%s' had no valid servers to query PAM users from.", service->name());
+        }
+        MXS_FREE(pw_clear);
     }
     return rval;
 }
 
+void PamInstance::fill_user_arrays(QResult user_res, QResult db_res, QResult roles_mapping_res)
+{
+    m_sqlite->exec("BEGIN");
+    // Delete any previous data.
+    const char delete_fmt[] = "DELETE FROM %s;";
+    for (const auto& tbl : {TABLE_USER, TABLE_DB, TABLE_ROLES_MAPPING})
+    {
+        string query = mxb::string_printf(delete_fmt, tbl.c_str());
+        m_sqlite->exec(query);
+    }
+
+    // TODO: use prepared stmt:s
+    if (user_res)
+    {
+        auto get_bool_enum = [&user_res](int64_t col_ind) {
+            string val = user_res->get_string(col_ind);
+            return (val == "Y" || val == "y");
+        };
+
+        auto get_bool_any = [&get_bool_enum](int64_t col_ind_min, int64_t col_ind_max) {
+            bool rval = false;
+            for (auto i = col_ind_min; i <= col_ind_max && !rval; i++)
+            {
+                bool val = get_bool_enum(i);
+                if (val)
+                {
+                    rval = true;
+                }
+            }
+            return rval;
+        };
+        // Input data order is: 0=user, 1=host, 2=select_priv, 3=insert_priv, 4=update_priv, 5=delete_priv,
+        // 6=authentication_string, 7=default_role, 8=is_role
+
+        // Output data order is: user, host, authentication_string, default_role, anydb, is_role, has_proxy.
+        // The proxy-part is sorted out later.
+        string insert_fmt = "INSERT INTO " + TABLE_USER + " VALUES ('%s', '%s', '%s', '%s', %i, %i, 0);";
+        while (user_res->next_row())
+        {
+            auto username = user_res->get_string(0);
+            auto host = user_res->get_string(1);
+            bool has_global_priv = get_bool_any(2, 5);
+            auto auth_string = user_res->get_string(6);
+            string default_role = user_res->get_string(7);
+            bool is_role = get_bool_enum(8);
+
+            m_sqlite->exec(mxb::string_printf(insert_fmt.c_str(), username.c_str(), host.c_str(),
+                                              auth_string.c_str(), default_role.c_str(), has_global_priv,
+                                              is_role));
+        }
+    }
+
+    if (db_res)
+    {
+        string insert_db_fmt = "INSERT INTO " + TABLE_DB + " VALUES ('%s', '%s', '%s');";
+        while (db_res->next_row())
+        {
+            auto username = db_res->get_string(0);
+            auto host = db_res->get_string(1);
+            auto datab = db_res->get_string(2);
+            m_sqlite->exec(mxb::string_printf(insert_db_fmt.c_str(),
+                           username.c_str(), host.c_str(), datab.c_str()));
+        }
+    }
+
+    if (roles_mapping_res)
+    {
+        string insert_roles_fmt = "INSERT INTO " + TABLE_ROLES_MAPPING + " VALUES ('%s', '%s', '%s');";
+        while (roles_mapping_res->next_row())
+        {
+            auto username = roles_mapping_res->get_string(0);
+            auto host = roles_mapping_res->get_string(1);
+            auto role = roles_mapping_res->get_string(2);
+            m_sqlite->exec(mxb::string_printf(insert_roles_fmt.c_str(),
+                                              username.c_str(), host.c_str(), role.c_str()));
+        }
+    }
+    m_sqlite->exec("COMMIT");
+}
+
 void PamInstance::diagnostic(DCB* dcb)
 {
     json_t* array = diagnostic_json();
@@ -365,7 +503,7 @@ static int diag_cb_json(json_t* data, int columns, char** row, char** field_name
 json_t* PamInstance::diagnostic_json()
 {
     json_t* rval = json_array();
-    string select = "SELECT * FROM " + m_tablename + ";";
+    string select = "SELECT * FROM " + TABLE_USER + ";";
     if (!m_sqlite->exec(select, diag_cb_json, rval))
     {
         MXS_ERROR("Failed to print users: %s", m_sqlite->error());

server/modules/authenticator/PAM/PAMAuth/pam_instance.hh

@@ -14,6 +14,7 @@
 #include "pam_auth.hh"
 
 #include <string>
+#include <maxsql/queryresult.hh>
 #include <maxscale/service.hh>
 #include <maxscale/sqlite3.h>
 
@@ -31,9 +32,11 @@ public:
     json_t* diagnostic_json();
 
     const std::string m_dbname;     /**< Name of the in-memory database */
-    const std::string m_tablename;  /**< The table where users are stored */
+
 
 private:
+    using QResult = std::unique_ptr<mxq::QueryResult>;
+
     PamInstance(SQLite::SSQLite dbhandle, const std::string& dbname);
     bool prepare_tables();
 
@@ -41,6 +44,6 @@ private:
                       const char* pam_service, bool proxy);
     void delete_old_users();
     bool fetch_anon_proxy_users(SERVER* server, MYSQL* conn);
-
+    void fill_user_arrays(QResult user_res, QResult db_res, QResult roles_mapping_res);
     SQLite::SSQLite const m_sqlite;      /**< SQLite3 database handle */
 };