Use on-disk database for MySQLAuth users

The SQLite database is now always created on disk. This will remove the
need to dump the database users from the in-memory database to the
persisted on-disk database.

This change will also make the authentication compatible with older SQLite
implementations which lack the URI-based database strings found in newer
versions.

server/modules/authenticator/MySQLAuth/CMakeLists.txt

@@ -1,4 +1,8 @@
-add_library(MySQLAuth SHARED mysql_auth.c dbusers.c)
+if(SQLITE_VERSION VERSION_LESS 3.3)
-target_link_libraries(MySQLAuth maxscale-common MySQLCommon sqlite3)
+  message(FATAL_ERROR "SQLite version 3.3 or higher is required")
-set_target_properties(MySQLAuth PROPERTIES VERSION "1.0.0")
+else()
-install_module(MySQLAuth core)
+  add_library(MySQLAuth SHARED mysql_auth.c dbusers.c)
+  target_link_libraries(MySQLAuth maxscale-common MySQLCommon sqlite3)
+  set_target_properties(MySQLAuth PROPERTIES VERSION "1.0.0")
+  install_module(MySQLAuth core)
+endif()

server/modules/authenticator/MySQLAuth/dbusers.c

@@ -421,109 +421,6 @@ retblock:
     return rc;
 }
 
-int dump_user_cb(void *data, int fields, char **row, char **field_names)
-{
-    sqlite3 *handle = (sqlite3*)data;
-    add_mysql_user(handle, row[0], row[1], row[2], row[3] && strcmp(row[3], "1"), row[4]);
-    return 0;
-}
-
-int dump_database_cb(void *data, int fields, char **row, char **field_names)
-{
-    sqlite3 *handle = (sqlite3*)data;
-    add_database(handle, row[0]);
-    return 0;
-}
-
-static bool transfer_table_contents(sqlite3 *src, sqlite3 *dest)
-{
-    bool rval = true;
-    char *err;
-
-    /** Make sure the tables exist in both databases */
-    if (sqlite3_exec(src, users_create_sql, NULL, NULL, &err) != SQLITE_OK ||
-        sqlite3_exec(dest, users_create_sql, NULL, NULL, &err) != SQLITE_OK ||
-        sqlite3_exec(src, databases_create_sql, NULL, NULL, &err) != SQLITE_OK ||
-        sqlite3_exec(dest, databases_create_sql, NULL, NULL, &err) != SQLITE_OK)
-    {
-        MXS_ERROR("Failed to create tables: %s", err);
-        sqlite3_free(err);
-        rval = false;
-    }
-    else if (sqlite3_exec(dest, "BEGIN", NULL, NULL, &err) != SQLITE_OK)
-    {
-        MXS_ERROR("Failed to start transaction: %s", err);
-        sqlite3_free(err);
-        rval = false;
-    }
-    else
-    {
-        /** Transaction is open */
-        if (!delete_mysql_users(dest))
-        {
-            rval = false;
-        }
-        else if (sqlite3_exec(src, dump_users_query, dump_user_cb, dest, &err) != SQLITE_OK ||
-                 sqlite3_exec(src, dump_databases_query, dump_database_cb, dest, &err) != SQLITE_OK)
-        {
-            MXS_ERROR("Failed to load database contents: %s", err);
-            sqlite3_free(err);
-            rval = false;
-        }
-        if (rval)
-        {
-            if (sqlite3_exec(dest, "COMMIT", NULL, NULL, &err) != SQLITE_OK)
-            {
-                MXS_ERROR("Failed to commit transaction: %s", err);
-                sqlite3_free(err);
-                rval = false;
-            }
-        }
-        else
-        {
-            if (sqlite3_exec(dest, "ROLLBACK", NULL, NULL, &err) != SQLITE_OK)
-            {
-                MXS_ERROR("Failed to rollback transaction: %s", err);
-                sqlite3_free(err);
-                rval = false;
-            }
-        }
-    }
-    return rval;
-}
-
-bool dbusers_load(sqlite3 *dest, const char *filename)
-{
-    sqlite3 *src;
-
-    if (sqlite3_open_v2(filename, &src, db_flags, NULL) != SQLITE_OK)
-    {
-        MXS_ERROR("Failed to open persisted SQLite3 database.");
-        return false;
-    }
-
-    bool rval = transfer_table_contents(src, dest);
-    sqlite3_close_v2(src);
-
-    return rval;
-}
-
-bool dbusers_save(sqlite3 *src, const char *filename)
-{
-    sqlite3 *dest;
-
-    if (sqlite3_open_v2(filename, &dest, db_flags, NULL) != SQLITE_OK)
-    {
-        MXS_ERROR("Failed to open persisted SQLite3 database.");
-        return -1;
-    }
-
-    bool rval = transfer_table_contents(src, dest);
-    sqlite3_close_v2(dest);
-
-    return rval;
-}
-
 /**
  * @brief Check service permissions on one server
  *
@@ -752,9 +649,28 @@ static bool get_hostname(const char *ip_address, char *client_hostname)
     return false;
 }
 
+void start_sqlite_transaction(sqlite3 *handle)
+{
+    char *err;
+    if (sqlite3_exec(handle, "BEGIN", NULL, NULL, &err) != SQLITE_OK)
+    {
+        MXS_ERROR("Failed to start transaction: %s", err);
+        sqlite3_free(err);
+    }
+}
+
+void commit_sqlite_transaction(sqlite3 *handle)
+{
+    char *err;
+    if (sqlite3_exec(handle, "COMMIT", NULL, NULL, &err) != SQLITE_OK)
+    {
+        MXS_ERROR("Failed to commit transaction: %s", err);
+        sqlite3_free(err);
+    }
+}
+
 int get_users_from_server(MYSQL *con, SERVER_REF *server, SERVICE *service, SERV_LISTENER *listener)
 {
-
     if (server->server->server_string == NULL)
     {
         const char *server_string = mysql_get_server_info(con);
@@ -764,7 +680,6 @@ int get_users_from_server(MYSQL *con, SERVER_REF *server, SERVICE *service, SERV
         }
     }
 
-    /** Testing new users query */
     char *query = get_new_users_query(server->server->server_string, service->enable_root);
     MYSQL_AUTH *instance = (MYSQL_AUTH*)listener->auth_instance;
     bool anon_user = false;
@@ -778,7 +693,12 @@ int get_users_from_server(MYSQL *con, SERVER_REF *server, SERVICE *service, SERV
 
             if (result)
             {
+                start_sqlite_transaction(instance->handle);
+
+                /** Delete the old users */
+                delete_mysql_users(instance->handle);
                 MYSQL_ROW row;
+
                 while ((row = mysql_fetch_row(result)))
                 {
                     if (service->strip_db_esc)
@@ -798,6 +718,8 @@ int get_users_from_server(MYSQL *con, SERVER_REF *server, SERVICE *service, SERV
                     }
                 }
 
+                commit_sqlite_transaction(instance->handle);
+
                 mysql_free_result(result);
             }
         }

server/modules/authenticator/MySQLAuth/mysql_auth.c

@@ -106,6 +106,66 @@ MXS_MODULE* MXS_CREATE_MODULE()
     return &info;
 }
 
+static void get_database_path(SERV_LISTENER *port, char *dest)
+{
+    MYSQL_AUTH *instance = port->auth_instance;
+    SERVICE *service = port->service;
+
+    if (instance->cache_dir)
+    {
+        snprintf(dest, sizeof(dest) - sizeof(DBUSERS_FILE) - 1, "%s/", instance->cache_dir);
+    }
+    else
+    {
+        sprintf(dest, "%s/%s/%s/%s/", get_cachedir(), service->name, port->name, DBUSERS_DIR);
+    }
+
+    if (mxs_mkdir_all(dest, S_IRWXU))
+    {
+        strcat(dest, DBUSERS_FILE);
+    }
+}
+
+static bool open_instance_database(const char *path, sqlite3 **handle)
+{
+    if (sqlite3_open_v2(path, handle, db_flags, NULL) != SQLITE_OK)
+    {
+        MXS_ERROR("Failed to open SQLite3 handle.");
+        return false;
+    }
+
+    char *err;
+
+    if (sqlite3_exec(*handle, users_create_sql, NULL, NULL, &err) != SQLITE_OK ||
+        sqlite3_exec(*handle, databases_create_sql, NULL, NULL, &err) != SQLITE_OK ||
+        sqlite3_exec(*handle, pragma_sql, NULL, NULL, &err) != SQLITE_OK)
+    {
+        MXS_ERROR("Failed to create database: %s", err);
+        sqlite3_free(err);
+        sqlite3_close_v2(*handle);
+        return false;
+    }
+
+    return true;
+}
+
+static bool open_client_database(const char *path, sqlite3 **handle)
+{
+    bool rval = false;
+
+    if (sqlite3_open_v2(path, handle, db_flags, NULL) == SQLITE_OK)
+    {
+        sqlite3_busy_timeout(*handle, MXS_SQLITE_BUSY_TIMEOUT);
+        rval = true;
+    }
+    else
+    {
+        MXS_ERROR("Failed to open SQLite3 handle.");
+    }
+
+    return rval;
+}
+
 /**
  * @brief Initialize the authenticator instance
  *
@@ -122,25 +182,7 @@ static void* mysql_auth_init(char **options)
         instance->cache_dir = NULL;
         instance->inject_service_user = true;
         instance->skip_auth = false;
-
+        instance->handle = NULL;
-        if (sqlite3_open_v2(MYSQLAUTH_DATABASE_NAME, &instance->handle, db_flags, NULL) != SQLITE_OK)
-        {
-            MXS_ERROR("Failed to open SQLite3 handle.");
-            MXS_FREE(instance);
-            return NULL;
-        }
-
-        char *err;
-
-        if (sqlite3_exec(instance->handle, users_create_sql, NULL, NULL, &err) != SQLITE_OK ||
-            sqlite3_exec(instance->handle, databases_create_sql, NULL, NULL, &err) != SQLITE_OK)
-        {
-            MXS_ERROR("Failed to create database: %s", err);
-            sqlite3_free(err);
-            sqlite3_close_v2(instance->handle);
-            MXS_FREE(instance);
-            return NULL;
-        }
 
         for (int i = 0; options[i]; i++)
         {
@@ -196,16 +238,7 @@ static void* mysql_auth_create(void *instance)
 
     if (rval)
     {
-        if (sqlite3_open_v2(MYSQLAUTH_DATABASE_NAME, &rval->handle, db_flags, NULL) == SQLITE_OK)
+        rval->handle = NULL;
-        {
-            sqlite3_busy_timeout(rval->handle, MXS_SQLITE_BUSY_TIMEOUT);
-        }
-        else
-        {
-            MXS_ERROR("Failed to open SQLite3 handle.");
-            MXS_FREE(rval);
-            rval = NULL;
-        }
     }
 
     return rval;
@@ -336,6 +369,18 @@ mysql_auth_set_protocol_data(DCB *dcb, GWBUF *buf)
     MySQLProtocol *protocol = NULL;
     MYSQL_session *client_data = NULL;
     int client_auth_packet_size = 0;
+    mysql_auth_t *auth_ses = (mysql_auth_t*)dcb->authenticator_data;
+
+    if (auth_ses->handle == NULL)
+    {
+        char path[PATH_MAX];
+        get_database_path(dcb->listener, path);
+
+        if (!open_client_database(path, &auth_ses->handle))
+        {
+            return MXS_AUTH_FAILED;
+        }
+    }
 
     protocol = DCB_PROTOCOL(dcb, MySQLProtocol);
     CHK_PROTOCOL(protocol);
@@ -543,38 +588,23 @@ static int mysql_auth_load_users(SERV_LISTENER *port)
         return MXS_AUTH_LOADUSERS_FATAL;
     }
 
-    int loaded = replace_mysql_users(port);
+    if (instance->handle == NULL)
+    {
         char path[PATH_MAX];
+        get_database_path(port, path);
+        if (!open_instance_database(path, &instance->handle))
+        {
+            return MXS_AUTH_LOADUSERS_FATAL;
+        }
+    }
 
-    if (instance->cache_dir)
+    int loaded = replace_mysql_users(port);
-    {
-        snprintf(path, sizeof(path) - sizeof(DBUSERS_FILE) - 1, "%s/", instance->cache_dir);
-    }
-    else
-    {
-        sprintf(path, "%s/%s/%s/%s/", get_cachedir(), service->name, port->name, DBUSERS_DIR);
-    }
 
     if (loaded < 0)
     {
         MXS_ERROR("[%s] Unable to load users for listener %s listening at %s:%d.", service->name,
                   port->name, port->address ? port->address : "0.0.0.0", port->port);
 
-        if (mxs_mkdir_all(path, S_IRWXU))
-        {
-            strcat(path, DBUSERS_FILE);
-
-            if (!dbusers_load(instance->handle, path))
-            {
-                MXS_ERROR("[%s] Failed to load cached users from '%s'.", service->name, path);
-                rc = MXS_AUTH_LOADUSERS_ERROR;
-            }
-            else
-            {
-                MXS_WARNING("[%s] Using cached credential information from '%s'.", service->name, path);
-            }
-        }
-
         if (instance->inject_service_user)
         {
             /** Inject the service user as a 'backup' user that's available
@@ -585,16 +615,6 @@ static int mysql_auth_load_users(SERV_LISTENER *port)
             }
         }
     }
-    else
-    {
-        /* Users loaded successfully, save authentication data to file cache */
-        if (mxs_mkdir_all(path, S_IRWXU))
-        {
-            strcat(path, DBUSERS_FILE);
-            dbusers_save(instance->handle, path);
-            MXS_INFO("[%s] Storing cached credential information at '%s'.", service->name, path);
-        }
-    }
 
     if (loaded == 0)
     {

server/modules/authenticator/MySQLAuth/mysql_auth.h

@@ -42,8 +42,6 @@ MXS_BEGIN_DECLS
 static const char DBUSERS_DIR[] = "cache";
 static const char DBUSERS_FILE[] = "dbusers.db";
 
-#define MYSQLAUTH_DATABASE_NAME "file:mysqlauth.db?mode=memory&cache=shared"
-
 /** The table name where we store the users */
 #define MYSQLAUTH_USERS_TABLE_NAME    "mysqlauth_users"
 
@@ -59,6 +57,9 @@ static const char users_create_sql[] =
 static const char databases_create_sql[] =
     "CREATE TABLE IF NOT EXISTS " MYSQLAUTH_DATABASES_TABLE_NAME "(db varchar(255))";
 
+/** PRAGMA configuration options for SQLite */
+static const char pragma_sql[] = "PRAGMA JOURNAL_MODE=MEMORY";
+
 /** Query that checks if there's a grant for the user being authenticated */
 static const char mysqlauth_validate_user_query[] =
     "SELECT password FROM " MYSQLAUTH_USERS_TABLE_NAME
@@ -95,7 +96,6 @@ static const char null_token[] = "NULL";
 /** Flags for sqlite3_open_v2() */
 static int db_flags = SQLITE_OPEN_READWRITE |
                       SQLITE_OPEN_CREATE |
-                      SQLITE_OPEN_URI |
                       SQLITE_OPEN_SHAREDCACHE;
 
 typedef struct mysql_auth