Updates to how shards are discovered while the module is running

SHOW DATABASES now takes notice of the state of the backends
Added scripts for sharding test preparation and local port blocking

server/modules/routing/dbshard/dbshard.c

@@ -328,8 +328,8 @@ static void* hfree(void* fval)
  */
 bool update_dbnames_hash(ROUTER_INSTANCE* inst,BACKEND** backends, HASHTABLE* hashtable)
 {
-	const unsigned int connect_timeout = 15;
+	const unsigned int connect_timeout = 1;
-	const unsigned int read_timeout = 10;
+	const unsigned int read_timeout = 1;
 	bool rval = true;
 	SERVER* server;
 	int i, rc, numfields;
@@ -349,6 +349,7 @@ bool update_dbnames_hash(ROUTER_INSTANCE* inst,BACKEND** backends, HASHTABLE* ha
 					"MySQL handle.")));
 			return false;
 		}
+
         rc = 0;
 		rc |= mysql_options(handle, 
 				    MYSQL_OPT_CONNECT_TIMEOUT, 
@@ -365,10 +366,16 @@ bool update_dbnames_hash(ROUTER_INSTANCE* inst,BACKEND** backends, HASHTABLE* ha
 			rval = false;
 			continue;
         }
+
 		server = backends[i]->backend_server;
 
 		ss_dassert(server != NULL);
 
+        if(!SERVER_IS_RUNNING(server))
+        {
+            continue;
+        }
+
 		if(server->monuser == NULL || server->monpw == NULL)
 		{
 			LOGIF(LE, (skygw_log_write_flush(
@@ -395,8 +402,9 @@ bool update_dbnames_hash(ROUTER_INSTANCE* inst,BACKEND** backends, HASHTABLE* ha
 				LOGIF(LE, (skygw_log_write_flush(
 						LOGFILE_ERROR,
 						"Error: failed to connect to backend "
-						"server '%s': %d %s",
+						"server '%s:%d': %d %s",
 						server->name,
+                        server->port,
 						mysql_errno(handle),
 						mysql_error(handle))));	
 			rval = false;
@@ -573,8 +581,13 @@ void* dbnames_hash_init(ROUTER_INSTANCE* inst,BACKEND** backends)
 	/**Update the new hashtable with the key-value pairs*/
 	if(!update_dbnames_hash(inst,backends,htbl))
 	{
-		hashtable_free(htbl);
+        /**
-		htbl = NULL;
+         * Log if there were some errors during the database configuration.
+         */
+
+		LOGIF(LE, (skygw_log_write_flush(
+				LOGFILE_ERROR,
+				"Warning : Errors occurred while resolving shard locations.")));
 	}
 	return htbl;
 }
@@ -1709,7 +1722,9 @@ GWBUF* gen_show_dbs_response(ROUTER_INSTANCE* router, ROUTER_CLIENT_SES* client)
 	GWBUF* rval = NULL;
 	HASHTABLE* ht = router->dbnames_hash;
 	HASHITERATOR* iter = hashtable_iterator(ht);
+    BACKEND** backends = router->servers;
 	unsigned int coldef_len = 0;
+    int j;
 	char dbname[MYSQL_DATABASE_MAXLEN+1];
 	char *value;
 	unsigned char* ptr;
@@ -1804,6 +1819,13 @@ GWBUF* gen_show_dbs_response(ROUTER_INSTANCE* router, ROUTER_CLIENT_SES* client)
 	unsigned int packet_num = 4;
 	
 	while((value = (char*)hashtable_next(iter)))
+	{
+        char* bend = hashtable_fetch(ht,value);
+        for(j = 0;backends[j];j++)
+        {
+            if(strcmp(backends[j]->backend_server->unique_name,bend) == 0)
+            {
+                if(SERVER_IS_RUNNING(backends[j]->backend_server))
                 {
                     GWBUF* temp;
                     int plen = strlen(value) + 1;
@@ -1821,7 +1843,10 @@ GWBUF* gen_show_dbs_response(ROUTER_INSTANCE* router, ROUTER_CLIENT_SES* client)
 		
                     /** Append the row*/
                     rval = gwbuf_append(rval,temp);
-		
+                }
+                break;
+            }
+        }
 	}
 
     eof[3] = packet_num;
@@ -2257,6 +2282,12 @@ static int routeQuery(
 
 		succp = get_shard_dcb(&target_dcb, router_cli_ses, tname);
 
+		if (!succp)
+		{			
+            update_dbnames_hash(inst,inst->servers,inst->dbnames_hash);
+            tname = get_shard_target_name(inst,router_cli_ses,querybuf,qtype);
+            succp = get_shard_dcb(&target_dcb, router_cli_ses, tname);
+            
             if (!succp)
             {
                 LOGIF(LT, (skygw_log_write(
@@ -2266,6 +2297,8 @@ static int routeQuery(
                         "suitable state.",
                         tname)));
             }
+
+		}
 	}
 
 	if (succp) /*< Have DCB of the target backend */
@@ -2989,25 +3022,25 @@ static bool connect_backend_servers(
 	}
 	else
 	{
-		LOGIF(LE, (skygw_log_write(
+		/* LOGIF(LE, (skygw_log_write( */
-			LOGFILE_ERROR,
+		/* 	LOGFILE_ERROR, */
-			"Warning : Couldn't connect to all available "
+		/* 	"Warning : Couldn't connect to all available " */
-			"servers. Session can't be created.")));
+		/* 	"servers. Session can't be created."))); */
 		
-		/** Clean up connections */
+		/* /\** Clean up connections *\/ */
-		for (i=0; i<router_nservers; i++)
+		/* for (i=0; i<router_nservers; i++) */
-		{
+		/* { */
-			if (BREF_IS_IN_USE((&backend_ref[i])))
+		/* 	if (BREF_IS_IN_USE((&backend_ref[i]))) */
-			{
+		/* 	{ */
-				ss_dassert(backend_ref[i].bref_backend->backend_conn_count > 0);
+		/* 		ss_dassert(backend_ref[i].bref_backend->backend_conn_count > 0); */
 				
-				/** disconnect opened connections */
+		/* 		/\** disconnect opened connections *\/ */
-				dcb_close(backend_ref[i].bref_dcb);
+		/* 		dcb_close(backend_ref[i].bref_dcb); */
-				bref_clear_state(&backend_ref[i], BREF_IN_USE);
+		/* 		bref_clear_state(&backend_ref[i], BREF_IN_USE); */
-				/** Decrease backend's connection counter. */
+		/* 		/\** Decrease backend's connection counter. *\/ */
-				atomic_add(&backend_ref[i].bref_backend->backend_conn_count, -1);
+		/* 		atomic_add(&backend_ref[i].bref_backend->backend_conn_count, -1); */
-			}
+		/* 	} */
-		}
+		/* } */
 	}
         return succp;
 }

server/modules/routing/dbshard/test/CMakeLists.txt

@@ -1,5 +1,8 @@
 if(MYSQLCLIENT_FOUND AND BUILD_TESTS)
   add_executable(testdbshard testdbshard.c)
   target_link_libraries(testdbshard ${MYSQLCLIENT_LIBRARIES} ssl crypto dl z m rt pthread)
-  add_test(NAME DBShardTest COMMAND $<TARGET_FILE:testdbshard> ${TEST_HOST} 3000 ${TEST_USER} ${TEST_PASSWORD})
+  foreach(VAR RANGE 3000 3003)
+    execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/prepare_shard.sh ${TEST_HOST} ${VAR} ${TEST_USER} ${TEST_PASSWORD})
+  endforeach()
+  add_test(NAME DBShardTest COMMAND $<TARGET_FILE:testdbshard> ${TEST_HOST} ${TEST_PORT_DB} ${TEST_USER} ${TEST_PASSWORD})
 endif()

server/modules/routing/dbshard/test/portblock.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+if [ $# -lt 1 ]
+then
+    echo "Usage $0 <port to block>"
+    exit 1
+fi
+sudo iptables -I INPUT 1 -i lo -p tcp --dport $1 -j DROP
+sudo iptables -I INPUT 1 -i lo -p tcp --sport $1 -j DROP
+echo "Traffic to port $1 blocked."

server/modules/routing/dbshard/test/prepare_shard.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+if [ $# -lt 3 ]
+then
+    echo "usage $0 <host> <port> <username> <password>"
+    exit 1
+fi
+HOST=$1
+PORT=$2
+USER=$3
+PW=$4
+SHD="shard$RANDOM"
+mysql -u $USER -p$PW -P $PORT -h $HOST -e "create database $SHD;"
+echo "Created database \"$SHD\" at $HOST:$PORT"

server/modules/routing/dbshard/test/testdbshard.c

@@ -39,6 +39,8 @@ int main(int argc, char** argv)
     password = strdup(argv[4]);
 	rval = 0;
 
+    printf("Connecting to %s:%d as %s/%s\n",host,port,username,password);
+    
 	if((server = mysql_init(NULL)) == NULL){
         fprintf(stderr,"Error : Initialization of MySQL client failed.\n");
 	    rval = 1;