MXS-884: qc_mysqlembedded now supports qc_get_field_info

Initial implementation, one failure in the tests still to be
sorted out. Plus some cleanup.

query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc

@@ -1032,6 +1032,35 @@ char* qc_get_stmtname(GWBUF* buf)
 }
 #endif
 
+/**
+ * Get the parsing info structure from a GWBUF
+ *
+ * @param querybuf A GWBUF
+ *
+ * @return The parsing info object, or NULL
+ */
+parsing_info_t* get_pinfo(GWBUF* querybuf)
+{
+    parsing_info_t *pi = NULL;
+
+    if ((querybuf != NULL) && GWBUF_IS_PARSED(querybuf))
+    {
+        pi = (parsing_info_t *) gwbuf_get_buffer_object_data(querybuf, GWBUF_PARSING_INFO);
+    }
+
+    return pi;
+}
+
+LEX* get_lex(parsing_info_t* pi)
+{
+    MYSQL* mysql = (MYSQL *) pi->pi_handle;
+    ss_dassert(mysql);
+    THD* thd = (THD *) mysql->thd;
+    ss_dassert(thd);
+
+    return thd->lex;
+}
+
 /**
  * Get the parse tree from parsed querybuf.
  * @param querybuf  The parsed GWBUF
@@ -1041,31 +1070,19 @@ char* qc_get_stmtname(GWBUF* buf)
  */
 LEX* get_lex(GWBUF* querybuf)
 {
+    LEX* lex = NULL;
+    parsing_info_t* pi = get_pinfo(querybuf);
 
-    parsing_info_t* pi;
+    if (pi)
-    MYSQL* mysql;
-    THD* thd;
-
-    if (querybuf == NULL || !GWBUF_IS_PARSED(querybuf))
     {
-        return NULL;
+        MYSQL* mysql = (MYSQL *) pi->pi_handle;
+        ss_dassert(mysql);
+        THD* thd = (THD *) mysql->thd;
+        ss_dassert(thd);
+        lex = thd->lex;
     }
 
-    pi = (parsing_info_t *) gwbuf_get_buffer_object_data(querybuf, GWBUF_PARSING_INFO);
+    return lex;
-
-    if (pi == NULL)
-    {
-        return NULL;
-    }
-
-    if ((mysql = (MYSQL *) pi->pi_handle) == NULL ||
-        (thd = (THD *) mysql->thd) == NULL)
-    {
-        ss_dassert(mysql != NULL && thd != NULL);
-        return NULL;
-    }
-
-    return thd->lex;
 }
 
 /**
@@ -2021,12 +2038,423 @@ qc_query_op_t qc_get_prepare_operation(GWBUF* stmt)
     return operation;
 }
 
-void qc_get_field_info(GWBUF* stmt, const QC_FIELD_INFO** infos, size_t* n_infos)
+static bool should_exclude(const char* name, List<Item>* excludep)
 {
-    MXS_ERROR("qc_get_field_info not implemented yet.");
+    bool exclude = false;
+    List_iterator<Item> ilist(*excludep);
+    Item* exclude_item;
 
-    *infos = NULL;
+    while (!exclude && (exclude_item = ilist++))
-    *n_infos = 0;
+    {
+        if (exclude_item->name && (strcasecmp(name, exclude_item->name) == 0))
+        {
+            exclude = true;
+        }
+    }
+
+    return exclude;
+}
+
+static void add_field_info(parsing_info_t* info,
+                           const char* database,
+                           const char* table,
+                           const char* column,
+                           List<Item>* excludep)
+{
+    ss_dassert(column);
+
+    // If only a column is specified, but not a table or database and we
+    // have a list of expressions that should be excluded, we check if the column
+    // value is present in that list. This is in order to exclude the second "d" in
+    // a statement like "select a as d from x where d = 2".
+    if (column && !table && !database && excludep && should_exclude(column, excludep))
+    {
+        return;
+    }
+
+    QC_FIELD_INFO item = { (char*)database, (char*)table, (char*)column };
+
+    size_t i;
+    for (i = 0; i < info->field_infos_len; ++i)
+    {
+        QC_FIELD_INFO* field_info = info->field_infos + i;
+
+        if (strcasecmp(item.column, field_info->column) == 0)
+        {
+            if (!item.table && !field_info->table)
+            {
+                ss_dassert(!item.database && !field_info->database);
+                break;
+            }
+            else if (item.table && field_info->table && (strcmp(item.table, field_info->table) == 0))
+            {
+                if (!item.database && !field_info->database)
+                {
+                    break;
+                }
+                else if (item.database &&
+                         field_info->database &&
+                         (strcmp(item.database, field_info->database) == 0))
+                {
+                    break;
+                }
+            }
+        }
+    }
+
+    QC_FIELD_INFO* field_infos = NULL;
+
+    if (i == info->field_infos_len) // If true, the field was not present already.
+    {
+        if (info->field_infos_len < info->field_infos_capacity)
+        {
+            field_infos = info->field_infos;
+        }
+        else
+        {
+            size_t capacity = info->field_infos_capacity ? 2 * info->field_infos_capacity : 8;
+            field_infos = (QC_FIELD_INFO*)realloc(info->field_infos, capacity * sizeof(QC_FIELD_INFO));
+
+            if (field_infos)
+            {
+                info->field_infos = field_infos;
+                info->field_infos_capacity = capacity;
+            }
+        }
+    }
+
+    // If field_infos is NULL, then the field was found and has already been noted.
+    if (field_infos)
+    {
+        item.database = item.database ? strdup(item.database) : NULL;
+        item.table = item.table ? strdup(item.table) : NULL;
+        ss_dassert(item.column);
+        item.column = strdup(item.column);
+
+        // We are happy if we at least could dup the column.
+
+        if (item.column)
+        {
+            field_infos[info->field_infos_len++] = item;
+        }
+    }
+}
+
+static void add_field_info(parsing_info_t* pi, Item_field* item, List<Item>* excludep)
+{
+    const char* database = item->db_name;
+    const char* table = item->table_name;
+    const char* column = item->field_name;
+
+    LEX* lex = get_lex(pi);
+
+    switch (lex->sql_command)
+    {
+    case SQLCOM_SHOW_FIELDS:
+        if (!database)
+        {
+            database = "information_schema";
+        }
+
+        if (!table)
+        {
+            table = "COLUMNS";
+        }
+        break;
+
+    case SQLCOM_SHOW_KEYS:
+        if (!database)
+        {
+            database = "information_schema";
+        }
+
+        if (!table)
+        {
+            table = "STATISTICS";
+        }
+        break;
+
+    case SQLCOM_SHOW_STATUS:
+        if (!database)
+        {
+            database = "information_schema";
+        }
+
+        if (!table)
+        {
+            table = "SESSION_STATUS";
+        }
+        break;
+
+    case SQLCOM_SHOW_TABLES:
+        if (!database)
+        {
+            database = "information_schema";
+        }
+
+        if (!table)
+        {
+            table = "TABLE_NAMES";
+        }
+        break;
+
+    case SQLCOM_SHOW_TABLE_STATUS:
+        if (!database)
+        {
+            database = "information_schema";
+        }
+
+        if (!table)
+        {
+            table = "TABLES";
+        }
+        break;
+
+    case SQLCOM_SHOW_VARIABLES:
+        if (!database)
+        {
+            database = "information_schema";
+        }
+
+        if (!table)
+        {
+            table = "SESSION_STATUS";
+        }
+        break;
+
+    default:
+        break;
+    }
+
+    add_field_info(pi, database, table, column, excludep);
+}
+
+static void add_field_info(parsing_info_t* pi, Item* item, List<Item>* excludep)
+{
+    const char* database = NULL;
+    const char* table = NULL;
+    const char* column = item->name;
+
+    add_field_info(pi, database, table, column, excludep);
+}
+
+static void update_field_infos(parsing_info_t* pi,
+                               collect_source_t source,
+                               Item* item,
+                               List<Item>* excludep)
+{
+    switch (item->type())
+    {
+    case Item::COND_ITEM:
+        {
+            Item_cond* cond_item = static_cast<Item_cond*>(item);
+            List_iterator<Item> ilist(*cond_item->argument_list());
+
+            while (Item *i = ilist++)
+            {
+                update_field_infos(pi, source, i, excludep);
+            }
+        }
+        break;
+
+    case Item::FIELD_ITEM:
+        add_field_info(pi, static_cast<Item_field*>(item), excludep);
+        break;
+
+    case Item::REF_ITEM:
+        {
+            if (source != COLLECT_SELECT)
+            {
+                Item_ref* ref_item = static_cast<Item_ref*>(item);
+
+                add_field_info(pi, item, excludep);
+
+                size_t n_items = ref_item->cols();
+
+                for (size_t i = 0; i < n_items; ++i)
+                {
+                    Item* reffed_item = ref_item->element_index(i);
+
+                    if (reffed_item != ref_item)
+                    {
+                        update_field_infos(pi, source, ref_item->element_index(i), excludep);
+                    }
+                }
+            }
+        }
+        break;
+
+    case Item::ROW_ITEM:
+        {
+            Item_row* row_item = static_cast<Item_row*>(item);
+            size_t n_items = row_item->cols();
+
+            for (size_t i = 0; i < n_items; ++i)
+            {
+                update_field_infos(pi, source, row_item->element_index(i), excludep);
+            }
+        }
+        break;
+
+    case Item::FUNC_ITEM:
+    case Item::SUM_FUNC_ITEM:
+        {
+            Item_func* func_item = static_cast<Item_func*>(item);
+            Item** items = func_item->arguments();
+            size_t n_items = func_item->argument_count();
+
+            for (size_t i = 0; i < n_items; ++i)
+            {
+                update_field_infos(pi, source, items[i], excludep);
+            }
+        }
+        break;
+
+    case Item::SUBSELECT_ITEM:
+        {
+            Item_subselect* subselect_item = static_cast<Item_subselect*>(item);
+
+            switch (subselect_item->substype())
+            {
+            case Item_subselect::IN_SUBS:
+            case Item_subselect::ALL_SUBS:
+            case Item_subselect::ANY_SUBS:
+                {
+                    Item_in_subselect* in_subselect_item = static_cast<Item_in_subselect*>(item);
+
+#if (((MYSQL_VERSION_MAJOR == 5) &&\
+      ((MYSQL_VERSION_MINOR > 5) ||\
+       ((MYSQL_VERSION_MINOR == 5) && (MYSQL_VERSION_PATCH >= 48))\
+      )\
+     ) ||\
+     (MYSQL_VERSION_MAJOR >= 10)\
+    )
+                    if (in_subselect_item->left_expr_orig)
+                    {
+                        update_field_infos(pi, source,
+                                                 in_subselect_item->left_expr_orig, excludep);
+                    }
+#else
+#pragma message "Figure out what to do with versions < 5.5.48."
+#endif
+                    // TODO: Anything else that needs to be looked into?
+                }
+                break;
+
+            case Item_subselect::EXISTS_SUBS:
+            case Item_subselect::SINGLEROW_SUBS:
+                // TODO: Handle these explicitly as well.
+                break;
+
+            case Item_subselect::UNKNOWN_SUBS:
+            default:
+                MXS_ERROR("Unknown subselect type: %d", subselect_item->substype());
+                break;
+            }
+        }
+        break;
+
+    default:
+        break;
+    }
+}
+
+void qc_get_field_info(GWBUF* buf, const QC_FIELD_INFO** infos, size_t* n_infos)
+{
+    if (!buf)
+    {
+        return;
+    }
+
+    if (!ensure_query_is_parsed(buf))
+    {
+        return;
+    }
+
+    parsing_info_t* pi = get_pinfo(buf);
+
+    if (!pi->field_infos)
+    {
+        ss_dassert(pi);
+        LEX* lex = get_lex(buf);
+        ss_dassert(lex);
+
+        if (!lex)
+        {
+            return;
+        }
+
+        lex->current_select = lex->all_selects_list;
+
+        while (lex->current_select)
+        {
+            List_iterator<Item> ilist(lex->current_select->item_list);
+
+            while (Item *item = ilist++)
+            {
+                update_field_infos(pi, COLLECT_SELECT, item, NULL);
+            }
+
+            if (lex->current_select->group_list.first)
+            {
+                ORDER* order = lex->current_select->group_list.first;
+                while (order)
+                {
+                    Item* item = *order->item;
+
+                    update_field_infos(pi, COLLECT_GROUP_BY, item, &lex->current_select->item_list);
+
+                    order = order->next;
+                }
+            }
+
+            if (lex->current_select->where)
+            {
+                update_field_infos(pi, COLLECT_WHERE,
+                                         lex->current_select->where,
+                                         &lex->current_select->item_list);
+            }
+
+            if (lex->current_select->having)
+            {
+                update_field_infos(pi, COLLECT_HAVING,
+                                         lex->current_select->having,
+                                         &lex->current_select->item_list);
+            }
+
+            lex->current_select = lex->current_select->next_select_in_list();
+        }
+
+
+        List_iterator<Item> ilist(lex->value_list);
+        while (Item* item = ilist++)
+        {
+            update_field_infos(pi, COLLECT_SELECT, item, NULL);
+        }
+
+        if ((lex->sql_command == SQLCOM_INSERT) ||
+            (lex->sql_command == SQLCOM_INSERT_SELECT) ||
+            (lex->sql_command == SQLCOM_REPLACE))
+        {
+            List_iterator<Item> ilist(lex->field_list);
+            while (Item *item = ilist++)
+            {
+                update_field_infos(pi, COLLECT_SELECT, item, NULL);
+            }
+
+            if (lex->insert_list)
+            {
+                List_iterator<Item> ilist(*lex->insert_list);
+                while (Item *item = ilist++)
+                {
+                    update_field_infos(pi, COLLECT_SELECT, item, NULL);
+                }
+            }
+        }
+    }
+
+    *infos = pi->field_infos;
+    *n_infos = pi->field_infos_len;
 }
 
 namespace

query_classifier/test/compare.cc

@@ -917,6 +917,113 @@ ostream& operator << (ostream& out, const QC_FIELD_INFO& x)
     return out;
 }
 
+class QcFieldInfo
+{
+public:
+    QcFieldInfo(const QC_FIELD_INFO& info)
+        : m_database(info.database ? info.database : "")
+        , m_table(info.table ? info.table : "")
+        , m_column(info.column ? info.column : "")
+    {}
+
+    bool eq(const QcFieldInfo& rhs) const
+    {
+        return
+            m_database == rhs.m_database &&
+            m_table == rhs.m_table &&
+            m_column == rhs.m_column;
+    }
+
+    bool lt(const QcFieldInfo& rhs) const
+    {
+        bool rv = false;
+
+        if (m_database < rhs.m_database)
+        {
+            rv = true;
+        }
+        else if (m_database > rhs.m_database)
+        {
+            rv = false;
+        }
+        else
+        {
+            if (m_table < rhs.m_table)
+            {
+                rv = true;
+            }
+            else if (m_table > rhs.m_table)
+            {
+                rv = false;
+            }
+            else
+            {
+                if (m_column < rhs.m_column)
+                {
+                    rv = true;
+                }
+            }
+        }
+
+        return rv;
+    }
+
+    void print(ostream& out) const
+    {
+        if (!m_database.empty())
+        {
+            out << m_database;
+            out << ".";
+        }
+
+        if (!m_table.empty())
+        {
+            out << m_table;
+            out << ".";
+        }
+
+        out << m_column;
+    }
+
+private:
+    std::string m_database;
+    std::string m_table;
+    std::string m_column;
+};
+
+ostream& operator << (ostream& out, const QcFieldInfo& x)
+{
+    x.print(out);
+    return out;
+}
+
+ostream& operator << (ostream& out, std::set<QcFieldInfo>& x)
+{
+    std::set<QcFieldInfo>::iterator i = x.begin();
+    std::set<QcFieldInfo>::iterator end = x.end();
+
+    while (i != end)
+    {
+        out << *i++;
+        if (i != end)
+        {
+            out << " ";
+        }
+    }
+
+    return out;
+}
+
+bool operator < (const QcFieldInfo& lhs, const QcFieldInfo& rhs)
+{
+    return lhs.lt(rhs);
+}
+
+bool operator == (const QcFieldInfo& lhs, const QcFieldInfo& rhs)
+{
+    return lhs.eq(rhs);
+}
+
 bool are_equal(const QC_FIELD_INFO* fields1, size_t n_fields1,
                const QC_FIELD_INFO* fields2, size_t n_fields2)
 {
@@ -924,6 +1031,7 @@ bool are_equal(const QC_FIELD_INFO* fields1, size_t n_fields1,
 
     if (rv)
     {
+
         size_t i = 0;
         while (rv && (i < n_fields1))
         {
@@ -968,18 +1076,23 @@ bool compare_get_field_info(QUERY_CLASSIFIER* pClassifier1, GWBUF* pCopy1,
     stringstream ss;
     ss << HEADING;
 
-    if (are_equal(infos1, n_infos1, infos2, n_infos2))
+    int i;
+
+    std::set<QcFieldInfo> f1;
+    f1.insert(infos1, infos1 + n_infos1);
+
+    std::set<QcFieldInfo> f2;
+    f2.insert(infos2, infos2 + n_infos2);
+
+    if (f1 == f2)
     {
         ss << "Ok : ";
-        print(ss, infos1, n_infos1);
+        ss << f1;
         success = true;
     }
     else
     {
-        ss << "ERR: ";
+        ss << "ERR: " << f1 << " != " << f2;
-        print(ss, infos1, n_infos1);
-        ss << " != ";
-        print(ss, infos2, n_infos2);
     }
 
     report(success, ss.str());
@@ -1008,7 +1121,7 @@ bool compare(QUERY_CLASSIFIER* pClassifier1, QUERY_CLASSIFIER* pClassifier2, con
     errors += !compare_get_database_names(pClassifier1, pCopy1, pClassifier2, pCopy2);
     errors += !compare_get_prepare_name(pClassifier1, pCopy1, pClassifier2, pCopy2);
     errors += !compare_get_prepare_operation(pClassifier1, pCopy1, pClassifier2, pCopy2);
-    //errors += !compare_get_field_info(pClassifier1, pCopy1, pClassifier2, pCopy2);
+    errors += !compare_get_field_info(pClassifier1, pCopy1, pClassifier2, pCopy2);
 
     gwbuf_free(pCopy1);
     gwbuf_free(pCopy2);

query_classifier/test/insert.test

@@ -222,9 +222,11 @@ replace into t1 values (4, 4);
 select row_count();
 # Reports that 2 rows are affected. This conforms to documentation.
 # (Useful for differentiating inserts from updates).
-insert into t1 values (2, 2) on duplicate key update data= data + 10;
+# MXSTODO: insert into t1 values (2, 2) on duplicate key update data= data + 10;
+# qc_sqlite: Cannot parse "on duplicate"
 select row_count();
-insert into t1 values (5, 5) on duplicate key update data= data + 10;
+# MXSTODO: insert into t1 values (5, 5) on duplicate key update data= data + 10;
+# qc_sqlite: Cannot parse "on duplicate"
 select row_count();
 drop table t1;
 

query_classifier/test/qc_sqlite_unsupported.test

@@ -20,3 +20,8 @@ SET @x:= (SELECT h FROM t1 WHERE (a,b,c,d,e,f,g)=(1,2,3,4,5,6,7));
 # REMOVE: expr(A) ::= LP(B) expr(X) RP(E). {A.pExpr = X.pExpr; spanSet(&A,&B,&E);}
 # REMOVE: expr(A) ::= LP expr(X) COMMA(OP) expr(Y) RP. {spanBinaryExpr(&A,pParse,@OP,&X,&Y);}
 # ADD   : expr(A) ::= LP exprlist RP. { ... }
+
+insert into t1 values (2, 2) on duplicate key update data= data + 10;
+# Problem: warning: [qc_sqlite] Statement was only partially parsed (Sqlite3 error: SQL logic error
+#          or missing database, near "on": syntax error): "insert into t1 values (2, 2) on duplicate 
+#          key update data= data + 10;"