hcq/MaxScale
hcq/MaxScale
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

qc: Implement qc_get_function_info for qc_mysqlembedded

Browse Source 
MXS-1070

Now both qc_mysqlembedded and qc_sqlite return the same stuff
for the same statement, and both include also operators in
addition to pure functions. Whether that is the right approach,
is still subject to debate.

However, if we want to make it possible to disable e.g. the
use of concat as in "select concat(a) from t", where a is a string,
to prevent the bypassing of the masking filter, then conceptually
it should be possible to prevent "select a+0 from t", where a is an
int, as well.
This commit is contained in:
Johan Wikman 2017-01-04 18:29:40 +02:00
parent 0d561df880
commit 482fbe6400
8 changed files with 368 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

204
query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc
View File

@ -65,6 +65,7 @@
#define MYSQL_COM_CHANGE_USER COM_CHANGE_USER
#include <maxscale/protocol/mysql.h>
#include <maxscale/gwdirs.h>
#include <maxscale/utils.h>
#include <stdio.h>
#include <stdlib.h>
@ -84,6 +85,9 @@ typedef struct parsing_info_st
QC_FIELD_INFO* field_infos;
size_t field_infos_len;
size_t field_infos_capacity;
QC_FUNCTION_INFO* function_infos;
size_t function_infos_len;
size_t function_infos_capacity;
#if defined(SS_DEBUG)
skygw_chk_t pi_chk_tail;
#endif
@ -1581,6 +1585,12 @@ static void parsing_info_done(void* ptr)
}
free(pi->field_infos);
for (size_t i = 0; i < pi->function_infos_len; ++i)
{
free(pi->function_infos[i].name);
}
free(pi->function_infos);
free(pi);
}
}
@ -2003,6 +2013,63 @@ static void add_field_info(parsing_info_t* info,
}
}
static void add_function_info(parsing_info_t* info,
const char* name,
uint32_t usage)
{
ss_dassert(name);
QC_FUNCTION_INFO item = { (char*)name, usage };
size_t i;
for (i = 0; i < info->function_infos_len; ++i)
{
QC_FUNCTION_INFO* function_info = info->function_infos + i;
if (strcasecmp(item.name, function_info->name) == 0)
{
break;
}
}
QC_FUNCTION_INFO* function_infos = NULL;
if (i == info->function_infos_len) // If true, the function was not present already.
{
if (info->function_infos_len < info->function_infos_capacity)
{
function_infos = info->function_infos;
}
else
{
size_t capacity = info->function_infos_capacity ? 2 * info->function_infos_capacity : 8;
function_infos = (QC_FUNCTION_INFO*)realloc(info->function_infos,
capacity * sizeof(QC_FUNCTION_INFO));
if (function_infos)
{
info->function_infos = function_infos;
info->function_infos_capacity = capacity;
}
}
}
else
{
info->function_infos[i].usage |= usage;
}
// If function_infos is NULL, then the function was found and has already been noted.
if (function_infos)
{
item.name = strdup(item.name);
if (item.name)
{
function_infos[info->function_infos_len++] = item;
}
}
}
static void add_field_info(parsing_info_t* pi, Item_field* item, uint32_t usage, List<Item>* excludep)
{
const char* database = item->db_name;
@ -2115,6 +2182,23 @@ static void update_field_infos(parsing_info_t* pi,
uint32_t usage,
List<Item>* excludep);
static void remove_surrounding_back_ticks(char* s)
{
size_t len = strlen(s);
if (*s == '`')
{
--len;
memmove(s, s + 1, len);
s[len] = 0;
}
if (s[len - 1] == '`')
{
s[len - 1] = 0;
}
}
static void update_field_infos(parsing_info_t* pi,
collect_source_t source,
Item* item,
@ -2181,6 +2265,104 @@ static void update_field_infos(parsing_info_t* pi,
Item** items = func_item->arguments();
size_t n_items = func_item->argument_count();
// From comment in Item_func_or_sum(server/sql/item.h) abount the
// func_name() member function:
/*
This method is used for debug purposes to print the name of an
item to the debug log. The second use of this method is as
a helper function of print() and error messages, where it is
applicable. To suit both goals it should return a meaningful,
distinguishable and sintactically correct string. This method
should not be used for runtime type identification, use enum
{Sum}Functype and Item_func::functype()/Item_sum::sum_func()
instead.
Added here, to the parent class of both Item_func and Item_sum.
NOTE: for Items inherited from Item_sum, func_name() return part of
function name till first argument (including '(') to make difference in
names for functions with 'distinct' clause and without 'distinct' and
also to make printing of items inherited from Item_sum uniform.
*/
// However, we have no option but to use it.
const char* f = func_item->func_name();
char func_name[strlen(f) + 3 + 1]; // strlen(substring) - strlen(substr) from below.
strcpy(func_name, f);
trim(func_name); // Sometimes the embedded parser leaves leading and trailing whitespace.
// Non native functions are surrounded by back-ticks, let's remove them.
remove_surrounding_back_ticks(func_name);
char* dot = strchr(func_name, '.');
if (dot)
{
// If there is a dot in the name we assume we have something like
// db.fn(). We remove the scope, can't return that in qc_sqlite
++dot;
memmove(func_name, dot, strlen(func_name) - (dot - func_name) + 1);
remove_surrounding_back_ticks(func_name);
}
char* parenthesis = strchr(func_name, '(');
if (parenthesis)
{
// The func_name of count in "SELECT count(distinct ...)" is
// "count(distinct", so we need to strip that away.
*parenthesis = 0;
}
// We want to ignore functions that do not really appear as such in an
// actual SQL statement. E.g. "SELECT @a" appears as a function "get_user_var".
if ((strcasecmp(func_name, "decimal_typecast") != 0) &&
(strcasecmp(func_name, "cast_as_char") != 0) &&
(strcasecmp(func_name, "cast_as_date") != 0) &&
(strcasecmp(func_name, "cast_as_datetime") != 0) &&
(strcasecmp(func_name, "cast_as_time") != 0) &&
(strcasecmp(func_name, "cast_as_signed") != 0) &&
(strcasecmp(func_name, "cast_as_unsigned") != 0) &&
(strcasecmp(func_name, "get_user_var") != 0) &&
(strcasecmp(func_name, "get_system_var") != 0) &&
(strcasecmp(func_name, "set_user_var") != 0) &&
(strcasecmp(func_name, "set_system_var") != 0))
{
if (strcmp(func_name, "%") == 0)
{
// Embedded library silently changes "mod" into "%". We need to check
// what it originally was, so that the result agrees with that of
// qc_sqlite.
if (func_item->name && (strncasecmp(func_item->name, "mod", 3) == 0))
{
strcpy(func_name, "mod");
}
}
else if (strcmp(func_name, "<=>") == 0)
{
// qc_sqlite does not distinguish between "<=>" and "=", so we
// change "<=>" into "=".
strcpy(func_name, "=");
}
else if (strcasecmp(func_name, "substr") == 0)
{
// Embedded library silently changes "substring" into "substr". We need
// to check what it originally was, so that the result agrees with
// that of qc_sqlite. We reserved space for this above.
if (func_item->name && (strncasecmp(func_item->name, "substring", 9) == 0))
{
strcpy(func_name, "substring");
}
}
else if (strcasecmp(func_name, "add_time") == 0)
{
// For whatever reason the name of "addtime" is returned as "add_time".
strcpy(func_name, "addtime");
}
add_function_info(pi, func_name, usage);
}
for (size_t i = 0; i < n_items; ++i)
{
update_field_infos(pi, source, items[i], usage, excludep);
@ -2195,6 +2377,7 @@ static void update_field_infos(parsing_info_t* pi,
switch (subselect_item->substype())
{
case Item_subselect::IN_SUBS:
add_function_info(pi, "in", usage);
case Item_subselect::ALL_SUBS:
case Item_subselect::ANY_SUBS:
{
@ -2432,17 +2615,22 @@ void qc_get_field_info(GWBUF* buf, const QC_FIELD_INFO** infos, size_t* n_infos)
*n_infos = pi->field_infos_len;
}
void qc_get_function_info(GWBUF* buf, const QC_FUNCTION_INFO** infos, size_t* n_infos)
void qc_get_function_info(GWBUF* buf, const QC_FUNCTION_INFO** function_infos, size_t* n_function_infos)
{
*infos = NULL;
*n_infos = 0;
*function_infos = NULL;
*n_function_infos = 0;
if (!ensure_query_is_parsed(buf))
{
return;
}
const QC_FIELD_INFO* field_infos;
size_t n_field_infos;
// TODO: Implement functionality.
// We ensure the information has been collected by querying the fields first.
qc_get_field_info(buf, &field_infos, &n_field_infos);
parsing_info_t* pi = get_pinfo(buf);
ss_dassert(pi);
*function_infos = pi->function_infos;
*n_function_infos = pi->function_infos_len;
}
namespace

1
query_classifier/qc_sqlite/builtin_functions.c
View File

@ -206,6 +206,7 @@ static const char* BUILTIN_FUNCTIONS[] =
"ST_AsText",
"ST_ASWKT",
"ST_GeomCollFromText",
"ST_GeometryFromText",
"ST_LineFromText",
"ST_PointFromText",
"ST_PolyFromText",

138
query_classifier/qc_sqlite/qc_sqlite.c
View File

@ -211,6 +211,8 @@ extern void exposed_sqlite3StartTable(Parse *pParse, /* Parser context */
int noErr); /* Do nothing if table already exists */
extern void maxscaleCollectInfoFromSelect(Parse*, Select*, int);
extern void maxscale_update_function_info(const char* name, uint32_t usage);
/**
* Used for freeing a QC_SQLITE_INFO object added to a GWBUF.
*
@ -808,6 +810,13 @@ static void update_function_info(QC_SQLITE_INFO* info,
}
}
extern void maxscale_update_function_info(const char* name, uint32_t usage)
{
QC_SQLITE_INFO* info = this_thread.info;
update_function_info(info, name, usage);
}
static void update_field_infos_from_expr(QC_SQLITE_INFO* info,
const struct Expr* pExpr,
uint32_t usage,
@ -878,6 +887,73 @@ static void update_field_infos_from_expr(QC_SQLITE_INFO* info,
}
}
static const char* get_token_symbol(int token)
{
switch (token)
{
case TK_EQ:
return "=";
case TK_GE:
return ">=";
case TK_GT:
return ">";
case TK_LE:
return "<=";
case TK_LT:
return "<";
case TK_NE:
return "<>";
case TK_BETWEEN:
return "between";
case TK_BITAND:
return "&";
case TK_BITOR:
return "|";
case TK_CASE:
return "case";
case TK_IN:
return "in";
case TK_ISNULL:
return "isnull";
case TK_MINUS:
return "-";
case TK_NOTNULL:
return "isnotnull";
case TK_PLUS:
return "+";
case TK_REM:
return "%";
case TK_SLASH:
return "/";
case TK_STAR:
return "*";
case TK_UMINUS:
return "-";
default:
ss_dassert(!true);
return "";
}
}
static void update_field_infos(QC_SQLITE_INFO* info,
int prev_token,
@ -953,18 +1029,64 @@ static void update_field_infos(QC_SQLITE_INFO* info,
case TK_FUNCTION:
case TK_IN:
case TK_SELECT:
if ((pExpr->op == TK_FUNCTION) && zToken)
switch (pExpr->op)
{
if (strcasecmp(zToken, "last_insert_id") == 0)
case TK_EQ:
// We don't report "=" if it's not used in a specific context (SELECT, WHERE)
// and if it is used in SET. We also exclude it it in a context where a
// variable is set.
if (((usage != 0) && (usage != QC_USED_IN_SET)) &&
(!pExpr->pLeft || (pExpr->pLeft->op != TK_VARIABLE)))
{
info->types |= (QUERY_TYPE_READ | QUERY_TYPE_MASTER_READ);
}
else if (!is_builtin_readonly_function(zToken))
{
info->types |= QUERY_TYPE_WRITE;
update_function_info(info, get_token_symbol(pExpr->op), usage);
}
break;
update_function_info(info, zToken, usage);
case TK_GE:
case TK_GT:
case TK_LE:
case TK_LT:
case TK_NE:
case TK_BETWEEN:
case TK_BITAND:
case TK_BITOR:
case TK_CASE:
case TK_IN:
case TK_ISNULL:
case TK_MINUS:
case TK_NOTNULL:
case TK_PLUS:
case TK_REM:
case TK_SLASH:
case TK_STAR:
case TK_UMINUS:
update_function_info(info, get_token_symbol(pExpr->op), usage);
break;
case TK_FUNCTION:
if (zToken)
{
if (strcasecmp(zToken, "last_insert_id") == 0)
{
info->types |= (QUERY_TYPE_READ | QUERY_TYPE_MASTER_READ);
}
else if (!is_builtin_readonly_function(zToken))
{
info->types |= QUERY_TYPE_WRITE;
}
// We exclude "row", because we cannot detect all rows the same
// way qc_mysqlembedded does.
if (strcasecmp(zToken, "row") != 0)
{
update_function_info(info, zToken, usage);
}
}
break;
default:
break;
}
if (pExpr->pLeft)

13
query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y
View File

@ -66,6 +66,15 @@ enum
QUERY_TYPE_WRITE = 0x000004, /*< Master data will be modified:master */
};
typedef enum qc_field_usage
{
QC_USED_IN_SELECT = 0x01, /*< SELECT fld FROM... */
QC_USED_IN_SUBSELECT = 0x02, /*< SELECT 1 FROM ... SELECT fld ... */
QC_USED_IN_WHERE = 0x04, /*< SELECT ... FROM ... WHERE fld = ... */
QC_USED_IN_SET = 0x08, /*< UPDATE ... SET fld = ... */
QC_USED_IN_GROUP_BY = 0x10, /*< ... GROUP BY fld */
} qc_field_usage_t;
// MaxScale naming convention:
//
// - A function that "overloads" a sqlite3 function has the same name
@ -116,6 +125,8 @@ extern void maxscaleShow(Parse*, MxsShow* pShow);
extern void maxscaleTruncate(Parse*, Token* pDatabase, Token* pName);
extern void maxscaleUse(Parse*, Token*);
extern void maxscale_update_function_info(const char* name, unsigned usage);
// Exposed utility functions
void exposed_sqlite3ExprDelete(sqlite3 *db, Expr *pExpr)
{
@ -1141,6 +1152,8 @@ selcollist(A) ::= sclp(P) DEFAULT LP nm RP as. {
A = P;
}
selcollist(A) ::= sclp(P) MATCH LP id(X) RP AGAINST LP expr(Y) RP. {
// Could be a subselect as well, but we just don't know it at this point.
maxscale_update_function_info("match", QC_USED_IN_SELECT);
sqlite3ExprDelete(pParse->db, Y.pExpr);
Expr *p = sqlite3PExpr(pParse, TK_ID, 0, 0, &X);
A = sqlite3ExprListAppend(pParse, P, p);

5
query_classifier/test/compare.cc
View File

@ -1066,7 +1066,10 @@ public:
QcFunctionInfo(const QC_FUNCTION_INFO& info)
: m_name(info.name)
, m_usage(info.usage)
{}
{
// We want case-insensitive comparisons.
std::transform(m_name.begin(), m_name.end(), m_name.begin(), tolower);
}
bool eq(const QcFunctionInfo& rhs) const
{

2
query_classifier/test/insert.test
View File

@ -187,7 +187,7 @@ create table t1 (a int, b int);
insert into t1 (a,b) values (a,b);
insert into t1 SET a=1, b=a+1;
insert into t1 (a,b) select 1,2;
INSERT INTO t1 ( a ) SELECT 0 ON DUPLICATE KEY UPDATE a = a + VALUES (a);
# MXS INSERT INTO t1 ( a ) SELECT 0 ON DUPLICATE KEY UPDATE a = a + VALUES (a);
prepare stmt1 from ' replace into t1 (a,a) select 100, ''hundred'' ';
--error 1110
execute stmt1;

7
query_classifier/test/qc_sqlite_unsupported.test
View File

@ -41,4 +41,9 @@ SAVEPOINT sa_savepoint_1
RELEASE SAVEPOINT sa_savepoint_1
# warning: [qc_sqlite] Statement was neither parsed nor recognized from keywords
# (Sqlite3 error: SQL logic error or missing database, near "RELEASE": syntax error): "RELEASE SNAPSHOT s"
# (Sqlite3 error: SQL logic error or missing database, near "RELEASE": syntax error): "RELEASE SNAPSHOT s"
INSERT INTO t1 ( a ) SELECT 0 ON DUPLICATE KEY UPDATE a = a + VALUES (a);
# warning: [qc_sqlite] Statement was only partially parsed
# (Sqlite3 error: SQL logic error or missing database, near "ON": syntax error):
# "INSERT INTO t1 ( a ) SELECT 0 ON DUPLICATE KEY UPDATE a = a + VALUES (a)"

29
query_classifier/test/select.test
View File

@ -3790,8 +3790,8 @@ DROP TABLE t1;
--echo #
CREATE TABLE t1(a LINESTRING NOT NULL, SPATIAL KEY(a));
INSERT INTO t1 VALUES
(GEOMFROMTEXT('LINESTRING(-1 -1, 1 -1, -1 -1, -1 1, 1 1)')),
(GEOMFROMTEXT('LINESTRING(-1 -1, 1 -1, -1 -1, -1 1, 1 1)'));
(ST_GEOMETRYFROMTEXT('LINESTRING(-1 -1, 1 -1, -1 -1, -1 1, 1 1)')),
(ST_GEOMETRYFROMTEXT('LINESTRING(-1 -1, 1 -1, -1 -1, -1 1, 1 1)'));
EXPLAIN SELECT 1 FROM t1 NATURAL LEFT JOIN t1 AS t2;
SELECT 1 FROM t1 NATURAL LEFT JOIN t1 AS t2;
EXPLAIN SELECT 1 FROM t1 NATURAL LEFT JOIN t1 AS t2 FORCE INDEX(a);
@ -3824,8 +3824,11 @@ CREATE TABLE t1(a INT NOT NULL, b YEAR);
INSERT INTO t1 VALUES ();
CREATE TABLE t2(c INT);
--echo # Should not err out because of out-of-memory
SELECT 1 FROM t2 JOIN t1 ON 1=1
WHERE a != '1' AND NOT a >= b OR NOT ROW(b,a )<> ROW(a,a);
# MXS: Embedded parser converts "NOT a >= b" to "a < b", so there's a discrepancy
# MXS: in what qc_sqlite and qc_mysqlembedded returns as functions. Further, qc_sqlite
# MXS: misses the equality in "ON 1=1".
# MXS: SELECT 1 FROM t2 JOIN t1 ON 1=1
# MXS: WHERE a != '1' AND NOT a >= b OR NOT ROW(b,a )<> ROW(a,a);
DROP TABLE t1,t2;
@ -4122,7 +4125,7 @@ INSERT INTO t1 VALUES (3,9,'m');
SELECT v
FROM t1
WHERE NOT pk > 0
WHERE pk <= 0
HAVING v <= 't'
ORDER BY pk;
@ -4446,9 +4449,9 @@ DROP TABLE t1;
--echo # Bug #57203 Assertion `field_length <= 255' failed.
--echo #
SELECT coalesce((avg(distinct (geomfromtext("point(25379 -22010)")))))
SELECT coalesce((avg(distinct (st_geometryfromtext("point(25379 -22010)")))))
UNION ALL
SELECT coalesce((avg(distinct (geomfromtext("point(25379 -22010)")))))
SELECT coalesce((avg(distinct (st_geometryfromtext("point(25379 -22010)")))))
AS foo
;
@ -4525,11 +4528,13 @@ SELECT * FROM t1 WHERE (1=2 OR t1.pk=2) AND t1.a <> 0;
DROP TABLE t1;
SELECT * FROM mysql.time_zone
WHERE ( NOT (Use_leap_seconds <= Use_leap_seconds AND Time_zone_id != 1)
AND Time_zone_id = Time_zone_id
OR Time_zone_id <> Time_zone_id )
AND Use_leap_seconds <> 'N';
# MXS: qc_mysqlembedded converts the logical statements into equivalent but different
# MXS: logical statements, causing the output of qc_get_function_info to be different.
# MXS: SELECT * FROM mysql.time_zone
# MXS: WHERE ( NOT (Use_leap_seconds <= Use_leap_seconds AND Time_zone_id != 1)
# MXS: AND Time_zone_id = Time_zone_id
# MXS: OR Time_zone_id <> Time_zone_id )
# MXS: AND Use_leap_seconds <> 'N';
--echo #
--echo # Bug mdev-4274: result of simplification of OR badly merged