/* * Copyright (c) 2020 Huawei Technologies Co.,Ltd. * * openGauss is licensed under Mulan PSL v2. * You can use this software according to the terms and conditions of the Mulan PSL v2. * You may obtain a copy of Mulan PSL v2 at: * * http://license.coscl.org.cn/MulanPSL2 * * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. * See the Mulan PSL v2 for more details. * ------------------------------------------------------------------------- * * index_advisor.cpp * * IDENTIFICATION * src/gausskernel/dbmind/kernel/index_advisor.cpp * * DESCRIPTION * The functions in this file are used to provide the built-in index recommendation function of the database, * and users can obtain better performance by creating indexes on the corresponding tables and columns. * ------------------------------------------------------------------------- */ #include "postgres.h" #include "access/tableam.h" #include "access/tupdesc.h" #include "catalog/indexing.h" #include "catalog/pg_attribute.h" #include "funcapi.h" #include "nodes/makefuncs.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "pg_config_manual.h" #include "parser/parser.h" #include "parser/analyze.h" #include "securec.h" #include "tcop/dest.h" #include "tcop/tcopprot.h" #include "tcop/utility.h" #include "tcop/pquery.h" #include "utils/builtins.h" #include "utils/elog.h" #include "utils/relcache.h" #include "utils/syscache.h" #include "utils/lsyscache.h" #include "utils/palloc.h" #include "utils/fmgroids.h" #include "utils/snapmgr.h" #define MAX_SAMPLE_ROWS 10000 /* sampling range for executing a query */ #define CARDINALITY_THRESHOLD 30 /* the threshold of index selection */ #define MAX_QUERY_LEN 256 typedef struct { char table[NAMEDATALEN]; char column[NAMEDATALEN]; } SuggestedIndex; typedef struct { DestReceiver pub; MemoryContext mxct; List *tuples; } StmtResult; typedef struct { char *table_name; List *alias_name; List *index; List *join_cond; List *index_print; } TableCell; typedef struct { char *index_name; uint4 cardinality; char *op; char *field_expr; } IndexCell; typedef struct { char *field; char *table; char *table_field; } JoinCell; THR_LOCAL List *g_stmt_list = NIL; THR_LOCAL List *g_tmp_table_list = NIL; THR_LOCAL List *g_table_list = NIL; THR_LOCAL List *g_drived_tables = NIL; THR_LOCAL TableCell *g_driver_table = NULL; static SuggestedIndex *suggest_index(const char *, _out_ int *); static StmtResult *execute_stmt(const char *query_string, bool need_result = false); static inline void analyze_tables(List *list); static List *get_table_indexes(Oid oid); static List *get_index_attname(Oid oid); static char *search_table_attname(Oid attrelid, int2 attnum); static DestReceiver *create_stmt_receiver(); static void receive(TupleTableSlot *slot, DestReceiver *self); static void shutdown(DestReceiver *self); static void startup(DestReceiver *self, int operation, TupleDesc typeinfo); static void destroy(DestReceiver *self); static void free_global_resource(); static void find_select_stmt(Node *); static void extract_stmt_from_clause(List *); static void extract_stmt_where_clause(Node *); static void parse_where_clause(Node *); static void field_value_trans(_out_ char *, A_Const *); static void parse_field_expr(List *, List *, List *); static inline uint4 tuple_to_uint(List *); static uint4 get_table_count(const char *); static uint4 calculate_field_cardinality(const char *, const char *); static bool is_tmp_table(const char *); static char *find_field_name(List *); static char *find_table_name(List *); static bool check_relation_type_valid(Oid); static TableCell *find_or_create_tblcell(char *, char *); static void add_index_from_field(char *, IndexCell *); static char *parse_group_clause(List *, List *); static char *parse_order_clause(List *, List *); static void add_index_from_group_order(TableCell *, List *, List *, bool); static void generate_final_index(TableCell *, Oid); static void parse_from_clause(List *); static void add_drived_tables(RangeVar *); static void parse_join_tree(JoinExpr *); static void add_join_cond(TableCell *, char *, TableCell *, char *); static void parse_join_expr(Node *); static void parse_join_expr(JoinExpr *); static void determine_driver_table(); static uint4 get_join_table_result_set(const char *, const char *); static void add_index_from_join(TableCell *, char *); static void add_index_for_drived_tables(); Datum gs_index_advise(PG_FUNCTION_ARGS) { FuncCallContext *func_ctx = NULL; SuggestedIndex *array = NULL; char *query = PG_GETARG_CSTRING(0); if (query == NULL) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("you must enter a query statement."))); } #define COLUMN_NUM 2 if (SRF_IS_FIRSTCALL()) { TupleDesc tup_desc; MemoryContext old_context; int max_calls = 0; /* create a function context for cross-call persistence */ func_ctx = SRF_FIRSTCALL_INIT(); /* * switch to memory context appropriate for multiple function calls */ old_context = MemoryContextSwitchTo(func_ctx->multi_call_memory_ctx); array = suggest_index(query, &max_calls); /* build tup_desc for result tuples */ tup_desc = CreateTemplateTupleDesc(COLUMN_NUM, false); TupleDescInitEntry(tup_desc, (AttrNumber)1, "table", TEXTOID, -1, 0); TupleDescInitEntry(tup_desc, (AttrNumber)2, "column", TEXTOID, -1, 0); func_ctx->tuple_desc = BlessTupleDesc(tup_desc); func_ctx->max_calls = max_calls; func_ctx->user_fctx = (void *)array; (void)MemoryContextSwitchTo(old_context); } /* stuff done on every call of the function */ func_ctx = SRF_PERCALL_SETUP(); array = (SuggestedIndex *)func_ctx->user_fctx; /* for each row */ if (func_ctx->call_cntr < func_ctx->max_calls) { Datum values[COLUMN_NUM]; bool nulls[COLUMN_NUM] = {false}; HeapTuple tuple = NULL; errno_t rc = EOK; rc = memset_s(values, sizeof(values), 0, sizeof(values)); securec_check(rc, "\0", "\0"); rc = memset_s(nulls, sizeof(nulls), 0, sizeof(nulls)); securec_check(rc, "\0", "\0"); int entry_index = func_ctx->call_cntr; SuggestedIndex *entry = array + entry_index; /* Locking is probably not really necessary */ values[0] = CStringGetTextDatum(entry->table); values[1] = CStringGetTextDatum(entry->column); tuple = heap_form_tuple(func_ctx->tuple_desc, values, nulls); SRF_RETURN_NEXT(func_ctx, HeapTupleGetDatum(tuple)); } else { pfree(array); SRF_RETURN_DONE(func_ctx); } } /* * suggest_index * Parse the given query and return the suggested indexes. The suggested * index consists of table names and column names. * * The main steps are summarized as follows: * 1. Get parse tree; * 2. Find and parse SelectStmt structures; * 3. Parse 'from' and 'where' clause, and add candidate indexes for tables; * 4. Determine the driver table; * 5. Parse 'group' and 'order' clause and add candidate indexes for tables; * 6. Add candidate indexes for drived tables according to the 'join' conditions. */ SuggestedIndex *suggest_index(const char *query_string, _out_ int *len) { g_stmt_list = NIL; g_tmp_table_list = NIL; g_table_list = NIL; g_drived_tables = NIL; g_driver_table = NULL; List *parse_tree_list = raw_parser(query_string); if (parse_tree_list == NULL || list_length(parse_tree_list) <= 0) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("can not parse query: %s.", query_string))); } if (list_length(parse_tree_list) > 1) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("can not advise for multiple queries."))); } Node *parsetree = (Node *)lfirst(list_head(parse_tree_list)); Node* parsetree_copy = (Node*)copyObject(parsetree); (void)parse_analyze(parsetree_copy, query_string, NULL, 0); find_select_stmt(parsetree); if (!g_stmt_list) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("can not advise for query: %s.", query_string))); } // Parse the SelectStmt structures ListCell *item = NULL; foreach (item, g_stmt_list) { SelectStmt *stmt = (SelectStmt *)lfirst(item); parse_from_clause(stmt->fromClause); if (g_table_list) { parse_where_clause(stmt->whereClause); determine_driver_table(); if (parse_group_clause(stmt->groupClause, stmt->targetList)) { add_index_from_group_order(g_driver_table, stmt->groupClause, stmt->targetList, true); } else if (parse_order_clause(stmt->sortClause, stmt->targetList)) { add_index_from_group_order(g_driver_table, stmt->sortClause, stmt->targetList, false); } if (g_table_list->length > 1 && g_driver_table) { add_index_for_drived_tables(); } // Generate the final index string for each table. ListCell *table_item = NULL; foreach (table_item, g_table_list) { TableCell *table = (TableCell *)lfirst(table_item); if (table->index != NIL) { RangeVar* rtable = makeRangeVar(NULL, table->table_name, -1); Oid table_oid = RangeVarGetRelid(rtable, NoLock, true); if (table_oid == InvalidOid) { continue; } generate_final_index(table, table_oid); } } g_driver_table = NULL; } } if (g_table_list == NIL) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("can not advise for query: %s.", query_string))); } // Format the returned result, e.g., 'table1, "(col1,col2),(col3)"'. int array_len = g_table_list->length; *len = array_len; SuggestedIndex *array = (SuggestedIndex *)palloc0(sizeof(SuggestedIndex) * array_len); errno_t rc = EOK; item = NULL; int i = 0; foreach (item, g_table_list) { TableCell *cur_table = (TableCell *)lfirst(item); List *index_list = cur_table->index_print; rc = strcpy_s((array + i)->table, NAMEDATALEN, cur_table->table_name); securec_check(rc, "\0", "\0"); if (!index_list) { rc = strcpy_s((array + i)->column, NAMEDATALEN, ""); securec_check(rc, "\0", "\0"); } else { ListCell *cur_index = NULL; int j = 0; foreach (cur_index, index_list) { if (strlen((char *)lfirst(cur_index)) + strlen((array + i)->column) + 3 > NAMEDATALEN) { continue; } if (j > 0) { rc = strcat_s((array + i)->column, NAMEDATALEN, ",("); } else { rc = strcpy_s((array + i)->column, NAMEDATALEN, "("); } securec_check(rc, "\0", "\0"); rc = strcat_s((array + i)->column, NAMEDATALEN, (char *)lfirst(cur_index)); securec_check(rc, "\0", "\0"); rc = strcat_s((array + i)->column, NAMEDATALEN, ")"); securec_check(rc, "\0", "\0"); j++; } } i++; list_free_deep(index_list); } // free resources list_free_deep(parse_tree_list); free_global_resource(); return array; } void free_global_resource() { list_free_deep(g_drived_tables); list_free_deep(g_table_list); list_free_deep(g_tmp_table_list); list_free(g_stmt_list); g_table_list = NIL; g_tmp_table_list = NIL; g_stmt_list = NIL; g_drived_tables = NIL; g_driver_table = NULL; } /* Update table statistics obtained from query tree by executing the 'analyze table' statement. */ inline void analyze_tables(List *list) { ListCell *item = NULL; foreach (item, list) { RangeTblEntry *entry = (RangeTblEntry *)lfirst(item); if (entry != NULL && entry->relname != NULL) { errno_t rc = EOK; char stmt[MAX_QUERY_LEN] = {0x00}; rc = sprintf_s(stmt, MAX_QUERY_LEN, "analyze %s;", entry->relname); securec_check_ss_c(rc, "\0", "\0"); (void)execute_stmt(stmt); } } } /* Search the oid of all indexes created on the table through the oid of the table, * and return the index oid list. * */ List *get_table_indexes(Oid oid) { Relation rel = heap_open(oid, NoLock); List *indexes = RelationGetIndexList(rel); heap_close(rel, NoLock); return indexes; } /* Search attribute name (column name) by the oid of the table * and the serial number of the column. */ char *search_table_attname(Oid attrelid, int2 attnum) { ScanKeyData key[2]; SysScanDesc scan = NULL; HeapTuple tup = NULL; char *name = NULL; bool isnull = true; Relation pg_attribute = heap_open(AttributeRelationId, NoLock); ScanKeyInit(&key[0], Anum_pg_attribute_attrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(attrelid)); ScanKeyInit(&key[1], Anum_pg_attribute_attnum, BTEqualStrategyNumber, F_INT2EQ, Int16GetDatum(attnum)); scan = systable_beginscan(pg_attribute, AttributeRelidNumIndexId, true, SnapshotNow, 2, key); tup = systable_getnext(scan); if (!HeapTupleIsValid(tup)) { systable_endscan(scan); heap_close(pg_attribute, NoLock); ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("attribute name lookup failed for table oid %u, attnum %d.", attrelid, attnum))); } TupleDesc desc = RelationGetDescr(pg_attribute); Datum datum = heap_getattr(tup, Anum_pg_attribute_attname, desc, &isnull); Assert(!isnull && datum); name = pstrdup(NameStr(*DatumGetName(datum))); systable_endscan(scan); heap_close(pg_attribute, NoLock); return name; } /* Return the names of all the columns involved in the index. */ List *get_index_attname(Oid index_oid) { HeapTuple index_tup = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(index_oid)); if (!HeapTupleIsValid(index_tup)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("cache lookup failed for index %u", index_oid))); Form_pg_index index_form = (Form_pg_index)GETSTRUCT(index_tup); int2vector *attnums = &(index_form->indkey); Oid indrelid = index_form->indrelid; // get attrnames from table oid. List *attnames = NIL; int2 attnum; int i; for (i = 0; i < attnums->dim1; i++) { attnum = attnums->values[i]; attnames = lappend(attnames, search_table_attname(indrelid, attnum)); } ReleaseSysCache(index_tup); return attnames; } // Execute an SQL statement and return its result. StmtResult *execute_stmt(const char *query_string, bool need_result) { int16 format = 0; List *parsetree_list = NULL; ListCell *parsetree_item = NULL; parsetree_list = pg_parse_query(query_string, NULL); Assert(list_length(parsetree_list) == 1); // ought to be one query parsetree_item = list_head(parsetree_list); Portal portal = NULL; DestReceiver *receiver = NULL; List *querytree_list = NULL; List *plantree_list = NULL; Node *parsetree = (Node *)lfirst(parsetree_item); const char *commandTag = CreateCommandTag(parsetree); querytree_list = pg_analyze_and_rewrite(parsetree, query_string, NULL, 0); plantree_list = pg_plan_queries(querytree_list, 0, NULL); portal = CreatePortal(query_string, true, true); portal->visible = false; PortalDefineQuery(portal, NULL, query_string, commandTag, plantree_list, NULL); if (need_result) receiver = create_stmt_receiver(); else receiver = CreateDestReceiver(DestNone); PortalStart(portal, NULL, 0, NULL); PortalSetResultFormat(portal, 1, &format); (void)PortalRun(portal, FETCH_ALL, true, receiver, receiver, NULL); PortalDrop(portal, false); return (StmtResult *)receiver; } /* create_stmt_receiver * * The existing code cannot store the execution results of SQL statements to the list structure. * Therefore, this routine implements the preceding function through a customized data structure -- StmtResult. * The following routines: receive, startup, shutdown and destroy are the methods of StmtResult. * */ DestReceiver *create_stmt_receiver() { StmtResult *self = (StmtResult *)palloc0(sizeof(StmtResult)); self->pub.receiveSlot = receive; self->pub.rStartup = startup; self->pub.rShutdown = shutdown; self->pub.rDestroy = destroy; self->pub.mydest = DestTuplestore; self->tuples = NIL; self->mxct = CurrentMemoryContext; return (DestReceiver *)self; } void receive(TupleTableSlot *slot, DestReceiver *self) { StmtResult *result = (StmtResult *)self; TupleDesc typeinfo = slot->tts_tupleDescriptor; int natts = typeinfo->natts; int i; Datum origattr, attr; char *value = NULL; bool isnull = false; Oid typoutput; bool typisvarlena = false; List *values = NIL; MemoryContext oldcontext = MemoryContextSwitchTo(result->mxct); for (i = 0; i < natts; ++i) { origattr = tableam_tslot_getattr(slot, i + 1, &isnull); if (isnull) { continue; } getTypeOutputInfo(typeinfo->attrs[i]->atttypid, &typoutput, &typisvarlena); if (typisvarlena) { attr = PointerGetDatum(PG_DETOAST_DATUM(origattr)); } else { attr = origattr; } value = OidOutputFunctionCall(typoutput, attr); values = lappend(values, value); /* Clean up detoasted copy, if any */ if (DatumGetPointer(attr) != DatumGetPointer(origattr)) { pfree(DatumGetPointer(attr)); } } result->tuples = lappend(result->tuples, values); MemoryContextSwitchTo(oldcontext); } void startup(DestReceiver *self, int operation, TupleDesc typeinfo) {} void shutdown(DestReceiver *self) {} void destroy(DestReceiver *self) { StmtResult *result = (StmtResult *)self; ListCell *item = NULL; List *tuples = result->tuples; foreach (item, tuples) { list_free_deep((List *)lfirst(item)); } list_free(tuples); pfree_ext(self); } /* * find_select_stmt * Recursively search for SelectStmt structures within a parse tree. * * The 'SelectStmt' structure may exist in set operations and the 'with', * 'from' and 'where' clauses, and it is added to the global variable * "g_stmt_list" only if it has no subquery structure. */ void find_select_stmt(Node *parsetree) { if (parsetree == NULL || nodeTag(parsetree) != T_SelectStmt) { return; } SelectStmt *stmt = (SelectStmt *)parsetree; g_stmt_list = lappend(g_stmt_list, stmt); switch (stmt->op) { case SETOP_UNION: { // analyze the set operation: union find_select_stmt((Node *)stmt->larg); find_select_stmt((Node *)stmt->rarg); break; } case SETOP_NONE: { // analyze the 'with' clause if (stmt->withClause) { List *cte_list = stmt->withClause->ctes; ListCell *item = NULL; foreach (item, cte_list) { CommonTableExpr *cte = (CommonTableExpr *)lfirst(item); g_tmp_table_list = lappend(g_tmp_table_list, cte->ctename); find_select_stmt(cte->ctequery); } break; } // analyze the 'from' clause if (stmt->fromClause) { extract_stmt_from_clause(stmt->fromClause); } // analyze the 'where' clause if (stmt->whereClause) { extract_stmt_where_clause(stmt->whereClause); } break; } default: { break; } } } void extract_stmt_from_clause(List *from_list) { ListCell *item = NULL; foreach (item, from_list) { Node *from = (Node *)lfirst(item); if (from && IsA(from, RangeSubselect)) { find_select_stmt(((RangeSubselect *)from)->subquery); } if (from && IsA(from, JoinExpr)) { Node *larg = ((JoinExpr *)from)->larg; Node *rarg = ((JoinExpr *)from)->rarg; if (larg && IsA(larg, RangeSubselect)) { find_select_stmt(((RangeSubselect *)larg)->subquery); } if (rarg && IsA(rarg, RangeSubselect)) { find_select_stmt(((RangeSubselect *)rarg)->subquery); } } } } void extract_stmt_where_clause(Node *item_where) { if (IsA(item_where, SubLink)) { find_select_stmt(((SubLink *)item_where)->subselect); } while (item_where && IsA(item_where, A_Expr)) { A_Expr *expr = (A_Expr *)item_where; Node *lexpr = expr->lexpr; Node *rexpr = expr->rexpr; if (lexpr && IsA(lexpr, SubLink)) { find_select_stmt(((SubLink *)lexpr)->subselect); } if (rexpr && IsA(rexpr, SubLink)) { find_select_stmt(((SubLink *)rexpr)->subselect); } item_where = lexpr; } } /* * generate_final_index * Concatenate the candidate indexes of the table into a string of * composite index. * * The composite index is generated only when it is not the same as the * previously suggested indexes or existing indexes. */ void generate_final_index(TableCell *table, Oid table_oid) { char *suggested_index = (char *)palloc0(NAMEDATALEN); ListCell *index = NULL; errno_t rc = EOK; int i = 0; // concatenate the candidate indexes into a string foreach (index, table->index) { char *index_name = ((IndexCell *)lfirst(index))->index_name; if (strlen(index_name) + strlen(suggested_index) + 1 > NAMEDATALEN) { break; } if (i == 0) { rc = strcpy_s(suggested_index, NAMEDATALEN, index_name); } else { rc = strcat_s(suggested_index, NAMEDATALEN, ","); securec_check(rc, "\0", "\0"); rc = strcat_s(suggested_index, NAMEDATALEN, index_name); } securec_check(rc, "\0", "\0"); i++; } list_free_deep(table->index); table->index = NIL; // check the previously suggested indexes ListCell *prev_index = NULL; foreach (prev_index, table->index_print) { if (strcasecmp((char *)lfirst(prev_index), suggested_index) == 0) { pfree(suggested_index); return; } } // check the existed indexes char *existed_index = (char *)palloc0(NAMEDATALEN); List *indexes = get_table_indexes(table_oid); index = NULL; foreach (index, indexes) { List *attnames = get_index_attname(lfirst_oid(index)); ListCell *item = NULL; i = 0; foreach (item, attnames) { if (i == 0) { rc = strcpy_s(existed_index, NAMEDATALEN, (char *)lfirst(item)); } else { rc = strcat_s(existed_index, NAMEDATALEN, ","); securec_check(rc, "\0", "\0"); rc = strcat_s(existed_index, NAMEDATALEN, (char *)lfirst(item)); } securec_check(rc, "\0", "\0"); i++; } // the suggested index has existed if (strcasecmp(existed_index, suggested_index) == 0) { pfree(suggested_index); pfree(existed_index); return; } } table->index_print = lappend(table->index_print, suggested_index); pfree(existed_index); return; } void parse_where_clause(Node *item_where) { if (item_where == NULL) { return; } switch (nodeTag(item_where)) { case T_A_Expr: { A_Expr *expr = (A_Expr *)item_where; Node *lexpr = expr->lexpr; Node *rexpr = expr->rexpr; List *field_value = NIL; if (lexpr == NULL || rexpr == NULL) { return; } switch (expr->kind) { case AEXPR_OP: { // normal operator if (IsA(lexpr, ColumnRef) && IsA(rexpr, ColumnRef)) { // where t1.c1 = t2.c2 parse_join_expr(item_where); } else if (IsA(lexpr, ColumnRef) && IsA(rexpr, A_Const)) { // where t1.c1... field_value = lappend(field_value, (A_Const *)rexpr); parse_field_expr(((ColumnRef *)lexpr)->fields, expr->name, field_value); } break; } case AEXPR_AND: { // where...and... if (IsA(lexpr, A_Expr)) { parse_where_clause(lexpr); } if (IsA(rexpr, A_Expr)) { parse_where_clause(rexpr); } break; } case AEXPR_IN: { // where...in... if (IsA(lexpr, ColumnRef) && IsA(rexpr, List)) { field_value = (List *)rexpr; parse_field_expr(((ColumnRef *)lexpr)->fields, expr->name, field_value); } break; } default: { break; } } } case T_NullTest: { break; } default: { break; } } } void parse_from_clause(List *from_list) { ListCell *from = NULL; foreach (from, from_list) { Node *item = (Node *)lfirst(from); if (nodeTag(item) == T_RangeVar) { // single table RangeVar *table = (RangeVar *)item; if (table->alias) { (void)find_or_create_tblcell(table->relname, table->alias->aliasname); } else { (void)find_or_create_tblcell(table->relname, NULL); } } else if (nodeTag(item) == T_JoinExpr) { // multi-table join parse_join_tree((JoinExpr *)item); } else { } } } void add_drived_tables(RangeVar *join_node) { TableCell *join_table = NULL; if (join_node->alias) { join_table = find_or_create_tblcell(join_node->relname, join_node->alias->aliasname); } else { join_table = find_or_create_tblcell(join_node->relname, NULL); } if (!join_table) { return; } if (!list_member(g_drived_tables, join_table)) { g_drived_tables = lappend(g_drived_tables, join_table); } } void parse_join_tree(JoinExpr *join_tree) { Node *larg = join_tree->larg; Node *rarg = join_tree->rarg; if (nodeTag(larg) == T_JoinExpr) { parse_join_tree((JoinExpr *)larg); } else if (nodeTag(larg) == T_RangeVar) { add_drived_tables((RangeVar *)larg); } if (nodeTag(rarg) == T_JoinExpr) { parse_join_tree((JoinExpr *)rarg); } else if (nodeTag(rarg) == T_RangeVar) { add_drived_tables((RangeVar *)rarg); } if (join_tree->isNatural == true) { return; } if (join_tree->usingClause) { parse_join_expr(join_tree); } else { parse_join_expr(join_tree->quals); } } void add_join_cond(TableCell *begin_table, char *begin_field, TableCell *end_table, char *end_field) { JoinCell *join_cond = NULL; join_cond = (JoinCell *)palloc0(sizeof(*join_cond)); join_cond->field = begin_field; join_cond->table = end_table->table_name; join_cond->table_field = end_field; begin_table->join_cond = lappend(begin_table->join_cond, join_cond); } // parse 'join on' void parse_join_expr(Node *item_join) { if (!item_join) { return; } A_Expr *join_cond = (A_Expr *)item_join; if (!join_cond->lexpr || !join_cond->rexpr) { return; } List *field_value = NIL; if (IsA(join_cond->lexpr, ColumnRef) && IsA(join_cond->rexpr, A_Const)){ // on t1.c1... field_value = lappend(field_value, (A_Const *)join_cond->rexpr); parse_field_expr(((ColumnRef *)join_cond->lexpr)->fields, join_cond->name, field_value); } else if (IsA(join_cond->rexpr, ColumnRef) && IsA(join_cond->lexpr, A_Const)){ // on ...t1.c1 field_value = lappend(field_value, (A_Const *)join_cond->lexpr); parse_field_expr(((ColumnRef *)join_cond->rexpr)->fields, join_cond->name, field_value); } else if (IsA(join_cond->lexpr, ColumnRef) && IsA(join_cond->rexpr, ColumnRef)){ // on t1.c1 = t2.c2 List *l_join_fields = ((ColumnRef *)(join_cond->lexpr))->fields; List *r_join_fields = ((ColumnRef *)(join_cond->rexpr))->fields; char *l_table_name = find_table_name(l_join_fields); char *r_table_name = find_table_name(r_join_fields); char *l_field_name = find_field_name(l_join_fields); char *r_field_name = find_field_name(r_join_fields); if (!l_table_name || !r_table_name || !l_field_name || !r_field_name) { return; } TableCell *ltable = find_or_create_tblcell(l_table_name, NULL); TableCell *rtable = find_or_create_tblcell(r_table_name, NULL); if (!ltable || !rtable) { return; } // add join conditons add_join_cond(ltable, l_field_name, rtable, r_field_name); add_join_cond(rtable, r_field_name, ltable, l_field_name); // add possible drived tables if (!list_member(g_drived_tables, ltable)) { g_drived_tables = lappend(g_drived_tables, ltable); } if (!list_member(g_drived_tables, rtable)) { g_drived_tables = lappend(g_drived_tables, rtable); } } } // parse 'join using' void parse_join_expr(JoinExpr *join_tree) { if (!join_tree) { return; } if (nodeTag(join_tree->larg) != T_RangeVar || nodeTag(join_tree->rarg) != T_RangeVar) { return; } List *join_fields = join_tree->usingClause; char *l_table_name = ((RangeVar *)(join_tree->larg))->relname; char *r_table_name = ((RangeVar *)(join_tree->rarg))->relname; TableCell *ltable = find_or_create_tblcell(l_table_name, NULL); TableCell *rtable = find_or_create_tblcell(r_table_name, NULL); if (!ltable || !rtable) { return; } // add join conditons ListCell *item = NULL; foreach (item, join_fields) { char *field_name = strVal(lfirst(item)); add_join_cond(ltable, field_name, rtable, field_name); add_join_cond(rtable, field_name, ltable, field_name); } } // convert field value to string void field_value_trans(_out_ char *target, A_Const *field_value) { Value value = field_value->val; if (value.type == T_Integer) { pg_itoa(value.val.ival, target); } else if (value.type == T_String) { errno_t rc = sprintf_s(target, MAX_QUERY_LEN, "'%s'", value.val.str); securec_check_ss_c(rc, "\0", "\0"); } } /* * parse_field_expr * Parse the field expression and add index. * * An index is added from the field only if the cardinality of the field * is greater than the threshold. */ void parse_field_expr(List *field, List *op, List *lfield_values) { errno_t rc = EOK; char *table_name = find_table_name(field); char *index_name = find_field_name(field); if (!table_name || !index_name) { return; } char *op_type = strVal(linitial(op)); char *field_expr = (char *)palloc0(MAX_QUERY_LEN); char *field_value = (char *)palloc0(MAX_QUERY_LEN); ListCell *item = NULL; int i = 0; // get field values foreach (item, lfield_values) { char *str = (char *)palloc0(MAX_QUERY_LEN); if (!IsA(lfirst(item), A_Const)) { continue; } field_value_trans(str, (A_Const *)lfirst(item)); if (i == 0) { rc = strcpy_s(field_value, MAX_QUERY_LEN, str); } else { rc = strcat_s(field_value, MAX_QUERY_LEN, ","); securec_check(rc, "\0", "\0"); rc = strcat_s(field_value, MAX_QUERY_LEN, str); } securec_check(rc, "\0", "\0"); i++; pfree(str); } if (i == 0) { pfree(field_value); pfree(field_expr); return; } // get field expression, e.g., 'id = 100' if (strcasecmp(op_type, "~~") == 0) { // ...like... if (field_value != NULL && field_value[1] == '%') { pfree(field_value); pfree(field_expr); return; } else { rc = sprintf_s(field_expr, MAX_QUERY_LEN, "%s like %s", index_name, field_value); } } else if (lfield_values->length > 1) { // ...(not)in... if (strcasecmp(op_type, "=") == 0) { rc = sprintf_s(field_expr, MAX_QUERY_LEN, "%s in (%s)", index_name, field_value); } else { rc = sprintf_s(field_expr, MAX_QUERY_LEN, "%s not in (%s)", index_name, field_value); } } else { // ...>=<... rc = sprintf_s(field_expr, MAX_QUERY_LEN, "%s %s %s", index_name, op_type, field_value); } securec_check_ss_c(rc, "\0", "\0"); pfree(field_value); uint4 cardinality = calculate_field_cardinality(table_name, field_expr); if (cardinality > CARDINALITY_THRESHOLD) { IndexCell *index = (IndexCell *)palloc0(sizeof(*index)); index->index_name = index_name; index->cardinality = cardinality; index->op = op_type; index->field_expr = field_expr; add_index_from_field(table_name, index); } } inline uint4 tuple_to_uint(List *tuples) { return pg_atoi((char *)linitial((List *)linitial(tuples)), sizeof(uint4), 0); } uint4 get_table_count(const char *table_name) { uint4 table_count = 0; char query_string[MAX_QUERY_LEN] = {0x00}; errno_t rc = sprintf_s(query_string, MAX_QUERY_LEN, "select reltuples from pg_class where relname='%s';", table_name); securec_check_ss_c(rc, "\0", "\0"); StmtResult *result = execute_stmt(query_string, true); table_count = tuple_to_uint(result->tuples); (*result->pub.rDestroy)((DestReceiver *)result); return table_count; } uint4 calculate_field_cardinality(const char *table_name, const char *field_expr) { const int sample_factor = 2; uint4 cardinality; uint4 table_count = get_table_count(table_name); uint4 sample_rows = (table_count / sample_factor) > MAX_SAMPLE_ROWS ? MAX_SAMPLE_ROWS : (table_count / sample_factor); char query_string[MAX_QUERY_LEN] = {0x00}; errno_t rc = sprintf_s(query_string, MAX_QUERY_LEN, "select count(*) from ( select * from %s limit %d) where %s", table_name, sample_rows, field_expr); securec_check_ss_c(rc, "\0", "\0"); StmtResult *result = execute_stmt(query_string, true); uint4 row = tuple_to_uint(result->tuples); (*result->pub.rDestroy)((DestReceiver *)result); cardinality = row == 0 ? sample_rows : (sample_rows / row); return cardinality; } char *find_field_name(List *fields) { if (!fields) { return NULL; } char *field_name = NULL; if (fields->length > 1) { field_name = strVal(lsecond(fields)); } else { field_name = strVal(linitial(fields)); } return field_name; } char *find_table_name(List *fields) { if (!fields) { return NULL; } char *table = NULL; ListCell *item = NULL; ListCell *sub_item = NULL; // if fields have table name if (fields->length > 1) { table = strVal(linitial(fields)); // check existed tables foreach (item, g_table_list) { TableCell *cur_table = (TableCell *)lfirst(item); if (strcasecmp(table, cur_table->table_name) == 0) { return cur_table->table_name; } foreach (sub_item, cur_table->alias_name) { char *cur_alias_name = (char *)lfirst(sub_item); if (strcasecmp(table, cur_alias_name) == 0) { return cur_table->table_name; } } } return NULL; } // if fields only have field name int cnt = 0; item = NULL; char query_string[MAX_QUERY_LEN] = {0x00}; char *column_name = strVal(linitial(fields)); // check which table has the given column_name foreach (item, g_table_list) { char *table_name = ((TableCell *)lfirst(item))->table_name; errno_t rc = sprintf_s(query_string, MAX_QUERY_LEN, "select count(*) from information_schema.columns where table_name = '%s' and column_name = '%s'", table_name, column_name); securec_check_ss_c(rc, "\0", "\0"); StmtResult *result = execute_stmt(query_string, true); if (result) { uint4 num = tuple_to_uint(result->tuples); if (num > 0) { cnt++; table = table_name; } (*result->pub.rDestroy)((DestReceiver *)result); } } if (cnt == 1) { return table; } return NULL; } // Check whether the table type is supported. bool check_relation_type_valid(Oid relid) { Relation relation; bool result = false; relation = heap_open(relid, AccessShareLock); if (RelationIsValid(relation) == false) { heap_close(relation, AccessShareLock); return result; } if (RelationIsRelation(relation) && RelationGetPartType(relation) == PARTTYPE_NON_PARTITIONED_RELATION && RelationGetRelPersistence(relation) == RELPERSISTENCE_PERMANENT) { const char *format = ((relation->rd_options) && (((StdRdOptions *)(relation->rd_options))->orientation)) ? ((char *)(relation->rd_options) + *(int *)&(((StdRdOptions *)(relation->rd_options))->orientation)) : ORIENTATION_ROW; if (pg_strcasecmp(format, ORIENTATION_ROW) == 0) { result = true; } } heap_close(relation, AccessShareLock); return result; } bool is_tmp_table(const char *table_name) { ListCell *item = NULL; foreach (item, g_tmp_table_list) { char *tmp_table_name = (char *)lfirst(item); if (strcasecmp(tmp_table_name, table_name) == 0) { return true; } } return false; } /* * find_or_create_tblcell * Find the TableCell that has given table_name(alias_name) among the global * variable 'g_table_list', and if not found, create a TableCell and return it. */ TableCell *find_or_create_tblcell(char *table_name, char *alias_name) { if (!table_name) { return NULL; } if (is_tmp_table(table_name)) { ereport(WARNING, (errmsg("can not advise for: %s.", table_name))); return NULL; } // seach the table among existed tables ListCell *item = NULL; ListCell *sub_item = NULL; if (g_table_list != NIL) { foreach (item, g_table_list) { TableCell *cur_table = (TableCell *)lfirst(item); char *cur_table_name = cur_table->table_name; if (strcasecmp(cur_table_name, table_name) == 0) { if (alias_name) { foreach (sub_item, cur_table->alias_name) { char *cur_alias_name = (char *)lfirst(sub_item); if (strcasecmp(cur_alias_name, alias_name) == 0) { return cur_table; } } cur_table->alias_name = lappend(cur_table->alias_name, alias_name); } return cur_table; } foreach (sub_item, cur_table->alias_name) { char *cur_alias_name = (char *)lfirst(sub_item); if (strcasecmp(cur_alias_name, table_name) == 0) { return cur_table; } } } } RangeVar* rtable = makeRangeVar(NULL, table_name, -1); Oid table_oid = RangeVarGetRelid(rtable, NoLock, true); if (check_relation_type_valid(table_oid) == false) { ereport(WARNING, (errmsg("can not advise for: %s.", table_name))); return NULL; } // create a new table TableCell *new_table = NULL; new_table = (TableCell *)palloc0(sizeof(*new_table)); new_table->table_name = table_name; new_table->alias_name = NIL; if (alias_name) { new_table->alias_name = lappend(new_table->alias_name, alias_name); } new_table->index = NIL; new_table->join_cond = NIL; new_table->index_print = NIL; g_table_list = lappend(g_table_list, new_table); return new_table; } /* * add_index_from_field * Add index from field for the table. * * The index sorting rule is as follows: the index with operator'=' is ranked first, * and the index with higher cardinality is ranked first. */ void add_index_from_field(char *table_name, IndexCell *index) { TableCell *table = find_or_create_tblcell(table_name, NULL); ListCell *table_index = NULL; if (table->index == NIL) { table->index = lappend(table->index, index); } else { ListCell *prev = NULL; foreach (table_index, table->index) { char *table_index_name = ((IndexCell *)lfirst(table_index))->index_name; char *table_index_op = ((IndexCell *)lfirst(table_index))->op; uint4 table_index_cardinality = ((IndexCell *)lfirst(table_index))->cardinality; if (strcasecmp(table_index_name, index->index_name) == 0) { break; } if (strcasecmp(table_index_op, "=") == 0) { if (strcasecmp(index->op, "=") == 0 && table_index_cardinality < index->cardinality) { if (prev == NULL) { table->index = lcons(index, table->index); } else { (void)lappend_cell(table->index, prev, index); } break; } else if (table_index->next == NULL) { table->index = lappend(table->index, index); break; } } else { if (strcasecmp(index->op, "=") == 0) { if (prev == NULL) { table->index = lcons(index, table->index); } else { (void)lappend_cell(table->index, prev, index); } break; } else if (table_index_cardinality < index->cardinality) { if (prev == NULL) { table->index = lcons(index, table->index); } else { (void)lappend_cell(table->index, prev, index); } break; } } prev = table_index; } } } // The table that has smallest result set is set as the driver table. void determine_driver_table() { if (g_table_list->length == 1) { g_driver_table = (TableCell *)linitial(g_table_list); } else { if (g_drived_tables != NIL) { uint4 small_result_set = UINT32_MAX; ListCell *item = NULL; foreach (item, g_drived_tables) { TableCell *table = (TableCell *)lfirst(item); uint4 result_set; if (table->index) { result_set = get_join_table_result_set(table->table_name, ((IndexCell *)linitial(table->index))->field_expr); } else { result_set = get_join_table_result_set(table->table_name, NULL); } if (result_set < small_result_set) { g_driver_table = table; small_result_set = result_set; } } } } list_free(g_drived_tables); g_drived_tables = NIL; } uint4 get_join_table_result_set(const char *table_name, const char *condition) { char query_string[MAX_QUERY_LEN] = {0x00}; errno_t rc = EOK; if (condition == NULL) { rc = sprintf_s(query_string, MAX_QUERY_LEN, "select * from %s;", table_name); } else { rc = sprintf_s(query_string, MAX_QUERY_LEN, "select * from %s where %s;", table_name, condition); } securec_check_ss_c(rc, "\0", "\0"); /* get query execution plan */ List *parsetree_list = pg_parse_query(query_string, NULL); ListCell *parsetree_item = list_head(parsetree_list); Node *parsetree = (Node *)lfirst(parsetree_item); List *querytree_list = pg_analyze_and_rewrite(parsetree, query_string, NULL, 0); List *plantree_list = pg_plan_queries(querytree_list, 0, NULL); PlannedStmt *plan_stmt = (PlannedStmt *)lfirst(list_head(plantree_list)); Scan *scan = (Scan *)(plan_stmt->planTree); return (uint4)scan->plan.plan_rows; } void add_index_from_join(TableCell *table, char *index_name) { ListCell *item = NULL; foreach (item, table->index) { char *table_index_name = ((IndexCell *)lfirst(item))->index_name; if (strcasecmp(table_index_name, index_name) == 0) { return; } } IndexCell *index = (IndexCell *)palloc0(sizeof(*index)); index->index_name = index_name; index->cardinality = 0; index->op = NULL; index->field_expr = NULL; table->index = lcons(index, table->index); } // Add index for the drived tables accordint to the join conditions. void add_index_for_drived_tables() { List *joined_tables = NIL; List *to_be_joined_tables = NIL; to_be_joined_tables = lappend(to_be_joined_tables, g_driver_table); TableCell *join_table; while (to_be_joined_tables != NIL) { join_table = (TableCell *)linitial(to_be_joined_tables); to_be_joined_tables = list_delete_cell2(to_be_joined_tables, list_head(to_be_joined_tables)); if (list_member(joined_tables, join_table)) { continue; } List *join_list = join_table->join_cond; ListCell *item = NULL; foreach (item, join_list) { JoinCell *join_cond = (JoinCell *)lfirst(item); TableCell *to_be_joined_table = find_or_create_tblcell(join_cond->table, NULL); if (list_member(joined_tables, to_be_joined_table)) { continue; } if (join_table != g_driver_table) { add_index_from_join(join_table, join_cond->field); } add_index_from_join(to_be_joined_table, join_cond->table_field); to_be_joined_tables = lappend(to_be_joined_tables, to_be_joined_table); } joined_tables = lappend(joined_tables, join_table); list_free_deep(join_list); join_table->join_cond = NIL; } list_free(joined_tables); list_free(to_be_joined_tables); } Node *transform_group_order_node(Node *node, List *target_list) { Value *val = &((A_Const *)node)->val; Assert(IsA(val, Integer)); long target_pos = intVal(val); Assert(target_pos <= list_length(target_list)); ResTarget *rt = (ResTarget *)list_nth(target_list, target_pos - 1); node = rt->val; return node; } char *parse_group_clause(List *group_clause, List *target_list) { if (group_clause == NULL) return NULL; bool is_only_one_table = true; ListCell *group_item = NULL; char *pre_table = NULL; foreach (group_item, group_clause) { Node *node = (Node *)lfirst(group_item); if (nodeTag(node) == T_A_Const) { node = transform_group_order_node(node, target_list); } if (nodeTag(node) != T_ColumnRef) break; List *fields = ((ColumnRef *)node)->fields; char *table_group = find_table_name(fields); if (!table_group) { return NULL; } if (pre_table != NULL && strcasecmp(table_group, pre_table) != 0) { is_only_one_table = false; break; } pre_table = table_group; } if (is_only_one_table && pre_table) { if (g_table_list->length == 1 || (g_driver_table && strcasecmp(pre_table, g_driver_table->table_name) == 0)) { return pre_table; } } return NULL; } char *parse_order_clause(List *order_clause, List *target_list) { if (order_clause == NULL) return NULL; bool is_only_one_table = true; ListCell *order_item = NULL; char *pre_table = NULL; SortByDir pre_dir; foreach (order_item, order_clause) { Node *node = ((SortBy *)lfirst(order_item))->node; SortByDir dir = ((SortBy *)(lfirst(order_item)))->sortby_dir; if (nodeTag(node) == T_A_Const) { node = transform_group_order_node(node, target_list); } if (nodeTag(node) != T_ColumnRef) break; List *fields = ((ColumnRef *)node)->fields; char *table_order = find_table_name(fields); if (!table_order) { return NULL; } if (pre_table != NULL && (strcasecmp(table_order, pre_table) != 0 || dir != pre_dir)) { is_only_one_table = false; break; } pre_table = table_order; pre_dir = dir; } if (is_only_one_table && pre_table) { if (g_table_list->length == 1 || (g_driver_table && strcasecmp(pre_table, g_driver_table->table_name) == 0)) { return pre_table; } } return NULL; } /* * add_index_from_group_order * Add index from the group or order clause. * * The index from goup or order clause is added after the index with operator '='. */ void add_index_from_group_order(TableCell *table, List *clause, List *target_list, bool flag_group_order) { ListCell *item = NULL; foreach (item, clause) { Node *node = NULL; List *fields = NULL; char *index_name = NULL; if (flag_group_order) { node = (Node *)lfirst(item); } else { node = ((SortBy *)lfirst(item))->node; } if (nodeTag(node) == T_A_Const) { node = transform_group_order_node(node, target_list); } if (nodeTag(node) != T_ColumnRef) break; fields = ((ColumnRef *)node)->fields; index_name = find_field_name(fields); IndexCell *index = (IndexCell *)palloc0(sizeof(*index)); index->index_name = index_name; index->cardinality = 0; index->op = NULL; index->field_expr = NULL; if (table->index == NIL) { table->index = lappend(table->index, index); } else { ListCell *cur = NULL; ListCell *prev = NULL; foreach (cur, table->index) { IndexCell *table_index = (IndexCell *)lfirst(cur); if (index->index_name == NULL || strcasecmp(table_index->index_name, index->index_name) == 0) { break; } if (table_index->op && strcasecmp(table_index->op, "=") != 0) { if (prev == NULL) { table->index = lcons(index, table->index); } else { (void)lappend_cell(table->index, prev, index); } break; } if (cur == list_tail(table->index)) { (void)lappend_cell(table->index, cur, index); break; } } } } }