From 656e011efdfbfa592247cf133aaa72330b8d4fd4 Mon Sep 17 00:00:00 2001 From: Mijamind Date: Tue, 17 Oct 2023 14:31:14 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=BA=90=E6=B1=A0=E5=8C=96?= =?UTF-8?q?=E5=A4=9A=E6=9C=BA=E5=B9=B6=E8=A1=8Cbugfix=E3=80=91=201.?= =?UTF-8?q?=E4=BF=AE=E5=A4=8Dspqplugin&dolphin=E7=BC=96=E8=AF=91=E5=86=B2?= =?UTF-8?q?=E7=AA=81=202.=E8=A7=A3=E5=86=B3orca=20memory=20leak=203.Q8?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E4=B8=8D=E4=B8=80=E8=87=B4=E9=97=AE=E9=A2=98?= =?UTF-8?q?=E8=A7=A3=E5=86=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- contrib/Makefile | 3 +- .../backend/pgxc_single/locator/locator.cpp | 20 +++++ src/common/backend/utils/cache/lsyscache.cpp | 5 +- src/gausskernel/optimizer/plan/planner.cpp | 85 +++++++++++++++++++ .../process/threadpool/knl_session.cpp | 3 + .../knl/knl_guc/knl_session_attr_spq.h | 3 - src/include/knl/knl_session.h | 12 +++ src/include/optimizer/planner.h | 5 ++ src/include/parser/parse_coerce.h | 3 + src/include/pgxc/locator.h | 5 ++ src/include/utils/lsyscache.h | 12 +-- 11 files changed, 141 insertions(+), 15 deletions(-) diff --git a/contrib/Makefile b/contrib/Makefile index 3ac21ab3d..f1f57913c 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -56,8 +56,7 @@ SUBDIRS = \ unaccent \ vacuumlo \ security_plugin \ - ndpplugin \ - spq_plugin + ndpplugin ifeq ($(with_openssl),yes) SUBDIRS += sslinfo diff --git a/src/common/backend/pgxc_single/locator/locator.cpp b/src/common/backend/pgxc_single/locator/locator.cpp index 088e4f737..ffb4c5362 100644 --- a/src/common/backend/pgxc_single/locator/locator.cpp +++ b/src/common/backend/pgxc_single/locator/locator.cpp @@ -1371,3 +1371,23 @@ int GetMinDnNum() return dataNodeNum; } + +#ifdef USE_SPQ +bool IsSpqTypeDistributable(Oid col_type) +{ + if (col_type == INT8OID || col_type == INT1OID || col_type == INT2OID || col_type == INT4OID || + col_type == NUMERICOID || col_type == CHAROID || col_type == BPCHAROID || col_type == VARCHAROID || + col_type == NVARCHAR2OID || col_type == DATEOID || col_type == TIMEOID || col_type == TIMESTAMPOID || + col_type == TIMESTAMPTZOID || col_type == INTERVALOID || col_type == TIMETZOID || + col_type == SMALLDATETIMEOID || col_type == TEXTOID || col_type == CLOBOID || col_type == UUIDOID) + return true; + /* SPQ support extended data types*/ + if (col_type == OIDOID || col_type == ABSTIMEOID || + col_type == RELTIMEOID || col_type == CASHOID || col_type == BYTEAOID || col_type == RAWOID || + col_type == BOOLOID || col_type == NAMEOID || col_type == INT2VECTOROID || col_type == OIDVECTOROID || + col_type == FLOAT4OID || col_type == FLOAT8OID || col_type == BYTEAWITHOUTORDERWITHEQUALCOLOID) + return true; + + return false; +} +#endif diff --git a/src/common/backend/utils/cache/lsyscache.cpp b/src/common/backend/utils/cache/lsyscache.cpp index 77ee10f32..9f8d942ea 100644 --- a/src/common/backend/utils/cache/lsyscache.cpp +++ b/src/common/backend/utils/cache/lsyscache.cpp @@ -91,6 +91,7 @@ #include "catalog/pg_inherits.h" #include "catalog/pg_aggregate.h" #include "catalog/pg_trigger.h" +#include "parser/parse_coerce.h" #endif /* ---------- AMOP CACHES ---------- */ @@ -5713,7 +5714,7 @@ bool type_exists(Oid oid) * get_comparison_type * Retrieve comparison type */ -CmpType get_comparison_type(Oid oidOp) +SPQCmpType get_comparison_type(Oid oidOp) { OpBtreeInterpretation *opBti; List *opBtis; @@ -5828,7 +5829,7 @@ char *get_check_constraint_name(Oid oidCheckconstraint) * get_comparison_operator * Retrieve comparison operator between given types */ -Oid get_comparison_operator(Oid oidLeft, Oid oidRight, CmpType cmpt) +Oid get_comparison_operator(Oid oidLeft, Oid oidRight, SPQCmpType cmpt) { int16 opstrat; HeapTuple ht; diff --git a/src/gausskernel/optimizer/plan/planner.cpp b/src/gausskernel/optimizer/plan/planner.cpp index 1433c864c..654ea5e04 100755 --- a/src/gausskernel/optimizer/plan/planner.cpp +++ b/src/gausskernel/optimizer/plan/planner.cpp @@ -15945,3 +15945,88 @@ adjust_plan_for_srfs(PlannerInfo *root, Plan *plan, List *targets, List *targets } return newplan; } + +#ifdef USE_SPQ +static Node* get_spq_multiple_from_expr( + PlannerInfo* root, Node* expr, double rows, double* skew_multiple, double* bias_multiple) +{ + List* groupExprs = NIL; + Oid datatype = exprType((Node*)(expr)); + bool use_skew_multiple = true; + + if (!OidIsValid(datatype) || !IsSpqTypeDistributable(datatype)) + return NULL; + + groupExprs = list_make1(expr); + get_multiple_from_exprlist(root, groupExprs, rows, &use_skew_multiple, true, skew_multiple, bias_multiple); + list_free_ext(groupExprs); + + return expr; +} + + +List* spq_get_distributekey_from_tlist( + PlannerInfo* root, List* tlist, List* groupcls, double rows, double* result_multiple, void* skew_info) +{ + ListCell* lcell = NULL; + List* distkey = NIL; + double multiple = 0.0; + double bias_multiple = 0.0; + double skew_multiple = 0.0; + List* exprMultipleList = NIL; + + foreach (lcell, groupcls) { + Node* expr = (Node*)lfirst(lcell); + + if (IsA(expr, SortGroupClause)) + expr = get_sortgroupclause_expr((SortGroupClause*)expr, tlist); + + expr = get_spq_multiple_from_expr(root, expr, rows, &skew_multiple, &bias_multiple); + if (expr != NULL) { + /* + * we can't estimate skew of grouping sets because there's + * null added, so just add all columns and set mutiple to 1 + */ + if (root->parse->groupingSets) { + distkey = lappend(distkey, expr); + *result_multiple = 1; + continue; + } + if ((skew_multiple == 1.0) && (bias_multiple <= 1.0)) { + *result_multiple = 1; + list_free_ext(exprMultipleList); + return list_make1(expr); + } else if ((u_sess->pgxc_cxt.NumDataNodes == skew_multiple) && + (u_sess->pgxc_cxt.NumDataNodes == + bias_multiple)) { /* All the expr are const, return the first expr. */ + if (distkey == NULL) + distkey = lappend(distkey, expr); + *result_multiple = u_sess->pgxc_cxt.NumDataNodes; + + continue; + } else { + if (skew_multiple == 1.0) { + /* + * If distinct num of multiple has no skew, we should use bias multiple to + * compute mix multiple. + */ + multiple = bias_multiple; + } + else if (bias_multiple <= 1.0) /* mcf has no skew, handle skew_multiple */ + multiple = skew_multiple; + else + multiple = Max(bias_multiple, skew_multiple); + + exprMultipleList = add_multiple_to_list(expr, multiple, exprMultipleList); + } + } + } + + if (exprMultipleList != NULL) { + distkey = get_mix_diskey_by_exprlist(root, exprMultipleList, rows, result_multiple, (AggSkewInfo*)skew_info); + list_free_ext(exprMultipleList); + } + + return distkey; +} +#endif diff --git a/src/gausskernel/process/threadpool/knl_session.cpp b/src/gausskernel/process/threadpool/knl_session.cpp index 6369903b2..941ac7027 100755 --- a/src/gausskernel/process/threadpool/knl_session.cpp +++ b/src/gausskernel/process/threadpool/knl_session.cpp @@ -1395,6 +1395,8 @@ static void knl_u_spq_init(knl_u_spq_context* spq_cxt) spq_cxt->m_pstrmap = NULL; spq_cxt->m_pxmlszmap = NULL; spq_cxt->m_mp = NULL; + spq_cxt->m_token_parse_handler_func_map = NULL; + spq_cxt->m_xform_mp = NULL; spq_cxt->m_memory_pool_mgr = NULL; spq_cxt->m_worker_pool_manager = NULL; spq_cxt->m_pcache = NULL; @@ -1404,6 +1406,7 @@ static void knl_u_spq_init(knl_u_spq_context* spq_cxt) spq_cxt->spq_worker_context = NULL; spq_cxt->spq_max_tuple_chunk_size = 0; spq_cxt->s_tupSerMemCtxt = NULL; + spq_cxt->spq_opt_initialized = false; } #endif diff --git a/src/include/knl/knl_guc/knl_session_attr_spq.h b/src/include/knl/knl_guc/knl_session_attr_spq.h index 5b4a6954b..8d80ed5d9 100644 --- a/src/include/knl/knl_guc/knl_session_attr_spq.h +++ b/src/include/knl/knl_guc/knl_session_attr_spq.h @@ -39,9 +39,6 @@ #define SRC_INCLUDE_KNL_KNL_SESSION_ATTR_MPP_H_ #include "knl/knl_guc/knl_guc_common.h" -#ifdef PGXC -#include "pgxc/nodemgr.h" -#endif struct NodeDefinition; diff --git a/src/include/knl/knl_session.h b/src/include/knl/knl_session.h index 09e586d42..2f4b69af2 100644 --- a/src/include/knl/knl_session.h +++ b/src/include/knl/knl_session.h @@ -2667,6 +2667,7 @@ typedef struct knl_u_mot_context { namespace spqdxl { class CDXLMemoryManager; class CDXLTokens; + class CParseHandlerFactory; } namespace spqos { @@ -2674,6 +2675,8 @@ namespace spqos { class CMemoryPoolManager; class CWorkerPoolManager; template class CCache; + class CCacheFactory; + class CMessageRepository; } namespace spqmd { @@ -2682,6 +2685,7 @@ namespace spqmd { namespace spqopt { class CMDKey; + class CXformFactory; } @@ -2694,6 +2698,7 @@ typedef struct knl_u_spq_context { uintptr_t m_ulpShutdownDXL; void *m_pstrmap; void *m_pxmlszmap; + void *m_token_parse_handler_func_map; spqos::CMemoryPool* m_mp; spqdxl::CDXLMemoryManager* m_dxl_memory_manager; /* memory pool manager */ @@ -2702,12 +2707,19 @@ typedef struct knl_u_spq_context { spqos::CWorkerPoolManager* m_worker_pool_manager; /* mdcache */ spqos::CCache *m_pcache; + /* cache factory */ + spqos::CCacheFactory* m_factory; + spqos::CMessageRepository *m_repository; + /* CXformFactory */ + spqos::CMemoryPool* m_xform_mp; + spqopt::CXformFactory* m_pxff; uint64 m_ullCacheQuota; int spq_node_all_configs_size; int spq_node_configs_size; MemoryContext spq_worker_context; MemoryContext s_tupSerMemCtxt; int32 spq_max_tuple_chunk_size; + bool spq_opt_initialized; } knl_u_spq_context; #endif diff --git a/src/include/optimizer/planner.h b/src/include/optimizer/planner.h index 667b057d8..1c7800c81 100644 --- a/src/include/optimizer/planner.h +++ b/src/include/optimizer/planner.h @@ -217,4 +217,9 @@ extern bool queryIsReadOnly(Query* query); typedef PlannedStmt* (*plannerFunc)(Query* parse, int cursorOptions, ParamListInfo boundParams); +#ifdef USE_SPQ +extern List* spq_get_distributekey_from_tlist( + PlannerInfo* root, List* tlist, List* groupcls, double rows, double* result_multiple, void* skew_info = NULL); +#endif + #endif /* PLANNER_H */ diff --git a/src/include/parser/parse_coerce.h b/src/include/parser/parse_coerce.h index 1fbd34b37..f709f9154 100644 --- a/src/include/parser/parse_coerce.h +++ b/src/include/parser/parse_coerce.h @@ -83,4 +83,7 @@ extern Node* coerce_to_target_charset(Node* expr, int target_charset, Oid target extern Node *transferConstToAconst(Node *node); extern Const* setValueToConstExpr(SetVariableExpr* set); +#ifdef USE_SPQ +extern bool get_cast_func(Oid oidSrc, Oid oidDest, bool *is_binary_coercible, Oid *oidCastFunc, CoercionPathType *pathtype); +#endif #endif /* PARSE_COERCE_H */ diff --git a/src/include/pgxc/locator.h b/src/include/pgxc/locator.h index 22712478e..0f548bfe6 100644 --- a/src/include/pgxc/locator.h +++ b/src/include/pgxc/locator.h @@ -249,4 +249,9 @@ extern void ConstructSliceBoundary(ExecNodes* en); extern Expr* pgxc_find_distcol_expr(void* query, Index varno, AttrNumber attrNum, Node* quals); extern bool IsFunctionShippable(Oid foid); + +#ifdef USE_SPQ +extern bool IsSpqTypeDistributable(Oid colType); +#endif + #endif /* LOCATOR_H */ diff --git a/src/include/utils/lsyscache.h b/src/include/utils/lsyscache.h index 8015eab2f..e957204f0 100644 --- a/src/include/utils/lsyscache.h +++ b/src/include/utils/lsyscache.h @@ -19,9 +19,6 @@ #include "catalog/pg_workload_group.h" #include "catalog/pg_app_workloadgroup_mapping.h" #include "catalog/pgxc_node.h" -#ifdef USE_SPQ -#include "parser/parse_coerce.h" -#endif /* Result list element for get_op_btree_interpretation */ typedef struct OpBtreeInterpretation { @@ -225,7 +222,7 @@ extern char get_typecategory(Oid typid); #ifdef USE_SPQ /* comparison types */ -typedef enum CmpType { +typedef enum SPQCmpType { CmptEq, // equality CmptNEq, // inequality CmptLT, // less than @@ -233,7 +230,7 @@ typedef enum CmpType { CmptGT, // greater than CmptGEq, // greater or equal to CmptOther // other operator -} CmpType; +} SPQCmpType; #define ATTSTATSSLOT_VALUES 0x01 #define ATTSTATSSLOT_NUMBERS 0x02 @@ -268,7 +265,6 @@ extern bool index_exists(Oid oid); extern bool aggregate_exists(Oid oid); extern Oid get_aggregate(const char *aggname, Oid oidType); extern bool function_exists(Oid oid); -extern bool get_cast_func(Oid oidSrc, Oid oidDest, bool *is_binary_coercible, Oid *oidCastFunc, CoercionPathType *pathtype); extern bool check_constraint_exists(Oid oidCheckconstraint); extern char *get_check_constraint_name(Oid oidCheckconstraint); extern Oid get_check_constraint_relid(Oid oidCheckconstraint); @@ -277,8 +273,8 @@ extern Node *get_check_constraint_expr_tree(Oid oidCheckconstraint); extern bool operator_exists(Oid oid); extern bool relation_exists(Oid oid); extern bool type_exists(Oid oid); -extern CmpType get_comparison_type(Oid oidOp); -extern Oid get_comparison_operator(Oid oidLeft, Oid oidRight, CmpType cmpt); +extern SPQCmpType get_comparison_type(Oid oidOp); +extern Oid get_comparison_operator(Oid oidLeft, Oid oidRight, SPQCmpType cmpt); extern bool has_subclass_slow(Oid relationId); extern List *get_operator_opfamilies(Oid opno); extern List *get_index_opfamilies(Oid oidIndex);