database/openGauss-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pgfdw support join\agg\sort\limit\lockrows

Browse Source
This commit is contained in:
gentle_hu
2022-11-15 15:31:38 +08:00
parent 0d771b4357
commit f995bd2209
58 changed files with 34971 additions and 1787 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/include/bulkload/foreignroutine.h
View File

@ -83,7 +83,7 @@ extern void ProcessDistImportOptions(DistImportPlanState *planstate, List *optio
extern void distImportGetRelSize(PlannerInfo* root, RelOptInfo* baserel, Oid foreigntableid);
extern void distImportGetPaths(PlannerInfo* root, RelOptInfo* baserel, Oid foreigntableid);
extern ForeignScan* distImportGetPlan(PlannerInfo* root, RelOptInfo* baserel, Oid foreigntableid,
ForeignPath* best_path, List* tlist, List* scan_clauses);
ForeignPath* best_path, List* tlist, List* scan_clauses, Plan *outer_plan);
extern void distImportExplain(ForeignScanState* node, ExplainState* es);

1
src/include/commands/explain.h
View File

@ -472,6 +472,7 @@ typedef struct ExplainState {
char* statement_id; /* statement_id for EXPLAIN PLAN */
bool is_explain_gplan;
char* opt_model_name;
ExplainFRSqlState es_frs; /* explain state for remote sql of foreign scan. */
} ExplainState;
/* Hook for plugins to get control in explain_get_index_name() */

17
src/include/foreign/fdwapi.h
View File

@ -30,7 +30,7 @@ typedef void (*GetForeignRelSize_function)(PlannerInfo* root, RelOptInfo* basere
typedef void (*GetForeignPaths_function)(PlannerInfo* root, RelOptInfo* baserel, Oid foreigntableid);
typedef ForeignScan* (*GetForeignPlan_function)(PlannerInfo* root, RelOptInfo* baserel, Oid foreigntableid,
ForeignPath* best_path, List* tlist, List* scan_clauses);
ForeignPath* best_path, List* tlist, List* scan_clauses, Plan *outer_plan);
typedef void (*BeginForeignScan_function)(ForeignScanState* node, int eflags);
@ -40,6 +40,11 @@ typedef void (*ReScanForeignScan_function)(ForeignScanState* node);
typedef void (*EndForeignScan_function)(ForeignScanState* node);
typedef void (*GetForeignJoinPaths_function)(PlannerInfo *root, RelOptInfo *joinrel, RelOptInfo *outerrel,
RelOptInfo *innerrel, JoinType jointype, SpecialJoinInfo* sjinfo, List* restrictlist);
typedef void (*GetForeignUpperPaths_function)(FDWUpperRelCxt* ufdwCxt, UpperRelationKind stage, Plan* mainPlan);
typedef void (*AddForeignUpdateTargets_function)(Query* parsetree, RangeTblEntry* target_rte, Relation target_relation);
typedef List* (*PlanForeignModify_function)(
@ -63,6 +68,8 @@ typedef int (*IsForeignRelUpdatable_function)(Relation rel);
typedef void (*ExplainForeignScan_function)(ForeignScanState* node, struct ExplainState* es);
typedef void (*ExplainForeignScanRemote_function)(ForeignScanState* node, struct ExplainState* es);
typedef void (*ExplainForeignModify_function)(
ModifyTableState* mtstate, ResultRelInfo* rinfo, List* fdw_private, int subplan_index, struct ExplainState* es);
@ -140,6 +147,12 @@ typedef struct FdwRoutine {
* These functions are optional. Set the pointer to NULL for any that are
* not provided.
*/
/* Functions for remote-join planning */
GetForeignJoinPaths_function GetForeignJoinPaths;
/* Functions for remote upper-relation (post scan/join) planning */
GetForeignUpperPaths_function GetForeignUpperPaths;
/* Functions for updating foreign tables */
AddForeignUpdateTargets_function AddForeignUpdateTargets;
@ -154,6 +167,7 @@ typedef struct FdwRoutine {
/* Support functions for EXPLAIN */
ExplainForeignScan_function ExplainForeignScan;
ExplainForeignModify_function ExplainForeignModify;
ExplainForeignScanRemote_function ExplainForeignScanRemote;
/* @hdfs Support functions for ANALYZE */
AnalyzeForeignTable_function AnalyzeForeignTable;
@ -202,5 +216,6 @@ extern FdwRoutine* GetFdwRoutine(Oid fdwhandler);
extern FdwRoutine* GetFdwRoutineByRelId(Oid relid, bool missHandlerOk = false);
extern FdwRoutine* GetFdwRoutineByServerId(Oid serverid);
extern FdwRoutine* GetFdwRoutineForRelation(Relation relation, bool makecopy);
extern Oid GetForeignServerIdByRelId(Oid relid);
#endif /* FDWAPI_H */

111
src/include/foreign/foreign.h
View File

@ -16,6 +16,7 @@
#include "access/obs/obs_am.h"
#include "commands/defrem.h"
#include "nodes/parsenodes.h"
#include "nodes/relation.h"
#ifndef OBS_SERVER
#define OBS_SERVER "obs"
@ -223,7 +224,7 @@ extern bool IsSpecifiedFDWFromRelid(Oid relId, const char* SepcifiedType);
* @in relId: The foreign table Oid.
* @return Rreturn true if the foreign table support those DML.
*/
extern bool CheckSupportedFDWType(Oid relId);
extern bool CheckSupportedFDWType(Oid oid, bool byServerId = false);
/**
* @Description: Get the all options for the OBS foreign table.
@ -304,11 +305,101 @@ void CheckGetServerIpAndPort(const char *Address, List **AddrList, bool IsCheck,
bool isWriteOnlyFt(Oid relid);
/*
* state for explain foreign remote sql
*/
typedef struct ExplainFRSqlState {
struct ExplainState* parent;
int node_num;
StringInfo str; /* output buffer */
} ExplainFRSqlState;
typedef struct SPJPathExtraData {
List *targetList;
} SPJPathExtraData;
/*
* Struct for extra information passed to subroutines of create_grouping_paths
*
* flags indicating what kinds of grouping are possible.
* partial_costs_set is true if the agg_partial_costs and agg_final_costs
* have been initialized.
* agg_partial_costs gives partial aggregation costs.
* agg_final_costs gives finalization costs.
* target_parallel_safe is true if target is parallel safe.
* havingQual gives list of quals to be applied after aggregation.
* targetList gives list of columns to be projected.
* patype is the type of partitionwise aggregation that is being performed.
*/
typedef struct GroupPathExtraData {
/* Data which remains constant once set. */
int flags;
bool partial_costs_set;
AggClauseCosts agg_partial_costs;
AggClauseCosts agg_final_costs;
/* Data which may differ across partitions. */
Node *havingQual;
List *targetList;
} GroupPathExtraData;
typedef double Cardinality; /* (estimated) number of rows or other integer count */
typedef struct OrderPathExtraData {
List *targetList;
List *irel_tel; // target entry list of lefttree
} OrderPathExtraData;
/*
* Struct for extra information passed to subroutines of grouping_planner
*
* limit_needed is true if we actually need a Limit plan node.
* limit_tuples is an estimated bound on the number of output tuples,
* or -1 if no LIMIT or couldn't estimate.
* count_est and offset_est are the estimated values of the LIMIT and OFFSET
* expressions computed by preprocess_limit() (see comments for
* preprocess_limit() for more information).
*/
typedef struct FinalPathExtraData {
bool limit_needed;
Cardinality limit_tuples;
int64 count_est;
int64 offset_est;
List* targetList;
} FinalPathExtraData;
typedef enum FDWUpperRelState {
FDW_UPPER_REL_INIT,
FDW_UPPER_REL_TRY,
FDW_UPPER_REL_END
} FDWUpperRelState;
typedef struct FDWUpperRelCxt {
FDWUpperRelState state;
PlannerInfo* root;
RelOptInfo* currentRel; // current rel to create path
RelOptInfo* upperRels[UPPERREL_FINAL + 1]; // the rel that we using to create foreign path
SPJPathExtraData* spjExtra;
GroupPathExtraData* groupExtra;
OrderPathExtraData* orderExtra;
FinalPathExtraData* finalExtra;
/* result */
Plan* resultPlan;
List* resultPathKeys;
} FDWUpperRelCxt;
#define FDWUpperPlanContinue(cxt) ((cxt) != NULL && (cxt)->stage != FDW_UPPER_REL_END)
extern FDWUpperRelCxt* InitFDWUpperPlan(PlannerInfo* root, RelOptInfo* baseRel, Plan* localPlan);
extern void AdvanceFDWUpperPlan(FDWUpperRelCxt* ufdwCxt, UpperRelationKind stage, Plan* localPlan);
#define isObsOrHdfsTableFormTblOid(relId) \
(isSpecifiedSrvTypeFromRelId(relId, HDFS) || isSpecifiedSrvTypeFromRelId(relId, OBS))
(OidIsValid(relId) && (isSpecifiedSrvTypeFromRelId(relId, HDFS) || isSpecifiedSrvTypeFromRelId(relId, OBS)))
#define isMOTFromTblOid(relId) \
(IsSpecifiedFDWFromRelid(relId, MOT_FDW))
(OidIsValid(relId) && IsSpecifiedFDWFromRelid(relId, MOT_FDW))
#define isObsOrHdfsTableFormSrvName(srvName) \
(isSpecifiedSrvTypeFromSrvName(srvName, HDFS) || isSpecifiedSrvTypeFromSrvName(srvName, OBS))
@ -320,16 +411,18 @@ bool isWriteOnlyFt(Oid relid);
(IsSpecifiedFDW(srvName, POSTGRES_FDW))
#define isMysqlFDWFromTblOid(relId) \
(IsSpecifiedFDWFromRelid(relId, MYSQL_FDW))
(OidIsValid(relId) && IsSpecifiedFDWFromRelid(relId, MYSQL_FDW))
#define isOracleFDWFromTblOid(relId) \
(IsSpecifiedFDWFromRelid(relId, ORACLE_FDW))
(OidIsValid(relId) && IsSpecifiedFDWFromRelid(relId, ORACLE_FDW))
#define isPostgresFDWFromTblOid(relId) \
(IsSpecifiedFDWFromRelid(relId, POSTGRES_FDW))
(OidIsValid(relId) && IsSpecifiedFDWFromRelid(relId, POSTGRES_FDW))
#define IS_OBS_CSV_TXT_FOREIGN_TABLE(relId) \
(IsSpecifiedFDWFromRelid(relId, DIST_FDW) && (is_obs_protocol(HdfsGetOptionValue(relId, optLocation))))
(OidIsValid(relId) && \
IsSpecifiedFDWFromRelid(relId, DIST_FDW) && \
(is_obs_protocol(HdfsGetOptionValue(relId, optLocation))))
#define CAN_BUILD_INFORMATIONAL_CONSTRAINT_BY_RELID(relId) \
(isObsOrHdfsTableFormTblOid(relId) || IS_OBS_CSV_TXT_FOREIGN_TABLE(relId))
@ -340,9 +433,9 @@ bool isWriteOnlyFt(Oid relid);
? false \
: is_obs_protocol(getFTOptionValue(stmt->options, optLocation))))
#define IS_LOGFDW_FOREIGN_TABLE(relId) (IsSpecifiedFDWFromRelid(relId, LOG_FDW))
#define IS_LOGFDW_FOREIGN_TABLE(relId) (OidIsValid(relId) && IsSpecifiedFDWFromRelid(relId, LOG_FDW))
#define IS_POSTGRESFDW_FOREIGN_TABLE(relId) (IsSpecifiedFDWFromRelid(relId, GC_FDW))
#define IS_POSTGRESFDW_FOREIGN_TABLE(relId) (OidIsValid(relId) && IsSpecifiedFDWFromRelid(relId, GC_FDW))
#define ENCRYPT_STR_PREFIX "encryptstr"
#define MIN_ENCRYPTED_PASSWORD_LENGTH 54

1
src/include/knl/knl_guc/knl_session_attr_common.h
View File

@ -209,6 +209,7 @@ typedef struct knl_session_attr_common {
bool enable_wdr_snapshot;
bool enable_set_variable_b_format;
bool enable_asp;
bool show_fdw_remote_plan;
int wdr_snapshot_interval;
int wdr_snapshot_retention_days;
int asp_sample_interval;

1
src/include/miscadmin.h
View File

@ -117,6 +117,7 @@ extern const uint32 STANDBY_STMTHIST_VERSION_NUM;
extern const uint32 PG_AUTHID_PASSWORDEXT_VERSION_NUM;
extern const uint32 MAT_VIEW_RECURSIVE_VERSION_NUM;
extern const uint32 SUPPORT_VIEW_AUTO_UPDATABLE;
extern const uint32 FDW_SUPPORT_JOIN_AGG_VERSION_NUM;
extern void register_backend_version(uint32 backend_version);
extern bool contain_backend_version(uint32 version_number);

1
src/include/nodes/execnodes.h
View File

@ -2007,6 +2007,7 @@ typedef struct WorkTableScanState {
*/
typedef struct ForeignScanState {
ScanState ss; /* its first field is NodeTag */
ExprState* fdw_recheck_quals; /* original quals not in ss.ps.qual */
/* use struct pointer to avoid including fdwapi.h here */
struct FdwRoutine* fdwroutine;
void* fdw_state; /* foreign-data wrapper can keep state here */

1
src/include/nodes/pg_list.h
View File

@ -251,6 +251,7 @@ extern List* lcons_int(int datum, List* list);
extern List* lcons_oid(Oid datum, List* list);
extern List* list_concat(List* list1, List* list2);
extern List* list_concat2(List* list1, List* list2);
extern List* list_truncate(List* list, int new_size);
extern void* list_nth(const List* list, int n);

11
src/include/nodes/plannodes.h
View File

@ -921,11 +921,10 @@ typedef struct WorkTableScan {
typedef struct ForeignScan {
Scan scan;
Oid scan_relid; /* Oid of the scan relation */
Oid scan_relid; /* Oid of the scan relation, InValidOid if this is a join\agg foreign scan. */
List* fdw_exprs; /* expressions that FDW may evaluate */
List* fdw_private; /* private data for FDW */
bool fsSystemCol; /* true if any "system column" is needed */
bool needSaveError;
ErrorCacheEntry* errCache; /* Error record cache */
@ -945,6 +944,14 @@ typedef struct ForeignScan {
*/
bool in_compute_pool;
bool not_use_bloomfilter; /* set true in ExecInitXXXX() of planrouter node */
// using for pg_fdw
CmdType operation; /* SELECT/INSERT/UPDATE/DELETE */
Index resultRelation; /* direct modification target's RT index */
Oid fs_server; /* OID of foreign server */
Bitmapset *fs_relids; /* RTIs generated by this scan */
List *fdw_scan_tlist; /* optional tlist describing scan tuple */
List *fdw_recheck_quals; /* original quals not in scan.plan.qual */
} ForeignScan;
/* ----------------

49
src/include/nodes/relation.h
View File

@ -85,6 +85,23 @@ typedef struct AggClauseCosts {
int aggWidth; /* total width of agg function */
} AggClauseCosts;
/*
* This enum identifies the different types of "upper" (post-scan/join)
* relations that we might deal with during planning.
*/
typedef enum UpperRelationKind {
UPPERREL_INIT, /* is a base rel */
UPPERREL_SETOP, /* result of UNION/INTERSECT/EXCEPT, if any */
UPPERREL_GROUP_AGG, /* result of grouping/aggregation, if any */
UPPERREL_WINDOW, /* result of window functions, if any */
UPPERREL_DISTINCT, /* result of "SELECT DISTINCT", if any */
UPPERREL_ORDERED, /* result of ORDER BY, if any */
UPPERREL_ROWMARKS, /* result of ROMARKS, if any */
UPPERREL_LIMIT, /* result of limit offset, if any */
UPPERREL_FINAL /* result of any remaining top-level actions */
/* NB: UPPERREL_FINAL must be last enum entry; it's used to size arrays */
} UpperRelationKind;
/*
* For global path optimization, we should keep all paths with interesting distribute
* keys. There are two kinds of such keys: super set (taking effect for intermediate
@ -465,9 +482,14 @@ typedef struct PlannerInfo {
*
* We also have "other rels", which are like base rels in that they refer to
* single RT indexes; but they are not part of the join tree, and are given
* a different RelOptKind to identify them. Lastly, there is a RelOptKind
* for "dead" relations, which are base rels that we have proven we don't
* need to join after all.
* a different RelOptKind to identify them.
* There is also a RelOptKind for "upper" relations, which are RelOptInfos
* that describe post-scan/join processing steps, such as aggregation.
* Many of the fields in these RelOptInfos are meaningless, but their Path
* fields always hold Paths showing ways to do that processing step, currently
* this kind is only used for fdw to search path.
* Lastly, there is a RelOptKind for "dead" relations, which are base rels
* that we have proven we don't need to join after all.
*
* Currently the only kind of otherrels are those made for member relations
* of an "append relation", that is an inheritance set or UNION ALL subquery.
@ -531,8 +553,6 @@ typedef struct PlannerInfo {
* allvisfrac - fraction of disk pages that are marked all-visible
* subplan - plan for subquery (NULL if it's not a subquery)
* subroot - PlannerInfo for subquery (NULL if it's not a subquery)
* fdwroutine - function hooks for FDW, if foreign table (else NULL)
* fdw_private - private state for FDW, if foreign table (else NULL)
*
* Note: for a subquery, tuples, subplan, subroot are not set immediately
* upon creation of the RelOptInfo object; they are filled in when
@ -541,7 +561,16 @@ typedef struct PlannerInfo {
*
* For otherrels that are appendrel members, these fields are filled
* in just as for a baserel.
* If the relation is either a foreign table or a join of foreign tables that
* all belong to the same foreign server and are assigned to the same user to
* check access permissions as (cf checkAsUser), these fields will be set:
*
* serverid - OID of foreign server, if foreign table (else InvalidOid)
* userid - OID of user to check access as (InvalidOid means current user)
* useridiscurrent - we've assumed that userid equals current user
* fdwroutine - function hooks for FDW, if foreign table (else NULL)
* fdw_private - private state for FDW, if foreign table (else NULL)
*
* The presence of the remaining fields depends on the restrictions
* and joins that the relation participates in:
*
@ -575,7 +604,7 @@ typedef struct PlannerInfo {
* and may need it multiple times to price index scans.
* ----------
*/
typedef enum RelOptKind { RELOPT_BASEREL, RELOPT_JOINREL, RELOPT_OTHER_MEMBER_REL, RELOPT_DEADREL } RelOptKind;
typedef enum RelOptKind { RELOPT_BASEREL, RELOPT_JOINREL, RELOPT_OTHER_MEMBER_REL, RELOPT_UPPER_REL, RELOPT_DEADREL } RelOptKind;
typedef enum PartitionFlag { PARTITION_NONE, PARTITION_REQURIED, PARTITION_ANCESOR } PartitionFlag;
@ -588,6 +617,9 @@ typedef enum PartitionFlag { PARTITION_NONE, PARTITION_REQURIED, PARTITION_ANCES
/* Is the given relation a join relation? */
#define IS_JOIN_REL(rel) ((rel)->reloptkind == RELOPT_JOINREL)
/* Is the given relation an upper relation? */
#define IS_UPPER_REL(rel) ((rel)->reloptkind == RELOPT_UPPER_REL)
typedef struct RelOptInfo {
NodeTag type;
@ -653,6 +685,11 @@ typedef struct RelOptInfo {
struct Plan* subplan; /* if subquery */
PlannerInfo* subroot; /* if subquery */
List *subplan_params; /* if subquery */
/* Information about foreign tables and foreign joins */
Oid serverid; /* identifies server for the table or join */
Oid userid; /* identifies user to check access as */
bool useridiscurrent; /* join is only valid for current user */
/* use "struct FdwRoutine" to avoid including fdwapi.h here */
struct FdwRoutine* fdwroutine; /* if foreign table */
void* fdw_private; /* if foreign table */

2
src/include/optimizer/pathnode.h
View File

@ -84,7 +84,7 @@ extern Path* create_valuesscan_path(PlannerInfo* root, RelOptInfo* rel, Relids r
extern Path* create_ctescan_path(PlannerInfo* root, RelOptInfo* rel);
extern Path* create_worktablescan_path(PlannerInfo* root, RelOptInfo* rel);
extern ForeignPath* create_foreignscan_path(PlannerInfo* root, RelOptInfo* rel, Cost startup_cost, Cost total_cost,
List* pathkeys, Relids required_outer, List* fdw_private, int dop = 1);
List* pathkeys, Relids required_outer, Path* fdw_outerpath, List* fdw_private, int dop = 1);
extern Relids calc_nestloop_required_outer(Path* outer_path, Path* inner_path);
extern Relids calc_non_nestloop_required_outer(Path* outer_path, Path* inner_path);

2
src/include/optimizer/paths.h
View File

@ -142,7 +142,7 @@ extern bool have_relevant_eclass_joinclause(PlannerInfo* root, RelOptInfo* rel1,
extern bool has_relevant_eclass_joinclause(PlannerInfo* root, RelOptInfo* rel1);
extern bool eclass_useful_for_merging(EquivalenceClass* eclass, RelOptInfo* rel);
extern bool is_redundant_derived_clause(RestrictInfo* rinfo, List* clauselist);
extern PathKey* make_canonical_pathkey(PlannerInfo* root, EquivalenceClass* eclass, Oid opfamily, int strategy, bool nulls_first);
/*
* pathkeys.c
* utilities for matching and building path keys

2
src/include/optimizer/planmain.h
View File

@ -71,7 +71,7 @@ extern void disuse_physical_tlist(Plan* plan, Path* path);
extern void copy_plan_costsize(Plan* dest, Plan* src);
extern SubqueryScan* make_subqueryscan(List* qptlist, List* qpqual, Index scanrelid, Plan* subplan);
extern ForeignScan* make_foreignscan(List* qptlist, List* qpqual, Index scanrelid, List* fdw_exprs, List* fdw_private,
RemoteQueryExecType type = EXEC_ON_ALL_NODES);
List *fdw_scan_tlist, List *fdw_recheck_quals, Plan *outer_plan, RemoteQueryExecType type = EXEC_ON_ALL_NODES);
extern Append* make_append(List* appendplans, List* tlist);
extern RecursiveUnion* make_recursive_union(
List* tlist, Plan* lefttree, Plan* righttree, int wtParam, List* distinctList, long numGroups);