From e8fb5abbc90aab1d4a573ace1e308f5c784c5fa6 Mon Sep 17 00:00:00 2001 From: yujiang Date: Fri, 11 Sep 2020 09:45:12 +0800 Subject: [PATCH] fix dump interval partitioned table issue Signed-off-by: yujiang --- src/bin/pg_dump/pg_dump.cpp | 81 ++++++++- .../hw_partition_interval_dump_restore.source | 112 ++++++++++++ .../hw_partition_interval_dump_restore.source | 172 ++++++++++++++++++ src/test/regress/parallel_schedule18 | 1 + 4 files changed, 364 insertions(+), 2 deletions(-) create mode 100644 src/test/regress/input/hw_partition_interval_dump_restore.source create mode 100644 src/test/regress/output/hw_partition_interval_dump_restore.source diff --git a/src/bin/pg_dump/pg_dump.cpp b/src/bin/pg_dump/pg_dump.cpp index f9d745f73..40a98ede8 100644 --- a/src/bin/pg_dump/pg_dump.cpp +++ b/src/bin/pg_dump/pg_dump.cpp @@ -156,6 +156,11 @@ typedef struct { int objsubid; /* subobject (table column #) */ } SecLabelItem; +typedef struct { + Oid oid; + char* name; +} TablespaceInfo; + /* global decls */ bool g_verbose; /* User wants verbose narration of our * activities. */ @@ -267,6 +272,9 @@ bool isSingleTableDump = false; static int include_nodes = 0; #endif +static int g_tablespaceNum = -1; +static TablespaceInfo* g_tablespaces = nullptr; + #define disconnect_and_exit(conn, code) \ do { \ if ((conn) != NULL) { \ @@ -15630,6 +15638,65 @@ static void binary_upgrade_set_pg_partition_oids( destroyPQExpBuffer(upgrade_query); } +static void LoadTablespaces(Archive* fout) +{ + PQExpBuffer query = createPQExpBuffer(); + appendPQExpBuffer(query, "select oid, spcname from pg_tablespace"); + PGresult* res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); + g_tablespaceNum = PQntuples(res); + Assert(g_tablespaceNum > 0); + g_tablespaces = (TablespaceInfo*)pg_malloc(g_tablespaceNum * sizeof(TablespaceInfo)); + for (int i = 0; i < g_tablespaceNum; ++i) { + g_tablespaces[i].oid = atooid(PQgetvalue(res, i, 0)); + g_tablespaces[i].name = gs_strdup(PQgetvalue(res, i, 1)); + } + destroyPQExpBuffer(query); + PQclear(res); +} + +static const char* GetTablespaceNameByOid(Archive* fout, Oid oid) +{ + if (g_tablespaceNum == -1) { + LoadTablespaces(fout); + } + + for (int i = 0; i < g_tablespaceNum; ++i) { + if (g_tablespaces[i].oid == oid) { + return g_tablespaces[i].name; + } + } + exit_horribly(NULL, "tablespace %u not exist\n", oid); +} + +static void OutputIntervalPartitionDef(Archive* fout, const PGresult* res, PQExpBuffer outputBuf) +{ + int intervalIdx = PQfnumber(res, "interval"); + appendPQExpBuffer(outputBuf, "INTERVAL('%s')", PQgetvalue(res, 0, intervalIdx)); + + int interTblSpcNumIdx = PQfnumber(res, "inttblspcnum"); + if (PQgetisnull(res, 0, interTblSpcNumIdx)) { + return; + } + + int interTblSpcNum = atoi(PQgetvalue(res, 0, interTblSpcNumIdx)); + if (interTblSpcNum <= 0) { + return; + } + appendPQExpBuffer(outputBuf, " STORE IN("); + + int interTblSpcIdx = PQfnumber(res, "intervaltablespace"); + Oid* interTblSpcs = (Oid*)pg_malloc((size_t)interTblSpcNum * sizeof(Oid)); + parseOidArray(PQgetvalue(res, 0, interTblSpcIdx), interTblSpcs, interTblSpcNum); + for (int i = 0; i < interTblSpcNum; ++i) { + if (i > 0) { + appendPQExpBuffer(outputBuf, ", "); + } + appendPQExpBuffer(outputBuf, "%s", GetTablespaceNameByOid(fout, interTblSpcs[i])); + } + appendPQExpBuffer(outputBuf, ")"); + free(interTblSpcs); +} + /* * createTablePartition * Write the declaration of partitioned table. @@ -15659,7 +15726,7 @@ static PQExpBuffer createTablePartition(Archive* fout, TableInfo* tbinfo) appendPQExpBuffer(defq, "SELECT partstrategy, interval[1], " "array_length(partkey, 1) AS partkeynum, partkey, " - "intspnum AS inttblspcnum, intervaltablespace " + "array_length(intervaltablespace, 1) AS inttblspcnum, intervaltablespace " "FROM pg_partition WHERE parentid = '%u' AND parttype = '%c'", tbinfo->dobj.catId.oid, PART_OBJ_TYPE_PARTED_TABLE); @@ -15670,7 +15737,11 @@ static PQExpBuffer createTablePartition(Archive* fout, TableInfo* tbinfo) partkeynum = atoi(PQgetvalue(res, 0, i_partkeynum)); partkeycols = (Oid*)pg_malloc((size_t)partkeynum * sizeof(Oid)); parseOidArray(PQgetvalue(res, 0, i_partkey), partkeycols, partkeynum); - PQclear(res); + int partStrategyIdx = PQfnumber(res, "partstrategy"); + char partStrategy = *PQgetvalue(res, 0, partStrategyIdx); + if (partStrategy != PART_STRATEGY_INTERVAL) { + PQclear(res); + } if (RELKIND_FOREIGN_TABLE == tbinfo->relkind) { int i_fdwname = -1; @@ -15714,6 +15785,12 @@ static PQExpBuffer createTablePartition(Archive* fout, TableInfo* tbinfo) } appendPQExpBuffer(result, ")\n"); + if (partStrategy == PART_STRATEGY_INTERVAL) { + OutputIntervalPartitionDef(fout, res, result); + appendPQExpBuffer(result, "\n"); + PQclear(res); + } + /* generate partition details */ if (isHDFSFTbl) { appendPQExpBuffer(result, "AUTOMAPPED"); diff --git a/src/test/regress/input/hw_partition_interval_dump_restore.source b/src/test/regress/input/hw_partition_interval_dump_restore.source new file mode 100644 index 000000000..1d06a234e --- /dev/null +++ b/src/test/regress/input/hw_partition_interval_dump_restore.source @@ -0,0 +1,112 @@ +-- +-- test interval partition without specified tablespaces +-- + +-- prepare: create interval partition table and load data +create database interval_db; +\c interval_db; + +drop table if exists t_part_auto; +create table t_part_auto(id int not null, name varchar, birthday timestamp not null, age int) +partition by range(birthday) interval ('1 month') ( + partition p1 values less than('2000-01-01 00:00:00'), + partition p2 values less than('2000-02-01 00:00:00') +); + +declare + v_sql varchar; + v_name varchar; + v_dt_begin timestamp; + v_dt timestamp; + v_age int; +begin + v_sql := 'insert into t_part_auto(id, name, birthday, age) values(:1, :2, :3, :4)'; + v_dt_begin := to_date('1999-10-01 00:00:00', 'yyyy-mm-dd hh24:mi:ss'); + for i in 1..200 loop + v_name := 'name_' || i; + v_dt := v_dt_begin + i*30; + v_age := mod(i,90) + 5; + execute immediate v_sql using i, v_name, v_dt, v_age; + end loop; +end; +/ + +\d t_part_auto + +select count(1) from t_part_auto; + +-- dump data +\! @abs_bindir@/gs_dump interval_db -p @portstring@ -t t_part_auto -f @abs_bindir@/backup.sql + +drop table t_part_auto; + +-- restore data +\! @abs_bindir@/gsql -p @portstring@ -d interval_db -f @abs_bindir@/backup.sql + +\d t_part_auto + +select count(1) from t_part_auto; + +drop table t_part_auto; + + +-- +-- test interval partition with specified tablespaces +-- + + +-- prepare: create interval partition table and load data +create tablespace interval_tsp1 location '@testtablespace@/interval_tsp1' maxsize '20M'; +create tablespace interval_tsp2 location '@testtablespace@/interval_tsp2' maxsize '20M'; +create tablespace interval_tsp3 location '@testtablespace@/interval_tsp3' maxsize '20M'; + +drop table if exists t_part_auto; +create table t_part_auto(id int not null, name varchar, birthday timestamp not null, age int) +partition by range(birthday) +interval ('1 month') store in(interval_tsp1, interval_tsp2, interval_tsp3) +( + partition p1 values less than('2000-01-01 00:00:00'), + partition p2 values less than('2000-02-01 00:00:00') +); + +declare + v_sql varchar; + v_name varchar; + v_dt_begin timestamp; + v_dt timestamp; + v_age int; +begin + v_sql := 'insert into t_part_auto(id, name, birthday, age) values(:1, :2, :3, :4)'; + v_dt_begin := to_date('1999-10-01 00:00:00', 'yyyy-mm-dd hh24:mi:ss'); + for i in 1..200 loop + v_name := 'name_' || i; + v_dt := v_dt_begin + i*30; + v_age := mod(i,90) + 5; + execute immediate v_sql using i, v_name, v_dt, v_age; + end loop; +end; +/ + +-- dump data +\! @abs_bindir@/gs_dump interval_db -p @portstring@ -t t_part_auto -f @abs_bindir@/backup.sql + +drop table t_part_auto; + +-- restore data +\! @abs_bindir@/gsql -p @portstring@ -d interval_db -f @abs_bindir@/backup.sql + +\d t_part_auto + +select count(1) from t_part_auto; + +-- check tablespace +select spcname from pg_tablespace where oid in(select intervaltablespace[0] from pg_partition where relname = 't_part_auto' union select intervaltablespace[1] from pg_partition where relname = 't_part_auto' union select intervaltablespace[2] from pg_partition where relname = 't_part_auto') order by spcname; + +drop table t_part_auto; +drop tablespace interval_tsp1; +drop tablespace interval_tsp2; +drop tablespace interval_tsp3; + +\c regression +drop database interval_db; + diff --git a/src/test/regress/output/hw_partition_interval_dump_restore.source b/src/test/regress/output/hw_partition_interval_dump_restore.source new file mode 100644 index 000000000..c15c71032 --- /dev/null +++ b/src/test/regress/output/hw_partition_interval_dump_restore.source @@ -0,0 +1,172 @@ +-- +-- test interval partition without specified tablespaces +-- +-- prepare: create interval partition table and load data +create database interval_db; +\c interval_db; +drop table if exists t_part_auto; +NOTICE: table "t_part_auto" does not exist, skipping +create table t_part_auto(id int not null, name varchar, birthday timestamp not null, age int) +partition by range(birthday) interval ('1 month') ( + partition p1 values less than('2000-01-01 00:00:00'), + partition p2 values less than('2000-02-01 00:00:00') +); +declare + v_sql varchar; + v_name varchar; + v_dt_begin timestamp; + v_dt timestamp; + v_age int; +begin + v_sql := 'insert into t_part_auto(id, name, birthday, age) values(:1, :2, :3, :4)'; + v_dt_begin := to_date('1999-10-01 00:00:00', 'yyyy-mm-dd hh24:mi:ss'); + for i in 1..200 loop + v_name := 'name_' || i; + v_dt := v_dt_begin + i*30; + v_age := mod(i,90) + 5; + execute immediate v_sql using i, v_name, v_dt, v_age; + end loop; +end; +/ +\d t_part_auto + Table "public.t_part_auto" + Column | Type | Modifiers +----------+-----------------------------+----------- + id | integer | not null + name | character varying | + birthday | timestamp without time zone | not null + age | integer | +Interval partition by(birthday) +Number of partition: 195 (View pg_partition to check each partition range.) + +select count(1) from t_part_auto; + count +------- + 200 +(1 row) + +-- dump data +\! @abs_bindir@/gs_dump interval_db -p @portstring@ -t t_part_auto -f @abs_bindir@/backup.sql +--?gs_dump[port='@portstring@'][interval_db][.*]: The total objects number is 374. +--?gs_dump[port='@portstring@'][interval_db][.*]: [100.00%] 374 objects have been dumped. +--?gs_dump[port='@portstring@'][interval_db][.*]: dump database interval_db successfully +--?gs_dump[port='@portstring@'][interval_db][.*]: total time: .* ms +drop table t_part_auto; +-- restore data +\! @abs_bindir@/gsql -p @portstring@ -d interval_db -f @abs_bindir@/backup.sql +SET +SET +SET +SET +SET +SET +SET +SET +SET +CREATE TABLE +ALTER TABLE +--?total time: .* ms +\d t_part_auto + Table "public.t_part_auto" + Column | Type | Modifiers +----------+-----------------------------+----------- + id | integer | not null + name | character varying | + birthday | timestamp without time zone | not null + age | integer | +Interval partition by(birthday) +Number of partition: 195 (View pg_partition to check each partition range.) + +select count(1) from t_part_auto; + count +------- + 200 +(1 row) + +drop table t_part_auto; +-- +-- test interval partition with specified tablespaces +-- +-- prepare: create interval partition table and load data +create tablespace interval_tsp1 location '@testtablespace@/interval_tsp1' maxsize '20M'; +create tablespace interval_tsp2 location '@testtablespace@/interval_tsp2' maxsize '20M'; +create tablespace interval_tsp3 location '@testtablespace@/interval_tsp3' maxsize '20M'; +drop table if exists t_part_auto; +NOTICE: table "t_part_auto" does not exist, skipping +create table t_part_auto(id int not null, name varchar, birthday timestamp not null, age int) +partition by range(birthday) +interval ('1 month') store in(interval_tsp1, interval_tsp2, interval_tsp3) +( + partition p1 values less than('2000-01-01 00:00:00'), + partition p2 values less than('2000-02-01 00:00:00') +); +declare + v_sql varchar; + v_name varchar; + v_dt_begin timestamp; + v_dt timestamp; + v_age int; +begin + v_sql := 'insert into t_part_auto(id, name, birthday, age) values(:1, :2, :3, :4)'; + v_dt_begin := to_date('1999-10-01 00:00:00', 'yyyy-mm-dd hh24:mi:ss'); + for i in 1..200 loop + v_name := 'name_' || i; + v_dt := v_dt_begin + i*30; + v_age := mod(i,90) + 5; + execute immediate v_sql using i, v_name, v_dt, v_age; + end loop; +end; +/ +-- dump data +\! @abs_bindir@/gs_dump interval_db -p @portstring@ -t t_part_auto -f @abs_bindir@/backup.sql +--?gs_dump[port='@portstring@'][interval_db][.*]: The total objects number is 374. +--?gs_dump[port='@portstring@'][interval_db][.*]: [100.00%] 374 objects have been dumped. +--?gs_dump[port='@portstring@'][interval_db][.*]: dump database interval_db successfully +--?gs_dump[port='@portstring@'][interval_db][.*]: total time: .* ms +drop table t_part_auto; +-- restore data +\! @abs_bindir@/gsql -p @portstring@ -d interval_db -f @abs_bindir@/backup.sql +SET +SET +SET +SET +SET +SET +SET +SET +SET +CREATE TABLE +ALTER TABLE +--?total time: .* ms +\d t_part_auto + Table "public.t_part_auto" + Column | Type | Modifiers +----------+-----------------------------+----------- + id | integer | not null + name | character varying | + birthday | timestamp without time zone | not null + age | integer | +Interval partition by(birthday) +Number of partition: 195 (View pg_partition to check each partition range.) + +select count(1) from t_part_auto; + count +------- + 200 +(1 row) + +-- check tablespace +select spcname from pg_tablespace where oid in(select intervaltablespace[0] from pg_partition where relname = 't_part_auto' union select intervaltablespace[1] from pg_partition where relname = 't_part_auto' union select intervaltablespace[2] from pg_partition where relname = 't_part_auto') order by spcname; + spcname +--------------- + interval_tsp1 + interval_tsp2 + interval_tsp3 +(3 rows) + +drop table t_part_auto; +drop tablespace interval_tsp1; +drop tablespace interval_tsp2; +drop tablespace interval_tsp3; +\c regression +drop database interval_db; diff --git a/src/test/regress/parallel_schedule18 b/src/test/regress/parallel_schedule18 index e6c001309..98124f1c2 100644 --- a/src/test/regress/parallel_schedule18 +++ b/src/test/regress/parallel_schedule18 @@ -27,6 +27,7 @@ test: hw_partition_interval_check_syntax test: hw_partition_interval_split test: hw_partition_interval_merge test: hw_partition_interval_compatibility +test: hw_partition_interval_dump_restore # Global Partition index feature testcase # gpi create