From 96fc9c3c9220e03d6af5510851053733756f7476 Mon Sep 17 00:00:00 2001 From: John Naylor Date: Sat, 19 Jan 2019 18:41:58 -0500 Subject: [PATCH v16 2/2] During pg_upgrade, conditionally skip transfer of FSMs. If a heap on the old cluster has 4 pages or fewer, don't copy or link the FSM. This will reduce space usage for installations with large numbers of small tables. --- src/bin/pg_upgrade/info.c | 11 ++++-- src/bin/pg_upgrade/pg_upgrade.h | 6 +++- src/bin/pg_upgrade/relfilenode.c | 57 ++++++++++++++++++-------------- 3 files changed, 47 insertions(+), 27 deletions(-) diff --git a/src/bin/pg_upgrade/info.c b/src/bin/pg_upgrade/info.c index 2f925f086c..c7598b3de0 100644 --- a/src/bin/pg_upgrade/info.c +++ b/src/bin/pg_upgrade/info.c @@ -200,6 +200,7 @@ create_rel_filename_map(const char *old_data, const char *new_data, map->old_db_oid = old_db->db_oid; map->new_db_oid = new_db->db_oid; + map->relkind = old_rel->relkind; /* * old_relfilenode might differ from pg_class.oid (and hence @@ -418,6 +419,7 @@ get_rel_infos(ClusterInfo *cluster, DbInfo *dbinfo) char *nspname = NULL; char *relname = NULL; char *tablespace = NULL; + char *relkind = NULL; int i_spclocation, i_nspname, i_relname, @@ -425,7 +427,8 @@ get_rel_infos(ClusterInfo *cluster, DbInfo *dbinfo) i_indtable, i_toastheap, i_relfilenode, - i_reltablespace; + i_reltablespace, + i_relkind; char query[QUERY_ALLOC]; char *last_namespace = NULL, *last_tablespace = NULL; @@ -494,7 +497,7 @@ get_rel_infos(ClusterInfo *cluster, DbInfo *dbinfo) */ snprintf(query + strlen(query), sizeof(query) - strlen(query), "SELECT all_rels.*, n.nspname, c.relname, " - " c.relfilenode, c.reltablespace, %s " + " c.relfilenode, c.reltablespace, c.relkind, %s " "FROM (SELECT * FROM regular_heap " " UNION ALL " " SELECT * FROM toast_heap " @@ -526,6 +529,7 @@ get_rel_infos(ClusterInfo *cluster, DbInfo *dbinfo) i_relfilenode = PQfnumber(res, "relfilenode"); i_reltablespace = PQfnumber(res, "reltablespace"); i_spclocation = PQfnumber(res, "spclocation"); + i_relkind = PQfnumber(res, "relkind"); for (relnum = 0; relnum < ntups; relnum++) { @@ -555,6 +559,9 @@ get_rel_infos(ClusterInfo *cluster, DbInfo *dbinfo) relname = PQgetvalue(res, relnum, i_relname); curr->relname = pg_strdup(relname); + relkind = PQgetvalue(res, relnum, i_relkind); + curr->relkind = relkind[0]; + curr->relfilenode = atooid(PQgetvalue(res, relnum, i_relfilenode)); curr->tblsp_alloc = false; diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 2f67eee22b..6cd0e0967f 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -147,6 +147,7 @@ typedef struct char *tablespace; /* tablespace path; "" for cluster default */ bool nsp_alloc; /* should nspname be freed? */ bool tblsp_alloc; /* should tablespace be freed? */ + char relkind; /* relation relkind -- see pg_class.h */ } RelInfo; typedef struct @@ -173,9 +174,12 @@ typedef struct */ Oid old_relfilenode; Oid new_relfilenode; - /* the rest are used only for logging and error reporting */ + + /* These are used only for logging and error reporting. */ char *nspname; /* namespaces */ char *relname; + + char relkind; /* relation relkind -- see pg_class.h */ } FileNameMap; /* diff --git a/src/bin/pg_upgrade/relfilenode.c b/src/bin/pg_upgrade/relfilenode.c index 0c78073f0e..1cdda5f5f1 100644 --- a/src/bin/pg_upgrade/relfilenode.c +++ b/src/bin/pg_upgrade/relfilenode.c @@ -14,10 +14,11 @@ #include #include "catalog/pg_class_d.h" #include "access/transam.h" +#include "storage/freespace.h" static void transfer_single_new_db(FileNameMap *maps, int size, char *old_tablespace); -static void transfer_relfile(FileNameMap *map, const char *suffix, bool vm_must_add_frozenbit); +static Size transfer_relfile(FileNameMap *map, const char *suffix, bool vm_must_add_frozenbit); /* @@ -144,6 +145,7 @@ transfer_single_new_db(FileNameMap *maps, int size, char *old_tablespace) int mapnum; bool vm_crashsafe_match = true; bool vm_must_add_frozenbit = false; + Size first_seg_size = 0; /* * Do the old and new cluster disagree on the crash-safetiness of the vm @@ -165,18 +167,23 @@ transfer_single_new_db(FileNameMap *maps, int size, char *old_tablespace) if (old_tablespace == NULL || strcmp(maps[mapnum].old_tablespace, old_tablespace) == 0) { - /* transfer primary file */ - transfer_relfile(&maps[mapnum], "", vm_must_add_frozenbit); + /* Transfer main fork and return size of the first segment. */ + first_seg_size = transfer_relfile(&maps[mapnum], "", vm_must_add_frozenbit); /* fsm/vm files added in PG 8.4 */ if (GET_MAJOR_VERSION(old_cluster.major_version) >= 804) { /* - * Copy/link any fsm and vm files, if they exist + * Copy/link any fsm and vm files, if they exist and if they would + * be created in the new cluster. */ - transfer_relfile(&maps[mapnum], "_fsm", vm_must_add_frozenbit); + if ((maps[mapnum].relkind != RELKIND_RELATION && + maps[mapnum].relkind != RELKIND_TOASTVALUE) || + first_seg_size > HEAP_FSM_CREATION_THRESHOLD * BLCKSZ || + GET_MAJOR_VERSION(new_cluster.major_version) <= 1100) + (void) transfer_relfile(&maps[mapnum], "_fsm", vm_must_add_frozenbit); if (vm_crashsafe_match) - transfer_relfile(&maps[mapnum], "_vm", vm_must_add_frozenbit); + (void) transfer_relfile(&maps[mapnum], "_vm", vm_must_add_frozenbit); } } } @@ -190,7 +197,7 @@ transfer_single_new_db(FileNameMap *maps, int size, char *old_tablespace) * is true, visibility map forks are converted and rewritten, even in link * mode. */ -static void +static Size transfer_relfile(FileNameMap *map, const char *type_suffix, bool vm_must_add_frozenbit) { char old_file[MAXPGPATH]; @@ -198,6 +205,7 @@ transfer_relfile(FileNameMap *map, const char *type_suffix, bool vm_must_add_fro int segno; char extent_suffix[65]; struct stat statbuf; + Size first_seg_size = 0; /* * Now copy/link any related segments as well. Remember, PG breaks large @@ -226,26 +234,27 @@ transfer_relfile(FileNameMap *map, const char *type_suffix, bool vm_must_add_fro type_suffix, extent_suffix); - /* Is it an extent, fsm, or vm file? */ - if (type_suffix[0] != '\0' || segno != 0) + /* Did file open fail? */ + if (stat(old_file, &statbuf) != 0) { - /* Did file open fail? */ - if (stat(old_file, &statbuf) != 0) - { - /* File does not exist? That's OK, just return */ - if (errno == ENOENT) - return; - else - pg_fatal("error while checking for file existence \"%s.%s\" (\"%s\" to \"%s\"): %s\n", - map->nspname, map->relname, old_file, new_file, - strerror(errno)); - } - - /* If file is empty, just return */ - if (statbuf.st_size == 0) - return; + /* Extent, fsm, or vm does not exist? That's OK, just return */ + if (errno == ENOENT && + (type_suffix[0] != '\0' || segno != 0)) + return first_seg_size; + else + pg_fatal("error while checking for file existence \"%s.%s\" (\"%s\" to \"%s\"): %s\n", + map->nspname, map->relname, old_file, new_file, + strerror(errno)); } + /* Save the size of the first segment of the main fork. */ + else if (type_suffix[0] == '\0' && segno == 0) + first_seg_size = statbuf.st_size; + + /* If extent, fsm, or vm is empty, just return */ + else if (statbuf.st_size == 0) + return first_seg_size; + unlink(new_file); /* Copying files might take some time, so give feedback. */ -- 2.17.1