From a24aca9b9993129a74caa453def1126f890cae9e Mon Sep 17 00:00:00 2001 From: Paul Guo Date: Thu, 18 Apr 2019 18:55:09 +0800 Subject: [PATCH v7 1/3] Extact common recovery related functions from pg_basebackup into separate files for additional pg_rewind use. --- src/bin/pg_basebackup/pg_basebackup.c | 162 +------------------------ src/bin/pg_rewind/libpq_fetch.c | 3 +- src/bin/pg_rewind/pg_rewind.c | 9 ++ src/bin/pg_rewind/pg_rewind.h | 3 + src/fe_utils/Makefile | 2 +- src/fe_utils/recovery_gen.c | 165 ++++++++++++++++++++++++++ src/include/fe_utils/recovery_gen.h | 25 ++++ src/tools/msvc/Mkvcbuild.pm | 2 +- 8 files changed, 208 insertions(+), 163 deletions(-) create mode 100644 src/fe_utils/recovery_gen.c create mode 100644 src/include/fe_utils/recovery_gen.h diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 7986872f10..55ef13926d 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -31,6 +31,7 @@ #include "common/file_utils.h" #include "common/logging.h" #include "common/string.h" +#include "fe_utils/recovery_gen.h" #include "fe_utils/string_utils.h" #include "getopt_long.h" #include "libpq-fe.h" @@ -67,11 +68,6 @@ typedef struct TablespaceList */ #define MINIMUM_VERSION_FOR_TEMP_SLOTS 100000 -/* - * recovery.conf is integrated into postgresql.conf from version 12. - */ -#define MINIMUM_VERSION_FOR_RECOVERY_GUC 120000 - /* * Different ways to include WAL */ @@ -147,8 +143,6 @@ static void progress_report(int tablespacenum, const char *filename, bool force) static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum); static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum); -static void GenerateRecoveryConf(PGconn *conn); -static void WriteRecoveryConf(void); static void BaseBackup(void); static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline, @@ -1629,7 +1623,7 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum) PQfreemem(copybuf); if (basetablespace && writerecoveryconf) - WriteRecoveryConf(); + WriteRecoveryConfig(conn, basedir, recoveryconfcontents); /* * No data is synced here, everything is done for all tablespaces at the @@ -1637,156 +1631,6 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum) */ } -/* - * Escape a string so that it can be used as a value in a key-value pair - * a configuration file. - */ -static char * -escape_quotes(const char *src) -{ - char *result = escape_single_quotes_ascii(src); - - if (!result) - { - pg_log_error("out of memory"); - exit(1); - } - return result; -} - -/* - * Create a configuration file in memory using a PQExpBuffer - */ -static void -GenerateRecoveryConf(PGconn *conn) -{ - PQconninfoOption *connOptions; - PQconninfoOption *option; - PQExpBufferData conninfo_buf; - char *escaped; - - recoveryconfcontents = createPQExpBuffer(); - if (!recoveryconfcontents) - { - pg_log_error("out of memory"); - exit(1); - } - - /* - * In PostgreSQL 12 and newer versions, standby_mode is gone, replaced by - * standby.signal to trigger a standby state at recovery. - */ - if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_RECOVERY_GUC) - appendPQExpBufferStr(recoveryconfcontents, "standby_mode = 'on'\n"); - - connOptions = PQconninfo(conn); - if (connOptions == NULL) - { - pg_log_error("out of memory"); - exit(1); - } - - initPQExpBuffer(&conninfo_buf); - for (option = connOptions; option && option->keyword; option++) - { - /* Omit empty settings and those libpqwalreceiver overrides. */ - if (strcmp(option->keyword, "replication") == 0 || - strcmp(option->keyword, "dbname") == 0 || - strcmp(option->keyword, "fallback_application_name") == 0 || - (option->val == NULL) || - (option->val != NULL && option->val[0] == '\0')) - continue; - - /* Separate key-value pairs with spaces */ - if (conninfo_buf.len != 0) - appendPQExpBufferChar(&conninfo_buf, ' '); - - /* - * Write "keyword=value" pieces, the value string is escaped and/or - * quoted if necessary. - */ - appendPQExpBuffer(&conninfo_buf, "%s=", option->keyword); - appendConnStrVal(&conninfo_buf, option->val); - } - - /* - * Escape the connection string, so that it can be put in the config file. - * Note that this is different from the escaping of individual connection - * options above! - */ - escaped = escape_quotes(conninfo_buf.data); - appendPQExpBuffer(recoveryconfcontents, "primary_conninfo = '%s'\n", escaped); - free(escaped); - - if (replication_slot) - { - /* unescaped: ReplicationSlotValidateName allows [a-z0-9_] only */ - appendPQExpBuffer(recoveryconfcontents, "primary_slot_name = '%s'\n", - replication_slot); - } - - if (PQExpBufferBroken(recoveryconfcontents) || - PQExpBufferDataBroken(conninfo_buf)) - { - pg_log_error("out of memory"); - exit(1); - } - - termPQExpBuffer(&conninfo_buf); - - PQconninfoFree(connOptions); -} - - -/* - * Write the configuration file into the directory specified in basedir, - * with the contents already collected in memory appended. Then write - * the signal file into the basedir. If the server does not support - * recovery parameters as GUCs, the signal file is not necessary, and - * configuration is written to recovery.conf. - */ -static void -WriteRecoveryConf(void) -{ - char filename[MAXPGPATH]; - FILE *cf; - bool is_recovery_guc_supported = true; - - if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_RECOVERY_GUC) - is_recovery_guc_supported = false; - - snprintf(filename, MAXPGPATH, "%s/%s", basedir, - is_recovery_guc_supported ? "postgresql.auto.conf" : "recovery.conf"); - - cf = fopen(filename, is_recovery_guc_supported ? "a" : "w"); - if (cf == NULL) - { - pg_log_error("could not open file \"%s\": %m", filename); - exit(1); - } - - if (fwrite(recoveryconfcontents->data, recoveryconfcontents->len, 1, cf) != 1) - { - pg_log_error("could not write to file \"%s\": %m", filename); - exit(1); - } - - fclose(cf); - - if (is_recovery_guc_supported) - { - snprintf(filename, MAXPGPATH, "%s/%s", basedir, "standby.signal"); - cf = fopen(filename, "w"); - if (cf == NULL) - { - pg_log_error("could not create file \"%s\": %m", filename); - exit(1); - } - - fclose(cf); - } -} - static void BaseBackup(void) @@ -1843,7 +1687,7 @@ BaseBackup(void) * Build contents of configuration file if requested */ if (writerecoveryconf) - GenerateRecoveryConf(conn); + recoveryconfcontents = GenerateRecoveryConfig(conn, replication_slot); /* * Run IDENTIFY_SYSTEM so we can get the timeline diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c index 002776f696..f4ebf7d842 100644 --- a/src/bin/pg_rewind/libpq_fetch.c +++ b/src/bin/pg_rewind/libpq_fetch.c @@ -20,12 +20,11 @@ #include "file_ops.h" #include "filemap.h" -#include "libpq-fe.h" #include "catalog/pg_type_d.h" #include "fe_utils/connect.h" #include "port/pg_bswap.h" -static PGconn *conn = NULL; +PGconn *conn = NULL; /* * Files are fetched max CHUNKSIZE bytes at a time. diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 15e3eab550..c3903c5753 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -85,6 +85,12 @@ usage(const char *progname) printf(_("\nReport bugs to .\n")); } +static void +disconnect_atexit(void) +{ + if (conn != NULL) + PQfinish(conn); +} int main(int argc, char **argv) @@ -231,7 +237,10 @@ main(int argc, char **argv) /* Connect to remote server */ if (connstr_source) + { libpqConnect(connstr_source); + atexit(disconnect_atexit); + } /* * Ok, we have all the options and we're ready to start. Read in all the diff --git a/src/bin/pg_rewind/pg_rewind.h b/src/bin/pg_rewind/pg_rewind.h index 1125c7e60f..d23c855107 100644 --- a/src/bin/pg_rewind/pg_rewind.h +++ b/src/bin/pg_rewind/pg_rewind.h @@ -19,6 +19,8 @@ #include "common/logging.h" +#include "libpq-fe.h" + /* Configuration options */ extern char *datadir_target; extern char *datadir_source; @@ -26,6 +28,7 @@ extern char *connstr_source; extern bool showprogress; extern bool dry_run; extern int WalSegSz; +extern PGconn *conn; /* Target history */ extern TimeLineHistoryEntry *targetHistory; diff --git a/src/fe_utils/Makefile b/src/fe_utils/Makefile index 7d73800323..ae0f6c884b 100644 --- a/src/fe_utils/Makefile +++ b/src/fe_utils/Makefile @@ -19,7 +19,7 @@ include $(top_builddir)/src/Makefile.global override CPPFLAGS := -DFRONTEND -I$(libpq_srcdir) $(CPPFLAGS) -OBJS = mbprint.o print.o psqlscan.o simple_list.o string_utils.o conditional.o +OBJS = mbprint.o print.o psqlscan.o simple_list.o string_utils.o conditional.o recovery_gen.o all: libpgfeutils.a diff --git a/src/fe_utils/recovery_gen.c b/src/fe_utils/recovery_gen.c new file mode 100644 index 0000000000..4e0f6a154a --- /dev/null +++ b/src/fe_utils/recovery_gen.c @@ -0,0 +1,165 @@ +/*------------------------------------------------------------------------- + * + * recovery_gen.c - general code that is used in both pg_basebackup and pg_rewind. + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" +#include "common/logging.h" +#include "fe_utils/string_utils.h" +#include "fe_utils/recovery_gen.h" + +/* + * Escape a string so that it can be used as a value in a key-value pair + * a configuration file. + */ +static char * +escape_quotes(const char *src) +{ + char *result = escape_single_quotes_ascii(src); + + if (!result) + { + pg_log_error("out of memory"); + exit(1); + } + return result; +} + +/* + * Create recovery configuration contents using a PQExpBuffer + */ +PQExpBuffer +GenerateRecoveryConfig(PGconn *pgconn, char *pg_replication_slot) +{ + PQconninfoOption *connOptions; + PQconninfoOption *option; + PQExpBufferData conninfo_buf; + char *escaped; + PQExpBuffer contents; + + contents = createPQExpBuffer(); + if (!contents) + { + pg_log_error("out of memory"); + exit(1); + } + + /* + * In PostgreSQL 12 and newer versions, standby_mode is gone, replaced by + * standby.signal to trigger a standby state at recovery. + */ + if (pgconn && PQserverVersion(pgconn) < MINIMUM_VERSION_FOR_RECOVERY_GUC) + appendPQExpBufferStr(contents, "standby_mode = 'on'\n"); + + connOptions = PQconninfo(pgconn); + if (connOptions == NULL) + { + pg_log_error("out of memory"); + exit(1); + } + + initPQExpBuffer(&conninfo_buf); + for (option = connOptions; option && option->keyword; option++) + { + /* Omit empty settings and those libpqwalreceiver overrides. */ + if (strcmp(option->keyword, "replication") == 0 || + strcmp(option->keyword, "dbname") == 0 || + strcmp(option->keyword, "fallback_application_name") == 0 || + (option->val == NULL) || + (option->val != NULL && option->val[0] == '\0')) + continue; + + /* Separate key-value pairs with spaces */ + if (conninfo_buf.len != 0) + appendPQExpBufferChar(&conninfo_buf, ' '); + + /* + * Write "keyword=value" pieces, the value string is escaped and/or + * quoted if necessary. + */ + appendPQExpBuffer(&conninfo_buf, "%s=", option->keyword); + appendConnStrVal(&conninfo_buf, option->val); + } + + /* + * Escape the connection string, so that it can be put in the config file. + * Note that this is different from the escaping of individual connection + * options above! + */ + escaped = escape_quotes(conninfo_buf.data); + appendPQExpBuffer(contents, "primary_conninfo = '%s'\n", escaped); + free(escaped); + + if (pg_replication_slot) + { + /* unescaped: ReplicationSlotValidateName allows [a-z0-9_] only */ + appendPQExpBuffer(contents, "primary_slot_name = '%s'\n", + pg_replication_slot); + } + + if (PQExpBufferBroken(contents) || + PQExpBufferDataBroken(conninfo_buf)) + { + pg_log_error("out of memory"); + exit(1); + } + + termPQExpBuffer(&conninfo_buf); + + PQconninfoFree(connOptions); + + return contents; +} + +/* + * Write the configuration file in the directory specified in target_dir, + * with the contents already collected in memory appended. Then write + * the signal file into the target_dir. If the server does not support + * recovery parameters as GUCs, the signal file is not necessary, and + * configuration is written to recovery.conf. + */ +void +WriteRecoveryConfig(PGconn *pgconn, char *target_dir, PQExpBuffer contents) +{ + char filename[MAXPGPATH]; + FILE *cf; + bool is_recovery_guc_supported = true; + + if (pgconn && PQserverVersion(pgconn) < MINIMUM_VERSION_FOR_RECOVERY_GUC) + is_recovery_guc_supported = false; + + snprintf(filename, MAXPGPATH, "%s/%s", target_dir, + is_recovery_guc_supported ? "postgresql.auto.conf" : "recovery.conf"); + + cf = fopen(filename, is_recovery_guc_supported ? "a" : "w"); + if (cf == NULL) + { + pg_log_error("could not open file \"%s\": %m", filename); + exit(1); + } + + if (fwrite(contents->data, contents->len, 1, cf) != 1) + { + pg_log_error("could not write to file \"%s\": %m", filename); + exit(1); + } + + fclose(cf); + + if (is_recovery_guc_supported) + { + snprintf(filename, MAXPGPATH, "%s/%s", target_dir, "standby.signal"); + cf = fopen(filename, "w"); + if (cf == NULL) + { + pg_log_error("could not create file \"%s\": %m", filename); + exit(1); + } + + fclose(cf); + } +} diff --git a/src/include/fe_utils/recovery_gen.h b/src/include/fe_utils/recovery_gen.h new file mode 100644 index 0000000000..7a79bf110b --- /dev/null +++ b/src/include/fe_utils/recovery_gen.h @@ -0,0 +1,25 @@ +/*------------------------------------------------------------------------- + * + * recovery_gen.h - general code that is used in both pg_basebackup and pg_rewind. + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ + +#ifndef RECOVERY_GEN_H +#define RECOVERY_GEN_H + +#include "libpq-fe.h" +#include "pqexpbuffer.h" + +extern PQExpBuffer GenerateRecoveryConfig(PGconn *pgconn, char *pg_replication_slot); +extern void WriteRecoveryConfig(PGconn *pgconn, char *target_dir, PQExpBuffer contents); + +/* + * recovery.conf is integrated into postgresql.conf from version 12. + */ +#define MINIMUM_VERSION_FOR_RECOVERY_GUC 120000 + + +#endif /* RECOVERY_GEN_H */ diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 239f13cc12..cd6bb7eef6 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -141,7 +141,7 @@ sub mkvcbuild our @pgcommonbkndfiles = @pgcommonallfiles; our @pgfeutilsfiles = qw( - conditional.c mbprint.c print.c psqlscan.l psqlscan.c simple_list.c string_utils.c); + conditional.c mbprint.c print.c psqlscan.l psqlscan.c simple_list.c string_utils.c recovery_gen.c); $libpgport = $solution->AddProject('libpgport', 'lib', 'misc'); $libpgport->AddDefine('FRONTEND'); -- 2.17.2