Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * backup_manifest.c
4 : * code for generating and sending a backup manifest
5 : *
6 : * Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/backend/replication/backup_manifest.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 : #include "postgres.h"
14 :
15 : #include "access/timeline.h"
16 : #include "libpq/libpq.h"
17 : #include "libpq/pqformat.h"
18 : #include "mb/pg_wchar.h"
19 : #include "replication/backup_manifest.h"
20 : #include "utils/builtins.h"
21 : #include "utils/json.h"
22 :
23 : static void AppendStringToManifest(backup_manifest_info *manifest, char *s);
24 :
25 : /*
26 : * Does the user want a backup manifest?
27 : *
28 : * It's simplest to always have a manifest_info object, so that we don't need
29 : * checks for NULL pointers in too many places. However, if the user doesn't
30 : * want a manifest, we set manifest->buffile to NULL.
31 : */
32 : static inline bool
33 0 : IsManifestEnabled(backup_manifest_info *manifest)
34 : {
35 0 : return (manifest->buffile != NULL);
36 : }
37 :
38 : /*
39 : * Convenience macro for appending data to the backup manifest.
40 : */
41 : #define AppendToManifest(manifest, ...) \
42 : { \
43 : char *_manifest_s = psprintf(__VA_ARGS__); \
44 : AppendStringToManifest(manifest, _manifest_s); \
45 : pfree(_manifest_s); \
46 : }
47 :
48 : /*
49 : * Initialize state so that we can construct a backup manifest.
50 : *
51 : * NB: Although the checksum type for the data files is configurable, the
52 : * checksum for the manifest itself always uses SHA-256. See comments in
53 : * SendBackupManifest.
54 : */
55 : void
56 0 : InitializeBackupManifest(backup_manifest_info *manifest,
57 : backup_manifest_option want_manifest,
58 : pg_checksum_type manifest_checksum_type)
59 : {
60 0 : if (want_manifest == MANIFEST_OPTION_NO)
61 0 : manifest->buffile = NULL;
62 : else
63 0 : manifest->buffile = BufFileCreateTemp(false);
64 0 : manifest->checksum_type = manifest_checksum_type;
65 0 : pg_sha256_init(&manifest->manifest_ctx);
66 0 : manifest->manifest_size = UINT64CONST(0);
67 0 : manifest->force_encode = (want_manifest == MANIFEST_OPTION_FORCE_ENCODE);
68 0 : manifest->first_file = true;
69 0 : manifest->still_checksumming = true;
70 :
71 0 : if (want_manifest != MANIFEST_OPTION_NO)
72 0 : AppendToManifest(manifest,
73 : "{ \"PostgreSQL-Backup-Manifest-Version\": 1,\n"
74 : "\"Files\": [");
75 0 : }
76 :
77 : /*
78 : * Add an entry to the backup manifest for a file.
79 : */
80 : void
81 0 : AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid,
82 : const char *pathname, size_t size, pg_time_t mtime,
83 : pg_checksum_context *checksum_ctx)
84 : {
85 : char pathbuf[MAXPGPATH];
86 : int pathlen;
87 : StringInfoData buf;
88 :
89 0 : if (!IsManifestEnabled(manifest))
90 0 : return;
91 :
92 : /*
93 : * If this file is part of a tablespace, the pathname passed to this
94 : * function will be relative to the tar file that contains it. We want the
95 : * pathname relative to the data directory (ignoring the intermediate
96 : * symlink traversal).
97 : */
98 0 : if (spcoid != NULL)
99 : {
100 0 : snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%s/%s", spcoid,
101 : pathname);
102 0 : pathname = pathbuf;
103 : }
104 :
105 : /*
106 : * Each file's entry needs to be separated from any entry that follows by
107 : * a comma, but there's no comma before the first one or after the last
108 : * one. To make that work, adding a file to the manifest starts by
109 : * terminating the most recently added line, with a comma if appropriate,
110 : * but does not terminate the line inserted for this file.
111 : */
112 0 : initStringInfo(&buf);
113 0 : if (manifest->first_file)
114 : {
115 0 : appendStringInfoChar(&buf, '\n');
116 0 : manifest->first_file = false;
117 : }
118 : else
119 0 : appendStringInfoString(&buf, ",\n");
120 :
121 : /*
122 : * Write the relative pathname to this file out to the manifest. The
123 : * manifest is always stored in UTF-8, so we have to encode paths that are
124 : * not valid in that encoding.
125 : */
126 0 : pathlen = strlen(pathname);
127 0 : if (!manifest->force_encode &&
128 0 : pg_verify_mbstr(PG_UTF8, pathname, pathlen, true))
129 : {
130 0 : appendStringInfoString(&buf, "{ \"Path\": ");
131 0 : escape_json(&buf, pathname);
132 0 : appendStringInfoString(&buf, ", ");
133 : }
134 : else
135 : {
136 0 : appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
137 0 : enlargeStringInfo(&buf, 2 * pathlen);
138 0 : buf.len += hex_encode(pathname, pathlen,
139 0 : &buf.data[buf.len]);
140 0 : appendStringInfoString(&buf, "\", ");
141 : }
142 :
143 0 : appendStringInfo(&buf, "\"Size\": %zu, ", size);
144 :
145 : /*
146 : * Convert last modification time to a string and append it to the
147 : * manifest. Since it's not clear what time zone to use and since time
148 : * zone definitions can change, possibly causing confusion, use GMT
149 : * always.
150 : */
151 0 : appendStringInfoString(&buf, "\"Last-Modified\": \"");
152 0 : enlargeStringInfo(&buf, 128);
153 0 : buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z",
154 0 : pg_gmtime(&mtime));
155 0 : appendStringInfoChar(&buf, '"');
156 :
157 : /* Add checksum information. */
158 0 : if (checksum_ctx->type != CHECKSUM_TYPE_NONE)
159 : {
160 : uint8 checksumbuf[PG_CHECKSUM_MAX_LENGTH];
161 : int checksumlen;
162 :
163 0 : checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
164 :
165 0 : appendStringInfo(&buf,
166 : ", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
167 : pg_checksum_type_name(checksum_ctx->type));
168 0 : enlargeStringInfo(&buf, 2 * checksumlen);
169 0 : buf.len += hex_encode((char *) checksumbuf, checksumlen,
170 0 : &buf.data[buf.len]);
171 0 : appendStringInfoChar(&buf, '"');
172 : }
173 :
174 : /* Close out the object. */
175 0 : appendStringInfoString(&buf, " }");
176 :
177 : /* OK, add it to the manifest. */
178 0 : AppendStringToManifest(manifest, buf.data);
179 :
180 : /* Avoid leaking memory. */
181 0 : pfree(buf.data);
182 : }
183 :
184 : /*
185 : * Add information about the WAL that will need to be replayed when restoring
186 : * this backup to the manifest.
187 : */
188 : void
189 0 : AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr,
190 : TimeLineID starttli, XLogRecPtr endptr,
191 : TimeLineID endtli)
192 : {
193 : List *timelines;
194 : ListCell *lc;
195 0 : bool first_wal_range = true;
196 0 : bool found_start_timeline = false;
197 :
198 0 : if (!IsManifestEnabled(manifest))
199 0 : return;
200 :
201 : /* Terminate the list of files. */
202 0 : AppendStringToManifest(manifest, "\n],\n");
203 :
204 : /* Read the timeline history for the ending timeline. */
205 0 : timelines = readTimeLineHistory(endtli);
206 :
207 : /* Start a list of LSN ranges. */
208 0 : AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n");
209 :
210 0 : foreach(lc, timelines)
211 : {
212 0 : TimeLineHistoryEntry *entry = lfirst(lc);
213 : XLogRecPtr tl_beginptr;
214 :
215 : /*
216 : * We only care about timelines that were active during the backup.
217 : * Skip any that ended before the backup started. (Note that if
218 : * entry->end is InvalidXLogRecPtr, it means that the timeline has not
219 : * yet ended.)
220 : */
221 0 : if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr)
222 0 : continue;
223 :
224 : /*
225 : * Because the timeline history file lists newer timelines before
226 : * older ones, the first timeline we encounter that is new enough to
227 : * matter ought to match the ending timeline of the backup.
228 : */
229 0 : if (first_wal_range && endtli != entry->tli)
230 0 : ereport(ERROR,
231 : errmsg("expected end timeline %u but found timeline %u",
232 : starttli, entry->tli));
233 :
234 0 : if (!XLogRecPtrIsInvalid(entry->begin))
235 0 : tl_beginptr = entry->begin;
236 : else
237 : {
238 0 : tl_beginptr = startptr;
239 :
240 : /*
241 : * If we reach a TLI that has no valid beginning LSN, there can't
242 : * be any more timelines in the history after this point, so we'd
243 : * better have arrived at the expected starting TLI. If not,
244 : * something's gone horribly wrong.
245 : */
246 0 : if (starttli != entry->tli)
247 0 : ereport(ERROR,
248 : errmsg("expected start timeline %u but found timeline %u",
249 : starttli, entry->tli));
250 : }
251 :
252 0 : AppendToManifest(manifest,
253 : "%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
254 : first_wal_range ? "" : ",\n",
255 : entry->tli,
256 : (uint32) (tl_beginptr >> 32), (uint32) tl_beginptr,
257 : (uint32) (endptr >> 32), (uint32) endptr);
258 :
259 0 : if (starttli == entry->tli)
260 : {
261 0 : found_start_timeline = true;
262 0 : break;
263 : }
264 :
265 0 : endptr = entry->begin;
266 0 : first_wal_range = false;
267 : }
268 :
269 : /*
270 : * The last entry in the timeline history for the ending timeline should
271 : * be the ending timeline itself. Verify that this is what we observed.
272 : */
273 0 : if (!found_start_timeline)
274 0 : ereport(ERROR,
275 : errmsg("start timeline %u not found in history of timeline %u",
276 : starttli, endtli));
277 :
278 : /* Terminate the list of WAL ranges. */
279 0 : AppendStringToManifest(manifest, "\n],\n");
280 : }
281 :
282 : /*
283 : * Finalize the backup manifest, and send it to the client.
284 : */
285 : void
286 0 : SendBackupManifest(backup_manifest_info *manifest)
287 : {
288 : StringInfoData protobuf;
289 : uint8 checksumbuf[PG_SHA256_DIGEST_LENGTH];
290 : char checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH];
291 0 : size_t manifest_bytes_done = 0;
292 :
293 0 : if (!IsManifestEnabled(manifest))
294 0 : return;
295 :
296 : /*
297 : * Append manifest checksum, so that the problems with the manifest itself
298 : * can be detected.
299 : *
300 : * We always use SHA-256 for this, regardless of what algorithm is chosen
301 : * for checksumming the files. If we ever want to make the checksum
302 : * algorithm used for the manifest file variable, the client will need a
303 : * way to figure out which algorithm to use as close to the beginning of
304 : * the manifest file as possible, to avoid having to read the whole thing
305 : * twice.
306 : */
307 0 : manifest->still_checksumming = false;
308 0 : pg_sha256_final(&manifest->manifest_ctx, checksumbuf);
309 0 : AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
310 0 : hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
311 0 : checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0';
312 0 : AppendStringToManifest(manifest, checksumstringbuf);
313 0 : AppendStringToManifest(manifest, "\"}\n");
314 :
315 : /*
316 : * We've written all the data to the manifest file. Rewind the file so
317 : * that we can read it all back.
318 : */
319 0 : if (BufFileSeek(manifest->buffile, 0, 0L, SEEK_SET))
320 0 : ereport(ERROR,
321 : (errcode_for_file_access(),
322 : errmsg("could not rewind temporary file")));
323 :
324 : /* Send CopyOutResponse message */
325 0 : pq_beginmessage(&protobuf, 'H');
326 0 : pq_sendbyte(&protobuf, 0); /* overall format */
327 0 : pq_sendint16(&protobuf, 0); /* natts */
328 0 : pq_endmessage(&protobuf);
329 :
330 : /*
331 : * Send CopyData messages.
332 : *
333 : * We choose to read back the data from the temporary file in chunks of
334 : * size BLCKSZ; this isn't necessary, but buffile.c uses that as the I/O
335 : * size, so it seems to make sense to match that value here.
336 : */
337 0 : while (manifest_bytes_done < manifest->manifest_size)
338 : {
339 : char manifestbuf[BLCKSZ];
340 : size_t bytes_to_read;
341 : size_t rc;
342 :
343 0 : bytes_to_read = Min(sizeof(manifestbuf),
344 : manifest->manifest_size - manifest_bytes_done);
345 0 : rc = BufFileRead(manifest->buffile, manifestbuf, bytes_to_read);
346 0 : if (rc != bytes_to_read)
347 0 : ereport(ERROR,
348 : (errcode_for_file_access(),
349 : errmsg("could not read from temporary file: %m")));
350 0 : pq_putmessage('d', manifestbuf, bytes_to_read);
351 0 : manifest_bytes_done += bytes_to_read;
352 : }
353 :
354 : /* No more data, so send CopyDone message */
355 0 : pq_putemptymessage('c');
356 :
357 : /* Release resources */
358 0 : BufFileClose(manifest->buffile);
359 : }
360 :
361 : /*
362 : * Append a cstring to the manifest.
363 : */
364 : static void
365 0 : AppendStringToManifest(backup_manifest_info *manifest, char *s)
366 : {
367 0 : int len = strlen(s);
368 :
369 0 : Assert(manifest != NULL);
370 0 : if (manifest->still_checksumming)
371 0 : pg_sha256_update(&manifest->manifest_ctx, (uint8 *) s, len);
372 0 : BufFileWrite(manifest->buffile, s, len);
373 0 : manifest->manifest_size += len;
374 0 : }
|