1/*-------------------------------------------------------------------------
2 *
3 * pgarch.c
4 *
5 * PostgreSQL WAL archiver
6 *
7 * All functions relating to archiver are included here
8 *
9 * - All functions executed by archiver process
10 *
11 * - archiver is forked from postmaster, and the two
12 * processes then communicate using signals. All functions
13 * executed by postmaster are included in this file.
14 *
15 * Initial author: Simon Riggs simon@2ndquadrant.com
16 *
17 * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
18 * Portions Copyright (c) 1994, Regents of the University of California
19 *
20 *
21 * IDENTIFICATION
22 * src/backend/postmaster/pgarch.c
23 *
24 *-------------------------------------------------------------------------
25 */
26#include "postgres.h"
27
28#include <fcntl.h>
29#include <signal.h>
30#include <time.h>
31#include <sys/stat.h>
32#include <sys/time.h>
33#include <sys/wait.h>
34#include <unistd.h>
35
36#include "access/xlog.h"
37#include "access/xlog_internal.h"
38#include "libpq/pqsignal.h"
39#include "miscadmin.h"
40#include "pgstat.h"
41#include "postmaster/fork_process.h"
42#include "postmaster/pgarch.h"
43#include "postmaster/postmaster.h"
44#include "storage/dsm.h"
45#include "storage/fd.h"
46#include "storage/ipc.h"
47#include "storage/latch.h"
48#include "storage/pg_shmem.h"
49#include "storage/pmsignal.h"
50#include "utils/guc.h"
51#include "utils/ps_status.h"
52
53
54/* ----------
55 * Timer definitions.
56 * ----------
57 */
58#define PGARCH_AUTOWAKE_INTERVAL 60 /* How often to force a poll of the
59 * archive status directory; in seconds. */
60#define PGARCH_RESTART_INTERVAL 10 /* How often to attempt to restart a
61 * failed archiver; in seconds. */
62
63/*
64 * Maximum number of retries allowed when attempting to archive a WAL
65 * file.
66 */
67#define NUM_ARCHIVE_RETRIES 3
68
69/*
70 * Maximum number of retries allowed when attempting to remove an
71 * orphan archive status file.
72 */
73#define NUM_ORPHAN_CLEANUP_RETRIES 3
74
75
76/* ----------
77 * Local data
78 * ----------
79 */
80static time_t last_pgarch_start_time;
81static time_t last_sigterm_time = 0;
82
83/*
84 * Flags set by interrupt handlers for later service in the main loop.
85 */
86static volatile sig_atomic_t got_SIGHUP = false;
87static volatile sig_atomic_t got_SIGTERM = false;
88static volatile sig_atomic_t wakened = false;
89static volatile sig_atomic_t ready_to_stop = false;
90
91/* ----------
92 * Local function forward declarations
93 * ----------
94 */
95#ifdef EXEC_BACKEND
96static pid_t pgarch_forkexec(void);
97#endif
98
99NON_EXEC_STATIC void PgArchiverMain(int argc, char *argv[]) pg_attribute_noreturn();
100static void pgarch_exit(SIGNAL_ARGS);
101static void ArchSigHupHandler(SIGNAL_ARGS);
102static void ArchSigTermHandler(SIGNAL_ARGS);
103static void pgarch_waken(SIGNAL_ARGS);
104static void pgarch_waken_stop(SIGNAL_ARGS);
105static void pgarch_MainLoop(void);
106static void pgarch_ArchiverCopyLoop(void);
107static bool pgarch_archiveXlog(char *xlog);
108static bool pgarch_readyXlog(char *xlog);
109static void pgarch_archiveDone(char *xlog);
110
111
112/* ------------------------------------------------------------
113 * Public functions called from postmaster follow
114 * ------------------------------------------------------------
115 */
116
117/*
118 * pgarch_start
119 *
120 * Called from postmaster at startup or after an existing archiver
121 * died. Attempt to fire up a fresh archiver process.
122 *
123 * Returns PID of child process, or 0 if fail.
124 *
125 * Note: if fail, we will be called again from the postmaster main loop.
126 */
127int
128pgarch_start(void)
129{
130 time_t curtime;
131 pid_t pgArchPid;
132
133 /*
134 * Do nothing if no archiver needed
135 */
136 if (!XLogArchivingActive())
137 return 0;
138
139 /*
140 * Do nothing if too soon since last archiver start. This is a safety
141 * valve to protect against continuous respawn attempts if the archiver is
142 * dying immediately at launch. Note that since we will be re-called from
143 * the postmaster main loop, we will get another chance later.
144 */
145 curtime = time(NULL);
146 if ((unsigned int) (curtime - last_pgarch_start_time) <
147 (unsigned int) PGARCH_RESTART_INTERVAL)
148 return 0;
149 last_pgarch_start_time = curtime;
150
151#ifdef EXEC_BACKEND
152 switch ((pgArchPid = pgarch_forkexec()))
153#else
154 switch ((pgArchPid = fork_process()))
155#endif
156 {
157 case -1:
158 ereport(LOG,
159 (errmsg("could not fork archiver: %m")));
160 return 0;
161
162#ifndef EXEC_BACKEND
163 case 0:
164 /* in postmaster child ... */
165 InitPostmasterChild();
166
167 /* Close the postmaster's sockets */
168 ClosePostmasterPorts(false);
169
170 /* Drop our connection to postmaster's shared memory, as well */
171 dsm_detach_all();
172 PGSharedMemoryDetach();
173
174 PgArchiverMain(0, NULL);
175 break;
176#endif
177
178 default:
179 return (int) pgArchPid;
180 }
181
182 /* shouldn't get here */
183 return 0;
184}
185
186/* ------------------------------------------------------------
187 * Local functions called by archiver follow
188 * ------------------------------------------------------------
189 */
190
191
192#ifdef EXEC_BACKEND
193
194/*
195 * pgarch_forkexec() -
196 *
197 * Format up the arglist for, then fork and exec, archive process
198 */
199static pid_t
200pgarch_forkexec(void)
201{
202 char *av[10];
203 int ac = 0;
204
205 av[ac++] = "postgres";
206
207 av[ac++] = "--forkarch";
208
209 av[ac++] = NULL; /* filled in by postmaster_forkexec */
210
211 av[ac] = NULL;
212 Assert(ac < lengthof(av));
213
214 return postmaster_forkexec(ac, av);
215}
216#endif /* EXEC_BACKEND */
217
218
219/*
220 * PgArchiverMain
221 *
222 * The argc/argv parameters are valid only in EXEC_BACKEND case. However,
223 * since we don't use 'em, it hardly matters...
224 */
225NON_EXEC_STATIC void
226PgArchiverMain(int argc, char *argv[])
227{
228 /*
229 * Ignore all signals usually bound to some action in the postmaster,
230 * except for SIGHUP, SIGTERM, SIGUSR1, SIGUSR2, and SIGQUIT.
231 */
232 pqsignal(SIGHUP, ArchSigHupHandler);
233 pqsignal(SIGINT, SIG_IGN);
234 pqsignal(SIGTERM, ArchSigTermHandler);
235 pqsignal(SIGQUIT, pgarch_exit);
236 pqsignal(SIGALRM, SIG_IGN);
237 pqsignal(SIGPIPE, SIG_IGN);
238 pqsignal(SIGUSR1, pgarch_waken);
239 pqsignal(SIGUSR2, pgarch_waken_stop);
240 /* Reset some signals that are accepted by postmaster but not here */
241 pqsignal(SIGCHLD, SIG_DFL);
242 PG_SETMASK(&UnBlockSig);
243
244 /*
245 * Identify myself via ps
246 */
247 init_ps_display("archiver", "", "", "");
248
249 pgarch_MainLoop();
250
251 exit(0);
252}
253
254/* SIGQUIT signal handler for archiver process */
255static void
256pgarch_exit(SIGNAL_ARGS)
257{
258 /* SIGQUIT means curl up and die ... */
259 exit(1);
260}
261
262/* SIGHUP signal handler for archiver process */
263static void
264ArchSigHupHandler(SIGNAL_ARGS)
265{
266 int save_errno = errno;
267
268 /* set flag to re-read config file at next convenient time */
269 got_SIGHUP = true;
270 SetLatch(MyLatch);
271
272 errno = save_errno;
273}
274
275/* SIGTERM signal handler for archiver process */
276static void
277ArchSigTermHandler(SIGNAL_ARGS)
278{
279 int save_errno = errno;
280
281 /*
282 * The postmaster never sends us SIGTERM, so we assume that this means
283 * that init is trying to shut down the whole system. If we hang around
284 * too long we'll get SIGKILL'd. Set flag to prevent starting any more
285 * archive commands.
286 */
287 got_SIGTERM = true;
288 SetLatch(MyLatch);
289
290 errno = save_errno;
291}
292
293/* SIGUSR1 signal handler for archiver process */
294static void
295pgarch_waken(SIGNAL_ARGS)
296{
297 int save_errno = errno;
298
299 /* set flag that there is work to be done */
300 wakened = true;
301 SetLatch(MyLatch);
302
303 errno = save_errno;
304}
305
306/* SIGUSR2 signal handler for archiver process */
307static void
308pgarch_waken_stop(SIGNAL_ARGS)
309{
310 int save_errno = errno;
311
312 /* set flag to do a final cycle and shut down afterwards */
313 ready_to_stop = true;
314 SetLatch(MyLatch);
315
316 errno = save_errno;
317}
318
319/*
320 * pgarch_MainLoop
321 *
322 * Main loop for archiver
323 */
324static void
325pgarch_MainLoop(void)
326{
327 pg_time_t last_copy_time = 0;
328 bool time_to_stop;
329
330 /*
331 * We run the copy loop immediately upon entry, in case there are
332 * unarchived files left over from a previous database run (or maybe the
333 * archiver died unexpectedly). After that we wait for a signal or
334 * timeout before doing more.
335 */
336 wakened = true;
337
338 /*
339 * There shouldn't be anything for the archiver to do except to wait for a
340 * signal ... however, the archiver exists to protect our data, so she
341 * wakes up occasionally to allow herself to be proactive.
342 */
343 do
344 {
345 ResetLatch(MyLatch);
346
347 /* When we get SIGUSR2, we do one more archive cycle, then exit */
348 time_to_stop = ready_to_stop;
349
350 /* Check for config update */
351 if (got_SIGHUP)
352 {
353 got_SIGHUP = false;
354 ProcessConfigFile(PGC_SIGHUP);
355 }
356
357 /*
358 * If we've gotten SIGTERM, we normally just sit and do nothing until
359 * SIGUSR2 arrives. However, that means a random SIGTERM would
360 * disable archiving indefinitely, which doesn't seem like a good
361 * idea. If more than 60 seconds pass since SIGTERM, exit anyway, so
362 * that the postmaster can start a new archiver if needed.
363 */
364 if (got_SIGTERM)
365 {
366 time_t curtime = time(NULL);
367
368 if (last_sigterm_time == 0)
369 last_sigterm_time = curtime;
370 else if ((unsigned int) (curtime - last_sigterm_time) >=
371 (unsigned int) 60)
372 break;
373 }
374
375 /* Do what we're here for */
376 if (wakened || time_to_stop)
377 {
378 wakened = false;
379 pgarch_ArchiverCopyLoop();
380 last_copy_time = time(NULL);
381 }
382
383 /*
384 * Sleep until a signal is received, or until a poll is forced by
385 * PGARCH_AUTOWAKE_INTERVAL having passed since last_copy_time, or
386 * until postmaster dies.
387 */
388 if (!time_to_stop) /* Don't wait during last iteration */
389 {
390 pg_time_t curtime = (pg_time_t) time(NULL);
391 int timeout;
392
393 timeout = PGARCH_AUTOWAKE_INTERVAL - (curtime - last_copy_time);
394 if (timeout > 0)
395 {
396 int rc;
397
398 rc = WaitLatch(MyLatch,
399 WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
400 timeout * 1000L,
401 WAIT_EVENT_ARCHIVER_MAIN);
402 if (rc & WL_TIMEOUT)
403 wakened = true;
404 if (rc & WL_POSTMASTER_DEATH)
405 time_to_stop = true;
406 }
407 else
408 wakened = true;
409 }
410
411 /*
412 * The archiver quits either when the postmaster dies (not expected)
413 * or after completing one more archiving cycle after receiving
414 * SIGUSR2.
415 */
416 } while (!time_to_stop);
417}
418
419/*
420 * pgarch_ArchiverCopyLoop
421 *
422 * Archives all outstanding xlogs then returns
423 */
424static void
425pgarch_ArchiverCopyLoop(void)
426{
427 char xlog[MAX_XFN_CHARS + 1];
428
429 /*
430 * loop through all xlogs with archive_status of .ready and archive
431 * them...mostly we expect this to be a single file, though it is possible
432 * some backend will add files onto the list of those that need archiving
433 * while we are still copying earlier archives
434 */
435 while (pgarch_readyXlog(xlog))
436 {
437 int failures = 0;
438 int failures_orphan = 0;
439
440 for (;;)
441 {
442 struct stat stat_buf;
443 char pathname[MAXPGPATH];
444
445 /*
446 * Do not initiate any more archive commands after receiving
447 * SIGTERM, nor after the postmaster has died unexpectedly. The
448 * first condition is to try to keep from having init SIGKILL the
449 * command, and the second is to avoid conflicts with another
450 * archiver spawned by a newer postmaster.
451 */
452 if (got_SIGTERM || !PostmasterIsAlive())
453 return;
454
455 /*
456 * Check for config update. This is so that we'll adopt a new
457 * setting for archive_command as soon as possible, even if there
458 * is a backlog of files to be archived.
459 */
460 if (got_SIGHUP)
461 {
462 got_SIGHUP = false;
463 ProcessConfigFile(PGC_SIGHUP);
464 }
465
466 /* can't do anything if no command ... */
467 if (!XLogArchiveCommandSet())
468 {
469 ereport(WARNING,
470 (errmsg("archive_mode enabled, yet archive_command is not set")));
471 return;
472 }
473
474 /*
475 * Since archive status files are not removed in a durable manner,
476 * a system crash could leave behind .ready files for WAL segments
477 * that have already been recycled or removed. In this case,
478 * simply remove the orphan status file and move on. unlink() is
479 * used here as even on subsequent crashes the same orphan files
480 * would get removed, so there is no need to worry about
481 * durability.
482 */
483 snprintf(pathname, MAXPGPATH, XLOGDIR "/%s", xlog);
484 if (stat(pathname, &stat_buf) != 0 && errno == ENOENT)
485 {
486 char xlogready[MAXPGPATH];
487
488 StatusFilePath(xlogready, xlog, ".ready");
489 if (unlink(xlogready) == 0)
490 {
491 ereport(WARNING,
492 (errmsg("removed orphan archive status file \"%s\"",
493 xlogready)));
494
495 /* leave loop and move to the next status file */
496 break;
497 }
498
499 if (++failures_orphan >= NUM_ORPHAN_CLEANUP_RETRIES)
500 {
501 ereport(WARNING,
502 (errmsg("removal of orphan archive status file \"%s\" failed too many times, will try again later",
503 xlogready)));
504
505 /* give up cleanup of orphan status files */
506 return;
507 }
508
509 /* wait a bit before retrying */
510 pg_usleep(1000000L);
511 continue;
512 }
513
514 if (pgarch_archiveXlog(xlog))
515 {
516 /* successful */
517 pgarch_archiveDone(xlog);
518
519 /*
520 * Tell the collector about the WAL file that we successfully
521 * archived
522 */
523 pgstat_send_archiver(xlog, false);
524
525 break; /* out of inner retry loop */
526 }
527 else
528 {
529 /*
530 * Tell the collector about the WAL file that we failed to
531 * archive
532 */
533 pgstat_send_archiver(xlog, true);
534
535 if (++failures >= NUM_ARCHIVE_RETRIES)
536 {
537 ereport(WARNING,
538 (errmsg("archiving write-ahead log file \"%s\" failed too many times, will try again later",
539 xlog)));
540 return; /* give up archiving for now */
541 }
542 pg_usleep(1000000L); /* wait a bit before retrying */
543 }
544 }
545 }
546}
547
548/*
549 * pgarch_archiveXlog
550 *
551 * Invokes system(3) to copy one archive file to wherever it should go
552 *
553 * Returns true if successful
554 */
555static bool
556pgarch_archiveXlog(char *xlog)
557{
558 char xlogarchcmd[MAXPGPATH];
559 char pathname[MAXPGPATH];
560 char activitymsg[MAXFNAMELEN + 16];
561 char *dp;
562 char *endp;
563 const char *sp;
564 int rc;
565
566 snprintf(pathname, MAXPGPATH, XLOGDIR "/%s", xlog);
567
568 /*
569 * construct the command to be executed
570 */
571 dp = xlogarchcmd;
572 endp = xlogarchcmd + MAXPGPATH - 1;
573 *endp = '\0';
574
575 for (sp = XLogArchiveCommand; *sp; sp++)
576 {
577 if (*sp == '%')
578 {
579 switch (sp[1])
580 {
581 case 'p':
582 /* %p: relative path of source file */
583 sp++;
584 strlcpy(dp, pathname, endp - dp);
585 make_native_path(dp);
586 dp += strlen(dp);
587 break;
588 case 'f':
589 /* %f: filename of source file */
590 sp++;
591 strlcpy(dp, xlog, endp - dp);
592 dp += strlen(dp);
593 break;
594 case '%':
595 /* convert %% to a single % */
596 sp++;
597 if (dp < endp)
598 *dp++ = *sp;
599 break;
600 default:
601 /* otherwise treat the % as not special */
602 if (dp < endp)
603 *dp++ = *sp;
604 break;
605 }
606 }
607 else
608 {
609 if (dp < endp)
610 *dp++ = *sp;
611 }
612 }
613 *dp = '\0';
614
615 ereport(DEBUG3,
616 (errmsg_internal("executing archive command \"%s\"",
617 xlogarchcmd)));
618
619 /* Report archive activity in PS display */
620 snprintf(activitymsg, sizeof(activitymsg), "archiving %s", xlog);
621 set_ps_display(activitymsg, false);
622
623 rc = system(xlogarchcmd);
624 if (rc != 0)
625 {
626 /*
627 * If either the shell itself, or a called command, died on a signal,
628 * abort the archiver. We do this because system() ignores SIGINT and
629 * SIGQUIT while waiting; so a signal is very likely something that
630 * should have interrupted us too. Also die if the shell got a hard
631 * "command not found" type of error. If we overreact it's no big
632 * deal, the postmaster will just start the archiver again.
633 */
634 int lev = wait_result_is_any_signal(rc, true) ? FATAL : LOG;
635
636 if (WIFEXITED(rc))
637 {
638 ereport(lev,
639 (errmsg("archive command failed with exit code %d",
640 WEXITSTATUS(rc)),
641 errdetail("The failed archive command was: %s",
642 xlogarchcmd)));
643 }
644 else if (WIFSIGNALED(rc))
645 {
646#if defined(WIN32)
647 ereport(lev,
648 (errmsg("archive command was terminated by exception 0x%X",
649 WTERMSIG(rc)),
650 errhint("See C include file \"ntstatus.h\" for a description of the hexadecimal value."),
651 errdetail("The failed archive command was: %s",
652 xlogarchcmd)));
653#else
654 ereport(lev,
655 (errmsg("archive command was terminated by signal %d: %s",
656 WTERMSIG(rc), pg_strsignal(WTERMSIG(rc))),
657 errdetail("The failed archive command was: %s",
658 xlogarchcmd)));
659#endif
660 }
661 else
662 {
663 ereport(lev,
664 (errmsg("archive command exited with unrecognized status %d",
665 rc),
666 errdetail("The failed archive command was: %s",
667 xlogarchcmd)));
668 }
669
670 snprintf(activitymsg, sizeof(activitymsg), "failed on %s", xlog);
671 set_ps_display(activitymsg, false);
672
673 return false;
674 }
675 elog(DEBUG1, "archived write-ahead log file \"%s\"", xlog);
676
677 snprintf(activitymsg, sizeof(activitymsg), "last was %s", xlog);
678 set_ps_display(activitymsg, false);
679
680 return true;
681}
682
683/*
684 * pgarch_readyXlog
685 *
686 * Return name of the oldest xlog file that has not yet been archived.
687 * No notification is set that file archiving is now in progress, so
688 * this would need to be extended if multiple concurrent archival
689 * tasks were created. If a failure occurs, we will completely
690 * re-copy the file at the next available opportunity.
691 *
692 * It is important that we return the oldest, so that we archive xlogs
693 * in order that they were written, for two reasons:
694 * 1) to maintain the sequential chain of xlogs required for recovery
695 * 2) because the oldest ones will sooner become candidates for
696 * recycling at time of checkpoint
697 *
698 * NOTE: the "oldest" comparison will consider any .history file to be older
699 * than any other file except another .history file. Segments on a timeline
700 * with a smaller ID will be older than all segments on a timeline with a
701 * larger ID; the net result being that past timelines are given higher
702 * priority for archiving. This seems okay, or at least not obviously worth
703 * changing.
704 */
705static bool
706pgarch_readyXlog(char *xlog)
707{
708 /*
709 * open xlog status directory and read through list of xlogs that have the
710 * .ready suffix, looking for earliest file. It is possible to optimise
711 * this code, though only a single file is expected on the vast majority
712 * of calls, so....
713 */
714 char XLogArchiveStatusDir[MAXPGPATH];
715 DIR *rldir;
716 struct dirent *rlde;
717 bool found = false;
718 bool historyFound = false;
719
720 snprintf(XLogArchiveStatusDir, MAXPGPATH, XLOGDIR "/archive_status");
721 rldir = AllocateDir(XLogArchiveStatusDir);
722
723 while ((rlde = ReadDir(rldir, XLogArchiveStatusDir)) != NULL)
724 {
725 int basenamelen = (int) strlen(rlde->d_name) - 6;
726 char basename[MAX_XFN_CHARS + 1];
727 bool ishistory;
728
729 /* Ignore entries with unexpected number of characters */
730 if (basenamelen < MIN_XFN_CHARS ||
731 basenamelen > MAX_XFN_CHARS)
732 continue;
733
734 /* Ignore entries with unexpected characters */
735 if (strspn(rlde->d_name, VALID_XFN_CHARS) < basenamelen)
736 continue;
737
738 /* Ignore anything not suffixed with .ready */
739 if (strcmp(rlde->d_name + basenamelen, ".ready") != 0)
740 continue;
741
742 /* Truncate off the .ready */
743 memcpy(basename, rlde->d_name, basenamelen);
744 basename[basenamelen] = '\0';
745
746 /* Is this a history file? */
747 ishistory = IsTLHistoryFileName(basename);
748
749 /*
750 * Consume the file to archive. History files have the highest
751 * priority. If this is the first file or the first history file
752 * ever, copy it. In the presence of a history file already chosen as
753 * target, ignore all other files except history files which have been
754 * generated for an older timeline than what is already chosen as
755 * target to archive.
756 */
757 if (!found || (ishistory && !historyFound))
758 {
759 strcpy(xlog, basename);
760 found = true;
761 historyFound = ishistory;
762 }
763 else if (ishistory || !historyFound)
764 {
765 if (strcmp(basename, xlog) < 0)
766 strcpy(xlog, basename);
767 }
768 }
769 FreeDir(rldir);
770
771 return found;
772}
773
774/*
775 * pgarch_archiveDone
776 *
777 * Emit notification that an xlog file has been successfully archived.
778 * We do this by renaming the status file from NNN.ready to NNN.done.
779 * Eventually, a checkpoint process will notice this and delete both the
780 * NNN.done file and the xlog file itself.
781 */
782static void
783pgarch_archiveDone(char *xlog)
784{
785 char rlogready[MAXPGPATH];
786 char rlogdone[MAXPGPATH];
787
788 StatusFilePath(rlogready, xlog, ".ready");
789 StatusFilePath(rlogdone, xlog, ".done");
790 (void) durable_rename(rlogready, rlogdone, WARNING);
791}
792