1/*-------------------------------------------------------------------------
2 *
3 * pg_backup_tar.c
4 *
5 * This file is copied from the 'files' format file, but dumps data into
6 * one temp file then sends it to the output TAR archive.
7 *
8 * The tar format also includes a 'restore.sql' script which is there for
9 * the benefit of humans. This script is never used by pg_restore.
10 *
11 * NOTE: If you untar the created 'tar' file, the resulting files are
12 * compatible with the 'directory' format. Please keep the two formats in
13 * sync.
14 *
15 * See the headers to pg_backup_directory & pg_restore for more details.
16 *
17 * Copyright (c) 2000, Philip Warner
18 * Rights are granted to use this software in any way so long
19 * as this notice is not removed.
20 *
21 * The author is not responsible for loss or damages that may
22 * result from its use.
23 *
24 *
25 * IDENTIFICATION
26 * src/bin/pg_dump/pg_backup_tar.c
27 *
28 *-------------------------------------------------------------------------
29 */
30#include "postgres_fe.h"
31
32#include "pg_backup_archiver.h"
33#include "pg_backup_tar.h"
34#include "pg_backup_utils.h"
35#include "pgtar.h"
36#include "common/file_utils.h"
37#include "fe_utils/string_utils.h"
38
39#include <sys/stat.h>
40#include <ctype.h>
41#include <limits.h>
42#include <unistd.h>
43
44static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
45static void _StartData(ArchiveHandle *AH, TocEntry *te);
46static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
47static void _EndData(ArchiveHandle *AH, TocEntry *te);
48static int _WriteByte(ArchiveHandle *AH, const int i);
49static int _ReadByte(ArchiveHandle *);
50static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
51static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
52static void _CloseArchive(ArchiveHandle *AH);
53static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
54static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
55static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
56static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
57
58static void _StartBlobs(ArchiveHandle *AH, TocEntry *te);
59static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
60static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
61static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
62
63#define K_STD_BUF_SIZE 1024
64
65
66typedef struct
67{
68#ifdef HAVE_LIBZ
69 gzFile zFH;
70#else
71 FILE *zFH;
72#endif
73 FILE *nFH;
74 FILE *tarFH;
75 FILE *tmpFH;
76 char *targetFile;
77 char mode;
78 pgoff_t pos;
79 pgoff_t fileLen;
80 ArchiveHandle *AH;
81} TAR_MEMBER;
82
83typedef struct
84{
85 int hasSeek;
86 pgoff_t filePos;
87 TAR_MEMBER *blobToc;
88 FILE *tarFH;
89 pgoff_t tarFHpos;
90 pgoff_t tarNextMember;
91 TAR_MEMBER *FH;
92 int isSpecialScript;
93 TAR_MEMBER *scriptTH;
94} lclContext;
95
96typedef struct
97{
98 TAR_MEMBER *TH;
99 char *filename;
100} lclTocEntry;
101
102static void _LoadBlobs(ArchiveHandle *AH);
103
104static TAR_MEMBER *tarOpen(ArchiveHandle *AH, const char *filename, char mode);
105static void tarClose(ArchiveHandle *AH, TAR_MEMBER *TH);
106
107#ifdef __NOT_USED__
108static char *tarGets(char *buf, size_t len, TAR_MEMBER *th);
109#endif
110static int tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(3, 4);
111
112static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th);
113static TAR_MEMBER *_tarPositionTo(ArchiveHandle *AH, const char *filename);
114static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th);
115static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th);
116static void _tarWriteHeader(TAR_MEMBER *th);
117static int _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th);
118static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh);
119
120static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len);
121
122/*
123 * Initializer
124 */
125void
126InitArchiveFmt_Tar(ArchiveHandle *AH)
127{
128 lclContext *ctx;
129
130 /* Assuming static functions, this can be copied for each format. */
131 AH->ArchiveEntryPtr = _ArchiveEntry;
132 AH->StartDataPtr = _StartData;
133 AH->WriteDataPtr = _WriteData;
134 AH->EndDataPtr = _EndData;
135 AH->WriteBytePtr = _WriteByte;
136 AH->ReadBytePtr = _ReadByte;
137 AH->WriteBufPtr = _WriteBuf;
138 AH->ReadBufPtr = _ReadBuf;
139 AH->ClosePtr = _CloseArchive;
140 AH->ReopenPtr = NULL;
141 AH->PrintTocDataPtr = _PrintTocData;
142 AH->ReadExtraTocPtr = _ReadExtraToc;
143 AH->WriteExtraTocPtr = _WriteExtraToc;
144 AH->PrintExtraTocPtr = _PrintExtraToc;
145
146 AH->StartBlobsPtr = _StartBlobs;
147 AH->StartBlobPtr = _StartBlob;
148 AH->EndBlobPtr = _EndBlob;
149 AH->EndBlobsPtr = _EndBlobs;
150 AH->ClonePtr = NULL;
151 AH->DeClonePtr = NULL;
152
153 AH->WorkerJobDumpPtr = NULL;
154 AH->WorkerJobRestorePtr = NULL;
155
156 /*
157 * Set up some special context used in compressing data.
158 */
159 ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
160 AH->formatData = (void *) ctx;
161 ctx->filePos = 0;
162 ctx->isSpecialScript = 0;
163
164 /* Initialize LO buffering */
165 AH->lo_buf_size = LOBBUFSIZE;
166 AH->lo_buf = (void *) pg_malloc(LOBBUFSIZE);
167
168 /*
169 * Now open the tar file, and load the TOC if we're in read mode.
170 */
171 if (AH->mode == archModeWrite)
172 {
173 if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
174 {
175 ctx->tarFH = fopen(AH->fSpec, PG_BINARY_W);
176 if (ctx->tarFH == NULL)
177 fatal("could not open TOC file \"%s\" for output: %m",
178 AH->fSpec);
179 }
180 else
181 {
182 ctx->tarFH = stdout;
183 if (ctx->tarFH == NULL)
184 fatal("could not open TOC file for output: %m");
185 }
186
187 ctx->tarFHpos = 0;
188
189 /*
190 * Make unbuffered since we will dup() it, and the buffers screw each
191 * other
192 */
193 /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
194
195 ctx->hasSeek = checkSeek(ctx->tarFH);
196
197 /*
198 * We don't support compression because reading the files back is not
199 * possible since gzdopen uses buffered IO which totally screws file
200 * positioning.
201 */
202 if (AH->compression != 0)
203 fatal("compression is not supported by tar archive format");
204 }
205 else
206 { /* Read Mode */
207 if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
208 {
209 ctx->tarFH = fopen(AH->fSpec, PG_BINARY_R);
210 if (ctx->tarFH == NULL)
211 fatal("could not open TOC file \"%s\" for input: %m",
212 AH->fSpec);
213 }
214 else
215 {
216 ctx->tarFH = stdin;
217 if (ctx->tarFH == NULL)
218 fatal("could not open TOC file for input: %m");
219 }
220
221 /*
222 * Make unbuffered since we will dup() it, and the buffers screw each
223 * other
224 */
225 /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
226
227 ctx->tarFHpos = 0;
228
229 ctx->hasSeek = checkSeek(ctx->tarFH);
230
231 /*
232 * Forcibly unmark the header as read since we use the lookahead
233 * buffer
234 */
235 AH->readHeader = 0;
236
237 ctx->FH = (void *) tarOpen(AH, "toc.dat", 'r');
238 ReadHead(AH);
239 ReadToc(AH);
240 tarClose(AH, ctx->FH); /* Nothing else in the file... */
241 }
242}
243
244/*
245 * - Start a new TOC entry
246 * Setup the output file name.
247 */
248static void
249_ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
250{
251 lclTocEntry *ctx;
252 char fn[K_STD_BUF_SIZE];
253
254 ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
255 if (te->dataDumper != NULL)
256 {
257#ifdef HAVE_LIBZ
258 if (AH->compression == 0)
259 sprintf(fn, "%d.dat", te->dumpId);
260 else
261 sprintf(fn, "%d.dat.gz", te->dumpId);
262#else
263 sprintf(fn, "%d.dat", te->dumpId);
264#endif
265 ctx->filename = pg_strdup(fn);
266 }
267 else
268 {
269 ctx->filename = NULL;
270 ctx->TH = NULL;
271 }
272 te->formatData = (void *) ctx;
273}
274
275static void
276_WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
277{
278 lclTocEntry *ctx = (lclTocEntry *) te->formatData;
279
280 if (ctx->filename)
281 WriteStr(AH, ctx->filename);
282 else
283 WriteStr(AH, "");
284}
285
286static void
287_ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
288{
289 lclTocEntry *ctx = (lclTocEntry *) te->formatData;
290
291 if (ctx == NULL)
292 {
293 ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
294 te->formatData = (void *) ctx;
295 }
296
297 ctx->filename = ReadStr(AH);
298 if (strlen(ctx->filename) == 0)
299 {
300 free(ctx->filename);
301 ctx->filename = NULL;
302 }
303 ctx->TH = NULL;
304}
305
306static void
307_PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
308{
309 lclTocEntry *ctx = (lclTocEntry *) te->formatData;
310
311 if (AH->public.verbose && ctx->filename != NULL)
312 ahprintf(AH, "-- File: %s\n", ctx->filename);
313}
314
315static void
316_StartData(ArchiveHandle *AH, TocEntry *te)
317{
318 lclTocEntry *tctx = (lclTocEntry *) te->formatData;
319
320 tctx->TH = tarOpen(AH, tctx->filename, 'w');
321}
322
323static TAR_MEMBER *
324tarOpen(ArchiveHandle *AH, const char *filename, char mode)
325{
326 lclContext *ctx = (lclContext *) AH->formatData;
327 TAR_MEMBER *tm;
328
329#ifdef HAVE_LIBZ
330 char fmode[14];
331#endif
332
333 if (mode == 'r')
334 {
335 tm = _tarPositionTo(AH, filename);
336 if (!tm) /* Not found */
337 {
338 if (filename)
339 {
340 /*
341 * Couldn't find the requested file. Future: do SEEK(0) and
342 * retry.
343 */
344 fatal("could not find file \"%s\" in archive", filename);
345 }
346 else
347 {
348 /* Any file OK, none left, so return NULL */
349 return NULL;
350 }
351 }
352
353#ifdef HAVE_LIBZ
354
355 if (AH->compression == 0)
356 tm->nFH = ctx->tarFH;
357 else
358 fatal("compression is not supported by tar archive format");
359 /* tm->zFH = gzdopen(dup(fileno(ctx->tarFH)), "rb"); */
360#else
361 tm->nFH = ctx->tarFH;
362#endif
363 }
364 else
365 {
366 int old_umask;
367
368 tm = pg_malloc0(sizeof(TAR_MEMBER));
369
370 /*
371 * POSIX does not require, but permits, tmpfile() to restrict file
372 * permissions. Given an OS crash after we write data, the filesystem
373 * might retain the data but forget tmpfile()'s unlink(). If so, the
374 * file mode protects confidentiality of the data written.
375 */
376 old_umask = umask(S_IRWXG | S_IRWXO);
377
378#ifndef WIN32
379 tm->tmpFH = tmpfile();
380#else
381
382 /*
383 * On WIN32, tmpfile() generates a filename in the root directory,
384 * which requires administrative permissions on certain systems. Loop
385 * until we find a unique file name we can create.
386 */
387 while (1)
388 {
389 char *name;
390 int fd;
391
392 name = _tempnam(NULL, "pg_temp_");
393 if (name == NULL)
394 break;
395 fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY |
396 O_TEMPORARY, S_IRUSR | S_IWUSR);
397 free(name);
398
399 if (fd != -1) /* created a file */
400 {
401 tm->tmpFH = fdopen(fd, "w+b");
402 break;
403 }
404 else if (errno != EEXIST) /* failure other than file exists */
405 break;
406 }
407#endif
408
409 if (tm->tmpFH == NULL)
410 fatal("could not generate temporary file name: %m");
411
412 umask(old_umask);
413
414#ifdef HAVE_LIBZ
415
416 if (AH->compression != 0)
417 {
418 sprintf(fmode, "wb%d", AH->compression);
419 tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode);
420 if (tm->zFH == NULL)
421 fatal("could not open temporary file");
422 }
423 else
424 tm->nFH = tm->tmpFH;
425#else
426
427 tm->nFH = tm->tmpFH;
428#endif
429
430 tm->AH = AH;
431 tm->targetFile = pg_strdup(filename);
432 }
433
434 tm->mode = mode;
435 tm->tarFH = ctx->tarFH;
436
437 return tm;
438}
439
440static void
441tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
442{
443 /*
444 * Close the GZ file since we dup'd. This will flush the buffers.
445 */
446 if (AH->compression != 0)
447 if (GZCLOSE(th->zFH) != 0)
448 fatal("could not close tar member");
449
450 if (th->mode == 'w')
451 _tarAddFile(AH, th); /* This will close the temp file */
452
453 /*
454 * else Nothing to do for normal read since we don't dup() normal file
455 * handle, and we don't use temp files.
456 */
457
458 if (th->targetFile)
459 free(th->targetFile);
460
461 th->nFH = NULL;
462 th->zFH = NULL;
463}
464
465#ifdef __NOT_USED__
466static char *
467tarGets(char *buf, size_t len, TAR_MEMBER *th)
468{
469 char *s;
470 size_t cnt = 0;
471 char c = ' ';
472 int eof = 0;
473
474 /* Can't read past logical EOF */
475 if (len > (th->fileLen - th->pos))
476 len = th->fileLen - th->pos;
477
478 while (cnt < len && c != '\n')
479 {
480 if (_tarReadRaw(th->AH, &c, 1, th, NULL) <= 0)
481 {
482 eof = 1;
483 break;
484 }
485 buf[cnt++] = c;
486 }
487
488 if (eof && cnt == 0)
489 s = NULL;
490 else
491 {
492 buf[cnt++] = '\0';
493 s = buf;
494 }
495
496 if (s)
497 {
498 len = strlen(s);
499 th->pos += len;
500 }
501
502 return s;
503}
504#endif
505
506/*
507 * Just read bytes from the archive. This is the low level read routine
508 * that is used for ALL reads on a tar file.
509 */
510static size_t
511_tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
512{
513 lclContext *ctx = (lclContext *) AH->formatData;
514 size_t avail;
515 size_t used = 0;
516 size_t res = 0;
517
518 avail = AH->lookaheadLen - AH->lookaheadPos;
519 if (avail > 0)
520 {
521 /* We have some lookahead bytes to use */
522 if (avail >= len) /* Just use the lookahead buffer */
523 used = len;
524 else
525 used = avail;
526
527 /* Copy, and adjust buffer pos */
528 memcpy(buf, AH->lookahead + AH->lookaheadPos, used);
529 AH->lookaheadPos += used;
530
531 /* Adjust required length */
532 len -= used;
533 }
534
535 /* Read the file if len > 0 */
536 if (len > 0)
537 {
538 if (fh)
539 {
540 res = fread(&((char *) buf)[used], 1, len, fh);
541 if (res != len && !feof(fh))
542 READ_ERROR_EXIT(fh);
543 }
544 else if (th)
545 {
546 if (th->zFH)
547 {
548 res = GZREAD(&((char *) buf)[used], 1, len, th->zFH);
549 if (res != len && !GZEOF(th->zFH))
550 {
551#ifdef HAVE_LIBZ
552 int errnum;
553 const char *errmsg = gzerror(th->zFH, &errnum);
554
555 fatal("could not read from input file: %s",
556 errnum == Z_ERRNO ? strerror(errno) : errmsg);
557#else
558 fatal("could not read from input file: %s",
559 strerror(errno));
560#endif
561 }
562 }
563 else
564 {
565 res = fread(&((char *) buf)[used], 1, len, th->nFH);
566 if (res != len && !feof(th->nFH))
567 READ_ERROR_EXIT(th->nFH);
568 }
569 }
570 else
571 fatal("internal error -- neither th nor fh specified in tarReadRaw()");
572 }
573
574 ctx->tarFHpos += res + used;
575
576 return (res + used);
577}
578
579static size_t
580tarRead(void *buf, size_t len, TAR_MEMBER *th)
581{
582 size_t res;
583
584 if (th->pos + len > th->fileLen)
585 len = th->fileLen - th->pos;
586
587 if (len <= 0)
588 return 0;
589
590 res = _tarReadRaw(th->AH, buf, len, th, NULL);
591
592 th->pos += res;
593
594 return res;
595}
596
597static size_t
598tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
599{
600 size_t res;
601
602 if (th->zFH != NULL)
603 res = GZWRITE(buf, 1, len, th->zFH);
604 else
605 res = fwrite(buf, 1, len, th->nFH);
606
607 th->pos += res;
608 return res;
609}
610
611static void
612_WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
613{
614 lclTocEntry *tctx = (lclTocEntry *) AH->currToc->formatData;
615
616 if (tarWrite(data, dLen, tctx->TH) != dLen)
617 WRITE_ERROR_EXIT;
618
619 return;
620}
621
622static void
623_EndData(ArchiveHandle *AH, TocEntry *te)
624{
625 lclTocEntry *tctx = (lclTocEntry *) te->formatData;
626
627 /* Close the file */
628 tarClose(AH, tctx->TH);
629 tctx->TH = NULL;
630}
631
632/*
633 * Print data for a given file
634 */
635static void
636_PrintFileData(ArchiveHandle *AH, char *filename)
637{
638 lclContext *ctx = (lclContext *) AH->formatData;
639 char buf[4096];
640 size_t cnt;
641 TAR_MEMBER *th;
642
643 if (!filename)
644 return;
645
646 th = tarOpen(AH, filename, 'r');
647 ctx->FH = th;
648
649 while ((cnt = tarRead(buf, 4095, th)) > 0)
650 {
651 buf[cnt] = '\0';
652 ahwrite(buf, 1, cnt, AH);
653 }
654
655 tarClose(AH, th);
656}
657
658
659/*
660 * Print data for a given TOC entry
661*/
662static void
663_PrintTocData(ArchiveHandle *AH, TocEntry *te)
664{
665 lclContext *ctx = (lclContext *) AH->formatData;
666 lclTocEntry *tctx = (lclTocEntry *) te->formatData;
667 int pos1;
668
669 if (!tctx->filename)
670 return;
671
672 /*
673 * If we're writing the special restore.sql script, emit a suitable
674 * command to include each table's data from the corresponding file.
675 *
676 * In the COPY case this is a bit klugy because the regular COPY command
677 * was already printed before we get control.
678 */
679 if (ctx->isSpecialScript)
680 {
681 if (te->copyStmt)
682 {
683 /* Abort the COPY FROM stdin */
684 ahprintf(AH, "\\.\n");
685
686 /*
687 * The COPY statement should look like "COPY ... FROM stdin;\n",
688 * see dumpTableData().
689 */
690 pos1 = (int) strlen(te->copyStmt) - 13;
691 if (pos1 < 6 || strncmp(te->copyStmt, "COPY ", 5) != 0 ||
692 strcmp(te->copyStmt + pos1, " FROM stdin;\n") != 0)
693 fatal("unexpected COPY statement syntax: \"%s\"",
694 te->copyStmt);
695
696 /* Emit all but the FROM part ... */
697 ahwrite(te->copyStmt, 1, pos1, AH);
698 /* ... and insert modified FROM */
699 ahprintf(AH, " FROM '$$PATH$$/%s';\n\n", tctx->filename);
700 }
701 else
702 {
703 /* --inserts mode, no worries, just include the data file */
704 ahprintf(AH, "\\i $$PATH$$/%s\n\n", tctx->filename);
705 }
706
707 return;
708 }
709
710 if (strcmp(te->desc, "BLOBS") == 0)
711 _LoadBlobs(AH);
712 else
713 _PrintFileData(AH, tctx->filename);
714}
715
716static void
717_LoadBlobs(ArchiveHandle *AH)
718{
719 Oid oid;
720 lclContext *ctx = (lclContext *) AH->formatData;
721 TAR_MEMBER *th;
722 size_t cnt;
723 bool foundBlob = false;
724 char buf[4096];
725
726 StartRestoreBlobs(AH);
727
728 th = tarOpen(AH, NULL, 'r'); /* Open next file */
729 while (th != NULL)
730 {
731 ctx->FH = th;
732
733 if (strncmp(th->targetFile, "blob_", 5) == 0)
734 {
735 oid = atooid(&th->targetFile[5]);
736 if (oid != 0)
737 {
738 pg_log_info("restoring large object with OID %u", oid);
739
740 StartRestoreBlob(AH, oid, AH->public.ropt->dropSchema);
741
742 while ((cnt = tarRead(buf, 4095, th)) > 0)
743 {
744 buf[cnt] = '\0';
745 ahwrite(buf, 1, cnt, AH);
746 }
747 EndRestoreBlob(AH, oid);
748 foundBlob = true;
749 }
750 tarClose(AH, th);
751 }
752 else
753 {
754 tarClose(AH, th);
755
756 /*
757 * Once we have found the first blob, stop at the first non-blob
758 * entry (which will be 'blobs.toc'). This coding would eat all
759 * the rest of the archive if there are no blobs ... but this
760 * function shouldn't be called at all in that case.
761 */
762 if (foundBlob)
763 break;
764 }
765
766 th = tarOpen(AH, NULL, 'r');
767 }
768 EndRestoreBlobs(AH);
769}
770
771
772static int
773_WriteByte(ArchiveHandle *AH, const int i)
774{
775 lclContext *ctx = (lclContext *) AH->formatData;
776 char b = i; /* Avoid endian problems */
777
778 if (tarWrite(&b, 1, ctx->FH) != 1)
779 WRITE_ERROR_EXIT;
780
781 ctx->filePos += 1;
782 return 1;
783}
784
785static int
786_ReadByte(ArchiveHandle *AH)
787{
788 lclContext *ctx = (lclContext *) AH->formatData;
789 size_t res;
790 unsigned char c;
791
792 res = tarRead(&c, 1, ctx->FH);
793 if (res != 1)
794 /* We already would have exited for errors on reads, must be EOF */
795 fatal("could not read from input file: end of file");
796 ctx->filePos += 1;
797 return c;
798}
799
800static void
801_WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
802{
803 lclContext *ctx = (lclContext *) AH->formatData;
804
805 if (tarWrite(buf, len, ctx->FH) != len)
806 WRITE_ERROR_EXIT;
807
808 ctx->filePos += len;
809}
810
811static void
812_ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
813{
814 lclContext *ctx = (lclContext *) AH->formatData;
815
816 if (tarRead(buf, len, ctx->FH) != len)
817 /* We already would have exited for errors on reads, must be EOF */
818 fatal("could not read from input file: end of file");
819
820 ctx->filePos += len;
821 return;
822}
823
824static void
825_CloseArchive(ArchiveHandle *AH)
826{
827 lclContext *ctx = (lclContext *) AH->formatData;
828 TAR_MEMBER *th;
829 RestoreOptions *ropt;
830 RestoreOptions *savRopt;
831 DumpOptions *savDopt;
832 int savVerbose,
833 i;
834
835 if (AH->mode == archModeWrite)
836 {
837 /*
838 * Write the Header & TOC to the archive FIRST
839 */
840 th = tarOpen(AH, "toc.dat", 'w');
841 ctx->FH = th;
842 WriteHead(AH);
843 WriteToc(AH);
844 tarClose(AH, th); /* Not needed any more */
845
846 /*
847 * Now send the data (tables & blobs)
848 */
849 WriteDataChunks(AH, NULL);
850
851 /*
852 * Now this format wants to append a script which does a full restore
853 * if the files have been extracted.
854 */
855 th = tarOpen(AH, "restore.sql", 'w');
856
857 tarPrintf(AH, th, "--\n"
858 "-- NOTE:\n"
859 "--\n"
860 "-- File paths need to be edited. Search for $$PATH$$ and\n"
861 "-- replace it with the path to the directory containing\n"
862 "-- the extracted data files.\n"
863 "--\n");
864
865 AH->CustomOutPtr = _scriptOut;
866
867 ctx->isSpecialScript = 1;
868 ctx->scriptTH = th;
869
870 ropt = NewRestoreOptions();
871 memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
872 ropt->filename = NULL;
873 ropt->dropSchema = 1;
874 ropt->compression = 0;
875 ropt->superuser = NULL;
876 ropt->suppressDumpWarnings = true;
877
878 savDopt = AH->public.dopt;
879 savRopt = AH->public.ropt;
880
881 SetArchiveOptions((Archive *) AH, NULL, ropt);
882
883 savVerbose = AH->public.verbose;
884 AH->public.verbose = 0;
885
886 RestoreArchive((Archive *) AH);
887
888 SetArchiveOptions((Archive *) AH, savDopt, savRopt);
889
890 AH->public.verbose = savVerbose;
891
892 tarClose(AH, th);
893
894 ctx->isSpecialScript = 0;
895
896 /*
897 * EOF marker for tar files is two blocks of NULLs.
898 */
899 for (i = 0; i < 512 * 2; i++)
900 {
901 if (fputc(0, ctx->tarFH) == EOF)
902 WRITE_ERROR_EXIT;
903 }
904
905 /* Sync the output file if one is defined */
906 if (AH->dosync && AH->fSpec)
907 (void) fsync_fname(AH->fSpec, false);
908 }
909
910 AH->FH = NULL;
911}
912
913static size_t
914_scriptOut(ArchiveHandle *AH, const void *buf, size_t len)
915{
916 lclContext *ctx = (lclContext *) AH->formatData;
917
918 return tarWrite(buf, len, ctx->scriptTH);
919}
920
921/*
922 * BLOB support
923 */
924
925/*
926 * Called by the archiver when starting to save all BLOB DATA (not schema).
927 * This routine should save whatever format-specific information is needed
928 * to read the BLOBs back into memory.
929 *
930 * It is called just prior to the dumper's DataDumper routine.
931 *
932 * Optional, but strongly recommended.
933 *
934 */
935static void
936_StartBlobs(ArchiveHandle *AH, TocEntry *te)
937{
938 lclContext *ctx = (lclContext *) AH->formatData;
939 char fname[K_STD_BUF_SIZE];
940
941 sprintf(fname, "blobs.toc");
942 ctx->blobToc = tarOpen(AH, fname, 'w');
943}
944
945/*
946 * Called by the archiver when the dumper calls StartBlob.
947 *
948 * Mandatory.
949 *
950 * Must save the passed OID for retrieval at restore-time.
951 */
952static void
953_StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
954{
955 lclContext *ctx = (lclContext *) AH->formatData;
956 lclTocEntry *tctx = (lclTocEntry *) te->formatData;
957 char fname[255];
958 char *sfx;
959
960 if (oid == 0)
961 fatal("invalid OID for large object (%u)", oid);
962
963 if (AH->compression != 0)
964 sfx = ".gz";
965 else
966 sfx = "";
967
968 sprintf(fname, "blob_%u.dat%s", oid, sfx);
969
970 tarPrintf(AH, ctx->blobToc, "%u %s\n", oid, fname);
971
972 tctx->TH = tarOpen(AH, fname, 'w');
973}
974
975/*
976 * Called by the archiver when the dumper calls EndBlob.
977 *
978 * Optional.
979 *
980 */
981static void
982_EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
983{
984 lclTocEntry *tctx = (lclTocEntry *) te->formatData;
985
986 tarClose(AH, tctx->TH);
987}
988
989/*
990 * Called by the archiver when finishing saving all BLOB DATA.
991 *
992 * Optional.
993 *
994 */
995static void
996_EndBlobs(ArchiveHandle *AH, TocEntry *te)
997{
998 lclContext *ctx = (lclContext *) AH->formatData;
999
1000 /* Write out a fake zero OID to mark end-of-blobs. */
1001 /* WriteInt(AH, 0); */
1002
1003 tarClose(AH, ctx->blobToc);
1004}
1005
1006
1007
1008/*------------
1009 * TAR Support
1010 *------------
1011 */
1012
1013static int
1014tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...)
1015{
1016 int save_errno = errno;
1017 char *p;
1018 size_t len = 128; /* initial assumption about buffer size */
1019 size_t cnt;
1020
1021 for (;;)
1022 {
1023 va_list args;
1024
1025 /* Allocate work buffer. */
1026 p = (char *) pg_malloc(len);
1027
1028 /* Try to format the data. */
1029 errno = save_errno;
1030 va_start(args, fmt);
1031 cnt = pvsnprintf(p, len, fmt, args);
1032 va_end(args);
1033
1034 if (cnt < len)
1035 break; /* success */
1036
1037 /* Release buffer and loop around to try again with larger len. */
1038 free(p);
1039 len = cnt;
1040 }
1041
1042 cnt = tarWrite(p, cnt, th);
1043 free(p);
1044 return (int) cnt;
1045}
1046
1047bool
1048isValidTarHeader(char *header)
1049{
1050 int sum;
1051 int chk = tarChecksum(header);
1052
1053 sum = read_tar_number(&header[148], 8);
1054
1055 if (sum != chk)
1056 return false;
1057
1058 /* POSIX tar format */
1059 if (memcmp(&header[257], "ustar\0", 6) == 0 &&
1060 memcmp(&header[263], "00", 2) == 0)
1061 return true;
1062 /* GNU tar format */
1063 if (memcmp(&header[257], "ustar \0", 8) == 0)
1064 return true;
1065 /* not-quite-POSIX format written by pre-9.3 pg_dump */
1066 if (memcmp(&header[257], "ustar00\0", 8) == 0)
1067 return true;
1068
1069 return false;
1070}
1071
1072/* Given the member, write the TAR header & copy the file */
1073static void
1074_tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th)
1075{
1076 lclContext *ctx = (lclContext *) AH->formatData;
1077 FILE *tmp = th->tmpFH; /* Grab it for convenience */
1078 char buf[32768];
1079 size_t cnt;
1080 pgoff_t len = 0;
1081 size_t res;
1082 size_t i,
1083 pad;
1084
1085 /*
1086 * Find file len & go back to start.
1087 */
1088 fseeko(tmp, 0, SEEK_END);
1089 th->fileLen = ftello(tmp);
1090 if (th->fileLen < 0)
1091 fatal("could not determine seek position in archive file: %m");
1092 fseeko(tmp, 0, SEEK_SET);
1093
1094 _tarWriteHeader(th);
1095
1096 while ((cnt = fread(buf, 1, sizeof(buf), tmp)) > 0)
1097 {
1098 if ((res = fwrite(buf, 1, cnt, th->tarFH)) != cnt)
1099 WRITE_ERROR_EXIT;
1100 len += res;
1101 }
1102 if (!feof(tmp))
1103 READ_ERROR_EXIT(tmp);
1104
1105 if (fclose(tmp) != 0) /* This *should* delete it... */
1106 fatal("could not close temporary file: %m");
1107
1108 if (len != th->fileLen)
1109 {
1110 char buf1[32],
1111 buf2[32];
1112
1113 snprintf(buf1, sizeof(buf1), INT64_FORMAT, (int64) len);
1114 snprintf(buf2, sizeof(buf2), INT64_FORMAT, (int64) th->fileLen);
1115 fatal("actual file length (%s) does not match expected (%s)",
1116 buf1, buf2);
1117 }
1118
1119 pad = ((len + 511) & ~511) - len;
1120 for (i = 0; i < pad; i++)
1121 {
1122 if (fputc('\0', th->tarFH) == EOF)
1123 WRITE_ERROR_EXIT;
1124 }
1125
1126 ctx->tarFHpos += len + pad;
1127}
1128
1129/* Locate the file in the archive, read header and position to data */
1130static TAR_MEMBER *
1131_tarPositionTo(ArchiveHandle *AH, const char *filename)
1132{
1133 lclContext *ctx = (lclContext *) AH->formatData;
1134 TAR_MEMBER *th = pg_malloc0(sizeof(TAR_MEMBER));
1135 char c;
1136 char header[512];
1137 size_t i,
1138 len,
1139 blks;
1140 int id;
1141
1142 th->AH = AH;
1143
1144 /* Go to end of current file, if any */
1145 if (ctx->tarFHpos != 0)
1146 {
1147 char buf1[100],
1148 buf2[100];
1149
1150 snprintf(buf1, sizeof(buf1), INT64_FORMAT, (int64) ctx->tarFHpos);
1151 snprintf(buf2, sizeof(buf2), INT64_FORMAT, (int64) ctx->tarNextMember);
1152 pg_log_debug("moving from position %s to next member at file position %s",
1153 buf1, buf2);
1154
1155 while (ctx->tarFHpos < ctx->tarNextMember)
1156 _tarReadRaw(AH, &c, 1, NULL, ctx->tarFH);
1157 }
1158
1159 {
1160 char buf[100];
1161
1162 snprintf(buf, sizeof(buf), INT64_FORMAT, (int64) ctx->tarFHpos);
1163 pg_log_debug("now at file position %s", buf);
1164 }
1165
1166 /* We are at the start of the file, or at the next member */
1167
1168 /* Get the header */
1169 if (!_tarGetHeader(AH, th))
1170 {
1171 if (filename)
1172 fatal("could not find header for file \"%s\" in tar archive", filename);
1173 else
1174 {
1175 /*
1176 * We're just scanning the archive for the next file, so return
1177 * null
1178 */
1179 free(th);
1180 return NULL;
1181 }
1182 }
1183
1184 while (filename != NULL && strcmp(th->targetFile, filename) != 0)
1185 {
1186 pg_log_debug("skipping tar member %s", th->targetFile);
1187
1188 id = atoi(th->targetFile);
1189 if ((TocIDRequired(AH, id) & REQ_DATA) != 0)
1190 fatal("restoring data out of order is not supported in this archive format: "
1191 "\"%s\" is required, but comes before \"%s\" in the archive file.",
1192 th->targetFile, filename);
1193
1194 /* Header doesn't match, so read to next header */
1195 len = ((th->fileLen + 511) & ~511); /* Padded length */
1196 blks = len >> 9; /* # of 512 byte blocks */
1197
1198 for (i = 0; i < blks; i++)
1199 _tarReadRaw(AH, &header[0], 512, NULL, ctx->tarFH);
1200
1201 if (!_tarGetHeader(AH, th))
1202 fatal("could not find header for file \"%s\" in tar archive", filename);
1203 }
1204
1205 ctx->tarNextMember = ctx->tarFHpos + ((th->fileLen + 511) & ~511);
1206 th->pos = 0;
1207
1208 return th;
1209}
1210
1211/* Read & verify a header */
1212static int
1213_tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
1214{
1215 lclContext *ctx = (lclContext *) AH->formatData;
1216 char h[512];
1217 char tag[100 + 1];
1218 int sum,
1219 chk;
1220 pgoff_t len;
1221 pgoff_t hPos;
1222 bool gotBlock = false;
1223
1224 while (!gotBlock)
1225 {
1226 /* Save the pos for reporting purposes */
1227 hPos = ctx->tarFHpos;
1228
1229 /* Read a 512 byte block, return EOF, exit if short */
1230 len = _tarReadRaw(AH, h, 512, NULL, ctx->tarFH);
1231 if (len == 0) /* EOF */
1232 return 0;
1233
1234 if (len != 512)
1235 fatal(ngettext("incomplete tar header found (%lu byte)",
1236 "incomplete tar header found (%lu bytes)",
1237 len),
1238 (unsigned long) len);
1239
1240 /* Calc checksum */
1241 chk = tarChecksum(h);
1242 sum = read_tar_number(&h[148], 8);
1243
1244 /*
1245 * If the checksum failed, see if it is a null block. If so, silently
1246 * continue to the next block.
1247 */
1248 if (chk == sum)
1249 gotBlock = true;
1250 else
1251 {
1252 int i;
1253
1254 for (i = 0; i < 512; i++)
1255 {
1256 if (h[i] != 0)
1257 {
1258 gotBlock = true;
1259 break;
1260 }
1261 }
1262 }
1263 }
1264
1265 /* Name field is 100 bytes, might not be null-terminated */
1266 strlcpy(tag, &h[0], 100 + 1);
1267
1268 len = read_tar_number(&h[124], 12);
1269
1270 {
1271 char posbuf[32];
1272 char lenbuf[32];
1273
1274 snprintf(posbuf, sizeof(posbuf), UINT64_FORMAT, (uint64) hPos);
1275 snprintf(lenbuf, sizeof(lenbuf), UINT64_FORMAT, (uint64) len);
1276 pg_log_debug("TOC Entry %s at %s (length %s, checksum %d)",
1277 tag, posbuf, lenbuf, sum);
1278 }
1279
1280 if (chk != sum)
1281 {
1282 char posbuf[32];
1283
1284 snprintf(posbuf, sizeof(posbuf), UINT64_FORMAT,
1285 (uint64) ftello(ctx->tarFH));
1286 fatal("corrupt tar header found in %s (expected %d, computed %d) file position %s",
1287 tag, sum, chk, posbuf);
1288 }
1289
1290 th->targetFile = pg_strdup(tag);
1291 th->fileLen = len;
1292
1293 return 1;
1294}
1295
1296
1297static void
1298_tarWriteHeader(TAR_MEMBER *th)
1299{
1300 char h[512];
1301
1302 tarCreateHeader(h, th->targetFile, NULL, th->fileLen,
1303 0600, 04000, 02000, time(NULL));
1304
1305 /* Now write the completed header. */
1306 if (fwrite(h, 1, 512, th->tarFH) != 512)
1307 WRITE_ERROR_EXIT;
1308}
1309