1/*-------------------------------------------------------------------------
2 *
3 * parsexlog.c
4 * Functions for reading Write-Ahead-Log
5 *
6 * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
7 * Portions Copyright (c) 1994, Regents of the University of California
8 *
9 *-------------------------------------------------------------------------
10 */
11
12#include "postgres_fe.h"
13
14#include <unistd.h>
15
16#include "pg_rewind.h"
17#include "filemap.h"
18
19#include "access/rmgr.h"
20#include "access/xlog_internal.h"
21#include "access/xlogreader.h"
22#include "catalog/pg_control.h"
23#include "catalog/storage_xlog.h"
24#include "commands/dbcommands_xlog.h"
25
26
27/*
28 * RmgrNames is an array of resource manager names, to make error messages
29 * a bit nicer.
30 */
31#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \
32 name,
33
34static const char *RmgrNames[RM_MAX_ID + 1] = {
35#include "access/rmgrlist.h"
36};
37
38static void extractPageInfo(XLogReaderState *record);
39
40static int xlogreadfd = -1;
41static XLogSegNo xlogreadsegno = -1;
42static char xlogfpath[MAXPGPATH];
43
44typedef struct XLogPageReadPrivate
45{
46 const char *datadir;
47 int tliIndex;
48} XLogPageReadPrivate;
49
50static int SimpleXLogPageRead(XLogReaderState *xlogreader,
51 XLogRecPtr targetPagePtr,
52 int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
53 TimeLineID *pageTLI);
54
55/*
56 * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
57 * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of
58 * the data blocks touched by the WAL records, and return them in a page map.
59 */
60void
61extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
62 XLogRecPtr endpoint)
63{
64 XLogRecord *record;
65 XLogReaderState *xlogreader;
66 char *errormsg;
67 XLogPageReadPrivate private;
68
69 private.datadir = datadir;
70 private.tliIndex = tliIndex;
71 xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
72 &private);
73 if (xlogreader == NULL)
74 pg_fatal("out of memory");
75
76 do
77 {
78 record = XLogReadRecord(xlogreader, startpoint, &errormsg);
79
80 if (record == NULL)
81 {
82 XLogRecPtr errptr;
83
84 errptr = startpoint ? startpoint : xlogreader->EndRecPtr;
85
86 if (errormsg)
87 pg_fatal("could not read WAL record at %X/%X: %s",
88 (uint32) (errptr >> 32), (uint32) (errptr),
89 errormsg);
90 else
91 pg_fatal("could not read WAL record at %X/%X",
92 (uint32) (errptr >> 32), (uint32) (errptr));
93 }
94
95 extractPageInfo(xlogreader);
96
97 startpoint = InvalidXLogRecPtr; /* continue reading at next record */
98
99 } while (xlogreader->ReadRecPtr != endpoint);
100
101 XLogReaderFree(xlogreader);
102 if (xlogreadfd != -1)
103 {
104 close(xlogreadfd);
105 xlogreadfd = -1;
106 }
107}
108
109/*
110 * Reads one WAL record. Returns the end position of the record, without
111 * doing anything with the record itself.
112 */
113XLogRecPtr
114readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
115{
116 XLogRecord *record;
117 XLogReaderState *xlogreader;
118 char *errormsg;
119 XLogPageReadPrivate private;
120 XLogRecPtr endptr;
121
122 private.datadir = datadir;
123 private.tliIndex = tliIndex;
124 xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
125 &private);
126 if (xlogreader == NULL)
127 pg_fatal("out of memory");
128
129 record = XLogReadRecord(xlogreader, ptr, &errormsg);
130 if (record == NULL)
131 {
132 if (errormsg)
133 pg_fatal("could not read WAL record at %X/%X: %s",
134 (uint32) (ptr >> 32), (uint32) (ptr), errormsg);
135 else
136 pg_fatal("could not read WAL record at %X/%X",
137 (uint32) (ptr >> 32), (uint32) (ptr));
138 }
139 endptr = xlogreader->EndRecPtr;
140
141 XLogReaderFree(xlogreader);
142 if (xlogreadfd != -1)
143 {
144 close(xlogreadfd);
145 xlogreadfd = -1;
146 }
147
148 return endptr;
149}
150
151/*
152 * Find the previous checkpoint preceding given WAL location.
153 */
154void
155findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
156 XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
157 XLogRecPtr *lastchkptredo)
158{
159 /* Walk backwards, starting from the given record */
160 XLogRecord *record;
161 XLogRecPtr searchptr;
162 XLogReaderState *xlogreader;
163 char *errormsg;
164 XLogPageReadPrivate private;
165
166 /*
167 * The given fork pointer points to the end of the last common record,
168 * which is not necessarily the beginning of the next record, if the
169 * previous record happens to end at a page boundary. Skip over the page
170 * header in that case to find the next record.
171 */
172 if (forkptr % XLOG_BLCKSZ == 0)
173 {
174 if (XLogSegmentOffset(forkptr, WalSegSz) == 0)
175 forkptr += SizeOfXLogLongPHD;
176 else
177 forkptr += SizeOfXLogShortPHD;
178 }
179
180 private.datadir = datadir;
181 private.tliIndex = tliIndex;
182 xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
183 &private);
184 if (xlogreader == NULL)
185 pg_fatal("out of memory");
186
187 searchptr = forkptr;
188 for (;;)
189 {
190 uint8 info;
191
192 record = XLogReadRecord(xlogreader, searchptr, &errormsg);
193
194 if (record == NULL)
195 {
196 if (errormsg)
197 pg_fatal("could not find previous WAL record at %X/%X: %s",
198 (uint32) (searchptr >> 32), (uint32) (searchptr),
199 errormsg);
200 else
201 pg_fatal("could not find previous WAL record at %X/%X",
202 (uint32) (searchptr >> 32), (uint32) (searchptr));
203 }
204
205 /*
206 * Check if it is a checkpoint record. This checkpoint record needs to
207 * be the latest checkpoint before WAL forked and not the checkpoint
208 * where the master has been stopped to be rewinded.
209 */
210 info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
211 if (searchptr < forkptr &&
212 XLogRecGetRmid(xlogreader) == RM_XLOG_ID &&
213 (info == XLOG_CHECKPOINT_SHUTDOWN ||
214 info == XLOG_CHECKPOINT_ONLINE))
215 {
216 CheckPoint checkPoint;
217
218 memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint));
219 *lastchkptrec = searchptr;
220 *lastchkpttli = checkPoint.ThisTimeLineID;
221 *lastchkptredo = checkPoint.redo;
222 break;
223 }
224
225 /* Walk backwards to previous record. */
226 searchptr = record->xl_prev;
227 }
228
229 XLogReaderFree(xlogreader);
230 if (xlogreadfd != -1)
231 {
232 close(xlogreadfd);
233 xlogreadfd = -1;
234 }
235}
236
237/* XLogreader callback function, to read a WAL page */
238static int
239SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
240 int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
241 TimeLineID *pageTLI)
242{
243 XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
244 uint32 targetPageOff;
245 XLogRecPtr targetSegEnd;
246 XLogSegNo targetSegNo;
247 int r;
248
249 XLByteToSeg(targetPagePtr, targetSegNo, WalSegSz);
250 XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, WalSegSz, targetSegEnd);
251 targetPageOff = XLogSegmentOffset(targetPagePtr, WalSegSz);
252
253 /*
254 * See if we need to switch to a new segment because the requested record
255 * is not in the currently open one.
256 */
257 if (xlogreadfd >= 0 &&
258 !XLByteInSeg(targetPagePtr, xlogreadsegno, WalSegSz))
259 {
260 close(xlogreadfd);
261 xlogreadfd = -1;
262 }
263
264 XLByteToSeg(targetPagePtr, xlogreadsegno, WalSegSz);
265
266 if (xlogreadfd < 0)
267 {
268 char xlogfname[MAXFNAMELEN];
269
270 /*
271 * Since incomplete segments are copied into next timelines, switch to
272 * the timeline holding the required segment. Assuming this scan can
273 * be done both forward and backward, consider also switching timeline
274 * accordingly.
275 */
276 while (private->tliIndex < targetNentries - 1 &&
277 targetHistory[private->tliIndex].end < targetSegEnd)
278 private->tliIndex++;
279 while (private->tliIndex > 0 &&
280 targetHistory[private->tliIndex].begin >= targetSegEnd)
281 private->tliIndex--;
282
283 XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
284 xlogreadsegno, WalSegSz);
285
286 snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s", private->datadir, xlogfname);
287
288 xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
289
290 if (xlogreadfd < 0)
291 {
292 pg_log_error("could not open file \"%s\": %m", xlogfpath);
293 return -1;
294 }
295 }
296
297 /*
298 * At this point, we have the right segment open.
299 */
300 Assert(xlogreadfd != -1);
301
302 /* Read the requested page */
303 if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
304 {
305 pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
306 return -1;
307 }
308
309
310 r = read(xlogreadfd, readBuf, XLOG_BLCKSZ);
311 if (r != XLOG_BLCKSZ)
312 {
313 if (r < 0)
314 pg_log_error("could not read file \"%s\": %m", xlogfpath);
315 else
316 pg_log_error("could not read file \"%s\": read %d of %zu",
317 xlogfpath, r, (Size) XLOG_BLCKSZ);
318
319 return -1;
320 }
321
322 Assert(targetSegNo == xlogreadsegno);
323
324 *pageTLI = targetHistory[private->tliIndex].tli;
325 return XLOG_BLCKSZ;
326}
327
328/*
329 * Extract information on which blocks the current record modifies.
330 */
331static void
332extractPageInfo(XLogReaderState *record)
333{
334 int block_id;
335 RmgrId rmid = XLogRecGetRmid(record);
336 uint8 info = XLogRecGetInfo(record);
337 uint8 rminfo = info & ~XLR_INFO_MASK;
338
339 /* Is this a special record type that I recognize? */
340
341 if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE)
342 {
343 /*
344 * New databases can be safely ignored. It won't be present in the
345 * source system, so it will be deleted. There's one corner-case,
346 * though: if a new, different, database is also created in the source
347 * system, we'll see that the files already exist and not copy them.
348 * That's OK, though; WAL replay of creating the new database, from
349 * the source systems's WAL, will re-copy the new database,
350 * overwriting the database created in the target system.
351 */
352 }
353 else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP)
354 {
355 /*
356 * An existing database was dropped. We'll see that the files don't
357 * exist in the target data dir, and copy them in toto from the source
358 * system. No need to do anything special here.
359 */
360 }
361 else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE)
362 {
363 /*
364 * We can safely ignore these. The file will be removed from the
365 * target, if it doesn't exist in source system. If a file with same
366 * name is created in source system, too, there will be WAL records
367 * for all the blocks in it.
368 */
369 }
370 else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE)
371 {
372 /*
373 * We can safely ignore these. When we compare the sizes later on,
374 * we'll notice that they differ, and copy the missing tail from
375 * source system.
376 */
377 }
378 else if (info & XLR_SPECIAL_REL_UPDATE)
379 {
380 /*
381 * This record type modifies a relation file in some special way, but
382 * we don't recognize the type. That's bad - we don't know how to
383 * track that change.
384 */
385 pg_fatal("WAL record modifies a relation, but record type is not recognized: "
386 "lsn: %X/%X, rmgr: %s, info: %02X",
387 (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr),
388 RmgrNames[rmid], info);
389 }
390
391 for (block_id = 0; block_id <= record->max_block_id; block_id++)
392 {
393 RelFileNode rnode;
394 ForkNumber forknum;
395 BlockNumber blkno;
396
397 if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno))
398 continue;
399
400 /* We only care about the main fork; others are copied in toto */
401 if (forknum != MAIN_FORKNUM)
402 continue;
403
404 process_block_change(forknum, rnode, blkno);
405 }
406}
407