1/*-------------------------------------------------------------------------
2 *
3 * copy_fetch.c
4 * Functions for using a data directory as the source.
5 *
6 * Portions Copyright (c) 2013-2019, PostgreSQL Global Development Group
7 *
8 *-------------------------------------------------------------------------
9 */
10#include "postgres_fe.h"
11
12#include <sys/stat.h>
13#include <dirent.h>
14#include <fcntl.h>
15#include <unistd.h>
16
17#include "datapagemap.h"
18#include "fetch.h"
19#include "file_ops.h"
20#include "filemap.h"
21#include "pg_rewind.h"
22
23static void recurse_dir(const char *datadir, const char *path,
24 process_file_callback_t callback);
25
26static void execute_pagemap(datapagemap_t *pagemap, const char *path);
27
28/*
29 * Traverse through all files in a data directory, calling 'callback'
30 * for each file.
31 */
32void
33traverse_datadir(const char *datadir, process_file_callback_t callback)
34{
35 recurse_dir(datadir, NULL, callback);
36}
37
38/*
39 * recursive part of traverse_datadir
40 *
41 * parentpath is the current subdirectory's path relative to datadir,
42 * or NULL at the top level.
43 */
44static void
45recurse_dir(const char *datadir, const char *parentpath,
46 process_file_callback_t callback)
47{
48 DIR *xldir;
49 struct dirent *xlde;
50 char fullparentpath[MAXPGPATH];
51
52 if (parentpath)
53 snprintf(fullparentpath, MAXPGPATH, "%s/%s", datadir, parentpath);
54 else
55 snprintf(fullparentpath, MAXPGPATH, "%s", datadir);
56
57 xldir = opendir(fullparentpath);
58 if (xldir == NULL)
59 pg_fatal("could not open directory \"%s\": %m",
60 fullparentpath);
61
62 while (errno = 0, (xlde = readdir(xldir)) != NULL)
63 {
64 struct stat fst;
65 char fullpath[MAXPGPATH * 2];
66 char path[MAXPGPATH * 2];
67
68 if (strcmp(xlde->d_name, ".") == 0 ||
69 strcmp(xlde->d_name, "..") == 0)
70 continue;
71
72 snprintf(fullpath, sizeof(fullpath), "%s/%s", fullparentpath, xlde->d_name);
73
74 if (lstat(fullpath, &fst) < 0)
75 {
76 if (errno == ENOENT)
77 {
78 /*
79 * File doesn't exist anymore. This is ok, if the new master
80 * is running and the file was just removed. If it was a data
81 * file, there should be a WAL record of the removal. If it
82 * was something else, it couldn't have been anyway.
83 *
84 * TODO: But complain if we're processing the target dir!
85 */
86 }
87 else
88 pg_fatal("could not stat file \"%s\": %m",
89 fullpath);
90 }
91
92 if (parentpath)
93 snprintf(path, sizeof(path), "%s/%s", parentpath, xlde->d_name);
94 else
95 snprintf(path, sizeof(path), "%s", xlde->d_name);
96
97 if (S_ISREG(fst.st_mode))
98 callback(path, FILE_TYPE_REGULAR, fst.st_size, NULL);
99 else if (S_ISDIR(fst.st_mode))
100 {
101 callback(path, FILE_TYPE_DIRECTORY, 0, NULL);
102 /* recurse to handle subdirectories */
103 recurse_dir(datadir, path, callback);
104 }
105#ifndef WIN32
106 else if (S_ISLNK(fst.st_mode))
107#else
108 else if (pgwin32_is_junction(fullpath))
109#endif
110 {
111#if defined(HAVE_READLINK) || defined(WIN32)
112 char link_target[MAXPGPATH];
113 int len;
114
115 len = readlink(fullpath, link_target, sizeof(link_target));
116 if (len < 0)
117 pg_fatal("could not read symbolic link \"%s\": %m",
118 fullpath);
119 if (len >= sizeof(link_target))
120 pg_fatal("symbolic link \"%s\" target is too long",
121 fullpath);
122 link_target[len] = '\0';
123
124 callback(path, FILE_TYPE_SYMLINK, 0, link_target);
125
126 /*
127 * If it's a symlink within pg_tblspc, we need to recurse into it,
128 * to process all the tablespaces. We also follow a symlink if
129 * it's for pg_wal. Symlinks elsewhere are ignored.
130 */
131 if ((parentpath && strcmp(parentpath, "pg_tblspc") == 0) ||
132 strcmp(path, "pg_wal") == 0)
133 recurse_dir(datadir, path, callback);
134#else
135 pg_fatal("\"%s\" is a symbolic link, but symbolic links are not supported on this platform",
136 fullpath);
137#endif /* HAVE_READLINK */
138 }
139 }
140
141 if (errno)
142 pg_fatal("could not read directory \"%s\": %m",
143 fullparentpath);
144
145 if (closedir(xldir))
146 pg_fatal("could not close directory \"%s\": %m",
147 fullparentpath);
148}
149
150/*
151 * Copy a file from source to target, between 'begin' and 'end' offsets.
152 *
153 * If 'trunc' is true, any existing file with the same name is truncated.
154 */
155static void
156rewind_copy_file_range(const char *path, off_t begin, off_t end, bool trunc)
157{
158 PGAlignedBlock buf;
159 char srcpath[MAXPGPATH];
160 int srcfd;
161
162 snprintf(srcpath, sizeof(srcpath), "%s/%s", datadir_source, path);
163
164 srcfd = open(srcpath, O_RDONLY | PG_BINARY, 0);
165 if (srcfd < 0)
166 pg_fatal("could not open source file \"%s\": %m",
167 srcpath);
168
169 if (lseek(srcfd, begin, SEEK_SET) == -1)
170 pg_fatal("could not seek in source file: %m");
171
172 open_target_file(path, trunc);
173
174 while (end - begin > 0)
175 {
176 int readlen;
177 int len;
178
179 if (end - begin > sizeof(buf))
180 len = sizeof(buf);
181 else
182 len = end - begin;
183
184 readlen = read(srcfd, buf.data, len);
185
186 if (readlen < 0)
187 pg_fatal("could not read file \"%s\": %m",
188 srcpath);
189 else if (readlen == 0)
190 pg_fatal("unexpected EOF while reading file \"%s\"", srcpath);
191
192 write_target_range(buf.data, begin, readlen);
193 begin += readlen;
194 }
195
196 if (close(srcfd) != 0)
197 pg_fatal("could not close file \"%s\": %m", srcpath);
198}
199
200/*
201 * Copy all relation data files from datadir_source to datadir_target, which
202 * are marked in the given data page map.
203 */
204void
205copy_executeFileMap(filemap_t *map)
206{
207 file_entry_t *entry;
208 int i;
209
210 for (i = 0; i < map->narray; i++)
211 {
212 entry = map->array[i];
213 execute_pagemap(&entry->pagemap, entry->path);
214
215 switch (entry->action)
216 {
217 case FILE_ACTION_NONE:
218 /* ok, do nothing.. */
219 break;
220
221 case FILE_ACTION_COPY:
222 rewind_copy_file_range(entry->path, 0, entry->newsize, true);
223 break;
224
225 case FILE_ACTION_TRUNCATE:
226 truncate_target_file(entry->path, entry->newsize);
227 break;
228
229 case FILE_ACTION_COPY_TAIL:
230 rewind_copy_file_range(entry->path, entry->oldsize,
231 entry->newsize, false);
232 break;
233
234 case FILE_ACTION_CREATE:
235 create_target(entry);
236 break;
237
238 case FILE_ACTION_REMOVE:
239 remove_target(entry);
240 break;
241 }
242 }
243
244 close_target_file();
245}
246
247static void
248execute_pagemap(datapagemap_t *pagemap, const char *path)
249{
250 datapagemap_iterator_t *iter;
251 BlockNumber blkno;
252 off_t offset;
253
254 iter = datapagemap_iterate(pagemap);
255 while (datapagemap_next(iter, &blkno))
256 {
257 offset = blkno * BLCKSZ;
258 rewind_copy_file_range(path, offset, offset + BLCKSZ, false);
259 /* Ok, this block has now been copied from new data dir to old */
260 }
261 pg_free(iter);
262}
263