1/******************************************************
2hot backup tool for InnoDB
3(c) 2009-2015 Percona LLC and/or its affiliates
4(c) 2017 MariaDB
5Originally Created 3/3/2009 Yasufumi Kinoshita
6Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
7Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
8
9This program is free software; you can redistribute it and/or modify
10it under the terms of the GNU General Public License as published by
11the Free Software Foundation; version 2 of the License.
12
13This program is distributed in the hope that it will be useful,
14but WITHOUT ANY WARRANTY; without even the implied warranty of
15MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16GNU General Public License for more details.
17
18You should have received a copy of the GNU General Public License
19along with this program; if not, write to the Free Software
20Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
21
22*******************************************************
23
24This file incorporates work covered by the following copyright and
25permission notice:
26
27Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.
28
29This program is free software; you can redistribute it and/or modify it under
30the terms of the GNU General Public License as published by the Free Software
31Foundation; version 2 of the License.
32
33This program is distributed in the hope that it will be useful, but WITHOUT
34ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
35FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
36
37You should have received a copy of the GNU General Public License along with
38this program; if not, write to the Free Software Foundation, Inc., 59 Temple
39Place, Suite 330, Boston, MA 02111-1307 USA
40
41*******************************************************/
42
43#include <my_global.h>
44#include <os0file.h>
45#include <my_dir.h>
46#include <ut0mem.h>
47#include <srv0start.h>
48#include <fil0fil.h>
49#include <trx0sys.h>
50#include <set>
51#include <string>
52#include <mysqld.h>
53#include <sstream>
54#include "fil_cur.h"
55#include "xtrabackup.h"
56#include "common.h"
57#include "backup_copy.h"
58#include "backup_mysql.h"
59#include <btr0btr.h>
60
61/* list of files to sync for --rsync mode */
62static std::set<std::string> rsync_list;
63/* locations of tablespaces read from .isl files */
64static std::map<std::string, std::string> tablespace_locations;
65
66/* Whether LOCK BINLOG FOR BACKUP has been issued during backup */
67bool binlog_locked;
68
69/************************************************************************
70Struct represents file or directory. */
71struct datadir_node_t {
72 ulint dbpath_len;
73 char *filepath;
74 ulint filepath_len;
75 char *filepath_rel;
76 ulint filepath_rel_len;
77 bool is_empty_dir;
78 bool is_file;
79};
80
81/************************************************************************
82Holds the state needed to enumerate files in MySQL data directory. */
83struct datadir_iter_t {
84 char *datadir_path;
85 char *dbpath;
86 ulint dbpath_len;
87 char *filepath;
88 ulint filepath_len;
89 char *filepath_rel;
90 ulint filepath_rel_len;
91 pthread_mutex_t mutex;
92 os_file_dir_t dir;
93 os_file_dir_t dbdir;
94 os_file_stat_t dbinfo;
95 os_file_stat_t fileinfo;
96 dberr_t err;
97 bool is_empty_dir;
98 bool is_file;
99 bool skip_first_level;
100};
101
102
103/************************************************************************
104Represents the context of the thread processing MySQL data directory. */
105struct datadir_thread_ctxt_t {
106 datadir_iter_t *it;
107 uint n_thread;
108 uint *count;
109 pthread_mutex_t count_mutex;
110 os_thread_id_t id;
111 bool ret;
112};
113
114static bool backup_files_from_datadir(const char *dir_path);
115
116/************************************************************************
117Retirn true if character if file separator */
118bool
119is_path_separator(char c)
120{
121 return(c == FN_LIBCHAR || c == FN_LIBCHAR2);
122}
123
124
125/************************************************************************
126Fill the node struct. Memory for node need to be allocated and freed by
127the caller. It is caller responsibility to initialize node with
128datadir_node_init and cleanup the memory with datadir_node_free.
129Node can not be shared between threads. */
130static
131void
132datadir_node_fill(datadir_node_t *node, datadir_iter_t *it)
133{
134 if (node->filepath_len < it->filepath_len) {
135 free(node->filepath);
136 node->filepath = (char*)(malloc(it->filepath_len));
137 node->filepath_len = it->filepath_len;
138 }
139 if (node->filepath_rel_len < it->filepath_rel_len) {
140 free(node->filepath_rel);
141 node->filepath_rel = (char*)(malloc(it->filepath_rel_len));
142 node->filepath_rel_len = it->filepath_rel_len;
143 }
144
145 strcpy(node->filepath, it->filepath);
146 strcpy(node->filepath_rel, it->filepath_rel);
147 node->is_empty_dir = it->is_empty_dir;
148 node->is_file = it->is_file;
149}
150
151static
152void
153datadir_node_free(datadir_node_t *node)
154{
155 free(node->filepath);
156 free(node->filepath_rel);
157 memset(node, 0, sizeof(datadir_node_t));
158}
159
160static
161void
162datadir_node_init(datadir_node_t *node)
163{
164 memset(node, 0, sizeof(datadir_node_t));
165}
166
167
168/************************************************************************
169Create the MySQL data directory iterator. Memory needs to be released
170with datadir_iter_free. Position should be advanced with
171datadir_iter_next_file. Iterator can be shared between multiple
172threads. It is guaranteed that each thread receives unique file from
173data directory into its local node struct. */
174static
175datadir_iter_t *
176datadir_iter_new(const char *path, bool skip_first_level = true)
177{
178 datadir_iter_t *it;
179
180 it = static_cast<datadir_iter_t *>(malloc(sizeof(datadir_iter_t)));
181 memset(it, 0, sizeof(datadir_iter_t));
182
183 pthread_mutex_init(&it->mutex, NULL);
184 it->datadir_path = strdup(path);
185
186 it->dir = os_file_opendir(it->datadir_path, TRUE);
187
188 if (it->dir == NULL) {
189
190 goto error;
191 }
192
193 it->err = DB_SUCCESS;
194
195 it->dbpath_len = FN_REFLEN;
196 it->dbpath = static_cast<char*>(malloc(it->dbpath_len));
197
198 it->filepath_len = FN_REFLEN;
199 it->filepath = static_cast<char*>(malloc(it->filepath_len));
200
201 it->filepath_rel_len = FN_REFLEN;
202 it->filepath_rel = static_cast<char*>(malloc(it->filepath_rel_len));
203
204 it->skip_first_level = skip_first_level;
205
206 return(it);
207
208error:
209 free(it);
210
211 return(NULL);
212}
213
214static
215bool
216datadir_iter_next_database(datadir_iter_t *it)
217{
218 if (it->dbdir != NULL) {
219 if (os_file_closedir(it->dbdir) != 0) {
220
221 msg("Warning: could not"
222 " close database directory %s\n", it->dbpath);
223
224 it->err = DB_ERROR;
225
226 }
227 it->dbdir = NULL;
228 }
229
230 while (os_file_readdir_next_file(it->datadir_path,
231 it->dir, &it->dbinfo) == 0) {
232 ulint len;
233
234 if ((it->dbinfo.type == OS_FILE_TYPE_FILE
235 && it->skip_first_level)
236 || it->dbinfo.type == OS_FILE_TYPE_UNKNOWN) {
237
238 continue;
239 }
240
241 /* We found a symlink or a directory; try opening it to see
242 if a symlink is a directory */
243
244 len = strlen(it->datadir_path)
245 + strlen (it->dbinfo.name) + 2;
246 if (len > it->dbpath_len) {
247 it->dbpath_len = len;
248 free(it->dbpath);
249
250 it->dbpath = static_cast<char*>(
251 malloc(it->dbpath_len));
252 }
253 snprintf(it->dbpath, it->dbpath_len, "%s/%s",
254 it->datadir_path, it->dbinfo.name);
255 os_normalize_path(it->dbpath);
256
257 if (it->dbinfo.type == OS_FILE_TYPE_FILE) {
258 it->is_file = true;
259 return(true);
260 }
261
262 if (check_if_skip_database_by_path(it->dbpath)) {
263 msg("Skipping db: %s\n", it->dbpath);
264 continue;
265 }
266
267 /* We want wrong directory permissions to be a fatal error for
268 XtraBackup. */
269 it->dbdir = os_file_opendir(it->dbpath, TRUE);
270
271 if (it->dbdir != NULL) {
272
273 it->is_file = false;
274 return(true);
275 }
276
277 }
278
279 return(false);
280}
281
282/************************************************************************
283Concatenate n parts into single path */
284static
285void
286make_path_n(int n, char **path, ulint *path_len, ...)
287{
288 ulint len_needed = n + 1;
289 char *p;
290 int i;
291 va_list vl;
292
293 ut_ad(n > 0);
294
295 va_start(vl, path_len);
296 for (i = 0; i < n; i++) {
297 p = va_arg(vl, char*);
298 len_needed += strlen(p);
299 }
300 va_end(vl);
301
302 if (len_needed < *path_len) {
303 free(*path);
304 *path = static_cast<char*>(malloc(len_needed));
305 }
306
307 va_start(vl, path_len);
308 p = va_arg(vl, char*);
309 strcpy(*path, p);
310 for (i = 1; i < n; i++) {
311 size_t plen;
312 p = va_arg(vl, char*);
313 plen = strlen(*path);
314 if (!is_path_separator((*path)[plen - 1])) {
315 (*path)[plen] = FN_LIBCHAR;
316 (*path)[plen + 1] = 0;
317 }
318 strcat(*path + plen, p);
319 }
320 va_end(vl);
321}
322
323static
324bool
325datadir_iter_next_file(datadir_iter_t *it)
326{
327 if (it->is_file && it->dbpath) {
328 make_path_n(2, &it->filepath, &it->filepath_len,
329 it->datadir_path, it->dbinfo.name);
330
331 make_path_n(1, &it->filepath_rel, &it->filepath_rel_len,
332 it->dbinfo.name);
333
334 it->is_empty_dir = false;
335 it->is_file = false;
336
337 return(true);
338 }
339
340 if (!it->dbpath || !it->dbdir) {
341
342 return(false);
343 }
344
345 while (os_file_readdir_next_file(it->dbpath, it->dbdir,
346 &it->fileinfo) == 0) {
347
348 if (it->fileinfo.type == OS_FILE_TYPE_DIR) {
349
350 continue;
351 }
352
353 /* We found a symlink or a file */
354 make_path_n(3, &it->filepath, &it->filepath_len,
355 it->datadir_path, it->dbinfo.name,
356 it->fileinfo.name);
357
358 make_path_n(2, &it->filepath_rel, &it->filepath_rel_len,
359 it->dbinfo.name, it->fileinfo.name);
360
361 it->is_empty_dir = false;
362
363 return(true);
364 }
365
366 return(false);
367}
368
369static
370bool
371datadir_iter_next(datadir_iter_t *it, datadir_node_t *node)
372{
373 bool ret = true;
374
375 pthread_mutex_lock(&it->mutex);
376
377 if (datadir_iter_next_file(it)) {
378
379 datadir_node_fill(node, it);
380
381 goto done;
382 }
383
384 while (datadir_iter_next_database(it)) {
385
386 if (datadir_iter_next_file(it)) {
387
388 datadir_node_fill(node, it);
389
390 goto done;
391 }
392
393 make_path_n(2, &it->filepath, &it->filepath_len,
394 it->datadir_path, it->dbinfo.name);
395
396 make_path_n(1, &it->filepath_rel, &it->filepath_rel_len,
397 it->dbinfo.name);
398
399 it->is_empty_dir = true;
400
401 datadir_node_fill(node, it);
402
403 goto done;
404 }
405
406 /* nothing found */
407 ret = false;
408
409done:
410 pthread_mutex_unlock(&it->mutex);
411
412 return(ret);
413}
414
415/************************************************************************
416Interface to read MySQL data file sequentially. One should open file
417with datafile_open to get cursor and close the cursor with
418datafile_close. Cursor can not be shared between multiple
419threads. */
420static
421void
422datadir_iter_free(datadir_iter_t *it)
423{
424 pthread_mutex_destroy(&it->mutex);
425
426 if (it->dbdir) {
427
428 os_file_closedir(it->dbdir);
429 }
430
431 if (it->dir) {
432
433 os_file_closedir(it->dir);
434 }
435
436 free(it->dbpath);
437 free(it->filepath);
438 free(it->filepath_rel);
439 free(it->datadir_path);
440 free(it);
441}
442
443
444/************************************************************************
445Holds the state needed to copy single data file. */
446struct datafile_cur_t {
447 pfs_os_file_t file;
448 char rel_path[FN_REFLEN];
449 char abs_path[FN_REFLEN];
450 MY_STAT statinfo;
451 uint thread_n;
452 byte* orig_buf;
453 byte* buf;
454 size_t buf_size;
455 size_t buf_read;
456 size_t buf_offset;
457};
458
459static
460void
461datafile_close(datafile_cur_t *cursor)
462{
463 if (cursor->file != OS_FILE_CLOSED) {
464 os_file_close(cursor->file);
465 }
466 free(cursor->buf);
467}
468
469static
470bool
471datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n)
472{
473 bool success;
474
475 memset(cursor, 0, sizeof(datafile_cur_t));
476
477 strncpy(cursor->abs_path, file, sizeof(cursor->abs_path));
478
479 /* Get the relative path for the destination tablespace name, i.e. the
480 one that can be appended to the backup root directory. Non-system
481 tablespaces may have absolute paths for remote tablespaces in MySQL
482 5.6+. We want to make "local" copies for the backup. */
483 strncpy(cursor->rel_path,
484 xb_get_relative_path(cursor->abs_path, FALSE),
485 sizeof(cursor->rel_path));
486
487 cursor->file = os_file_create_simple_no_error_handling(
488 0, cursor->abs_path,
489 OS_FILE_OPEN, OS_FILE_READ_ALLOW_DELETE, true, &success);
490 if (!success) {
491 /* The following call prints an error message */
492 os_file_get_last_error(TRUE);
493
494 msg("[%02u] error: cannot open "
495 "file %s\n",
496 thread_n, cursor->abs_path);
497
498 return(false);
499 }
500
501 if (!my_stat(cursor->abs_path, &cursor->statinfo, 0)) {
502 msg("[%02u] error: cannot stat %s\n",
503 thread_n, cursor->abs_path);
504
505 datafile_close(cursor);
506
507 return(false);
508 }
509
510 posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL);
511
512 cursor->buf_size = 10 * 1024 * 1024;
513 cursor->buf = static_cast<byte *>(malloc((ulint)cursor->buf_size));
514
515 return(true);
516}
517
518
519static
520xb_fil_cur_result_t
521datafile_read(datafile_cur_t *cursor)
522{
523 ulint to_read;
524
525 xtrabackup_io_throttling();
526
527 to_read = (ulint)MY_MIN(cursor->statinfo.st_size - cursor->buf_offset,
528 cursor->buf_size);
529
530 if (to_read == 0) {
531 return(XB_FIL_CUR_EOF);
532 }
533
534 if (!os_file_read(IORequestRead,
535 cursor->file, cursor->buf, cursor->buf_offset,
536 to_read)) {
537 return(XB_FIL_CUR_ERROR);
538 }
539
540 posix_fadvise(cursor->file, cursor->buf_offset, to_read,
541 POSIX_FADV_DONTNEED);
542
543 cursor->buf_read = to_read;
544 cursor->buf_offset += to_read;
545
546 return(XB_FIL_CUR_SUCCESS);
547}
548
549
550
551/************************************************************************
552Check to see if a file exists.
553Takes name of the file to check.
554@return true if file exists. */
555static
556bool
557file_exists(const char *filename)
558{
559 MY_STAT stat_arg;
560
561 if (!my_stat(filename, &stat_arg, MYF(0))) {
562
563 return(false);
564 }
565
566 return(true);
567}
568
569/************************************************************************
570Trim leading slashes from absolute path so it becomes relative */
571static
572const char *
573trim_dotslash(const char *path)
574{
575 while (*path) {
576 if (is_path_separator(*path)) {
577 ++path;
578 continue;
579 }
580 if (*path == '.' && is_path_separator(path[1])) {
581 path += 2;
582 continue;
583 }
584 break;
585 }
586
587 return(path);
588}
589
590
591
592/************************************************************************
593Check if string ends with given suffix.
594@return true if string ends with given suffix. */
595bool
596ends_with(const char *str, const char *suffix)
597{
598 size_t suffix_len = strlen(suffix);
599 size_t str_len = strlen(str);
600 return(str_len >= suffix_len
601 && strcmp(str + str_len - suffix_len, suffix) == 0);
602}
603
604static bool starts_with(const char *str, const char *prefix)
605{
606 return strncmp(str, prefix, strlen(prefix)) == 0;
607}
608
609/************************************************************************
610Create directories recursively.
611@return 0 if directories created successfully. */
612static
613int
614mkdirp(const char *pathname, int Flags, myf MyFlags)
615{
616 char parent[PATH_MAX], *p;
617
618 /* make a parent directory path */
619 strncpy(parent, pathname, sizeof(parent));
620 parent[sizeof(parent) - 1] = 0;
621
622 for (p = parent + strlen(parent);
623 !is_path_separator(*p) && p != parent; p--);
624
625 *p = 0;
626
627 /* try to make parent directory */
628 if (p != parent && mkdirp(parent, Flags, MyFlags) != 0) {
629 return(-1);
630 }
631
632 /* make this one if parent has been made */
633 if (my_mkdir(pathname, Flags, MyFlags) == 0) {
634 return(0);
635 }
636
637 /* if it already exists that is fine */
638 if (errno == EEXIST) {
639 return(0);
640 }
641
642 return(-1);
643}
644
645/************************************************************************
646Return true if first and second arguments are the same path. */
647bool
648equal_paths(const char *first, const char *second)
649{
650#ifdef HAVE_REALPATH
651 char real_first[PATH_MAX];
652 char real_second[PATH_MAX];
653
654 if (realpath(first, real_first) == NULL) {
655 return false;
656 }
657 if (realpath(second, real_second) == NULL) {
658 return false;
659 }
660
661 return (strcmp(real_first, real_second) == 0);
662#else
663 return strcmp(first, second) == 0;
664#endif
665}
666
667/************************************************************************
668Check if directory exists. Optionally create directory if doesn't
669exist.
670@return true if directory exists and if it was created successfully. */
671bool
672directory_exists(const char *dir, bool create)
673{
674 os_file_dir_t os_dir;
675 MY_STAT stat_arg;
676 char errbuf[MYSYS_STRERROR_SIZE];
677
678 if (my_stat(dir, &stat_arg, MYF(0)) == NULL) {
679
680 if (!create) {
681 return(false);
682 }
683
684 if (mkdirp(dir, 0777, MYF(0)) < 0) {
685 my_strerror(errbuf, sizeof(errbuf), my_errno);
686 msg("Can not create directory %s: %s\n", dir, errbuf);
687 return(false);
688
689 }
690 }
691
692 /* could be symlink */
693 os_dir = os_file_opendir(dir, FALSE);
694
695 if (os_dir == NULL) {
696 my_strerror(errbuf, sizeof(errbuf), my_errno);
697 msg("Can not open directory %s: %s\n", dir,
698 errbuf);
699
700 return(false);
701 }
702
703 os_file_closedir(os_dir);
704
705 return(true);
706}
707
708/************************************************************************
709Check that directory exists and it is empty. */
710static
711bool
712directory_exists_and_empty(const char *dir, const char *comment)
713{
714 os_file_dir_t os_dir;
715 dberr_t err;
716 os_file_stat_t info;
717 bool empty;
718
719 if (!directory_exists(dir, true)) {
720 return(false);
721 }
722
723 os_dir = os_file_opendir(dir, FALSE);
724
725 if (os_dir == NULL) {
726 msg("%s can not open directory %s\n", comment, dir);
727 return(false);
728 }
729
730 empty = (fil_file_readdir_next_file(&err, dir, os_dir, &info) != 0);
731
732 os_file_closedir(os_dir);
733
734 if (!empty) {
735 msg("%s directory %s is not empty!\n", comment, dir);
736 }
737
738 return(empty);
739}
740
741
742/************************************************************************
743Check if file name ends with given set of suffixes.
744@return true if it does. */
745static
746bool
747filename_matches(const char *filename, const char **ext_list)
748{
749 const char **ext;
750
751 for (ext = ext_list; *ext; ext++) {
752 if (ends_with(filename, *ext)) {
753 return(true);
754 }
755 }
756
757 return(false);
758}
759
760
761/************************************************************************
762Copy data file for backup. Also check if it is allowed to copy by
763comparing its name to the list of known data file types and checking
764if passes the rules for partial backup.
765@return true if file backed up or skipped successfully. */
766static
767bool
768datafile_copy_backup(const char *filepath, uint thread_n)
769{
770 const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
771 "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
772 NULL};
773
774 /* Get the name and the path for the tablespace. node->name always
775 contains the path (which may be absolute for remote tablespaces in
776 5.6+). space->name contains the tablespace name in the form
777 "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a
778 multi-node shared tablespace, space->name contains the name of the first
779 node, but that's irrelevant, since we only need node_name to match them
780 against filters, and the shared tablespace is always copied regardless
781 of the filters value. */
782
783 if (check_if_skip_table(filepath)) {
784 msg_ts("[%02u] Skipping %s.\n", thread_n, filepath);
785 return(true);
786 }
787
788 if (filename_matches(filepath, ext_list)) {
789 return copy_file(ds_data, filepath, filepath, thread_n);
790 }
791
792 return(true);
793}
794
795
796/************************************************************************
797Same as datafile_copy_backup, but put file name into the list for
798rsync command. */
799static
800bool
801datafile_rsync_backup(const char *filepath, bool save_to_list, FILE *f)
802{
803 const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
804 "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
805 NULL};
806
807 /* Get the name and the path for the tablespace. node->name always
808 contains the path (which may be absolute for remote tablespaces in
809 5.6+). space->name contains the tablespace name in the form
810 "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a
811 multi-node shared tablespace, space->name contains the name of the first
812 node, but that's irrelevant, since we only need node_name to match them
813 against filters, and the shared tablespace is always copied regardless
814 of the filters value. */
815
816 if (check_if_skip_table(filepath)) {
817 return(true);
818 }
819
820 if (filename_matches(filepath, ext_list)) {
821 fprintf(f, "%s\n", filepath);
822 if (save_to_list) {
823 rsync_list.insert(filepath);
824 }
825 }
826
827 return(true);
828}
829
830
831static
832bool
833backup_file_vprintf(const char *filename, const char *fmt, va_list ap)
834{
835 ds_file_t *dstfile = NULL;
836 MY_STAT stat; /* unused for now */
837 char *buf = 0;
838 int buf_len;
839 const char *action;
840
841 memset(&stat, 0, sizeof(stat));
842
843 buf_len = vasprintf(&buf, fmt, ap);
844
845 stat.st_size = buf_len;
846 stat.st_mtime = my_time(0);
847
848 dstfile = ds_open(ds_data, filename, &stat);
849 if (dstfile == NULL) {
850 msg("[%02u] error: "
851 "cannot open the destination stream for %s\n",
852 0, filename);
853 goto error;
854 }
855
856 action = xb_get_copy_action("Writing");
857 msg_ts("[%02u] %s %s\n", 0, action, filename);
858
859 if (buf_len == -1) {
860 goto error;
861 }
862
863 if (ds_write(dstfile, buf, buf_len)) {
864 goto error;
865 }
866
867 /* close */
868 msg_ts("[%02u] ...done\n", 0);
869 free(buf);
870
871 if (ds_close(dstfile)) {
872 goto error_close;
873 }
874
875 return(true);
876
877error:
878 free(buf);
879 if (dstfile != NULL) {
880 ds_close(dstfile);
881 }
882
883error_close:
884 msg("[%02u] Error: backup file failed.\n", 0);
885 return(false); /*ERROR*/
886}
887
888
889bool
890backup_file_printf(const char *filename, const char *fmt, ...)
891{
892 bool result;
893 va_list ap;
894
895 va_start(ap, fmt);
896
897 result = backup_file_vprintf(filename, fmt, ap);
898
899 va_end(ap);
900
901 return(result);
902}
903
904static
905bool
906run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n)
907{
908 datadir_thread_ctxt_t *data_threads;
909 uint i, count;
910 pthread_mutex_t count_mutex;
911 bool ret;
912
913 data_threads = (datadir_thread_ctxt_t*)
914 malloc(sizeof(datadir_thread_ctxt_t) * n);
915
916 pthread_mutex_init(&count_mutex, NULL);
917 count = n;
918
919 for (i = 0; i < n; i++) {
920 data_threads[i].it = it;
921 data_threads[i].n_thread = i + 1;
922 data_threads[i].count = &count;
923 data_threads[i].count_mutex = count_mutex;
924 os_thread_create(func, data_threads + i, &data_threads[i].id);
925 }
926
927 /* Wait for threads to exit */
928 while (1) {
929 os_thread_sleep(100000);
930 pthread_mutex_lock(&count_mutex);
931 if (count == 0) {
932 pthread_mutex_unlock(&count_mutex);
933 break;
934 }
935 pthread_mutex_unlock(&count_mutex);
936 }
937
938 pthread_mutex_destroy(&count_mutex);
939
940 ret = true;
941 for (i = 0; i < n; i++) {
942 ret = data_threads[i].ret && ret;
943 if (!data_threads[i].ret) {
944 msg("Error: thread %u failed.\n", i);
945 }
946 }
947
948 free(data_threads);
949
950 return(ret);
951}
952
953
954/************************************************************************
955Copy file for backup/restore.
956@return true in case of success. */
957bool
958copy_file(ds_ctxt_t *datasink,
959 const char *src_file_path,
960 const char *dst_file_path,
961 uint thread_n)
962{
963 char dst_name[FN_REFLEN];
964 ds_file_t *dstfile = NULL;
965 datafile_cur_t cursor;
966 xb_fil_cur_result_t res;
967 const char *dst_path =
968 (xtrabackup_copy_back || xtrabackup_move_back)?
969 dst_file_path : trim_dotslash(dst_file_path);
970
971 if (!datafile_open(src_file_path, &cursor, thread_n)) {
972 goto error_close;
973 }
974
975 strncpy(dst_name, cursor.rel_path, sizeof(dst_name));
976
977 dstfile = ds_open(datasink, dst_path, &cursor.statinfo);
978 if (dstfile == NULL) {
979 msg("[%02u] error: "
980 "cannot open the destination stream for %s\n",
981 thread_n, dst_name);
982 goto error;
983 }
984
985 msg_ts("[%02u] %s %s to %s\n",
986 thread_n, xb_get_copy_action(), src_file_path, dstfile->path);
987
988 /* The main copy loop */
989 while ((res = datafile_read(&cursor)) == XB_FIL_CUR_SUCCESS) {
990
991 if (ds_write(dstfile, cursor.buf, cursor.buf_read)) {
992 goto error;
993 }
994 }
995
996 if (res == XB_FIL_CUR_ERROR) {
997 goto error;
998 }
999
1000 /* close */
1001 msg_ts("[%02u] ...done\n", thread_n);
1002 datafile_close(&cursor);
1003 if (ds_close(dstfile)) {
1004 goto error_close;
1005 }
1006 return(true);
1007
1008error:
1009 datafile_close(&cursor);
1010 if (dstfile != NULL) {
1011 ds_close(dstfile);
1012 }
1013
1014error_close:
1015 msg("[%02u] Error: copy_file() failed.\n", thread_n);
1016 return(false); /*ERROR*/
1017}
1018
1019
1020/************************************************************************
1021Try to move file by renaming it. If source and destination are on
1022different devices fall back to copy and unlink.
1023@return true in case of success. */
1024static
1025bool
1026move_file(ds_ctxt_t *datasink,
1027 const char *src_file_path,
1028 const char *dst_file_path,
1029 const char *dst_dir, uint thread_n)
1030{
1031 char errbuf[MYSYS_STRERROR_SIZE];
1032 char dst_file_path_abs[FN_REFLEN];
1033 char dst_dir_abs[FN_REFLEN];
1034 size_t dirname_length;
1035
1036 snprintf(dst_file_path_abs, sizeof(dst_file_path_abs),
1037 "%s/%s", dst_dir, dst_file_path);
1038
1039 dirname_part(dst_dir_abs, dst_file_path_abs, &dirname_length);
1040
1041 if (!directory_exists(dst_dir_abs, true)) {
1042 return(false);
1043 }
1044
1045 if (file_exists(dst_file_path_abs)) {
1046 msg("Error: Move file %s to %s failed: Destination "
1047 "file exists\n",
1048 src_file_path, dst_file_path_abs);
1049 return(false);
1050 }
1051
1052 msg_ts("[%02u] Moving %s to %s\n",
1053 thread_n, src_file_path, dst_file_path_abs);
1054
1055 if (my_rename(src_file_path, dst_file_path_abs, MYF(0)) != 0) {
1056 if (my_errno == EXDEV) {
1057 bool ret;
1058 ret = copy_file(datasink, src_file_path,
1059 dst_file_path, thread_n);
1060 msg_ts("[%02u] Removing %s\n", thread_n, src_file_path);
1061 if (unlink(src_file_path) != 0) {
1062 my_strerror(errbuf, sizeof(errbuf), errno);
1063 msg("Error: unlink %s failed: %s\n",
1064 src_file_path,
1065 errbuf);
1066 }
1067 return(ret);
1068 }
1069 my_strerror(errbuf, sizeof(errbuf), my_errno);
1070 msg("Can not move file %s to %s: %s\n",
1071 src_file_path, dst_file_path_abs,
1072 errbuf);
1073 return(false);
1074 }
1075
1076 msg_ts("[%02u] ...done\n", thread_n);
1077
1078 return(true);
1079}
1080
1081
1082/************************************************************************
1083Read link from .isl file if any and store it in the global map associated
1084with given tablespace. */
1085static
1086void
1087read_link_file(const char *ibd_filepath, const char *link_filepath)
1088{
1089 char *filepath= NULL;
1090
1091 FILE *file = fopen(link_filepath, "r+b");
1092 if (file) {
1093 filepath = static_cast<char*>(malloc(OS_FILE_MAX_PATH));
1094
1095 os_file_read_string(file, filepath, OS_FILE_MAX_PATH);
1096 fclose(file);
1097
1098 if (strlen(filepath)) {
1099 /* Trim whitespace from end of filepath */
1100 ulint lastch = strlen(filepath) - 1;
1101 while (lastch > 4 && filepath[lastch] <= 0x20) {
1102 filepath[lastch--] = 0x00;
1103 }
1104 os_normalize_path(filepath);
1105 }
1106
1107 tablespace_locations[ibd_filepath] = filepath;
1108 }
1109 free(filepath);
1110}
1111
1112
1113/************************************************************************
1114Return the location of given .ibd if it was previously read
1115from .isl file.
1116@return NULL or destination .ibd file path. */
1117static
1118const char *
1119tablespace_filepath(const char *ibd_filepath)
1120{
1121 std::map<std::string, std::string>::iterator it;
1122
1123 it = tablespace_locations.find(ibd_filepath);
1124
1125 if (it != tablespace_locations.end()) {
1126 return it->second.c_str();
1127 }
1128
1129 return NULL;
1130}
1131
1132
1133/************************************************************************
1134Copy or move file depending on current mode.
1135@return true in case of success. */
1136static
1137bool
1138copy_or_move_file(const char *src_file_path,
1139 const char *dst_file_path,
1140 const char *dst_dir,
1141 uint thread_n)
1142{
1143 ds_ctxt_t *datasink = ds_data; /* copy to datadir by default */
1144 char filedir[FN_REFLEN];
1145 size_t filedir_len;
1146 bool ret;
1147
1148 /* read the link from .isl file */
1149 if (ends_with(src_file_path, ".isl")) {
1150 char *ibd_filepath;
1151
1152 ibd_filepath = strdup(src_file_path);
1153 strcpy(ibd_filepath + strlen(ibd_filepath) - 3, "ibd");
1154
1155 read_link_file(ibd_filepath, src_file_path);
1156
1157 free(ibd_filepath);
1158 }
1159
1160 /* check if there is .isl file */
1161 if (ends_with(src_file_path, ".ibd")) {
1162 char *link_filepath;
1163 const char *filepath;
1164
1165 link_filepath = strdup(src_file_path);
1166 strcpy(link_filepath + strlen(link_filepath) - 3, "isl");
1167
1168 read_link_file(src_file_path, link_filepath);
1169
1170 filepath = tablespace_filepath(src_file_path);
1171
1172 if (filepath != NULL) {
1173 dirname_part(filedir, filepath, &filedir_len);
1174
1175 dst_file_path = filepath + filedir_len;
1176 dst_dir = filedir;
1177
1178 if (!directory_exists(dst_dir, true)) {
1179 ret = false;
1180 goto cleanup;
1181 }
1182
1183 datasink = ds_create(dst_dir, DS_TYPE_LOCAL);
1184 }
1185
1186 free(link_filepath);
1187 }
1188
1189 ret = (xtrabackup_copy_back ?
1190 copy_file(datasink, src_file_path, dst_file_path, thread_n) :
1191 move_file(datasink, src_file_path, dst_file_path,
1192 dst_dir, thread_n));
1193
1194cleanup:
1195
1196 if (datasink != ds_data) {
1197 ds_destroy(datasink);
1198 }
1199
1200 return(ret);
1201}
1202
1203
1204
1205
1206bool
1207backup_files(const char *from, bool prep_mode)
1208{
1209 char rsync_tmpfile_name[FN_REFLEN];
1210 FILE *rsync_tmpfile = NULL;
1211 datadir_iter_t *it;
1212 datadir_node_t node;
1213 bool ret = true;
1214
1215 if (prep_mode && !opt_rsync) {
1216 return(true);
1217 }
1218
1219 if (opt_rsync) {
1220 snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name),
1221 "%s/%s%d", opt_mysql_tmpdir,
1222 "xtrabackup_rsyncfiles_pass",
1223 prep_mode ? 1 : 2);
1224 rsync_tmpfile = fopen(rsync_tmpfile_name, "w");
1225 if (rsync_tmpfile == NULL) {
1226 msg("Error: can't create file %s\n",
1227 rsync_tmpfile_name);
1228 return(false);
1229 }
1230 }
1231
1232 msg_ts("Starting %s non-InnoDB tables and files\n",
1233 prep_mode ? "prep copy of" : "to backup");
1234
1235 datadir_node_init(&node);
1236 it = datadir_iter_new(from);
1237
1238 while (datadir_iter_next(it, &node)) {
1239
1240 if (!node.is_empty_dir) {
1241 if (opt_rsync) {
1242 ret = datafile_rsync_backup(node.filepath,
1243 !prep_mode, rsync_tmpfile);
1244 } else {
1245 ret = datafile_copy_backup(node.filepath, 1);
1246 }
1247 if (!ret) {
1248 msg("Failed to copy file %s\n", node.filepath);
1249 goto out;
1250 }
1251 } else if (!prep_mode) {
1252 /* backup fake file into empty directory */
1253 char path[FN_REFLEN];
1254 snprintf(path, sizeof(path),
1255 "%s/db.opt", node.filepath);
1256 if (!(ret = backup_file_printf(
1257 trim_dotslash(path), "%s", ""))) {
1258 msg("Failed to create file %s\n", path);
1259 goto out;
1260 }
1261 }
1262 }
1263
1264 if (opt_rsync) {
1265 std::stringstream cmd;
1266 int err;
1267
1268 if (buffer_pool_filename && file_exists(buffer_pool_filename)) {
1269 fprintf(rsync_tmpfile, "%s\n", buffer_pool_filename);
1270 rsync_list.insert(buffer_pool_filename);
1271 }
1272 if (file_exists("ib_lru_dump")) {
1273 fprintf(rsync_tmpfile, "%s\n", "ib_lru_dump");
1274 rsync_list.insert("ib_lru_dump");
1275 }
1276
1277 fclose(rsync_tmpfile);
1278 rsync_tmpfile = NULL;
1279
1280 cmd << "rsync -t . --files-from=" << rsync_tmpfile_name
1281 << " " << xtrabackup_target_dir;
1282
1283 msg_ts("Starting rsync as: %s\n", cmd.str().c_str());
1284 if ((err = system(cmd.str().c_str()) && !prep_mode) != 0) {
1285 msg_ts("Error: rsync failed with error code %d\n", err);
1286 ret = false;
1287 goto out;
1288 }
1289 msg_ts("rsync finished successfully.\n");
1290
1291 if (!prep_mode && !opt_no_lock) {
1292 char path[FN_REFLEN];
1293 char dst_path[FN_REFLEN];
1294 char *newline;
1295
1296 /* Remove files that have been removed between first and
1297 second passes. Cannot use "rsync --delete" because it
1298 does not work with --files-from. */
1299 snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name),
1300 "%s/%s", opt_mysql_tmpdir,
1301 "xtrabackup_rsyncfiles_pass1");
1302
1303 rsync_tmpfile = fopen(rsync_tmpfile_name, "r");
1304 if (rsync_tmpfile == NULL) {
1305 msg("Error: can't open file %s\n",
1306 rsync_tmpfile_name);
1307 return(false);
1308 }
1309
1310 while (fgets(path, sizeof(path), rsync_tmpfile)) {
1311
1312 newline = strchr(path, '\n');
1313 if (newline) {
1314 *newline = 0;
1315 }
1316 if (rsync_list.count(path) < 1) {
1317 snprintf(dst_path, sizeof(dst_path),
1318 "%s/%s", xtrabackup_target_dir,
1319 path);
1320 msg_ts("Removing %s\n", dst_path);
1321 unlink(dst_path);
1322 }
1323 }
1324
1325 fclose(rsync_tmpfile);
1326 rsync_tmpfile = NULL;
1327 }
1328 }
1329
1330 msg_ts("Finished %s non-InnoDB tables and files\n",
1331 prep_mode ? "a prep copy of" : "backing up");
1332
1333out:
1334 datadir_iter_free(it);
1335 datadir_node_free(&node);
1336
1337 if (rsync_tmpfile != NULL) {
1338 fclose(rsync_tmpfile);
1339 }
1340
1341 return(ret);
1342}
1343
1344/** Start --backup */
1345bool backup_start()
1346{
1347 if (!opt_no_lock) {
1348 if (opt_safe_slave_backup) {
1349 if (!wait_for_safe_slave(mysql_connection)) {
1350 return(false);
1351 }
1352 }
1353
1354 if (!backup_files(fil_path_to_mysql_datadir, true)) {
1355 return(false);
1356 }
1357
1358 history_lock_time = time(NULL);
1359
1360 if (!lock_tables(mysql_connection)) {
1361 return(false);
1362 }
1363 }
1364
1365 if (!backup_files(fil_path_to_mysql_datadir, false)) {
1366 return(false);
1367 }
1368
1369 if (!backup_files_from_datadir(fil_path_to_mysql_datadir)) {
1370 return false;
1371 }
1372
1373 // There is no need to stop slave thread before coping non-Innodb data when
1374 // --no-lock option is used because --no-lock option requires that no DDL or
1375 // DML to non-transaction tables can occur.
1376 if (opt_no_lock) {
1377 if (opt_safe_slave_backup) {
1378 if (!wait_for_safe_slave(mysql_connection)) {
1379 return(false);
1380 }
1381 }
1382 }
1383
1384 if (opt_slave_info) {
1385 lock_binlog_maybe(mysql_connection);
1386
1387 if (!write_slave_info(mysql_connection)) {
1388 return(false);
1389 }
1390 }
1391
1392 /* The only reason why Galera/binlog info is written before
1393 wait_for_ibbackup_log_copy_finish() is that after that call the xtrabackup
1394 binary will start streamig a temporary copy of REDO log to stdout and
1395 thus, any streaming from innobackupex would interfere. The only way to
1396 avoid that is to have a single process, i.e. merge innobackupex and
1397 xtrabackup. */
1398 if (opt_galera_info) {
1399 if (!write_galera_info(mysql_connection)) {
1400 return(false);
1401 }
1402 write_current_binlog_file(mysql_connection);
1403 }
1404
1405 if (opt_binlog_info == BINLOG_INFO_ON) {
1406
1407 lock_binlog_maybe(mysql_connection);
1408 write_binlog_info(mysql_connection);
1409 }
1410
1411 if (have_flush_engine_logs) {
1412 msg_ts("Executing FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS...\n");
1413 xb_mysql_query(mysql_connection,
1414 "FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS", false);
1415 }
1416
1417 return(true);
1418}
1419
1420/** Release resources after backup_start() */
1421void backup_release()
1422{
1423 /* release all locks */
1424 if (!opt_no_lock) {
1425 unlock_all(mysql_connection);
1426 history_lock_time = 0;
1427 } else {
1428 history_lock_time = time(NULL) - history_lock_time;
1429 }
1430
1431 if (opt_lock_ddl_per_table) {
1432 mdl_unlock_all();
1433 }
1434
1435 if (opt_safe_slave_backup && sql_thread_started) {
1436 msg("Starting slave SQL thread\n");
1437 xb_mysql_query(mysql_connection,
1438 "START SLAVE SQL_THREAD", false);
1439 }
1440}
1441
1442/** Finish after backup_start() and backup_release() */
1443bool backup_finish()
1444{
1445 /* Copy buffer pool dump or LRU dump */
1446 if (!opt_rsync) {
1447 if (buffer_pool_filename && file_exists(buffer_pool_filename)) {
1448 const char *dst_name;
1449
1450 dst_name = trim_dotslash(buffer_pool_filename);
1451 copy_file(ds_data, buffer_pool_filename, dst_name, 0);
1452 }
1453 if (file_exists("ib_lru_dump")) {
1454 copy_file(ds_data, "ib_lru_dump", "ib_lru_dump", 0);
1455 }
1456 }
1457
1458 msg_ts("Backup created in directory '%s'\n", xtrabackup_target_dir);
1459 if (mysql_binlog_position != NULL) {
1460 msg("MySQL binlog position: %s\n", mysql_binlog_position);
1461 }
1462 if (mysql_slave_position && opt_slave_info) {
1463 msg("MySQL slave binlog position: %s\n",
1464 mysql_slave_position);
1465 }
1466
1467 if (!write_backup_config_file()) {
1468 return(false);
1469 }
1470
1471 if (!write_xtrabackup_info(mysql_connection, XTRABACKUP_INFO, opt_history != 0)) {
1472 return(false);
1473 }
1474
1475 return(true);
1476}
1477
1478bool
1479ibx_copy_incremental_over_full()
1480{
1481 const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
1482 "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
1483 NULL};
1484 const char *sup_files[] = {"xtrabackup_binlog_info",
1485 "xtrabackup_galera_info",
1486 "xtrabackup_slave_info",
1487 "xtrabackup_info",
1488 "ib_lru_dump",
1489 NULL};
1490 datadir_iter_t *it = NULL;
1491 datadir_node_t node;
1492 bool ret = true;
1493 char path[FN_REFLEN];
1494 int i;
1495
1496 datadir_node_init(&node);
1497
1498 /* If we were applying an incremental change set, we need to make
1499 sure non-InnoDB files and xtrabackup_* metainfo files are copied
1500 to the full backup directory. */
1501
1502 if (xtrabackup_incremental) {
1503
1504 ds_data = ds_create(xtrabackup_target_dir, DS_TYPE_LOCAL);
1505
1506 it = datadir_iter_new(xtrabackup_incremental_dir);
1507
1508 while (datadir_iter_next(it, &node)) {
1509
1510 /* copy only non-innodb files */
1511
1512 if (node.is_empty_dir
1513 || !filename_matches(node.filepath, ext_list)) {
1514 continue;
1515 }
1516
1517 if (file_exists(node.filepath_rel)) {
1518 unlink(node.filepath_rel);
1519 }
1520
1521 if (!(ret = copy_file(ds_data, node.filepath,
1522 node.filepath_rel, 1))) {
1523 msg("Failed to copy file %s\n",
1524 node.filepath);
1525 goto cleanup;
1526 }
1527 }
1528
1529 /* copy buffer pool dump */
1530 if (innobase_buffer_pool_filename) {
1531 const char *src_name;
1532
1533 src_name = trim_dotslash(innobase_buffer_pool_filename);
1534
1535 snprintf(path, sizeof(path), "%s/%s",
1536 xtrabackup_incremental_dir,
1537 src_name);
1538
1539 if (file_exists(path)) {
1540 copy_file(ds_data, path,
1541 innobase_buffer_pool_filename, 0);
1542 }
1543 }
1544
1545 /* copy supplementary files */
1546
1547 for (i = 0; sup_files[i]; i++) {
1548 snprintf(path, sizeof(path), "%s/%s",
1549 xtrabackup_incremental_dir,
1550 sup_files[i]);
1551
1552 if (file_exists(path))
1553 {
1554 if (file_exists(sup_files[i])) {
1555 unlink(sup_files[i]);
1556 }
1557 copy_file(ds_data, path, sup_files[i], 0);
1558 }
1559 }
1560
1561 }
1562
1563cleanup:
1564 if (it != NULL) {
1565 datadir_iter_free(it);
1566 }
1567
1568 if (ds_data != NULL) {
1569 ds_destroy(ds_data);
1570 }
1571
1572 datadir_node_free(&node);
1573
1574 return(ret);
1575}
1576
1577bool
1578ibx_cleanup_full_backup()
1579{
1580 const char *ext_list[] = {"delta", "meta", "ibd", NULL};
1581 datadir_iter_t *it = NULL;
1582 datadir_node_t node;
1583 bool ret = true;
1584
1585 datadir_node_init(&node);
1586
1587 /* If we are applying an incremental change set, we need to make
1588 sure non-InnoDB files are cleaned up from full backup dir before
1589 we copy files from incremental dir. */
1590
1591 it = datadir_iter_new(xtrabackup_target_dir);
1592
1593 while (datadir_iter_next(it, &node)) {
1594
1595 if (node.is_empty_dir) {
1596#ifdef _WIN32
1597 DeleteFile(node.filepath);
1598#else
1599 rmdir(node.filepath);
1600#endif
1601 }
1602
1603 if (xtrabackup_incremental && !node.is_empty_dir
1604 && !filename_matches(node.filepath, ext_list)) {
1605 unlink(node.filepath);
1606 }
1607 }
1608
1609 datadir_iter_free(it);
1610
1611 datadir_node_free(&node);
1612
1613 return(ret);
1614}
1615
1616bool
1617apply_log_finish()
1618{
1619 if (!ibx_cleanup_full_backup()
1620 || !ibx_copy_incremental_over_full()) {
1621 return(false);
1622 }
1623
1624 return(true);
1625}
1626
1627bool
1628copy_back()
1629{
1630 bool ret;
1631 datadir_iter_t *it = NULL;
1632 datadir_node_t node;
1633 char *dst_dir;
1634
1635 memset(&node, 0, sizeof(node));
1636
1637 if (!opt_force_non_empty_dirs) {
1638 if (!directory_exists_and_empty(mysql_data_home,
1639 "Original data")) {
1640 return(false);
1641 }
1642 } else {
1643 if (!directory_exists(mysql_data_home, true)) {
1644 return(false);
1645 }
1646 }
1647 if (srv_undo_dir && *srv_undo_dir
1648 && !directory_exists(srv_undo_dir, true)) {
1649 return(false);
1650 }
1651 if (innobase_data_home_dir && *innobase_data_home_dir
1652 && !directory_exists(innobase_data_home_dir, true)) {
1653 return(false);
1654 }
1655 if (srv_log_group_home_dir && *srv_log_group_home_dir
1656 && !directory_exists(srv_log_group_home_dir, true)) {
1657 return(false);
1658 }
1659
1660 /* cd to backup directory */
1661 if (my_setwd(xtrabackup_target_dir, MYF(MY_WME)))
1662 {
1663 msg("cannot my_setwd %s\n", xtrabackup_target_dir);
1664 return(false);
1665 }
1666
1667 /* parse data file path */
1668
1669 if (!innobase_data_file_path) {
1670 innobase_data_file_path = (char*) "ibdata1:10M:autoextend";
1671 }
1672
1673 srv_sys_space.set_path(".");
1674
1675 if (!srv_sys_space.parse_params(innobase_data_file_path, true)) {
1676 msg("syntax error in innodb_data_file_path\n");
1677 return(false);
1678 }
1679
1680 srv_max_n_threads = 1000;
1681 sync_check_init();
1682 ut_crc32_init();
1683
1684 /* copy undo tablespaces */
1685
1686
1687 dst_dir = (srv_undo_dir && *srv_undo_dir)
1688 ? srv_undo_dir : mysql_data_home;
1689
1690 ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
1691
1692 for (uint i = 1; i <= TRX_SYS_MAX_UNDO_SPACES; i++) {
1693 char filename[20];
1694 sprintf(filename, "undo%03u", i);
1695 if (!file_exists(filename)) {
1696 break;
1697 }
1698 if (!(ret = copy_or_move_file(filename, filename,
1699 dst_dir, 1))) {
1700 goto cleanup;
1701 }
1702 }
1703
1704 ds_destroy(ds_data);
1705 ds_data = NULL;
1706
1707 /* copy redo logs */
1708
1709 dst_dir = (srv_log_group_home_dir && *srv_log_group_home_dir)
1710 ? srv_log_group_home_dir : mysql_data_home;
1711
1712 /* --backup generates a single ib_logfile0, which we must copy
1713 if it exists. */
1714
1715 ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
1716 MY_STAT stat_arg;
1717 if (!my_stat("ib_logfile0", &stat_arg, MYF(0)) || !stat_arg.st_size) {
1718 /* After completed --prepare, redo log files are redundant.
1719 We must delete any redo logs at the destination, so that
1720 the database will not jump to a different log sequence number
1721 (LSN). */
1722
1723 for (uint i = 0; i <= SRV_N_LOG_FILES_MAX + 1; i++) {
1724 char filename[FN_REFLEN];
1725 snprintf(filename, sizeof filename, "%s/ib_logfile%u",
1726 dst_dir, i);
1727 unlink(filename);
1728 }
1729 } else if (!(ret = copy_or_move_file("ib_logfile0", "ib_logfile0",
1730 dst_dir, 1))) {
1731 goto cleanup;
1732 }
1733 ds_destroy(ds_data);
1734
1735 /* copy innodb system tablespace(s) */
1736
1737 dst_dir = (innobase_data_home_dir && *innobase_data_home_dir)
1738 ? innobase_data_home_dir : mysql_data_home;
1739
1740 ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
1741
1742 for (Tablespace::const_iterator iter(srv_sys_space.begin()),
1743 end(srv_sys_space.end());
1744 iter != end;
1745 ++iter) {
1746 const char *filename = base_name(iter->name());
1747
1748 if (!(ret = copy_or_move_file(filename, iter->name(),
1749 dst_dir, 1))) {
1750 goto cleanup;
1751 }
1752 }
1753
1754 ds_destroy(ds_data);
1755
1756 /* copy the rest of tablespaces */
1757 ds_data = ds_create(mysql_data_home, DS_TYPE_LOCAL);
1758
1759 it = datadir_iter_new(".", false);
1760
1761 datadir_node_init(&node);
1762
1763 while (datadir_iter_next(it, &node)) {
1764 const char *ext_list[] = {"backup-my.cnf",
1765 "xtrabackup_binary", "xtrabackup_binlog_info",
1766 "xtrabackup_checkpoints", ".qp", ".pmap", ".tmp",
1767 NULL};
1768 const char *filename;
1769 char c_tmp;
1770 int i_tmp;
1771 bool is_ibdata_file;
1772
1773 /* create empty directories */
1774 if (node.is_empty_dir) {
1775 char path[FN_REFLEN];
1776
1777 snprintf(path, sizeof(path), "%s/%s",
1778 mysql_data_home, node.filepath_rel);
1779
1780 msg_ts("[%02u] Creating directory %s\n", 1, path);
1781
1782 if (mkdirp(path, 0777, MYF(0)) < 0) {
1783 char errbuf[MYSYS_STRERROR_SIZE];
1784 my_strerror(errbuf, sizeof(errbuf), my_errno);
1785 msg("Can not create directory %s: %s\n",
1786 path, errbuf);
1787 ret = false;
1788
1789 goto cleanup;
1790
1791 }
1792
1793 msg_ts("[%02u] ...done.", 1);
1794
1795 continue;
1796 }
1797
1798 filename = base_name(node.filepath);
1799
1800 /* skip .qp files */
1801 if (filename_matches(filename, ext_list)) {
1802 continue;
1803 }
1804
1805 /* skip undo tablespaces */
1806 if (sscanf(filename, "undo%d%c", &i_tmp, &c_tmp) == 1) {
1807 continue;
1808 }
1809
1810 /* skip the redo log (it was already copied) */
1811 if (!strcmp(filename, "ib_logfile0")) {
1812 continue;
1813 }
1814
1815 /* skip innodb data files */
1816 is_ibdata_file = false;
1817 for (Tablespace::const_iterator iter(srv_sys_space.begin()),
1818 end(srv_sys_space.end()); iter != end; ++iter) {
1819 const char *ibfile = base_name(iter->name());
1820 if (strcmp(ibfile, filename) == 0) {
1821 is_ibdata_file = true;
1822 break;
1823 }
1824 }
1825 if (is_ibdata_file) {
1826 continue;
1827 }
1828
1829 if (!(ret = copy_or_move_file(node.filepath, node.filepath_rel,
1830 mysql_data_home, 1))) {
1831 goto cleanup;
1832 }
1833 }
1834
1835 /* copy buffer pool dump */
1836
1837 if (innobase_buffer_pool_filename) {
1838 const char *src_name;
1839 char path[FN_REFLEN];
1840
1841 src_name = trim_dotslash(innobase_buffer_pool_filename);
1842
1843 snprintf(path, sizeof(path), "%s/%s",
1844 mysql_data_home,
1845 src_name);
1846
1847 /* could be already copied with other files
1848 from data directory */
1849 if (file_exists(src_name) &&
1850 !file_exists(innobase_buffer_pool_filename)) {
1851 copy_or_move_file(src_name,
1852 innobase_buffer_pool_filename,
1853 mysql_data_home, 0);
1854 }
1855 }
1856
1857cleanup:
1858 if (it != NULL) {
1859 datadir_iter_free(it);
1860 }
1861
1862 datadir_node_free(&node);
1863
1864 if (ds_data != NULL) {
1865 ds_destroy(ds_data);
1866 }
1867
1868 ds_data = NULL;
1869
1870 sync_check_close();
1871 return(ret);
1872}
1873
1874bool
1875decrypt_decompress_file(const char *filepath, uint thread_n)
1876{
1877 std::stringstream cmd, message;
1878 char *dest_filepath = strdup(filepath);
1879 bool needs_action = false;
1880
1881 cmd << IF_WIN("type ","cat ") << filepath;
1882
1883 if (opt_decompress
1884 && ends_with(filepath, ".qp")) {
1885 cmd << " | qpress -dio ";
1886 dest_filepath[strlen(dest_filepath) - 3] = 0;
1887 if (needs_action) {
1888 message << " and ";
1889 }
1890 message << "decompressing";
1891 needs_action = true;
1892 }
1893
1894 cmd << " > " << dest_filepath;
1895 message << " " << filepath;
1896
1897 free(dest_filepath);
1898
1899 if (needs_action) {
1900
1901 msg_ts("[%02u] %s\n", thread_n, message.str().c_str());
1902
1903 if (system(cmd.str().c_str()) != 0) {
1904 return(false);
1905 }
1906
1907 if (opt_remove_original) {
1908 msg_ts("[%02u] removing %s\n", thread_n, filepath);
1909 if (my_delete(filepath, MYF(MY_WME)) != 0) {
1910 return(false);
1911 }
1912 }
1913 }
1914
1915 return(true);
1916}
1917
1918static
1919os_thread_ret_t STDCALL
1920decrypt_decompress_thread_func(void *arg)
1921{
1922 bool ret = true;
1923 datadir_node_t node;
1924 datadir_thread_ctxt_t *ctxt = (datadir_thread_ctxt_t *)(arg);
1925
1926 datadir_node_init(&node);
1927
1928 while (datadir_iter_next(ctxt->it, &node)) {
1929
1930 /* skip empty directories in backup */
1931 if (node.is_empty_dir) {
1932 continue;
1933 }
1934
1935 if (!ends_with(node.filepath, ".qp")) {
1936 continue;
1937 }
1938
1939 if (!(ret = decrypt_decompress_file(node.filepath,
1940 ctxt->n_thread))) {
1941 goto cleanup;
1942 }
1943 }
1944
1945cleanup:
1946
1947 datadir_node_free(&node);
1948
1949 pthread_mutex_lock(&ctxt->count_mutex);
1950 --(*ctxt->count);
1951 pthread_mutex_unlock(&ctxt->count_mutex);
1952
1953 ctxt->ret = ret;
1954
1955 os_thread_exit();
1956 OS_THREAD_DUMMY_RETURN;
1957}
1958
1959bool
1960decrypt_decompress()
1961{
1962 bool ret;
1963 datadir_iter_t *it = NULL;
1964
1965 srv_max_n_threads = 1000;
1966 sync_check_init();
1967
1968 /* cd to backup directory */
1969 if (my_setwd(xtrabackup_target_dir, MYF(MY_WME)))
1970 {
1971 msg("cannot my_setwd %s\n", xtrabackup_target_dir);
1972 return(false);
1973 }
1974
1975 /* copy the rest of tablespaces */
1976 ds_data = ds_create(".", DS_TYPE_LOCAL);
1977
1978 it = datadir_iter_new(".", false);
1979
1980 ut_a(xtrabackup_parallel >= 0);
1981
1982 ret = run_data_threads(it, decrypt_decompress_thread_func,
1983 xtrabackup_parallel ? xtrabackup_parallel : 1);
1984
1985 if (it != NULL) {
1986 datadir_iter_free(it);
1987 }
1988
1989 if (ds_data != NULL) {
1990 ds_destroy(ds_data);
1991 }
1992
1993 ds_data = NULL;
1994
1995 sync_check_close();
1996
1997 return(ret);
1998}
1999
2000/*
2001 Copy some files from top level datadir.
2002 Do not copy the Innodb files (ibdata1, redo log files),
2003 as this is done in a separate step.
2004*/
2005static bool backup_files_from_datadir(const char *dir_path)
2006{
2007 os_file_dir_t dir = os_file_opendir(dir_path, TRUE);
2008 os_file_stat_t info;
2009 bool ret = true;
2010 while (os_file_readdir_next_file(dir_path, dir, &info) == 0) {
2011
2012 if (info.type != OS_FILE_TYPE_FILE)
2013 continue;
2014
2015 const char *pname = strrchr(info.name, IF_WIN('\\', '/'));
2016 if (!pname)
2017 pname = info.name;
2018
2019 /* Copy aria log files, and aws keys for encryption plugins.*/
2020 const char *prefixes[] = { "aria_log", "aws-kms-key" };
2021 for (size_t i = 0; i < array_elements(prefixes); i++) {
2022 if (starts_with(pname, prefixes[i])) {
2023 ret = copy_file(ds_data, info.name, info.name, 1);
2024 if (!ret) {
2025 break;
2026 }
2027 }
2028 }
2029 }
2030 os_file_closedir(dir);
2031 return ret;
2032}
2033