1/******************************************************
2MariaBackup: hot backup tool for InnoDB
3(c) 2009-2017 Percona LLC and/or its affiliates
4Originally Created 3/3/2009 Yasufumi Kinoshita
5Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
6Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
7(c) 2017, 2018, MariaDB Corporation.
8Portions written by Marko Mäkelä.
9
10This program is free software; you can redistribute it and/or modify
11it under the terms of the GNU General Public License as published by
12the Free Software Foundation; version 2 of the License.
13
14This program is distributed in the hope that it will be useful,
15but WITHOUT ANY WARRANTY; without even the implied warranty of
16MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17GNU General Public License for more details.
18
19You should have received a copy of the GNU General Public License
20along with this program; if not, write to the Free Software
21Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
22
23*******************************************************
24
25This file incorporates work covered by the following copyright and
26permission notice:
27
28Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.
29
30This program is free software; you can redistribute it and/or modify it under
31the terms of the GNU General Public License as published by the Free Software
32Foundation; version 2 of the License.
33
34This program is distributed in the hope that it will be useful, but WITHOUT
35ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
36FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
37
38You should have received a copy of the GNU General Public License along with
39this program; if not, write to the Free Software Foundation, Inc., 59 Temple
40Place, Suite 330, Boston, MA 02111-1307 USA
41
42*******************************************************/
43
44//#define XTRABACKUP_TARGET_IS_PLUGIN
45
46#include <my_global.h>
47#include <my_config.h>
48#include <unireg.h>
49#include <mysql_version.h>
50#include <my_base.h>
51#include <my_getopt.h>
52#include <mysql_com.h>
53#include <my_default.h>
54#include <mysqld.h>
55
56#include <fcntl.h>
57#include <string.h>
58
59#ifdef __linux__
60# include <sys/prctl.h>
61#include <sys/resource.h>
62#endif
63
64
65#include <btr0sea.h>
66#include <dict0priv.h>
67#include <lock0lock.h>
68#include <log0recv.h>
69#include <log0crypt.h>
70#include <row0mysql.h>
71#include <row0quiesce.h>
72#include <srv0start.h>
73#include "trx0sys.h"
74#include <buf0dblwr.h>
75
76#include <list>
77#include <sstream>
78#include <set>
79#include <mysql.h>
80
81#define G_PTR uchar*
82
83#include "common.h"
84#include "datasink.h"
85
86#include "xb_regex.h"
87#include "fil_cur.h"
88#include "write_filt.h"
89#include "xtrabackup.h"
90#include "ds_buffer.h"
91#include "ds_tmpfile.h"
92#include "xbstream.h"
93#include "changed_page_bitmap.h"
94#include "read_filt.h"
95#include "backup_wsrep.h"
96#include "innobackupex.h"
97#include "backup_mysql.h"
98#include "backup_copy.h"
99#include "backup_mysql.h"
100#include "encryption_plugin.h"
101#include <sql_plugin.h>
102#include <srv0srv.h>
103#include <crc_glue.h>
104#include <log.h>
105
106int sys_var_init();
107
108/* === xtrabackup specific options === */
109char xtrabackup_real_target_dir[FN_REFLEN] = "./xtrabackup_backupfiles/";
110char *xtrabackup_target_dir= xtrabackup_real_target_dir;
111static my_bool xtrabackup_version;
112my_bool xtrabackup_backup;
113my_bool xtrabackup_prepare;
114my_bool xtrabackup_copy_back;
115my_bool xtrabackup_move_back;
116my_bool xtrabackup_decrypt_decompress;
117my_bool xtrabackup_print_param;
118
119my_bool xtrabackup_export;
120
121longlong xtrabackup_use_memory;
122
123uint opt_protocol;
124long xtrabackup_throttle; /* 0:unlimited */
125static lint io_ticket;
126static os_event_t wait_throttle;
127static os_event_t log_copying_stop;
128
129char *xtrabackup_incremental;
130lsn_t incremental_lsn;
131lsn_t incremental_to_lsn;
132lsn_t incremental_last_lsn;
133xb_page_bitmap *changed_page_bitmap;
134
135char *xtrabackup_incremental_basedir; /* for --backup */
136char *xtrabackup_extra_lsndir; /* for --backup with --extra-lsndir */
137char *xtrabackup_incremental_dir; /* for --prepare */
138
139char xtrabackup_real_incremental_basedir[FN_REFLEN];
140char xtrabackup_real_extra_lsndir[FN_REFLEN];
141char xtrabackup_real_incremental_dir[FN_REFLEN];
142
143
144char *xtrabackup_tmpdir;
145
146char *xtrabackup_tables;
147char *xtrabackup_tables_file;
148char *xtrabackup_tables_exclude;
149
150typedef std::list<regex_t> regex_list_t;
151static regex_list_t regex_include_list;
152static regex_list_t regex_exclude_list;
153
154static hash_table_t* tables_include_hash = NULL;
155static hash_table_t* tables_exclude_hash = NULL;
156
157char *xtrabackup_databases = NULL;
158char *xtrabackup_databases_file = NULL;
159char *xtrabackup_databases_exclude = NULL;
160static hash_table_t* databases_include_hash = NULL;
161static hash_table_t* databases_exclude_hash = NULL;
162
163static hash_table_t* inc_dir_tables_hash;
164
165struct xb_filter_entry_struct{
166 char* name;
167 ibool has_tables;
168 hash_node_t name_hash;
169};
170typedef struct xb_filter_entry_struct xb_filter_entry_t;
171
172lsn_t checkpoint_lsn_start;
173lsn_t checkpoint_no_start;
174static lsn_t log_copy_scanned_lsn;
175static bool log_copying;
176static bool log_copying_running;
177static bool io_watching_thread_running;
178
179int xtrabackup_parallel;
180
181char *xtrabackup_stream_str = NULL;
182xb_stream_fmt_t xtrabackup_stream_fmt = XB_STREAM_FMT_NONE;
183ibool xtrabackup_stream = FALSE;
184
185const char *xtrabackup_compress_alg = NULL;
186ibool xtrabackup_compress = FALSE;
187uint xtrabackup_compress_threads;
188ulonglong xtrabackup_compress_chunk_size = 0;
189
190/* sleep interval beetween log copy iterations in log copying thread
191in milliseconds (default is 1 second) */
192ulint xtrabackup_log_copy_interval = 1000;
193static ulong max_buf_pool_modified_pct;
194
195/* Ignored option (--log) for MySQL option compatibility */
196static char* log_ignored_opt;
197
198
199extern my_bool opt_use_ssl;
200my_bool opt_ssl_verify_server_cert;
201
202/* === metadata of backup === */
203#define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints"
204char metadata_type[30] = ""; /*[full-backuped|log-applied|incremental]*/
205lsn_t metadata_from_lsn;
206lsn_t metadata_to_lsn;
207lsn_t metadata_last_lsn;
208
209static ds_file_t* dst_log_file;
210
211static char mysql_data_home_buff[2];
212
213const char *defaults_group = "mysqld";
214
215/* === static parameters in ha_innodb.cc */
216
217#define HA_INNOBASE_ROWS_IN_TABLE 10000 /* to get optimization right */
218#define HA_INNOBASE_RANGE_COUNT 100
219
220ulong innobase_large_page_size = 0;
221
222/* The default values for the following, type long or longlong, start-up
223parameters are declared in mysqld.cc: */
224
225long innobase_buffer_pool_awe_mem_mb = 0;
226long innobase_file_io_threads = 4;
227long innobase_read_io_threads = 4;
228long innobase_write_io_threads = 4;
229
230longlong innobase_page_size = (1LL << 14); /* 16KB */
231char* innobase_buffer_pool_filename = NULL;
232
233longlong innobase_buffer_pool_size = 8*1024*1024L;
234
235/* The default values for the following char* start-up parameters
236are determined in innobase_init below: */
237
238static char* innobase_ignored_opt;
239char* innobase_data_home_dir;
240char* innobase_data_file_path;
241
242my_bool innobase_use_doublewrite;
243my_bool innobase_use_large_pages;
244my_bool innobase_file_per_table;
245my_bool innobase_locks_unsafe_for_binlog;
246my_bool innobase_rollback_on_timeout;
247my_bool innobase_create_status_file;
248
249/* The following counter is used to convey information to InnoDB
250about server activity: in selects it is not sensible to call
251srv_active_wake_master_thread after each fetch or search, we only do
252it every INNOBASE_WAKE_INTERVAL'th step. */
253
254#define INNOBASE_WAKE_INTERVAL 32
255ulong innobase_active_counter = 0;
256
257#ifndef _WIN32
258static char *xtrabackup_debug_sync = NULL;
259#endif
260
261my_bool xtrabackup_incremental_force_scan = FALSE;
262
263/* The flushed lsn which is read from data files */
264lsn_t flushed_lsn= 0;
265
266ulong xb_open_files_limit= 0;
267char *xb_plugin_dir;
268char *xb_plugin_load;
269my_bool xb_close_files;
270
271/* Datasinks */
272ds_ctxt_t *ds_data = NULL;
273ds_ctxt_t *ds_meta = NULL;
274ds_ctxt_t *ds_redo = NULL;
275
276static bool innobackupex_mode = false;
277
278/* String buffer used by --print-param to accumulate server options as they are
279parsed from the defaults file */
280static std::ostringstream print_param_str;
281
282/* Set of specified parameters */
283std::set<std::string> param_set;
284
285static ulonglong global_max_value;
286
287extern "C" sig_handler handle_fatal_signal(int sig);
288extern LOGGER logger;
289
290my_bool opt_galera_info = FALSE;
291my_bool opt_slave_info = FALSE;
292my_bool opt_no_lock = FALSE;
293my_bool opt_safe_slave_backup = FALSE;
294my_bool opt_rsync = FALSE;
295my_bool opt_force_non_empty_dirs = FALSE;
296my_bool opt_noversioncheck = FALSE;
297my_bool opt_no_backup_locks = FALSE;
298my_bool opt_decompress = FALSE;
299my_bool opt_remove_original;
300
301my_bool opt_lock_ddl_per_table = FALSE;
302
303extern const char *innodb_checksum_algorithm_names[];
304extern TYPELIB innodb_checksum_algorithm_typelib;
305extern const char *innodb_flush_method_names[];
306extern TYPELIB innodb_flush_method_typelib;
307
308static const char *binlog_info_values[] = {"off", "lockless", "on", "auto",
309 NullS};
310static TYPELIB binlog_info_typelib = {array_elements(binlog_info_values)-1, "",
311 binlog_info_values, NULL};
312ulong opt_binlog_info;
313
314char *opt_incremental_history_name;
315char *opt_incremental_history_uuid;
316
317char *opt_user;
318char *opt_password;
319char *opt_host;
320char *opt_defaults_group;
321char *opt_socket;
322uint opt_port;
323char *opt_log_bin;
324
325const char *query_type_names[] = { "ALL", "UPDATE", "SELECT", NullS};
326
327TYPELIB query_type_typelib= {array_elements(query_type_names) - 1, "",
328 query_type_names, NULL};
329
330ulong opt_lock_wait_query_type;
331ulong opt_kill_long_query_type;
332
333uint opt_kill_long_queries_timeout = 0;
334uint opt_lock_wait_timeout = 0;
335uint opt_lock_wait_threshold = 0;
336uint opt_debug_sleep_before_unlock = 0;
337uint opt_safe_slave_backup_timeout = 0;
338
339const char *opt_history = NULL;
340
341
342char mariabackup_exe[FN_REFLEN];
343char orig_argv1[FN_REFLEN];
344
345/* Whether xtrabackup_binlog_info should be created on recovery */
346static bool recover_binlog_info;
347
348/* Simple datasink creation tracking...add datasinks in the reverse order you
349want them destroyed. */
350#define XTRABACKUP_MAX_DATASINKS 10
351static ds_ctxt_t *datasinks[XTRABACKUP_MAX_DATASINKS];
352static uint actual_datasinks = 0;
353static inline
354void
355xtrabackup_add_datasink(ds_ctxt_t *ds)
356{
357 xb_ad(actual_datasinks < XTRABACKUP_MAX_DATASINKS);
358 datasinks[actual_datasinks] = ds; actual_datasinks++;
359}
360
361
362typedef void (*process_single_tablespace_func_t)(const char *dirname, const char *filname, bool is_remote);
363static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback);
364
365
366/* ======== Datafiles iterator ======== */
367struct datafiles_iter_t {
368 fil_space_t *space;
369 fil_node_t *node;
370 ibool started;
371 pthread_mutex_t mutex;
372};
373
374/* ======== Datafiles iterator ======== */
375static
376datafiles_iter_t *
377datafiles_iter_new()
378{
379 datafiles_iter_t *it;
380
381 it = static_cast<datafiles_iter_t *>(malloc(sizeof(datafiles_iter_t)));
382 pthread_mutex_init(&it->mutex, NULL);
383
384 it->space = NULL;
385 it->node = NULL;
386 it->started = FALSE;
387
388 return it;
389}
390
391static
392fil_node_t *
393datafiles_iter_next(datafiles_iter_t *it)
394{
395 fil_node_t *new_node;
396
397 pthread_mutex_lock(&it->mutex);
398
399 if (it->node == NULL) {
400 if (it->started)
401 goto end;
402 it->started = TRUE;
403 } else {
404 it->node = UT_LIST_GET_NEXT(chain, it->node);
405 if (it->node != NULL)
406 goto end;
407 }
408
409 it->space = (it->space == NULL) ?
410 UT_LIST_GET_FIRST(fil_system.space_list) :
411 UT_LIST_GET_NEXT(space_list, it->space);
412
413 while (it->space != NULL &&
414 (it->space->purpose != FIL_TYPE_TABLESPACE ||
415 UT_LIST_GET_LEN(it->space->chain) == 0))
416 it->space = UT_LIST_GET_NEXT(space_list, it->space);
417 if (it->space == NULL)
418 goto end;
419
420 it->node = UT_LIST_GET_FIRST(it->space->chain);
421
422end:
423 new_node = it->node;
424 pthread_mutex_unlock(&it->mutex);
425
426 return new_node;
427}
428
429static
430void
431datafiles_iter_free(datafiles_iter_t *it)
432{
433 pthread_mutex_destroy(&it->mutex);
434 free(it);
435}
436
437#ifndef DBUG_OFF
438struct dbug_thread_param_t
439{
440 MYSQL *con;
441 const char *query;
442 int expect_err;
443 int expect_errno;
444 os_event_t done_event;
445};
446
447
448/* Thread procedure used in dbug_start_query_thread. */
449extern "C"
450os_thread_ret_t
451DECLARE_THREAD(dbug_execute_in_new_connection)(void *arg)
452{
453 mysql_thread_init();
454 dbug_thread_param_t *par= (dbug_thread_param_t *)arg;
455 int err = mysql_query(par->con, par->query);
456 int err_no = mysql_errno(par->con);
457 DBUG_ASSERT(par->expect_err == err);
458 if (err && par->expect_errno)
459 DBUG_ASSERT(err_no == par->expect_errno);
460 mysql_close(par->con);
461 mysql_thread_end();
462 os_event_t done = par->done_event;
463 delete par;
464 os_event_set(done);
465 os_thread_exit();
466 return os_thread_ret_t(0);
467}
468
469/*
470Execute query from a new connection, in own thread.
471
472@param query - query to be executed
473@param wait_state - if not NULL, wait until query from new connection
474 reaches this state (value of column State in I_S.PROCESSLIST)
475@param expected_err - if 0, query is supposed to finish successfully,
476 otherwise query should return error.
477@param expected_errno - if not 0, and query finished with error,
478 expected mysql_errno()
479*/
480static os_event_t dbug_start_query_thread(
481 const char *query,
482 const char *wait_state,
483 int expected_err,
484 int expected_errno)
485
486{
487 dbug_thread_param_t *par = new dbug_thread_param_t;
488 par->query = query;
489 par->expect_err = expected_err;
490 par->expect_errno = expected_errno;
491 par->done_event = os_event_create(0);
492 par->con = xb_mysql_connect();
493 os_thread_create(dbug_execute_in_new_connection, par, 0);
494
495 if (!wait_state)
496 return par->done_event;
497
498 char q[256];
499 snprintf(q, sizeof(q),
500 "SELECT 1 FROM INFORMATION_SCHEMA.PROCESSLIST where ID=%lu"
501 " AND Command='Query' AND State='%s'",
502 mysql_thread_id(par->con), wait_state);
503 for (;;) {
504 MYSQL_RES *result = xb_mysql_query(mysql_connection,q, true, true);
505 if (mysql_fetch_row(result)) {
506 goto end;
507 }
508 msg_ts("Waiting for query '%s' on connection %lu to "
509 " reach state '%s'", query, mysql_thread_id(par->con),
510 wait_state);
511 my_sleep(1000);
512 }
513end:
514 msg_ts("query '%s' on connection %lu reached state '%s'", query,
515 mysql_thread_id(par->con), wait_state);
516 return par->done_event;
517}
518
519os_event_t dbug_alter_thread_done;
520#endif
521
522void mdl_lock_all()
523{
524 mdl_lock_init();
525 datafiles_iter_t *it = datafiles_iter_new();
526 if (!it)
527 return;
528
529 while (fil_node_t *node = datafiles_iter_next(it)){
530 if (fil_is_user_tablespace_id(node->space->id)
531 && check_if_skip_table(node->space->name))
532 continue;
533
534 mdl_lock_table(node->space->id);
535 }
536 datafiles_iter_free(it);
537
538 DBUG_EXECUTE_IF("check_mdl_lock_works",
539 dbug_alter_thread_done =
540 dbug_start_query_thread("ALTER TABLE test.t ADD COLUMN mdl_lock_column int",
541 "Waiting for table metadata lock",1, ER_QUERY_INTERRUPTED););
542}
543
544/** Check if the space id belongs to the table which name should
545be skipped based on the --tables, --tables-file and --table-exclude
546options.
547@param[in] space_id space id to check
548@return true if the space id belongs to skip table/database list. */
549static bool backup_includes(space_id_t space_id)
550{
551 datafiles_iter_t *it = datafiles_iter_new();
552 if (!it)
553 return true;
554
555 while (fil_node_t *node = datafiles_iter_next(it)){
556 if (space_id == 0
557 || (node->space->id == space_id
558 && !check_if_skip_table(node->space->name))) {
559
560 msg("mariabackup: Unsupported redo log detected "
561 "and it belongs to %s\n",
562 space_id ? node->name: "the InnoDB system tablespace");
563
564 msg("mariabackup: ALTER TABLE or OPTIMIZE TABLE "
565 "was being executed during the backup.\n");
566
567 if (!opt_lock_ddl_per_table) {
568 msg("mariabackup: Use --lock-ddl-per-table "
569 "parameter to lock all the table before "
570 "backup operation.\n");
571 }
572
573 datafiles_iter_free(it);
574 return false;
575 }
576 }
577
578 datafiles_iter_free(it);
579 return true;
580}
581
582/* ======== Date copying thread context ======== */
583
584typedef struct {
585 datafiles_iter_t *it;
586 uint num;
587 uint *count;
588 pthread_mutex_t count_mutex;
589 os_thread_id_t id;
590} data_thread_ctxt_t;
591
592/* ======== for option and variables ======== */
593#include <../../client/client_priv.h>
594
595enum options_xtrabackup
596{
597 OPT_XTRA_TARGET_DIR = 1000, /* make sure it is larger
598 than OPT_MAX_CLIENT_OPTION */
599 OPT_XTRA_BACKUP,
600 OPT_XTRA_PREPARE,
601 OPT_XTRA_EXPORT,
602 OPT_XTRA_PRINT_PARAM,
603 OPT_XTRA_USE_MEMORY,
604 OPT_XTRA_THROTTLE,
605 OPT_XTRA_LOG_COPY_INTERVAL,
606 OPT_XTRA_INCREMENTAL,
607 OPT_XTRA_INCREMENTAL_BASEDIR,
608 OPT_XTRA_EXTRA_LSNDIR,
609 OPT_XTRA_INCREMENTAL_DIR,
610 OPT_XTRA_TABLES,
611 OPT_XTRA_TABLES_FILE,
612 OPT_XTRA_DATABASES,
613 OPT_XTRA_DATABASES_FILE,
614 OPT_XTRA_PARALLEL,
615 OPT_XTRA_STREAM,
616 OPT_XTRA_COMPRESS,
617 OPT_XTRA_COMPRESS_THREADS,
618 OPT_XTRA_COMPRESS_CHUNK_SIZE,
619 OPT_LOG,
620 OPT_INNODB,
621 OPT_INNODB_DATA_FILE_PATH,
622 OPT_INNODB_DATA_HOME_DIR,
623 OPT_INNODB_ADAPTIVE_HASH_INDEX,
624 OPT_INNODB_DOUBLEWRITE,
625 OPT_INNODB_FILE_PER_TABLE,
626 OPT_INNODB_FLUSH_METHOD,
627 OPT_INNODB_LOG_GROUP_HOME_DIR,
628 OPT_INNODB_MAX_DIRTY_PAGES_PCT,
629 OPT_INNODB_MAX_PURGE_LAG,
630 OPT_INNODB_STATUS_FILE,
631 OPT_INNODB_AUTOEXTEND_INCREMENT,
632 OPT_INNODB_BUFFER_POOL_SIZE,
633 OPT_INNODB_COMMIT_CONCURRENCY,
634 OPT_INNODB_CONCURRENCY_TICKETS,
635 OPT_INNODB_FILE_IO_THREADS,
636 OPT_INNODB_IO_CAPACITY,
637 OPT_INNODB_READ_IO_THREADS,
638 OPT_INNODB_WRITE_IO_THREADS,
639 OPT_INNODB_USE_NATIVE_AIO,
640 OPT_INNODB_PAGE_SIZE,
641 OPT_INNODB_BUFFER_POOL_FILENAME,
642 OPT_INNODB_LOCK_WAIT_TIMEOUT,
643 OPT_INNODB_LOG_BUFFER_SIZE,
644 OPT_INNODB_LOG_FILE_SIZE,
645 OPT_INNODB_LOG_FILES_IN_GROUP,
646 OPT_INNODB_OPEN_FILES,
647 OPT_XTRA_DEBUG_SYNC,
648 OPT_INNODB_CHECKSUM_ALGORITHM,
649 OPT_INNODB_UNDO_DIRECTORY,
650 OPT_INNODB_UNDO_TABLESPACES,
651 OPT_INNODB_LOG_CHECKSUMS,
652 OPT_XTRA_INCREMENTAL_FORCE_SCAN,
653 OPT_DEFAULTS_GROUP,
654 OPT_PLUGIN_LOAD,
655 OPT_INNODB_ENCRYPT_LOG,
656 OPT_CLOSE_FILES,
657 OPT_CORE_FILE,
658
659 OPT_COPY_BACK,
660 OPT_MOVE_BACK,
661 OPT_GALERA_INFO,
662 OPT_SLAVE_INFO,
663 OPT_NO_LOCK,
664 OPT_SAFE_SLAVE_BACKUP,
665 OPT_RSYNC,
666 OPT_FORCE_NON_EMPTY_DIRS,
667 OPT_NO_VERSION_CHECK,
668 OPT_NO_BACKUP_LOCKS,
669 OPT_DECOMPRESS,
670 OPT_INCREMENTAL_HISTORY_NAME,
671 OPT_INCREMENTAL_HISTORY_UUID,
672 OPT_REMOVE_ORIGINAL,
673 OPT_LOCK_WAIT_QUERY_TYPE,
674 OPT_KILL_LONG_QUERY_TYPE,
675 OPT_HISTORY,
676 OPT_KILL_LONG_QUERIES_TIMEOUT,
677 OPT_LOCK_WAIT_TIMEOUT,
678 OPT_LOCK_WAIT_THRESHOLD,
679 OPT_DEBUG_SLEEP_BEFORE_UNLOCK,
680 OPT_SAFE_SLAVE_BACKUP_TIMEOUT,
681 OPT_BINLOG_INFO,
682 OPT_XB_SECURE_AUTH,
683
684 OPT_XTRA_TABLES_EXCLUDE,
685 OPT_XTRA_DATABASES_EXCLUDE,
686 OPT_PROTOCOL,
687 OPT_LOCK_DDL_PER_TABLE
688};
689
690struct my_option xb_client_options[] =
691{
692 {"version", 'v', "print xtrabackup version information",
693 (G_PTR *) &xtrabackup_version, (G_PTR *) &xtrabackup_version, 0, GET_BOOL,
694 NO_ARG, 0, 0, 0, 0, 0, 0},
695 {"target-dir", OPT_XTRA_TARGET_DIR, "destination directory", (G_PTR*) &xtrabackup_target_dir,
696 (G_PTR*) &xtrabackup_target_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
697 {"backup", OPT_XTRA_BACKUP, "take backup to target-dir",
698 (G_PTR*) &xtrabackup_backup, (G_PTR*) &xtrabackup_backup,
699 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
700 {"prepare", OPT_XTRA_PREPARE, "prepare a backup for starting mysql server on the backup.",
701 (G_PTR*) &xtrabackup_prepare, (G_PTR*) &xtrabackup_prepare,
702 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
703 {"export", OPT_XTRA_EXPORT, "create files to import to another database when prepare.",
704 (G_PTR*) &xtrabackup_export, (G_PTR*) &xtrabackup_export,
705 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
706 {"print-param", OPT_XTRA_PRINT_PARAM, "print parameter of mysqld needed for copyback.",
707 (G_PTR*) &xtrabackup_print_param, (G_PTR*) &xtrabackup_print_param,
708 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
709 {"use-memory", OPT_XTRA_USE_MEMORY, "The value is used instead of buffer_pool_size",
710 (G_PTR*) &xtrabackup_use_memory, (G_PTR*) &xtrabackup_use_memory,
711 0, GET_LL, REQUIRED_ARG, 100*1024*1024L, 1024*1024L, LONGLONG_MAX, 0,
712 1024*1024L, 0},
713 {"throttle", OPT_XTRA_THROTTLE, "limit count of IO operations (pairs of read&write) per second to IOS values (for '--backup')",
714 (G_PTR*) &xtrabackup_throttle, (G_PTR*) &xtrabackup_throttle,
715 0, GET_LONG, REQUIRED_ARG, 0, 0, LONG_MAX, 0, 1, 0},
716 {"log", OPT_LOG, "Ignored option for MySQL option compatibility",
717 (G_PTR*) &log_ignored_opt, (G_PTR*) &log_ignored_opt, 0,
718 GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
719 {"log-copy-interval", OPT_XTRA_LOG_COPY_INTERVAL, "time interval between checks done by log copying thread in milliseconds (default is 1 second).",
720 (G_PTR*) &xtrabackup_log_copy_interval, (G_PTR*) &xtrabackup_log_copy_interval,
721 0, GET_LONG, REQUIRED_ARG, 1000, 0, LONG_MAX, 0, 1, 0},
722 {"extra-lsndir", OPT_XTRA_EXTRA_LSNDIR, "(for --backup): save an extra copy of the xtrabackup_checkpoints file in this directory.",
723 (G_PTR*) &xtrabackup_extra_lsndir, (G_PTR*) &xtrabackup_extra_lsndir,
724 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
725 {"incremental-lsn", OPT_XTRA_INCREMENTAL, "(for --backup): copy only .ibd pages newer than specified LSN 'high:low'. ##ATTENTION##: If a wrong LSN value is specified, it is impossible to diagnose this, causing the backup to be unusable. Be careful!",
726 (G_PTR*) &xtrabackup_incremental, (G_PTR*) &xtrabackup_incremental,
727 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
728 {"incremental-basedir", OPT_XTRA_INCREMENTAL_BASEDIR, "(for --backup): copy only .ibd pages newer than backup at specified directory.",
729 (G_PTR*) &xtrabackup_incremental_basedir, (G_PTR*) &xtrabackup_incremental_basedir,
730 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
731 {"incremental-dir", OPT_XTRA_INCREMENTAL_DIR, "(for --prepare): apply .delta files and logfile in the specified directory.",
732 (G_PTR*) &xtrabackup_incremental_dir, (G_PTR*) &xtrabackup_incremental_dir,
733 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
734 {"tables", OPT_XTRA_TABLES, "filtering by regexp for table names.",
735 (G_PTR*) &xtrabackup_tables, (G_PTR*) &xtrabackup_tables,
736 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
737 {"tables_file", OPT_XTRA_TABLES_FILE, "filtering by list of the exact database.table name in the file.",
738 (G_PTR*) &xtrabackup_tables_file, (G_PTR*) &xtrabackup_tables_file,
739 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
740 {"databases", OPT_XTRA_DATABASES, "filtering by list of databases.",
741 (G_PTR*) &xtrabackup_databases, (G_PTR*) &xtrabackup_databases,
742 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
743 {"databases_file", OPT_XTRA_DATABASES_FILE,
744 "filtering by list of databases in the file.",
745 (G_PTR*) &xtrabackup_databases_file, (G_PTR*) &xtrabackup_databases_file,
746 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
747 {"tables-exclude", OPT_XTRA_TABLES_EXCLUDE, "filtering by regexp for table names. "
748 "Operates the same way as --tables, but matched names are excluded from backup. "
749 "Note that this option has a higher priority than --tables.",
750 (G_PTR*) &xtrabackup_tables_exclude, (G_PTR*) &xtrabackup_tables_exclude,
751 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
752 {"databases-exclude", OPT_XTRA_DATABASES_EXCLUDE, "Excluding databases based on name, "
753 "Operates the same way as --databases, but matched names are excluded from backup. "
754 "Note that this option has a higher priority than --databases.",
755 (G_PTR*) &xtrabackup_databases_exclude, (G_PTR*) &xtrabackup_databases_exclude,
756 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
757
758 {"stream", OPT_XTRA_STREAM, "Stream all backup files to the standard output "
759 "in the specified format."
760 "Supported format is 'xbstream'."
761 ,
762 (G_PTR*) &xtrabackup_stream_str, (G_PTR*) &xtrabackup_stream_str, 0, GET_STR,
763 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
764
765 {"compress", OPT_XTRA_COMPRESS, "Compress individual backup files using the "
766 "specified compression algorithm. Currently the only supported algorithm "
767 "is 'quicklz'. It is also the default algorithm, i.e. the one used when "
768 "--compress is used without an argument.",
769 (G_PTR*) &xtrabackup_compress_alg, (G_PTR*) &xtrabackup_compress_alg, 0,
770 GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
771
772 {"compress-threads", OPT_XTRA_COMPRESS_THREADS,
773 "Number of threads for parallel data compression. The default value is 1.",
774 (G_PTR*) &xtrabackup_compress_threads, (G_PTR*) &xtrabackup_compress_threads,
775 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0},
776
777 {"compress-chunk-size", OPT_XTRA_COMPRESS_CHUNK_SIZE,
778 "Size of working buffer(s) for compression threads in bytes. The default value is 64K.",
779 (G_PTR*) &xtrabackup_compress_chunk_size, (G_PTR*) &xtrabackup_compress_chunk_size,
780 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0},
781
782 {"incremental-force-scan", OPT_XTRA_INCREMENTAL_FORCE_SCAN,
783 "Perform a full-scan incremental backup even in the presence of changed "
784 "page bitmap data",
785 (G_PTR*)&xtrabackup_incremental_force_scan,
786 (G_PTR*)&xtrabackup_incremental_force_scan, 0, GET_BOOL, NO_ARG,
787 0, 0, 0, 0, 0, 0},
788
789
790 {"close_files", OPT_CLOSE_FILES, "do not keep files opened. Use at your own "
791 "risk.", (G_PTR*) &xb_close_files, (G_PTR*) &xb_close_files, 0, GET_BOOL,
792 NO_ARG, 0, 0, 0, 0, 0, 0},
793
794 {"core-file", OPT_CORE_FILE, "Write core on fatal signals", 0, 0, 0,
795 GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
796
797
798 {"copy-back", OPT_COPY_BACK, "Copy all the files in a previously made "
799 "backup from the backup directory to their original locations.",
800 (uchar *) &xtrabackup_copy_back, (uchar *) &xtrabackup_copy_back, 0,
801 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
802
803 {"move-back", OPT_MOVE_BACK, "Move all the files in a previously made "
804 "backup from the backup directory to the actual datadir location. "
805 "Use with caution, as it removes backup files.",
806 (uchar *) &xtrabackup_move_back, (uchar *) &xtrabackup_move_back, 0,
807 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
808
809 {"galera-info", OPT_GALERA_INFO, "This options creates the "
810 "xtrabackup_galera_info file which contains the local node state at "
811 "the time of the backup. Option should be used when performing the "
812 "backup of MariaDB Galera Cluster. Has no effect when backup locks "
813 "are used to create the backup.",
814 (uchar *) &opt_galera_info, (uchar *) &opt_galera_info, 0,
815 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
816
817 {"slave-info", OPT_SLAVE_INFO, "This option is useful when backing "
818 "up a replication slave server. It prints the binary log position "
819 "and name of the master server. It also writes this information to "
820 "the \"xtrabackup_slave_info\" file as a \"CHANGE MASTER\" command. "
821 "A new slave for this master can be set up by starting a slave server "
822 "on this backup and issuing a \"CHANGE MASTER\" command with the "
823 "binary log position saved in the \"xtrabackup_slave_info\" file.",
824 (uchar *) &opt_slave_info, (uchar *) &opt_slave_info, 0,
825 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
826
827 {"no-lock", OPT_NO_LOCK, "Use this option to disable table lock "
828 "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your "
829 "tables are InnoDB and you DO NOT CARE about the binary log "
830 "position of the backup. This option shouldn't be used if there "
831 "are any DDL statements being executed or if any updates are "
832 "happening on non-InnoDB tables (this includes the system MyISAM "
833 "tables in the mysql database), otherwise it could lead to an "
834 "inconsistent backup. If you are considering to use --no-lock "
835 "because your backups are failing to acquire the lock, this could "
836 "be because of incoming replication events preventing the lock "
837 "from succeeding. Please try using --safe-slave-backup to "
838 "momentarily stop the replication slave thread, this may help "
839 "the backup to succeed and you then don't need to resort to "
840 "using this option.",
841 (uchar *) &opt_no_lock, (uchar *) &opt_no_lock, 0,
842 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
843
844 {"safe-slave-backup", OPT_SAFE_SLAVE_BACKUP, "Stop slave SQL thread "
845 "and wait to start backup until Slave_open_temp_tables in "
846 "\"SHOW STATUS\" is zero. If there are no open temporary tables, "
847 "the backup will take place, otherwise the SQL thread will be "
848 "started and stopped until there are no open temporary tables. "
849 "The backup will fail if Slave_open_temp_tables does not become "
850 "zero after --safe-slave-backup-timeout seconds. The slave SQL "
851 "thread will be restarted when the backup finishes.",
852 (uchar *) &opt_safe_slave_backup,
853 (uchar *) &opt_safe_slave_backup,
854 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
855
856 {"rsync", OPT_RSYNC, "Uses the rsync utility to optimize local file "
857 "transfers. When this option is specified, innobackupex uses rsync "
858 "to copy all non-InnoDB files instead of spawning a separate cp for "
859 "each file, which can be much faster for servers with a large number "
860 "of databases or tables. This option cannot be used together with "
861 "--stream.",
862 (uchar *) &opt_rsync, (uchar *) &opt_rsync,
863 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
864
865 {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS, "This "
866 "option, when specified, makes --copy-back or --move-back transfer "
867 "files to non-empty directories. Note that no existing files will be "
868 "overwritten. If --copy-back or --nove-back has to copy a file from "
869 "the backup directory which already exists in the destination "
870 "directory, it will still fail with an error.",
871 (uchar *) &opt_force_non_empty_dirs,
872 (uchar *) &opt_force_non_empty_dirs,
873 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
874
875 {"no-version-check", OPT_NO_VERSION_CHECK, "This option disables the "
876 "version check which is enabled by the --version-check option.",
877 (uchar *) &opt_noversioncheck,
878 (uchar *) &opt_noversioncheck,
879 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
880
881 {"no-backup-locks", OPT_NO_BACKUP_LOCKS, "This option controls if "
882 "backup locks should be used instead of FLUSH TABLES WITH READ LOCK "
883 "on the backup stage. The option has no effect when backup locks are "
884 "not supported by the server. This option is enabled by default, "
885 "disable with --no-backup-locks.",
886 (uchar *) &opt_no_backup_locks,
887 (uchar *) &opt_no_backup_locks,
888 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
889
890 {"decompress", OPT_DECOMPRESS, "Decompresses all files with the .qp "
891 "extension in a backup previously made with the --compress option.",
892 (uchar *) &opt_decompress,
893 (uchar *) &opt_decompress,
894 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
895
896 {"user", 'u', "This option specifies the MySQL username used "
897 "when connecting to the server, if that's not the current user. "
898 "The option accepts a string argument. See mysql --help for details.",
899 (uchar*) &opt_user, (uchar*) &opt_user, 0, GET_STR,
900 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
901
902 {"host", 'H', "This option specifies the host to use when "
903 "connecting to the database server with TCP/IP. The option accepts "
904 "a string argument. See mysql --help for details.",
905 (uchar*) &opt_host, (uchar*) &opt_host, 0, GET_STR,
906 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
907
908 {"port", 'P', "This option specifies the port to use when "
909 "connecting to the database server with TCP/IP. The option accepts "
910 "a string argument. See mysql --help for details.",
911 &opt_port, &opt_port, 0, GET_UINT, REQUIRED_ARG,
912 0, 0, 0, 0, 0, 0},
913
914 {"password", 'p', "This option specifies the password to use "
915 "when connecting to the database. It accepts a string argument. "
916 "See mysql --help for details.",
917 0, 0, 0, GET_STR,
918 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
919
920 {"protocol", OPT_PROTOCOL, "The protocol to use for connection (tcp, socket, pipe, memory).",
921 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
922
923 {"socket", 'S', "This option specifies the socket to use when "
924 "connecting to the local database server with a UNIX domain socket. "
925 "The option accepts a string argument. See mysql --help for details.",
926 (uchar*) &opt_socket, (uchar*) &opt_socket, 0, GET_STR,
927 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
928
929 {"incremental-history-name", OPT_INCREMENTAL_HISTORY_NAME,
930 "This option specifies the name of the backup series stored in the "
931 "PERCONA_SCHEMA.xtrabackup_history history record to base an "
932 "incremental backup on. Xtrabackup will search the history table "
933 "looking for the most recent (highest innodb_to_lsn), successful "
934 "backup in the series and take the to_lsn value to use as the "
935 "starting lsn for the incremental backup. This will be mutually "
936 "exclusive with --incremental-history-uuid, --incremental-basedir "
937 "and --incremental-lsn. If no valid lsn can be found (no series by "
938 "that name, no successful backups by that name) xtrabackup will "
939 "return with an error. It is used with the --incremental option.",
940 (uchar*) &opt_incremental_history_name,
941 (uchar*) &opt_incremental_history_name, 0, GET_STR,
942 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
943
944 {"incremental-history-uuid", OPT_INCREMENTAL_HISTORY_UUID,
945 "This option specifies the UUID of the specific history record "
946 "stored in the PERCONA_SCHEMA.xtrabackup_history to base an "
947 "incremental backup on. --incremental-history-name, "
948 "--incremental-basedir and --incremental-lsn. If no valid lsn can be "
949 "found (no success record with that uuid) xtrabackup will return "
950 "with an error. It is used with the --incremental option.",
951 (uchar*) &opt_incremental_history_uuid,
952 (uchar*) &opt_incremental_history_uuid, 0, GET_STR,
953 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
954
955 {"remove-original", OPT_REMOVE_ORIGINAL, "Remove .qp files after decompression.",
956 (uchar *) &opt_remove_original,
957 (uchar *) &opt_remove_original,
958 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
959
960 {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE,
961 "This option specifies which types of queries are allowed to complete "
962 "before innobackupex will issue the global lock. Default is all.",
963 (uchar*) &opt_lock_wait_query_type,
964 (uchar*) &opt_lock_wait_query_type, &query_type_typelib,
965 GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, 0},
966
967 {"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE,
968 "This option specifies which types of queries should be killed to "
969 "unblock the global lock. Default is \"all\".",
970 (uchar*) &opt_kill_long_query_type,
971 (uchar*) &opt_kill_long_query_type, &query_type_typelib,
972 GET_ENUM, REQUIRED_ARG, QUERY_TYPE_SELECT, 0, 0, 0, 0, 0},
973
974 {"history", OPT_HISTORY,
975 "This option enables the tracking of backup history in the "
976 "PERCONA_SCHEMA.xtrabackup_history table. An optional history "
977 "series name may be specified that will be placed with the history "
978 "record for the current backup being taken.",
979 NULL, NULL, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
980
981 {"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT,
982 "This option specifies the number of seconds innobackupex waits "
983 "between starting FLUSH TABLES WITH READ LOCK and killing those "
984 "queries that block it. Default is 0 seconds, which means "
985 "innobackupex will not attempt to kill any queries.",
986 (uchar*) &opt_kill_long_queries_timeout,
987 (uchar*) &opt_kill_long_queries_timeout, 0, GET_UINT,
988 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
989
990 {"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT,
991 "This option specifies time in seconds that innobackupex should wait "
992 "for queries that would block FTWRL before running it. If there are "
993 "still such queries when the timeout expires, innobackupex terminates "
994 "with an error. Default is 0, in which case innobackupex does not "
995 "wait for queries to complete and starts FTWRL immediately.",
996 (uchar*) &opt_lock_wait_timeout,
997 (uchar*) &opt_lock_wait_timeout, 0, GET_UINT,
998 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
999
1000 {"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD,
1001 "This option specifies the query run time threshold which is used by "
1002 "innobackupex to detect long-running queries with a non-zero value "
1003 "of --ftwrl-wait-timeout. FTWRL is not started until such "
1004 "long-running queries exist. This option has no effect if "
1005 "--ftwrl-wait-timeout is 0. Default value is 60 seconds.",
1006 (uchar*) &opt_lock_wait_threshold,
1007 (uchar*) &opt_lock_wait_threshold, 0, GET_UINT,
1008 REQUIRED_ARG, 60, 0, 0, 0, 0, 0},
1009
1010 {"debug-sleep-before-unlock", OPT_DEBUG_SLEEP_BEFORE_UNLOCK,
1011 "This is a debug-only option used by the XtraBackup test suite.",
1012 (uchar*) &opt_debug_sleep_before_unlock,
1013 (uchar*) &opt_debug_sleep_before_unlock, 0, GET_UINT,
1014 REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1015
1016 {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT,
1017 "How many seconds --safe-slave-backup should wait for "
1018 "Slave_open_temp_tables to become zero. (default 300)",
1019 (uchar*) &opt_safe_slave_backup_timeout,
1020 (uchar*) &opt_safe_slave_backup_timeout, 0, GET_UINT,
1021 REQUIRED_ARG, 300, 0, 0, 0, 0, 0},
1022
1023 {"binlog-info", OPT_BINLOG_INFO,
1024 "This option controls how XtraBackup should retrieve server's binary log "
1025 "coordinates corresponding to the backup. Possible values are OFF, ON, "
1026 "LOCKLESS and AUTO. See the XtraBackup manual for more information",
1027 &opt_binlog_info, &opt_binlog_info,
1028 &binlog_info_typelib, GET_ENUM, OPT_ARG, BINLOG_INFO_AUTO, 0, 0, 0, 0, 0},
1029
1030 {"secure-auth", OPT_XB_SECURE_AUTH, "Refuse client connecting to server if it"
1031 " uses old (pre-4.1.1) protocol.", &opt_secure_auth,
1032 &opt_secure_auth, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
1033#define MYSQL_CLIENT
1034#include "sslopt-longopts.h"
1035#undef MYSQL_CLIENT
1036
1037 { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
1038};
1039
1040uint xb_client_options_count = array_elements(xb_client_options);
1041
1042#ifndef DBUG_OFF
1043/** Parameters to DBUG */
1044static const char *dbug_option;
1045#endif
1046
1047struct my_option xb_server_options[] =
1048{
1049 {"datadir", 'h', "Path to the database root.", (G_PTR*) &mysql_data_home,
1050 (G_PTR*) &mysql_data_home, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1051 {"tmpdir", 't',
1052 "Path for temporary files. Several paths may be specified, separated by a "
1053#if defined(__WIN__) || defined(OS2) || defined(__NETWARE__)
1054 "semicolon (;)"
1055#else
1056 "colon (:)"
1057#endif
1058 ", in this case they are used in a round-robin fashion.",
1059 (G_PTR*) &opt_mysql_tmpdir,
1060 (G_PTR*) &opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1061 {"parallel", OPT_XTRA_PARALLEL,
1062 "Number of threads to use for parallel datafiles transfer. "
1063 "The default value is 1.",
1064 (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT,
1065 REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0},
1066
1067 {"log", OPT_LOG, "Ignored option for MySQL option compatibility",
1068 (G_PTR*) &log_ignored_opt, (G_PTR*) &log_ignored_opt, 0,
1069 GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
1070
1071 {"log_bin", OPT_LOG, "Base name for the log sequence",
1072 &opt_log_bin, &opt_log_bin, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
1073
1074 {"innodb", OPT_INNODB, "Ignored option for MySQL option compatibility",
1075 (G_PTR*) &innobase_ignored_opt, (G_PTR*) &innobase_ignored_opt, 0,
1076 GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
1077#ifdef BTR_CUR_HASH_ADAPT
1078 {"innodb_adaptive_hash_index", OPT_INNODB_ADAPTIVE_HASH_INDEX,
1079 "Enable InnoDB adaptive hash index (enabled by default). "
1080 "Disable with --skip-innodb-adaptive-hash-index.",
1081 &btr_search_enabled,
1082 &btr_search_enabled,
1083 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
1084#endif /* BTR_CUR_HASH_ADAPT */
1085 {"innodb_autoextend_increment", OPT_INNODB_AUTOEXTEND_INCREMENT,
1086 "Data file autoextend increment in megabytes",
1087 (G_PTR*) &sys_tablespace_auto_extend_increment,
1088 (G_PTR*) &sys_tablespace_auto_extend_increment,
1089 0, GET_ULONG, REQUIRED_ARG, 8L, 1L, 1000L, 0, 1L, 0},
1090 {"innodb_buffer_pool_size", OPT_INNODB_BUFFER_POOL_SIZE,
1091 "The size of the memory buffer InnoDB uses to cache data and indexes of its tables.",
1092 (G_PTR*) &innobase_buffer_pool_size, (G_PTR*) &innobase_buffer_pool_size, 0,
1093 GET_LL, REQUIRED_ARG, 8*1024*1024L, 1024*1024L, LONGLONG_MAX, 0,
1094 1024*1024L, 0},
1095 {"innodb_data_file_path", OPT_INNODB_DATA_FILE_PATH,
1096 "Path to individual files and their sizes.", &innobase_data_file_path,
1097 &innobase_data_file_path, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1098 {"innodb_data_home_dir", OPT_INNODB_DATA_HOME_DIR,
1099 "The common part for InnoDB table spaces.", &innobase_data_home_dir,
1100 &innobase_data_home_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1101 {"innodb_doublewrite", OPT_INNODB_DOUBLEWRITE,
1102 "Enable InnoDB doublewrite buffer during --prepare.",
1103 (G_PTR*) &innobase_use_doublewrite,
1104 (G_PTR*) &innobase_use_doublewrite, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1105 {"innodb_io_capacity", OPT_INNODB_IO_CAPACITY,
1106 "Number of IOPs the server can do. Tunes the background IO rate",
1107 (G_PTR*) &srv_io_capacity, (G_PTR*) &srv_io_capacity,
1108 0, GET_ULONG, OPT_ARG, 200, 100, ~0UL, 0, 0, 0},
1109 {"innodb_file_io_threads", OPT_INNODB_FILE_IO_THREADS,
1110 "Number of file I/O threads in InnoDB.", (G_PTR*) &innobase_file_io_threads,
1111 (G_PTR*) &innobase_file_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 4, 64, 0,
1112 1, 0},
1113 {"innodb_read_io_threads", OPT_INNODB_READ_IO_THREADS,
1114 "Number of background read I/O threads in InnoDB.", (G_PTR*) &innobase_read_io_threads,
1115 (G_PTR*) &innobase_read_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 1, 64, 0,
1116 1, 0},
1117 {"innodb_write_io_threads", OPT_INNODB_WRITE_IO_THREADS,
1118 "Number of background write I/O threads in InnoDB.", (G_PTR*) &innobase_write_io_threads,
1119 (G_PTR*) &innobase_write_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 1, 64, 0,
1120 1, 0},
1121 {"innodb_file_per_table", OPT_INNODB_FILE_PER_TABLE,
1122 "Stores each InnoDB table to an .ibd file in the database dir.",
1123 (G_PTR*) &innobase_file_per_table,
1124 (G_PTR*) &innobase_file_per_table, 0, GET_BOOL, NO_ARG,
1125 FALSE, 0, 0, 0, 0, 0},
1126
1127 {"innodb_flush_method", OPT_INNODB_FLUSH_METHOD,
1128 "With which method to flush data.",
1129 &srv_file_flush_method, &srv_file_flush_method,
1130 &innodb_flush_method_typelib, GET_ENUM, REQUIRED_ARG,
1131 IF_WIN(SRV_ALL_O_DIRECT_FSYNC, SRV_FSYNC), 0, 0, 0, 0, 0},
1132
1133 {"innodb_log_buffer_size", OPT_INNODB_LOG_BUFFER_SIZE,
1134 "The size of the buffer which InnoDB uses to write log to the log files on disk.",
1135 (G_PTR*) &srv_log_buffer_size, (G_PTR*) &srv_log_buffer_size, 0,
1136 GET_ULONG, REQUIRED_ARG, 1024*1024L, 256*1024L, LONG_MAX, 0, 1024, 0},
1137 {"innodb_log_file_size", OPT_INNODB_LOG_FILE_SIZE,
1138 "Ignored for mysqld option compatibility",
1139 (G_PTR*) &srv_log_file_size, (G_PTR*) &srv_log_file_size, 0,
1140 GET_ULL, REQUIRED_ARG, 48 << 20, 1 << 20, 512ULL << 30, 0,
1141 UNIV_PAGE_SIZE_MAX, 0},
1142 {"innodb_log_files_in_group", OPT_INNODB_LOG_FILES_IN_GROUP,
1143 "Ignored for mysqld option compatibility",
1144 &srv_n_log_files, &srv_n_log_files,
1145 0, GET_LONG, REQUIRED_ARG, 1, 1, 100, 0, 1, 0},
1146 {"innodb_log_group_home_dir", OPT_INNODB_LOG_GROUP_HOME_DIR,
1147 "Path to InnoDB log files.", &srv_log_group_home_dir,
1148 &srv_log_group_home_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1149 {"innodb_max_dirty_pages_pct", OPT_INNODB_MAX_DIRTY_PAGES_PCT,
1150 "Percentage of dirty pages allowed in bufferpool.", (G_PTR*) &srv_max_buf_pool_modified_pct,
1151 (G_PTR*) &srv_max_buf_pool_modified_pct, 0, GET_ULONG, REQUIRED_ARG, 90, 0, 100, 0, 0, 0},
1152 {"innodb_use_native_aio", OPT_INNODB_USE_NATIVE_AIO,
1153 "Use native AIO if supported on this platform.",
1154 (G_PTR*) &srv_use_native_aio,
1155 (G_PTR*) &srv_use_native_aio, 0, GET_BOOL, NO_ARG,
1156 FALSE, 0, 0, 0, 0, 0},
1157 {"innodb_page_size", OPT_INNODB_PAGE_SIZE,
1158 "The universal page size of the database.",
1159 (G_PTR*) &innobase_page_size, (G_PTR*) &innobase_page_size, 0,
1160 /* Use GET_LL to support numeric suffixes in 5.6 */
1161 GET_LL, REQUIRED_ARG,
1162 (1LL << 14), (1LL << 12), (1LL << UNIV_PAGE_SIZE_SHIFT_MAX), 0, 1L, 0},
1163 {"innodb_buffer_pool_filename", OPT_INNODB_BUFFER_POOL_FILENAME,
1164 "Ignored for mysqld option compatibility",
1165 (G_PTR*) &innobase_buffer_pool_filename,
1166 (G_PTR*) &innobase_buffer_pool_filename,
1167 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1168
1169#ifndef DBUG_OFF /* unfortunately "debug" collides with existing options */
1170 {"dbug", '#', "Built in DBUG debugger.",
1171 &dbug_option, &dbug_option, 0, GET_STR, OPT_ARG,
1172 0, 0, 0, 0, 0, 0},
1173#endif
1174#ifndef __WIN__
1175 {"debug-sync", OPT_XTRA_DEBUG_SYNC,
1176 "Debug sync point. This is only used by the xtrabackup test suite",
1177 (G_PTR*) &xtrabackup_debug_sync,
1178 (G_PTR*) &xtrabackup_debug_sync,
1179 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1180#endif
1181
1182 {"innodb_checksum_algorithm", OPT_INNODB_CHECKSUM_ALGORITHM,
1183 "The algorithm InnoDB uses for page checksumming. [CRC32, STRICT_CRC32, "
1184 "INNODB, STRICT_INNODB, NONE, STRICT_NONE]", &srv_checksum_algorithm,
1185 &srv_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM,
1186 REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_INNODB, 0, 0, 0, 0, 0},
1187
1188 {"innodb_undo_directory", OPT_INNODB_UNDO_DIRECTORY,
1189 "Directory where undo tablespace files live, this path can be absolute.",
1190 &srv_undo_dir, &srv_undo_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0,
1191 0},
1192
1193 {"innodb_undo_tablespaces", OPT_INNODB_UNDO_TABLESPACES,
1194 "Number of undo tablespaces to use.",
1195 (G_PTR*)&srv_undo_tablespaces, (G_PTR*)&srv_undo_tablespaces,
1196 0, GET_ULONG, REQUIRED_ARG, 0, 0, 126, 0, 1, 0},
1197
1198 {"defaults_group", OPT_DEFAULTS_GROUP, "defaults group in config file (default \"mysqld\").",
1199 (G_PTR*) &defaults_group, (G_PTR*) &defaults_group,
1200 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1201
1202 {"plugin-dir", OPT_PLUGIN_DIR,
1203 "Server plugin directory. Used to load encryption plugin during 'prepare' phase."
1204 "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)",
1205 &xb_plugin_dir, &xb_plugin_dir,
1206 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
1207
1208 { "plugin-load", OPT_PLUGIN_LOAD, "encrypton plugin to load during 'prepare' phase.",
1209 &xb_plugin_load, &xb_plugin_load,
1210 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
1211
1212 { "innodb-encrypt-log", OPT_INNODB_ENCRYPT_LOG, "encrypton plugin to load",
1213 &srv_encrypt_log, &srv_encrypt_log,
1214 0, GET_BOOL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
1215
1216 {"innodb-log-checksums", OPT_INNODB_LOG_CHECKSUMS,
1217 "Whether to require checksums for InnoDB redo log blocks",
1218 &innodb_log_checksums, &innodb_log_checksums,
1219 0, GET_BOOL, REQUIRED_ARG, 1, 0, 0, 0, 0, 0 },
1220
1221 {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file "
1222 "descriptors to reserve with setrlimit().",
1223 (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG,
1224 REQUIRED_ARG, 0, 0, UINT_MAX, 0, 1, 0},
1225
1226 {"lock-ddl-per-table", OPT_LOCK_DDL_PER_TABLE, "Lock DDL for each table "
1227 "before xtrabackup starts to copy it and until the backup is completed.",
1228 (uchar*) &opt_lock_ddl_per_table, (uchar*) &opt_lock_ddl_per_table, 0,
1229 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1230
1231 { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
1232};
1233
1234uint xb_server_options_count = array_elements(xb_server_options);
1235
1236#ifndef __WIN__
1237static int debug_sync_resumed;
1238
1239static void sigcont_handler(int sig);
1240
1241static void sigcont_handler(int sig __attribute__((unused)))
1242{
1243 debug_sync_resumed= 1;
1244}
1245#endif
1246
1247static inline
1248void
1249debug_sync_point(const char *name)
1250{
1251#ifndef __WIN__
1252 FILE *fp;
1253 pid_t pid;
1254 char pid_path[FN_REFLEN];
1255
1256 if (xtrabackup_debug_sync == NULL) {
1257 return;
1258 }
1259
1260 if (strcmp(xtrabackup_debug_sync, name)) {
1261 return;
1262 }
1263
1264 pid = getpid();
1265
1266 snprintf(pid_path, sizeof(pid_path), "%s/xtrabackup_debug_sync",
1267 xtrabackup_target_dir);
1268 fp = fopen(pid_path, "w");
1269 if (fp == NULL) {
1270 msg("mariabackup: Error: cannot open %s\n", pid_path);
1271 exit(EXIT_FAILURE);
1272 }
1273 fprintf(fp, "%u\n", (uint) pid);
1274 fclose(fp);
1275
1276 msg("mariabackup: DEBUG: Suspending at debug sync point '%s'. "
1277 "Resume with 'kill -SIGCONT %u'.\n", name, (uint) pid);
1278
1279 debug_sync_resumed= 0;
1280 kill(pid, SIGSTOP);
1281 while (!debug_sync_resumed) {
1282 sleep(1);
1283 }
1284
1285 /* On resume */
1286 msg("mariabackup: DEBUG: removing the pid file.\n");
1287 my_delete(pid_path, MYF(MY_WME));
1288#endif
1289}
1290
1291
1292static std::vector<std::string> tables_for_export;
1293
1294static void append_export_table(const char *dbname, const char *tablename, bool is_remote)
1295{
1296 if(dbname && tablename && !is_remote)
1297 {
1298 char buf[3*FN_REFLEN];
1299 snprintf(buf,sizeof(buf),"%s/%s",dbname, tablename);
1300 // trim .ibd
1301 char *p=strrchr(buf, '.');
1302 if (p) *p=0;
1303
1304 tables_for_export.push_back(ut_get_name(0,buf));
1305 }
1306}
1307
1308
1309#define BOOTSTRAP_FILENAME "mariabackup_prepare_for_export.sql"
1310
1311static int create_bootstrap_file()
1312{
1313 FILE *f= fopen(BOOTSTRAP_FILENAME,"wb");
1314 if(!f)
1315 return -1;
1316
1317 fputs("SET NAMES UTF8;\n",f);
1318 enumerate_ibd_files(append_export_table);
1319 for (size_t i= 0; i < tables_for_export.size(); i++)
1320 {
1321 const char *tab = tables_for_export[i].c_str();
1322 fprintf(f,
1323 "BEGIN NOT ATOMIC "
1324 "DECLARE CONTINUE HANDLER FOR NOT FOUND,SQLEXCEPTION BEGIN END;"
1325 "FLUSH TABLES %s FOR EXPORT;"
1326 "END;\n"
1327 "UNLOCK TABLES;\n",
1328 tab);
1329 }
1330 fclose(f);
1331 return 0;
1332}
1333
1334static int prepare_export()
1335{
1336 int err= -1;
1337
1338 char cmdline[2*FN_REFLEN];
1339 FILE *outf;
1340
1341 if (create_bootstrap_file())
1342 return -1;
1343
1344 // Process defaults-file , it can have some --lc-language stuff,
1345 // which is* unfortunately* still necessary to get mysqld up
1346 if (strncmp(orig_argv1,"--defaults-file=",16) == 0)
1347 {
1348 sprintf(cmdline,
1349 IF_WIN("\"","") "\"%s\" --mysqld \"%s\" "
1350 " --defaults-extra-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
1351 " --innodb --innodb-fast-shutdown=0"
1352 " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu"
1353 " --console --skip-log-error --bootstrap < " BOOTSTRAP_FILENAME IF_WIN("\"",""),
1354 mariabackup_exe,
1355 orig_argv1, (my_defaults_group_suffix?my_defaults_group_suffix:""),
1356 xtrabackup_use_memory);
1357 }
1358 else
1359 {
1360 sprintf(cmdline,
1361 IF_WIN("\"","") "\"%s\" --mysqld"
1362 " --defaults-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
1363 " --innodb --innodb-fast-shutdown=0"
1364 " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu"
1365 " --console --log-error= --bootstrap < " BOOTSTRAP_FILENAME IF_WIN("\"",""),
1366 mariabackup_exe,
1367 (my_defaults_group_suffix?my_defaults_group_suffix:""),
1368 xtrabackup_use_memory);
1369 }
1370
1371 msg("Prepare export : executing %s\n", cmdline);
1372 fflush(stderr);
1373
1374 outf= popen(cmdline,"r");
1375 if (!outf)
1376 goto end;
1377
1378 char outline[FN_REFLEN];
1379 while(fgets(outline, sizeof(outline)-1, outf))
1380 fprintf(stderr,"%s",outline);
1381
1382 err = pclose(outf);
1383end:
1384 unlink(BOOTSTRAP_FILENAME);
1385 return err;
1386}
1387
1388
1389static const char *xb_client_default_groups[]=
1390 { "xtrabackup", "mariabackup", "client", 0, 0, 0 };
1391
1392static const char *xb_server_default_groups[]=
1393 { "xtrabackup", "mariabackup", "mysqld", 0, 0, 0 };
1394
1395static void print_version(void)
1396{
1397 msg("%s based on MariaDB server %s %s (%s) \n",
1398 my_progname, MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE);
1399}
1400
1401static void usage(void)
1402{
1403 puts("Open source backup tool for InnoDB and XtraDB\n\
1404\n\
1405Copyright (C) 2009-2015 Percona LLC and/or its affiliates.\n\
1406Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.\n\
1407\n\
1408This program is free software; you can redistribute it and/or\n\
1409modify it under the terms of the GNU General Public License\n\
1410as published by the Free Software Foundation version 2\n\
1411of the License.\n\
1412\n\
1413This program is distributed in the hope that it will be useful,\n\
1414but WITHOUT ANY WARRANTY; without even the implied warranty of\n\
1415MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n\
1416GNU General Public License for more details.\n\
1417\n\
1418You can download full text of the license on http://www.gnu.org/licenses/gpl-2.0.txt\n");
1419
1420 printf("Usage: %s [--defaults-file=#] [--backup | --prepare | --copy-back | --move-back] [OPTIONS]\n",my_progname);
1421 print_defaults("my", xb_server_default_groups);
1422 my_print_help(xb_client_options);
1423 my_print_help(xb_server_options);
1424 my_print_variables(xb_server_options);
1425 my_print_variables(xb_client_options);
1426}
1427
1428#define ADD_PRINT_PARAM_OPT(value) \
1429 { \
1430 print_param_str << opt->name << "=" << value << "\n"; \
1431 param_set.insert(opt->name); \
1432 }
1433
1434/************************************************************************
1435Check if parameter is set in defaults file or via command line argument
1436@return true if parameter is set. */
1437bool
1438check_if_param_set(const char *param)
1439{
1440 return param_set.find(param) != param_set.end();
1441}
1442
1443my_bool
1444xb_get_one_option(int optid,
1445 const struct my_option *opt __attribute__((unused)),
1446 char *argument)
1447{
1448 switch(optid) {
1449 case 'h':
1450 strmake(mysql_real_data_home,argument, FN_REFLEN - 1);
1451 mysql_data_home= mysql_real_data_home;
1452
1453 ADD_PRINT_PARAM_OPT(mysql_real_data_home);
1454 break;
1455
1456 case 't':
1457
1458 ADD_PRINT_PARAM_OPT(opt_mysql_tmpdir);
1459 break;
1460
1461 case OPT_INNODB_DATA_HOME_DIR:
1462
1463 ADD_PRINT_PARAM_OPT(innobase_data_home_dir);
1464 break;
1465
1466 case OPT_INNODB_DATA_FILE_PATH:
1467
1468 ADD_PRINT_PARAM_OPT(innobase_data_file_path);
1469 break;
1470
1471 case OPT_INNODB_LOG_GROUP_HOME_DIR:
1472
1473 ADD_PRINT_PARAM_OPT(srv_log_group_home_dir);
1474 break;
1475
1476 case OPT_INNODB_LOG_FILES_IN_GROUP:
1477 case OPT_INNODB_LOG_FILE_SIZE:
1478 break;
1479
1480 case OPT_INNODB_FLUSH_METHOD:
1481 ut_a(srv_file_flush_method
1482 <= IF_WIN(SRV_ALL_O_DIRECT_FSYNC, SRV_O_DIRECT_NO_FSYNC));
1483 ADD_PRINT_PARAM_OPT(innodb_flush_method_names[srv_file_flush_method]);
1484 break;
1485
1486 case OPT_INNODB_PAGE_SIZE:
1487
1488 ADD_PRINT_PARAM_OPT(innobase_page_size);
1489 break;
1490
1491 case OPT_INNODB_UNDO_DIRECTORY:
1492
1493 ADD_PRINT_PARAM_OPT(srv_undo_dir);
1494 break;
1495
1496 case OPT_INNODB_UNDO_TABLESPACES:
1497
1498 ADD_PRINT_PARAM_OPT(srv_undo_tablespaces);
1499 break;
1500
1501 case OPT_INNODB_CHECKSUM_ALGORITHM:
1502
1503 ut_a(srv_checksum_algorithm <= SRV_CHECKSUM_ALGORITHM_STRICT_NONE);
1504
1505 ADD_PRINT_PARAM_OPT(innodb_checksum_algorithm_names[srv_checksum_algorithm]);
1506 break;
1507
1508 case OPT_INNODB_BUFFER_POOL_FILENAME:
1509
1510 ADD_PRINT_PARAM_OPT(innobase_buffer_pool_filename);
1511 break;
1512
1513 case OPT_XTRA_TARGET_DIR:
1514 strmake(xtrabackup_real_target_dir,argument, sizeof(xtrabackup_real_target_dir)-1);
1515 xtrabackup_target_dir= xtrabackup_real_target_dir;
1516 break;
1517 case OPT_XTRA_STREAM:
1518 if (!strcasecmp(argument, "xbstream"))
1519 xtrabackup_stream_fmt = XB_STREAM_FMT_XBSTREAM;
1520 else
1521 {
1522 msg("Invalid --stream argument: %s\n", argument);
1523 return 1;
1524 }
1525 xtrabackup_stream = TRUE;
1526 break;
1527 case OPT_XTRA_COMPRESS:
1528 if (argument == NULL)
1529 xtrabackup_compress_alg = "quicklz";
1530 else if (strcasecmp(argument, "quicklz"))
1531 {
1532 msg("Invalid --compress argument: %s\n", argument);
1533 return 1;
1534 }
1535 xtrabackup_compress = TRUE;
1536 break;
1537 case OPT_DECOMPRESS:
1538 opt_decompress = TRUE;
1539 xtrabackup_decrypt_decompress = true;
1540 break;
1541 case (int) OPT_CORE_FILE:
1542 test_flags |= TEST_CORE_ON_SIGNAL;
1543 break;
1544 case OPT_HISTORY:
1545 if (argument) {
1546 opt_history = argument;
1547 } else {
1548 opt_history = "";
1549 }
1550 break;
1551 case 'p':
1552 if (argument)
1553 {
1554 char *start= argument;
1555 my_free(opt_password);
1556 opt_password= my_strdup(argument, MYF(MY_FAE));
1557 while (*argument) *argument++= 'x'; // Destroy argument
1558 if (*start)
1559 start[1]=0 ;
1560 }
1561 break;
1562 case OPT_PROTOCOL:
1563 if (argument)
1564 {
1565 if ((opt_protocol= find_type_with_warning(argument, &sql_protocol_typelib,
1566 opt->name)) <= 0)
1567 {
1568 sf_leaking_memory= 1; /* no memory leak reports here */
1569 exit(1);
1570 }
1571 }
1572 break;
1573#define MYSQL_CLIENT
1574#include "sslopt-case.h"
1575#undef MYSQL_CLIENT
1576
1577 case '?':
1578 usage();
1579 exit(EXIT_SUCCESS);
1580 break;
1581 case 'v':
1582 print_version();
1583 exit(EXIT_SUCCESS);
1584 break;
1585 default:
1586 break;
1587 }
1588 return 0;
1589}
1590
1591static bool innodb_init_param()
1592{
1593 srv_is_being_started = TRUE;
1594 /* === some variables from mysqld === */
1595 memset((G_PTR) &mysql_tmpdir_list, 0, sizeof(mysql_tmpdir_list));
1596
1597 if (init_tmpdir(&mysql_tmpdir_list, opt_mysql_tmpdir))
1598 return true;
1599 xtrabackup_tmpdir = my_tmpdir(&mysql_tmpdir_list);
1600 /* dummy for initialize all_charsets[] */
1601 get_charset_name(0);
1602
1603 srv_page_size = 0;
1604 srv_page_size_shift = 0;
1605#ifdef BTR_CUR_HASH_ADAPT
1606 btr_ahi_parts = 1;
1607#endif /* BTR_CUR_HASH_ADAPT */
1608
1609 if (innobase_page_size != (1LL << 14)) {
1610 size_t n_shift = get_bit_shift(size_t(innobase_page_size));
1611
1612 if (n_shift >= 12 && n_shift <= UNIV_PAGE_SIZE_SHIFT_MAX) {
1613 srv_page_size_shift = ulong(n_shift);
1614 srv_page_size = 1U << n_shift;
1615 msg("InnoDB: The universal page size of the "
1616 "database is set to %lu.\n", srv_page_size);
1617 } else {
1618 msg("InnoDB: Error: invalid value of "
1619 "innobase_page_size: %lld", innobase_page_size);
1620 goto error;
1621 }
1622 } else {
1623 srv_page_size_shift = 14;
1624 srv_page_size = 1U << 14;
1625 }
1626
1627 /* Check that values don't overflow on 32-bit systems. */
1628 if (sizeof(ulint) == 4) {
1629 if (xtrabackup_use_memory > UINT_MAX32) {
1630 msg("mariabackup: use-memory can't be over 4GB"
1631 " on 32-bit systems\n");
1632 }
1633
1634 if (innobase_buffer_pool_size > UINT_MAX32) {
1635 msg("mariabackup: innobase_buffer_pool_size can't be "
1636 "over 4GB on 32-bit systems\n");
1637
1638 goto error;
1639 }
1640 }
1641
1642 static char default_path[2] = { FN_CURLIB, 0 };
1643 fil_path_to_mysql_datadir = default_path;
1644
1645 /* Set InnoDB initialization parameters according to the values
1646 read from MySQL .cnf file */
1647
1648 if (xtrabackup_backup) {
1649 msg("mariabackup: using the following InnoDB configuration:\n");
1650 } else {
1651 msg("mariabackup: using the following InnoDB configuration "
1652 "for recovery:\n");
1653 }
1654
1655 /*--------------- Data files -------------------------*/
1656
1657 /* The default dir for data files is the datadir of MySQL */
1658
1659 srv_data_home = (xtrabackup_backup && innobase_data_home_dir
1660 ? innobase_data_home_dir : default_path);
1661 msg("mariabackup: innodb_data_home_dir = %s\n", srv_data_home);
1662
1663 /* Set default InnoDB data file size to 10 MB and let it be
1664 auto-extending. Thus users can use InnoDB in >= 4.0 without having
1665 to specify any startup options. */
1666
1667 if (!innobase_data_file_path) {
1668 innobase_data_file_path = (char*) "ibdata1:10M:autoextend";
1669 }
1670 msg("mariabackup: innodb_data_file_path = %s\n",
1671 innobase_data_file_path);
1672
1673 /* This is the first time univ_page_size is used.
1674 It was initialized to 16k pages before srv_page_size was set */
1675 univ_page_size.copy_from(
1676 page_size_t(srv_page_size, srv_page_size, false));
1677
1678 srv_sys_space.set_space_id(TRX_SYS_SPACE);
1679 srv_sys_space.set_name("innodb_system");
1680 srv_sys_space.set_path(srv_data_home);
1681 srv_sys_space.set_flags(FSP_FLAGS_PAGE_SSIZE());
1682
1683 if (!srv_sys_space.parse_params(innobase_data_file_path, true)) {
1684 goto error;
1685 }
1686
1687 srv_sys_space.normalize_size();
1688 srv_lock_table_size = 5 * (srv_buf_pool_size >> srv_page_size_shift);
1689
1690 /* -------------- Log files ---------------------------*/
1691
1692 /* The default dir for log files is the datadir of MySQL */
1693
1694 if (!(xtrabackup_backup && srv_log_group_home_dir)) {
1695 srv_log_group_home_dir = default_path;
1696 }
1697 if (xtrabackup_prepare && xtrabackup_incremental_dir) {
1698 srv_log_group_home_dir = xtrabackup_incremental_dir;
1699 }
1700 msg("mariabackup: innodb_log_group_home_dir = %s\n",
1701 srv_log_group_home_dir);
1702
1703 os_normalize_path(srv_log_group_home_dir);
1704
1705 if (strchr(srv_log_group_home_dir, ';')) {
1706
1707 msg("syntax error in innodb_log_group_home_dir, ");
1708 goto error;
1709 }
1710
1711 srv_adaptive_flushing = FALSE;
1712
1713 /* We set srv_pool_size here in units of 1 kB. InnoDB internally
1714 changes the value so that it becomes the number of database pages. */
1715
1716 srv_buf_pool_size = (ulint) xtrabackup_use_memory;
1717 srv_buf_pool_chunk_unit = (ulong)srv_buf_pool_size;
1718 srv_buf_pool_instances = 1;
1719 srv_n_page_cleaners = 1;
1720
1721 srv_n_file_io_threads = (ulint) innobase_file_io_threads;
1722 srv_n_read_io_threads = (ulint) innobase_read_io_threads;
1723 srv_n_write_io_threads = (ulint) innobase_write_io_threads;
1724
1725 srv_use_doublewrite_buf = (ibool) innobase_use_doublewrite;
1726
1727 os_use_large_pages = (ibool) innobase_use_large_pages;
1728 os_large_page_size = (ulint) innobase_large_page_size;
1729 row_rollback_on_timeout = (ibool) innobase_rollback_on_timeout;
1730
1731 srv_file_per_table = (my_bool) innobase_file_per_table;
1732
1733 srv_locks_unsafe_for_binlog = (ibool) innobase_locks_unsafe_for_binlog;
1734
1735 srv_max_n_open_files = ULINT_UNDEFINED - 5;
1736 srv_innodb_status = (ibool) innobase_create_status_file;
1737
1738 srv_print_verbose_log = 1;
1739
1740 /* Store the default charset-collation number of this MySQL
1741 installation */
1742
1743 /* We cannot treat characterset here for now!! */
1744 data_mysql_default_charset_coll = (ulint)default_charset_info->number;
1745
1746 ut_ad(DATA_MYSQL_BINARY_CHARSET_COLL == my_charset_bin.number);
1747
1748#ifdef _WIN32
1749 srv_use_native_aio = TRUE;
1750
1751#elif defined(LINUX_NATIVE_AIO)
1752
1753 if (srv_use_native_aio) {
1754 ut_print_timestamp(stderr);
1755 msg(" InnoDB: Using Linux native AIO\n");
1756 }
1757#else
1758 /* Currently native AIO is supported only on windows and linux
1759 and that also when the support is compiled in. In all other
1760 cases, we ignore the setting of innodb_use_native_aio. */
1761 srv_use_native_aio = FALSE;
1762
1763#endif
1764
1765 /* Assign the default value to srv_undo_dir if it's not specified, as
1766 my_getopt does not support default values for string options. We also
1767 ignore the option and override innodb_undo_directory on --prepare,
1768 because separate undo tablespaces are copied to the root backup
1769 directory. */
1770
1771 if (!srv_undo_dir || !xtrabackup_backup) {
1772 srv_undo_dir = (char*) ".";
1773 }
1774
1775 log_checksum_algorithm_ptr = innodb_log_checksums || srv_encrypt_log
1776 ? log_block_calc_checksum_crc32
1777 : log_block_calc_checksum_none;
1778
1779#ifdef _WIN32
1780 srv_use_native_aio = TRUE;
1781#endif
1782 return false;
1783
1784error:
1785 msg("mariabackup: innodb_init_param(): Error occured.\n");
1786 return true;
1787}
1788
1789static bool innodb_init()
1790{
1791 bool create_new_db = false;
1792 /* Check if the data files exist or not. */
1793 dberr_t err = srv_sys_space.check_file_spec(&create_new_db, 5U << 20);
1794
1795 if (err == DB_SUCCESS) {
1796 err = srv_start(create_new_db);
1797 }
1798
1799 if (err != DB_SUCCESS) {
1800 msg("mariabackup: innodb_init() returned %d (%s).\n",
1801 err, ut_strerr(err));
1802 innodb_shutdown();
1803 return(TRUE);
1804 }
1805
1806 return(FALSE);
1807}
1808
1809/* ================= common ================= */
1810
1811/***********************************************************************
1812Read backup meta info.
1813@return TRUE on success, FALSE on failure. */
1814static
1815my_bool
1816xtrabackup_read_metadata(char *filename)
1817{
1818 FILE *fp;
1819 my_bool r = TRUE;
1820 int t;
1821
1822 fp = fopen(filename,"r");
1823 if(!fp) {
1824 msg("mariabackup: Error: cannot open %s\n", filename);
1825 return(FALSE);
1826 }
1827
1828 if (fscanf(fp, "backup_type = %29s\n", metadata_type)
1829 != 1) {
1830 r = FALSE;
1831 goto end;
1832 }
1833 /* Use UINT64PF instead of LSN_PF here, as we have to maintain the file
1834 format. */
1835 if (fscanf(fp, "from_lsn = " UINT64PF "\n", &metadata_from_lsn)
1836 != 1) {
1837 r = FALSE;
1838 goto end;
1839 }
1840 if (fscanf(fp, "to_lsn = " UINT64PF "\n", &metadata_to_lsn)
1841 != 1) {
1842 r = FALSE;
1843 goto end;
1844 }
1845 if (fscanf(fp, "last_lsn = " UINT64PF "\n", &metadata_last_lsn)
1846 != 1) {
1847 metadata_last_lsn = 0;
1848 }
1849 /* Optional fields */
1850
1851 if (fscanf(fp, "recover_binlog_info = %d\n", &t) == 1) {
1852 recover_binlog_info = (t == 1);
1853 }
1854end:
1855 fclose(fp);
1856
1857 return(r);
1858}
1859
1860/***********************************************************************
1861Print backup meta info to a specified buffer. */
1862static
1863void
1864xtrabackup_print_metadata(char *buf, size_t buf_len)
1865{
1866 /* Use UINT64PF instead of LSN_PF here, as we have to maintain the file
1867 format. */
1868 snprintf(buf, buf_len,
1869 "backup_type = %s\n"
1870 "from_lsn = " UINT64PF "\n"
1871 "to_lsn = " UINT64PF "\n"
1872 "last_lsn = " UINT64PF "\n"
1873 "recover_binlog_info = %d\n",
1874 metadata_type,
1875 metadata_from_lsn,
1876 metadata_to_lsn,
1877 metadata_last_lsn,
1878 MY_TEST(opt_binlog_info == BINLOG_INFO_LOCKLESS));
1879}
1880
1881/***********************************************************************
1882Stream backup meta info to a specified datasink.
1883@return TRUE on success, FALSE on failure. */
1884static
1885my_bool
1886xtrabackup_stream_metadata(ds_ctxt_t *ds_ctxt)
1887{
1888 char buf[1024];
1889 size_t len;
1890 ds_file_t *stream;
1891 MY_STAT mystat;
1892 my_bool rc = TRUE;
1893
1894 xtrabackup_print_metadata(buf, sizeof(buf));
1895
1896 len = strlen(buf);
1897
1898 mystat.st_size = len;
1899 mystat.st_mtime = my_time(0);
1900
1901 stream = ds_open(ds_ctxt, XTRABACKUP_METADATA_FILENAME, &mystat);
1902 if (stream == NULL) {
1903 msg("mariabackup: Error: cannot open output stream "
1904 "for %s\n", XTRABACKUP_METADATA_FILENAME);
1905 return(FALSE);
1906 }
1907
1908 if (ds_write(stream, buf, len)) {
1909 rc = FALSE;
1910 }
1911
1912 if (ds_close(stream)) {
1913 rc = FALSE;
1914 }
1915
1916 return(rc);
1917}
1918
1919/***********************************************************************
1920Write backup meta info to a specified file.
1921@return TRUE on success, FALSE on failure. */
1922static
1923my_bool
1924xtrabackup_write_metadata(const char *filepath)
1925{
1926 char buf[1024];
1927 size_t len;
1928 FILE *fp;
1929
1930 xtrabackup_print_metadata(buf, sizeof(buf));
1931
1932 len = strlen(buf);
1933
1934 fp = fopen(filepath, "w");
1935 if(!fp) {
1936 msg("mariabackup: Error: cannot open %s\n", filepath);
1937 return(FALSE);
1938 }
1939 if (fwrite(buf, len, 1, fp) < 1) {
1940 fclose(fp);
1941 return(FALSE);
1942 }
1943
1944 fclose(fp);
1945
1946 return(TRUE);
1947}
1948
1949/***********************************************************************
1950Read meta info for an incremental delta.
1951@return TRUE on success, FALSE on failure. */
1952static my_bool
1953xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info)
1954{
1955 FILE* fp;
1956 char key[51];
1957 char value[51];
1958 my_bool r = TRUE;
1959
1960 /* set defaults */
1961 ulint page_size = ULINT_UNDEFINED, zip_size = 0;
1962 info->space_id = ULINT_UNDEFINED;
1963
1964 fp = fopen(filepath, "r");
1965 if (!fp) {
1966 /* Meta files for incremental deltas are optional */
1967 return(TRUE);
1968 }
1969
1970 while (!feof(fp)) {
1971 if (fscanf(fp, "%50s = %50s\n", key, value) == 2) {
1972 if (strcmp(key, "page_size") == 0) {
1973 page_size = strtoul(value, NULL, 10);
1974 } else if (strcmp(key, "zip_size") == 0) {
1975 zip_size = strtoul(value, NULL, 10);
1976 } else if (strcmp(key, "space_id") == 0) {
1977 info->space_id = strtoul(value, NULL, 10);
1978 }
1979 }
1980 }
1981
1982 fclose(fp);
1983
1984 if (page_size == ULINT_UNDEFINED) {
1985 msg("mariabackup: page_size is required in %s\n", filepath);
1986 r = FALSE;
1987 } else {
1988 info->page_size = page_size_t(zip_size ? zip_size : page_size,
1989 page_size, zip_size != 0);
1990 }
1991
1992 if (info->space_id == ULINT_UNDEFINED) {
1993 msg("mariabackup: Warning: This backup was taken with XtraBackup 2.0.1 "
1994 "or earlier, some DDL operations between full and incremental "
1995 "backups may be handled incorrectly\n");
1996 }
1997
1998 return(r);
1999}
2000
2001/***********************************************************************
2002Write meta info for an incremental delta.
2003@return TRUE on success, FALSE on failure. */
2004my_bool
2005xb_write_delta_metadata(const char *filename, const xb_delta_info_t *info)
2006{
2007 ds_file_t *f;
2008 char buf[64];
2009 my_bool ret;
2010 size_t len;
2011 MY_STAT mystat;
2012
2013 snprintf(buf, sizeof(buf),
2014 "page_size = " ULINTPF "\n"
2015 "zip_size = " ULINTPF " \n"
2016 "space_id = " ULINTPF "\n",
2017 info->page_size.logical(),
2018 info->page_size.is_compressed()
2019 ? info->page_size.physical() : 0,
2020 info->space_id);
2021 len = strlen(buf);
2022
2023 mystat.st_size = len;
2024 mystat.st_mtime = my_time(0);
2025
2026 f = ds_open(ds_meta, filename, &mystat);
2027 if (f == NULL) {
2028 msg("mariabackup: Error: cannot open output stream for %s\n",
2029 filename);
2030 return(FALSE);
2031 }
2032
2033 ret = (ds_write(f, buf, len) == 0);
2034
2035 if (ds_close(f)) {
2036 ret = FALSE;
2037 }
2038
2039 return(ret);
2040}
2041
2042/* ================= backup ================= */
2043void
2044xtrabackup_io_throttling(void)
2045{
2046 if (xtrabackup_backup && xtrabackup_throttle && (io_ticket--) < 0) {
2047 os_event_reset(wait_throttle);
2048 os_event_wait(wait_throttle);
2049 }
2050}
2051
2052static
2053my_bool regex_list_check_match(
2054 const regex_list_t& list,
2055 const char* name)
2056{
2057 regmatch_t tables_regmatch[1];
2058 for (regex_list_t::const_iterator i = list.begin(), end = list.end();
2059 i != end; ++i) {
2060 const regex_t& regex = *i;
2061 int regres = regexec(&regex, name, 1, tables_regmatch, 0);
2062
2063 if (regres != REG_NOMATCH) {
2064 return(TRUE);
2065 }
2066 }
2067 return(FALSE);
2068}
2069
2070static
2071my_bool
2072find_filter_in_hashtable(
2073 const char* name,
2074 hash_table_t* table,
2075 xb_filter_entry_t** result
2076)
2077{
2078 xb_filter_entry_t* found = NULL;
2079 HASH_SEARCH(name_hash, table, ut_fold_string(name),
2080 xb_filter_entry_t*,
2081 found, (void) 0,
2082 !strcmp(found->name, name));
2083
2084 if (found && result) {
2085 *result = found;
2086 }
2087 return (found != NULL);
2088}
2089
2090/************************************************************************
2091Checks if a given table name matches any of specifications given in
2092regex_list or tables_hash.
2093
2094@return TRUE on match or both regex_list and tables_hash are empty.*/
2095static my_bool
2096check_if_table_matches_filters(const char *name,
2097 const regex_list_t& regex_list,
2098 hash_table_t* tables_hash)
2099{
2100 if (regex_list.empty() && !tables_hash) {
2101 return(FALSE);
2102 }
2103
2104 if (regex_list_check_match(regex_list, name)) {
2105 return(TRUE);
2106 }
2107
2108 if (tables_hash && find_filter_in_hashtable(name, tables_hash, NULL)) {
2109 return(TRUE);
2110 }
2111
2112 return FALSE;
2113}
2114
2115enum skip_database_check_result {
2116 DATABASE_SKIP,
2117 DATABASE_SKIP_SOME_TABLES,
2118 DATABASE_DONT_SKIP,
2119 DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED,
2120};
2121
2122/************************************************************************
2123Checks if a database specified by name should be skipped from backup based on
2124the --databases, --databases_file or --databases_exclude options.
2125
2126@return TRUE if entire database should be skipped,
2127 FALSE otherwise.
2128*/
2129static
2130skip_database_check_result
2131check_if_skip_database(
2132 const char* name /*!< in: path to the database */
2133)
2134{
2135 /* There are some filters for databases, check them */
2136 xb_filter_entry_t* database = NULL;
2137
2138 if (databases_exclude_hash &&
2139 find_filter_in_hashtable(name, databases_exclude_hash,
2140 &database) &&
2141 !database->has_tables) {
2142 /* Database is found and there are no tables specified,
2143 skip entire db. */
2144 return DATABASE_SKIP;
2145 }
2146
2147 if (databases_include_hash) {
2148 if (!find_filter_in_hashtable(name, databases_include_hash,
2149 &database)) {
2150 /* Database isn't found, skip the database */
2151 return DATABASE_SKIP;
2152 } else if (database->has_tables) {
2153 return DATABASE_SKIP_SOME_TABLES;
2154 } else {
2155 return DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED;
2156 }
2157 }
2158
2159 return DATABASE_DONT_SKIP;
2160}
2161
2162/************************************************************************
2163Checks if a database specified by path should be skipped from backup based on
2164the --databases, --databases_file or --databases_exclude options.
2165
2166@return TRUE if the table should be skipped. */
2167my_bool
2168check_if_skip_database_by_path(
2169 const char* path /*!< in: path to the db directory. */
2170)
2171{
2172 if (databases_include_hash == NULL &&
2173 databases_exclude_hash == NULL) {
2174 return(FALSE);
2175 }
2176
2177 const char* db_name = strrchr(path, OS_PATH_SEPARATOR);
2178 if (db_name == NULL) {
2179 db_name = path;
2180 } else {
2181 ++db_name;
2182 }
2183
2184 return check_if_skip_database(db_name) == DATABASE_SKIP;
2185}
2186
2187/************************************************************************
2188Checks if a table specified as a name in the form "database/name" (InnoDB 5.6)
2189or "./database/name.ibd" (InnoDB 5.5-) should be skipped from backup based on
2190the --tables or --tables-file options.
2191
2192@return TRUE if the table should be skipped. */
2193my_bool
2194check_if_skip_table(
2195/******************/
2196 const char* name) /*!< in: path to the table */
2197{
2198 char buf[FN_REFLEN];
2199 const char *dbname, *tbname;
2200 const char *ptr;
2201 char *eptr;
2202
2203 if (regex_exclude_list.empty() &&
2204 regex_include_list.empty() &&
2205 tables_include_hash == NULL &&
2206 tables_exclude_hash == NULL &&
2207 databases_include_hash == NULL &&
2208 databases_exclude_hash == NULL) {
2209 return(FALSE);
2210 }
2211
2212 dbname = NULL;
2213 tbname = name;
2214 while ((ptr = strchr(tbname, '/')) != NULL) {
2215 dbname = tbname;
2216 tbname = ptr + 1;
2217 }
2218
2219 if (dbname == NULL) {
2220 return(FALSE);
2221 }
2222
2223 strncpy(buf, dbname, FN_REFLEN);
2224 buf[tbname - 1 - dbname] = 0;
2225
2226 const skip_database_check_result skip_database =
2227 check_if_skip_database(buf);
2228 if (skip_database == DATABASE_SKIP) {
2229 return (TRUE);
2230 }
2231
2232 buf[FN_REFLEN - 1] = '\0';
2233 buf[tbname - 1 - dbname] = '.';
2234
2235 /* Check if there's a suffix in the table name. If so, truncate it. We
2236 rely on the fact that a dot cannot be a part of a table name (it is
2237 encoded by the server with the @NNNN syntax). */
2238 if ((eptr = strchr(&buf[tbname - dbname], '.')) != NULL) {
2239
2240 *eptr = '\0';
2241 }
2242
2243 /* For partitioned tables first try to match against the regexp
2244 without truncating the #P#... suffix so we can backup individual
2245 partitions with regexps like '^test[.]t#P#p5' */
2246 if (check_if_table_matches_filters(buf, regex_exclude_list,
2247 tables_exclude_hash)) {
2248 return(TRUE);
2249 }
2250 if (check_if_table_matches_filters(buf, regex_include_list,
2251 tables_include_hash)) {
2252 return(FALSE);
2253 }
2254 if ((eptr = strstr(buf, "#P#")) != NULL) {
2255 *eptr = 0;
2256
2257 if (check_if_table_matches_filters(buf, regex_exclude_list,
2258 tables_exclude_hash)) {
2259 return (TRUE);
2260 }
2261 if (check_if_table_matches_filters(buf, regex_include_list,
2262 tables_include_hash)) {
2263 return(FALSE);
2264 }
2265 }
2266
2267 if (skip_database == DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED) {
2268 /* Database is in include-list, and qualified name wasn't
2269 found in any of exclusion filters.*/
2270 return (FALSE);
2271 }
2272
2273 if (skip_database == DATABASE_SKIP_SOME_TABLES ||
2274 !regex_include_list.empty() ||
2275 tables_include_hash) {
2276
2277 /* Include lists are present, but qualified name
2278 failed to match any.*/
2279 return(TRUE);
2280 }
2281
2282 return(FALSE);
2283}
2284
2285const char*
2286xb_get_copy_action(const char *dflt)
2287{
2288 const char *action;
2289
2290 if (xtrabackup_stream) {
2291 if (xtrabackup_compress) {
2292 action = "Compressing and streaming";
2293 } else {
2294 action = "Streaming";
2295 }
2296 } else {
2297 if (xtrabackup_compress) {
2298 action = "Compressing";
2299 } else {
2300 action = dflt;
2301 }
2302 }
2303
2304 return(action);
2305}
2306
2307/* TODO: We may tune the behavior (e.g. by fil_aio)*/
2308
2309static
2310my_bool
2311xtrabackup_copy_datafile(fil_node_t* node, uint thread_n)
2312{
2313 char dst_name[FN_REFLEN];
2314 ds_file_t *dstfile = NULL;
2315 xb_fil_cur_t cursor;
2316 xb_fil_cur_result_t res;
2317 xb_write_filt_t *write_filter = NULL;
2318 xb_write_filt_ctxt_t write_filt_ctxt;
2319 const char *action;
2320 xb_read_filt_t *read_filter;
2321 my_bool rc = FALSE;
2322
2323 /* Get the name and the path for the tablespace. node->name always
2324 contains the path (which may be absolute for remote tablespaces in
2325 5.6+). space->name contains the tablespace name in the form
2326 "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a
2327 multi-node shared tablespace, space->name contains the name of the first
2328 node, but that's irrelevant, since we only need node_name to match them
2329 against filters, and the shared tablespace is always copied regardless
2330 of the filters value. */
2331
2332 const char* const node_name = node->space->name;
2333 const char* const node_path = node->name;
2334
2335 if (fil_is_user_tablespace_id(node->space->id)
2336 && check_if_skip_table(node_name)) {
2337 msg("[%02u] Skipping %s.\n", thread_n, node_name);
2338 return(FALSE);
2339 }
2340
2341 if (!changed_page_bitmap) {
2342 read_filter = &rf_pass_through;
2343 }
2344 else {
2345 read_filter = &rf_bitmap;
2346 }
2347 res = xb_fil_cur_open(&cursor, read_filter, node, thread_n);
2348 if (res == XB_FIL_CUR_SKIP) {
2349 goto skip;
2350 } else if (res == XB_FIL_CUR_ERROR) {
2351 goto error;
2352 }
2353
2354 strncpy(dst_name, cursor.rel_path, sizeof(dst_name));
2355
2356 /* Setup the page write filter */
2357 if (xtrabackup_incremental) {
2358 write_filter = &wf_incremental;
2359 } else {
2360 write_filter = &wf_write_through;
2361 }
2362
2363 memset(&write_filt_ctxt, 0, sizeof(xb_write_filt_ctxt_t));
2364 ut_a(write_filter->process != NULL);
2365
2366 if (write_filter->init != NULL &&
2367 !write_filter->init(&write_filt_ctxt, dst_name, &cursor)) {
2368 msg("[%02u] mariabackup: error: "
2369 "failed to initialize page write filter.\n", thread_n);
2370 goto error;
2371 }
2372
2373 dstfile = ds_open(ds_data, dst_name, &cursor.statinfo);
2374 if (dstfile == NULL) {
2375 msg("[%02u] mariabackup: error: "
2376 "cannot open the destination stream for %s\n",
2377 thread_n, dst_name);
2378 goto error;
2379 }
2380
2381 action = xb_get_copy_action();
2382
2383 if (xtrabackup_stream) {
2384 msg_ts("[%02u] %s %s\n", thread_n, action, node_path);
2385 } else {
2386 msg_ts("[%02u] %s %s to %s\n", thread_n, action,
2387 node_path, dstfile->path);
2388 }
2389
2390 /* The main copy loop */
2391 while ((res = xb_fil_cur_read(&cursor)) == XB_FIL_CUR_SUCCESS) {
2392 if (!write_filter->process(&write_filt_ctxt, dstfile)) {
2393 goto error;
2394 }
2395 }
2396
2397 if (res == XB_FIL_CUR_ERROR) {
2398 goto error;
2399 }
2400
2401 if (write_filter->finalize
2402 && !write_filter->finalize(&write_filt_ctxt, dstfile)) {
2403 goto error;
2404 }
2405
2406 /* close */
2407 msg_ts("[%02u] ...done\n", thread_n);
2408 xb_fil_cur_close(&cursor);
2409 if (ds_close(dstfile)) {
2410 rc = TRUE;
2411 }
2412 if (write_filter && write_filter->deinit) {
2413 write_filter->deinit(&write_filt_ctxt);
2414 }
2415 return(rc);
2416
2417error:
2418 xb_fil_cur_close(&cursor);
2419 if (dstfile != NULL) {
2420 ds_close(dstfile);
2421 }
2422 if (write_filter && write_filter->deinit) {
2423 write_filter->deinit(&write_filt_ctxt);;
2424 }
2425 msg("[%02u] mariabackup: Error: "
2426 "xtrabackup_copy_datafile() failed.\n", thread_n);
2427 return(TRUE); /*ERROR*/
2428
2429skip:
2430
2431 if (dstfile != NULL) {
2432 ds_close(dstfile);
2433 }
2434 if (write_filter && write_filter->deinit) {
2435 write_filter->deinit(&write_filt_ctxt);
2436 }
2437 msg("[%02u] mariabackup: Warning: We assume the "
2438 "table was dropped during xtrabackup execution "
2439 "and ignore the file.\n", thread_n);
2440 msg("[%02u] mariabackup: Warning: skipping tablespace %s.\n",
2441 thread_n, node_name);
2442 return(FALSE);
2443}
2444
2445/** How to copy a redo log segment in backup */
2446enum copy_logfile {
2447 /** Initial copying: copy at least one block */
2448 COPY_FIRST,
2449 /** Tracking while copying data files */
2450 COPY_ONLINE,
2451 /** Final copying: copy until the end of the log */
2452 COPY_LAST
2453};
2454
2455/** Copy redo log blocks to the data sink.
2456@param[in] copy how to copy the log
2457@param[in] start_lsn buffer start LSN
2458@param[in] end_lsn buffer end LSN
2459@return last scanned LSN (equals to last copied LSN if copy=COPY_LAST)
2460@retval 0 on failure */
2461static
2462lsn_t
2463xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn)
2464{
2465 lsn_t scanned_lsn = start_lsn;
2466 const byte* log_block = log_sys.buf;
2467 bool more_data = false;
2468
2469 for (ulint scanned_checkpoint = 0;
2470 scanned_lsn < end_lsn;
2471 log_block += OS_FILE_LOG_BLOCK_SIZE) {
2472 ulint checkpoint = log_block_get_checkpoint_no(log_block);
2473
2474 if (scanned_checkpoint > checkpoint
2475 && scanned_checkpoint - checkpoint >= 0x80000000UL) {
2476 /* Garbage from a log buffer flush which was made
2477 before the most recent database recovery */
2478 break;
2479 }
2480
2481 scanned_checkpoint = checkpoint;
2482
2483 ulint data_len = log_block_get_data_len(log_block);
2484
2485 more_data = recv_sys_add_to_parsing_buf(
2486 log_block,
2487 scanned_lsn + data_len);
2488
2489 recv_sys->scanned_lsn = scanned_lsn + data_len;
2490
2491 if (data_len == OS_FILE_LOG_BLOCK_SIZE) {
2492 /* We got a full log block. */
2493 scanned_lsn += data_len;
2494 } else if (data_len
2495 >= OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_TRL_SIZE
2496 || data_len <= LOG_BLOCK_HDR_SIZE) {
2497 /* We got a garbage block (abrupt end of the log). */
2498 break;
2499 } else {
2500 /* We got a partial block (abrupt end of the log). */
2501 scanned_lsn += data_len;
2502 break;
2503 }
2504 }
2505
2506 if (more_data && recv_parse_log_recs(0, STORE_NO, false)) {
2507
2508 msg("mariabackup: copying the log failed \n");
2509
2510 return(0);
2511 }
2512
2513 recv_sys_justify_left_parsing_buf();
2514
2515 log_sys.log.scanned_lsn = scanned_lsn;
2516
2517 end_lsn = copy == COPY_LAST
2518 ? ut_uint64_align_up(scanned_lsn, OS_FILE_LOG_BLOCK_SIZE)
2519 : scanned_lsn & ~lsn_t(OS_FILE_LOG_BLOCK_SIZE - 1);
2520
2521 if (ulint write_size = ulint(end_lsn - start_lsn)) {
2522 if (srv_encrypt_log) {
2523 log_crypt(log_sys.buf, start_lsn, write_size);
2524 }
2525
2526 if (ds_write(dst_log_file, log_sys.buf, write_size)) {
2527 msg("mariabackup: Error: "
2528 "write to logfile failed\n");
2529 return(0);
2530 }
2531 }
2532
2533 return(scanned_lsn);
2534}
2535
2536/** Copy redo log until the current end of the log is reached
2537@param copy how to copy the log
2538@return whether the operation failed */
2539static bool
2540xtrabackup_copy_logfile(copy_logfile copy)
2541{
2542 ut_a(dst_log_file != NULL);
2543 ut_ad(recv_sys != NULL);
2544
2545 lsn_t start_lsn;
2546 lsn_t end_lsn;
2547
2548 recv_sys->parse_start_lsn = log_copy_scanned_lsn;
2549 recv_sys->scanned_lsn = log_copy_scanned_lsn;
2550
2551 start_lsn = ut_uint64_align_down(log_copy_scanned_lsn,
2552 OS_FILE_LOG_BLOCK_SIZE);
2553 /* When copying the first or last part of the log, retry a few
2554 times to ensure that all log up to the last checkpoint will be
2555 read. */
2556 do {
2557 end_lsn = start_lsn + RECV_SCAN_SIZE;
2558
2559 xtrabackup_io_throttling();
2560
2561 log_mutex_enter();
2562
2563 lsn_t lsn= start_lsn;
2564 for(int retries= 0; retries < 100; retries++) {
2565 if (log_sys.log.read_log_seg(&lsn, end_lsn)) {
2566 break;
2567 }
2568 msg("Retrying read of a redo log block");
2569 my_sleep(1000);
2570 }
2571
2572 start_lsn = xtrabackup_copy_log(copy, start_lsn, lsn);
2573
2574 log_mutex_exit();
2575
2576 if (!start_lsn) {
2577 ds_close(dst_log_file);
2578 dst_log_file = NULL;
2579 msg("mariabackup: Error: xtrabackup_copy_logfile()"
2580 " failed.\n");
2581 return(true);
2582 }
2583 } while (start_lsn == end_lsn);
2584
2585 ut_ad(start_lsn == log_sys.log.scanned_lsn);
2586
2587 msg_ts(">> log scanned up to (" LSN_PF ")\n", start_lsn);
2588
2589 /* update global variable*/
2590 log_copy_scanned_lsn = start_lsn;
2591
2592 debug_sync_point("xtrabackup_copy_logfile_pause");
2593 return(false);
2594}
2595
2596static os_thread_ret_t DECLARE_THREAD(log_copying_thread)(void*)
2597{
2598 /*
2599 Initialize mysys thread-specific memory so we can
2600 use mysys functions in this thread.
2601 */
2602 my_thread_init();
2603
2604 do {
2605 os_event_reset(log_copying_stop);
2606 os_event_wait_time_low(log_copying_stop,
2607 xtrabackup_log_copy_interval * 1000ULL,
2608 0);
2609 } while (log_copying && xtrabackup_copy_logfile(COPY_ONLINE));
2610
2611 log_copying_running = false;
2612 my_thread_end();
2613 os_thread_exit();
2614
2615 return(0);
2616}
2617
2618/* io throttle watching (rough) */
2619static os_thread_ret_t DECLARE_THREAD(io_watching_thread)(void*)
2620{
2621 /* currently, for --backup only */
2622 ut_a(xtrabackup_backup);
2623
2624 while (log_copying) {
2625 os_thread_sleep(1000000); /*1 sec*/
2626 io_ticket = xtrabackup_throttle;
2627 os_event_set(wait_throttle);
2628 }
2629
2630 /* stop io throttle */
2631 xtrabackup_throttle = 0;
2632 os_event_set(wait_throttle);
2633
2634 io_watching_thread_running = false;
2635
2636 os_thread_exit();
2637
2638 return(0);
2639}
2640
2641/**************************************************************************
2642Datafiles copying thread.*/
2643static
2644os_thread_ret_t
2645DECLARE_THREAD(data_copy_thread_func)(
2646/*==================*/
2647 void *arg) /* thread context */
2648{
2649 data_thread_ctxt_t *ctxt = (data_thread_ctxt_t *) arg;
2650 uint num = ctxt->num;
2651 fil_node_t* node;
2652
2653 /*
2654 Initialize mysys thread-specific memory so we can
2655 use mysys functions in this thread.
2656 */
2657 my_thread_init();
2658
2659 debug_sync_point("data_copy_thread_func");
2660
2661 while ((node = datafiles_iter_next(ctxt->it)) != NULL) {
2662
2663 /* copy the datafile */
2664 if(xtrabackup_copy_datafile(node, num)) {
2665 msg("[%02u] mariabackup: Error: "
2666 "failed to copy datafile.\n", num);
2667 exit(EXIT_FAILURE);
2668 }
2669 }
2670
2671 pthread_mutex_lock(&ctxt->count_mutex);
2672 (*ctxt->count)--;
2673 pthread_mutex_unlock(&ctxt->count_mutex);
2674
2675 my_thread_end();
2676 os_thread_exit();
2677 OS_THREAD_DUMMY_RETURN;
2678}
2679
2680/************************************************************************
2681Initialize the appropriate datasink(s). Both local backups and streaming in the
2682'xbstream' format allow parallel writes so we can write directly.
2683
2684Otherwise (i.e. when streaming in the 'tar' format) we need 2 separate datasinks
2685for the data stream (and don't allow parallel data copying) and for metainfo
2686files (including ib_logfile0). The second datasink writes to temporary
2687files first, and then streams them in a serialized way when closed. */
2688static void
2689xtrabackup_init_datasinks(void)
2690{
2691 /* Start building out the pipelines from the terminus back */
2692 if (xtrabackup_stream) {
2693 /* All streaming goes to stdout */
2694 ds_data = ds_meta = ds_redo = ds_create(xtrabackup_target_dir,
2695 DS_TYPE_STDOUT);
2696 } else {
2697 /* Local filesystem */
2698 ds_data = ds_meta = ds_redo = ds_create(xtrabackup_target_dir,
2699 DS_TYPE_LOCAL);
2700 }
2701
2702 /* Track it for destruction */
2703 xtrabackup_add_datasink(ds_data);
2704
2705 /* Stream formatting */
2706 if (xtrabackup_stream) {
2707 ds_ctxt_t *ds;
2708
2709 ut_a(xtrabackup_stream_fmt == XB_STREAM_FMT_XBSTREAM);
2710 ds = ds_create(xtrabackup_target_dir, DS_TYPE_XBSTREAM);
2711
2712 xtrabackup_add_datasink(ds);
2713
2714 ds_set_pipe(ds, ds_data);
2715 ds_data = ds;
2716
2717
2718 ds_redo = ds_meta = ds_data;
2719 }
2720
2721 /* Compression for ds_data and ds_redo */
2722 if (xtrabackup_compress) {
2723 ds_ctxt_t *ds;
2724
2725 /* Use a 1 MB buffer for compressed output stream */
2726 ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER);
2727 ds_buffer_set_size(ds, 1024 * 1024);
2728 xtrabackup_add_datasink(ds);
2729 ds_set_pipe(ds, ds_data);
2730 if (ds_data != ds_redo) {
2731 ds_data = ds;
2732 ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER);
2733 ds_buffer_set_size(ds, 1024 * 1024);
2734 xtrabackup_add_datasink(ds);
2735 ds_set_pipe(ds, ds_redo);
2736 ds_redo = ds;
2737 } else {
2738 ds_redo = ds_data = ds;
2739 }
2740
2741 ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS);
2742 xtrabackup_add_datasink(ds);
2743 ds_set_pipe(ds, ds_data);
2744 if (ds_data != ds_redo) {
2745 ds_data = ds;
2746 ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS);
2747 xtrabackup_add_datasink(ds);
2748 ds_set_pipe(ds, ds_redo);
2749 ds_redo = ds;
2750 } else {
2751 ds_redo = ds_data = ds;
2752 }
2753 }
2754}
2755
2756/************************************************************************
2757Destroy datasinks.
2758
2759Destruction is done in the specific order to not violate their order in the
2760pipeline so that each datasink is able to flush data down the pipeline. */
2761static void xtrabackup_destroy_datasinks(void)
2762{
2763 for (uint i = actual_datasinks; i > 0; i--) {
2764 ds_destroy(datasinks[i-1]);
2765 datasinks[i-1] = NULL;
2766 }
2767 ds_data = NULL;
2768 ds_meta = NULL;
2769 ds_redo = NULL;
2770}
2771
2772#define SRV_MAX_N_PENDING_SYNC_IOS 100
2773
2774/** Initialize the tablespace cache subsystem. */
2775static
2776void
2777xb_fil_io_init()
2778{
2779 fil_system.create(srv_file_per_table ? 50000 : 5000);
2780}
2781
2782static
2783Datafile*
2784xb_new_datafile(const char *name, bool is_remote)
2785{
2786 if (is_remote) {
2787 RemoteDatafile *remote_file = new RemoteDatafile();
2788 remote_file->set_name(name);
2789 return(remote_file);
2790 } else {
2791 Datafile *file = new Datafile();
2792 file->set_name(name);
2793 file->make_filepath(".", name, IBD);
2794 return(file);
2795 }
2796}
2797
2798
2799static
2800void
2801xb_load_single_table_tablespace(
2802 const char *dirname,
2803 const char *filname,
2804 bool is_remote)
2805{
2806 ut_ad(srv_operation == SRV_OPERATION_BACKUP
2807 || srv_operation == SRV_OPERATION_RESTORE_DELTA);
2808 /* Ignore .isl files on XtraBackup recovery. All tablespaces must be
2809 local. */
2810 if (is_remote && srv_operation == SRV_OPERATION_RESTORE_DELTA) {
2811 return;
2812 }
2813 if (check_if_skip_table(filname)) {
2814 return;
2815 }
2816
2817 /* The name ends in .ibd or .isl;
2818 try opening the file */
2819 char* name;
2820 size_t dirlen = dirname == NULL ? 0 : strlen(dirname);
2821 size_t namelen = strlen(filname);
2822 ulint pathlen = dirname == NULL ? namelen + 1: dirlen + namelen + 2;
2823 lsn_t flush_lsn;
2824 dberr_t err;
2825 fil_space_t *space;
2826
2827 name = static_cast<char*>(ut_malloc_nokey(pathlen));
2828
2829 if (dirname != NULL) {
2830 snprintf(name, pathlen, "%s/%s", dirname, filname);
2831 name[pathlen - 5] = 0;
2832 } else {
2833 snprintf(name, pathlen, "%s", filname);
2834 name[pathlen - 5] = 0;
2835 }
2836
2837 Datafile *file = xb_new_datafile(name, is_remote);
2838
2839 if (file->open_read_only(true) != DB_SUCCESS) {
2840 ut_free(name);
2841 exit(EXIT_FAILURE);
2842 }
2843
2844 err = file->validate_first_page(&flush_lsn);
2845
2846 if (err == DB_SUCCESS && file->space_id() != SRV_TMP_SPACE_ID) {
2847 os_offset_t node_size = os_file_get_size(file->handle());
2848 os_offset_t n_pages;
2849
2850 ut_a(node_size != (os_offset_t) -1);
2851
2852 n_pages = node_size / page_size_t(file->flags()).physical();
2853
2854 space = fil_space_create(
2855 name, file->space_id(), file->flags(),
2856 FIL_TYPE_TABLESPACE, NULL/* TODO: crypt_data */);
2857
2858 ut_a(space != NULL);
2859
2860 if (!fil_node_create(file->filepath(), ulint(n_pages), space,
2861 false, false)) {
2862 ut_error;
2863 }
2864
2865 /* by opening the tablespace we forcing node and space objects
2866 in the cache to be populated with fields from space header */
2867 space->open();
2868
2869 if (srv_operation == SRV_OPERATION_RESTORE_DELTA
2870 || xb_close_files) {
2871 space->close();
2872 }
2873 }
2874
2875 ut_free(name);
2876
2877 delete file;
2878
2879 if (err != DB_SUCCESS && err != DB_CORRUPTION && xtrabackup_backup) {
2880 /* allow corrupted first page for xtrabackup, it could be just
2881 zero-filled page, which we restore from redo log later */
2882 exit(EXIT_FAILURE);
2883 }
2884}
2885
2886/** Scan the database directories under the MySQL datadir, looking for
2887.ibd files and determining the space id in each of them.
2888@return DB_SUCCESS or error number */
2889
2890static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback)
2891{
2892 int ret;
2893 char* dbpath = NULL;
2894 ulint dbpath_len = 100;
2895 os_file_dir_t dir;
2896 os_file_dir_t dbdir;
2897 os_file_stat_t dbinfo;
2898 os_file_stat_t fileinfo;
2899 dberr_t err = DB_SUCCESS;
2900 size_t len;
2901
2902 /* The datadir of MySQL is always the default directory of mysqld */
2903
2904 dir = os_file_opendir(fil_path_to_mysql_datadir, true);
2905
2906 if (dir == NULL) {
2907
2908 return(DB_ERROR);
2909 }
2910
2911 dbpath = static_cast<char*>(ut_malloc_nokey(dbpath_len));
2912
2913 /* Scan all directories under the datadir. They are the database
2914 directories of MySQL. */
2915
2916 ret = fil_file_readdir_next_file(&err, fil_path_to_mysql_datadir, dir,
2917 &dbinfo);
2918 while (ret == 0) {
2919
2920 /* General tablespaces are always at the first level of the
2921 data home dir */
2922 if (dbinfo.type == OS_FILE_TYPE_FILE) {
2923 bool is_isl = ends_with(dbinfo.name, ".isl");
2924 bool is_ibd = !is_isl && ends_with(dbinfo.name,".ibd");
2925
2926 if (is_isl || is_ibd) {
2927 (*callback)(NULL, dbinfo.name, is_isl);
2928 }
2929 }
2930
2931 if (dbinfo.type == OS_FILE_TYPE_FILE
2932 || dbinfo.type == OS_FILE_TYPE_UNKNOWN) {
2933
2934 goto next_datadir_item;
2935 }
2936
2937 /* We found a symlink or a directory; try opening it to see
2938 if a symlink is a directory */
2939
2940 len = strlen(fil_path_to_mysql_datadir)
2941 + strlen (dbinfo.name) + 2;
2942 if (len > dbpath_len) {
2943 dbpath_len = len;
2944
2945 if (dbpath) {
2946 ut_free(dbpath);
2947 }
2948
2949 dbpath = static_cast<char*>(ut_malloc_nokey(dbpath_len));
2950 }
2951 snprintf(dbpath, dbpath_len,
2952 "%s/%s", fil_path_to_mysql_datadir, dbinfo.name);
2953 os_normalize_path(dbpath);
2954
2955 if (check_if_skip_database_by_path(dbpath)) {
2956 fprintf(stderr, "Skipping db: %s\n", dbpath);
2957 goto next_datadir_item;
2958 }
2959
2960 /* We want wrong directory permissions to be a fatal error for
2961 XtraBackup. */
2962 dbdir = os_file_opendir(dbpath, true);
2963
2964 if (dbdir != NULL) {
2965
2966 /* We found a database directory; loop through it,
2967 looking for possible .ibd files in it */
2968
2969 for (ret = fil_file_readdir_next_file(&err, dbpath,
2970 dbdir,
2971 &fileinfo);
2972 ret == 0;
2973 ret = fil_file_readdir_next_file(&err, dbpath,
2974 dbdir,
2975 &fileinfo)) {
2976 if (fileinfo.type == OS_FILE_TYPE_DIR) {
2977 continue;
2978 }
2979
2980 /* We found a symlink or a file */
2981 if (strlen(fileinfo.name) > 4) {
2982 bool is_isl= false;
2983 if (ends_with(fileinfo.name, ".ibd") || ((is_isl = ends_with(fileinfo.name, ".isl"))))
2984 (*callback)(dbinfo.name, fileinfo.name, is_isl);
2985 }
2986 }
2987
2988 if (0 != os_file_closedir(dbdir)) {
2989 fprintf(stderr, "InnoDB: Warning: could not"
2990 " close database directory %s\n",
2991 dbpath);
2992
2993 err = DB_ERROR;
2994 }
2995
2996 } else {
2997
2998 err = DB_ERROR;
2999 break;
3000
3001 }
3002
3003next_datadir_item:
3004 ret = fil_file_readdir_next_file(&err,
3005 fil_path_to_mysql_datadir,
3006 dir, &dbinfo);
3007 }
3008
3009 ut_free(dbpath);
3010
3011 if (0 != os_file_closedir(dir)) {
3012 fprintf(stderr,
3013 "InnoDB: Error: could not close MySQL datadir\n");
3014
3015 return(DB_ERROR);
3016 }
3017
3018 return(err);
3019}
3020
3021/** Assign srv_undo_space_id_start variable if there are undo tablespace present.
3022Read the TRX_SYS page from ibdata1 file and get the minimum space id from
3023the first slot rollback segments of TRX_SYS_PAGE_NO.
3024@retval DB_ERROR if file open or page read failed.
3025@retval DB_SUCCESS if srv_undo_space_id assigned successfully. */
3026static dberr_t xb_assign_undo_space_start()
3027{
3028 ulint dirnamelen;
3029 char name[1000];
3030 pfs_os_file_t file;
3031 byte* buf;
3032 byte* page;
3033 bool ret;
3034 dberr_t error = DB_SUCCESS;
3035 ulint space, page_no __attribute__((unused));
3036
3037 if (srv_undo_tablespaces == 0) {
3038 return error;
3039 }
3040
3041 os_normalize_path(srv_data_home);
3042 dirnamelen = strlen(srv_data_home);
3043 memcpy(name, srv_data_home, dirnamelen);
3044
3045 if (dirnamelen && name[dirnamelen - 1] != OS_PATH_SEPARATOR) {
3046 name[dirnamelen++] = OS_PATH_SEPARATOR;
3047 }
3048
3049 snprintf(name + dirnamelen, (sizeof name) - dirnamelen,
3050 "%s", "ibdata1");
3051
3052 file = os_file_create(0, name, OS_FILE_OPEN,
3053 OS_FILE_NORMAL, OS_DATA_FILE, true, &ret);
3054
3055 if (!ret) {
3056 msg("mariabackup: Error in opening %s\n", name);
3057 return DB_ERROR;
3058 }
3059
3060 buf = static_cast<byte*>(ut_malloc_nokey(2U << srv_page_size_shift));
3061 page = static_cast<byte*>(ut_align(buf, srv_page_size));
3062
3063retry:
3064 if (!os_file_read(IORequestRead, file, page,
3065 TRX_SYS_PAGE_NO << srv_page_size_shift,
3066 srv_page_size)) {
3067 msg("mariabackup: Reading TRX_SYS page failed.\n");
3068 error = DB_ERROR;
3069 goto func_exit;
3070 }
3071
3072 /* TRX_SYS page can't be compressed or encrypted. */
3073 if (buf_page_is_corrupted(false, page, univ_page_size)) {
3074 goto retry;
3075 }
3076
3077 /* 0th slot always points to system tablespace.
3078 1st slot should point to first undotablespace which is minimum. */
3079
3080 page_no = mach_read_ulint(TRX_SYS + TRX_SYS_RSEGS
3081 + TRX_SYS_RSEG_SLOT_SIZE
3082 + TRX_SYS_RSEG_PAGE_NO + page, MLOG_4BYTES);
3083 ut_ad(page_no != FIL_NULL);
3084
3085 space = mach_read_ulint(TRX_SYS + TRX_SYS_RSEGS
3086 + TRX_SYS_RSEG_SLOT_SIZE
3087 + TRX_SYS_RSEG_SPACE + page, MLOG_4BYTES);
3088
3089 srv_undo_space_id_start = space;
3090
3091func_exit:
3092 ut_free(buf);
3093 ret = os_file_close(file);
3094 ut_a(ret);
3095
3096 return error;
3097}
3098
3099/****************************************************************************
3100Populates the tablespace memory cache by scanning for and opening data files.
3101@returns DB_SUCCESS or error code.*/
3102static
3103dberr_t
3104xb_load_tablespaces()
3105{
3106 bool create_new_db;
3107 dberr_t err;
3108 ulint sum_of_new_sizes;
3109 lsn_t flush_lsn;
3110
3111 ut_ad(srv_operation == SRV_OPERATION_BACKUP
3112 || srv_operation == SRV_OPERATION_RESTORE_DELTA);
3113
3114 err = srv_sys_space.check_file_spec(&create_new_db, 0);
3115
3116 /* create_new_db must not be true. */
3117 if (err != DB_SUCCESS || create_new_db) {
3118 msg("mariabackup: could not find data files at the "
3119 "specified datadir\n");
3120 return(DB_ERROR);
3121 }
3122
3123 err = srv_sys_space.open_or_create(false, false, &sum_of_new_sizes,
3124 &flush_lsn);
3125
3126 if (err != DB_SUCCESS) {
3127 msg("mariabackup: Could not open data files.\n");
3128 return(err);
3129 }
3130
3131 /* Add separate undo tablespaces to fil_system */
3132
3133 err = xb_assign_undo_space_start();
3134
3135 if (err != DB_SUCCESS) {
3136 return err;
3137 }
3138
3139 err = srv_undo_tablespaces_init(false);
3140
3141 if (err != DB_SUCCESS) {
3142 return(err);
3143 }
3144
3145 /* It is important to call xb_load_single_table_tablespaces() after
3146 srv_undo_tablespaces_init(), because fil_is_user_tablespace_id() *
3147 relies on srv_undo_tablespaces_open to be properly initialized */
3148
3149 msg("mariabackup: Generating a list of tablespaces\n");
3150
3151 err = enumerate_ibd_files(xb_load_single_table_tablespace);
3152 if (err != DB_SUCCESS) {
3153 return(err);
3154 }
3155
3156 debug_sync_point("xtrabackup_load_tablespaces_pause");
3157
3158 return(DB_SUCCESS);
3159}
3160
3161/************************************************************************
3162Initialize the tablespace memory cache and populate it by scanning for and
3163opening data files.
3164@returns DB_SUCCESS or error code.*/
3165static
3166dberr_t
3167xb_data_files_init()
3168{
3169 xb_fil_io_init();
3170
3171 return(xb_load_tablespaces());
3172}
3173
3174/************************************************************************
3175Destroy the tablespace memory cache. */
3176static
3177void
3178xb_data_files_close()
3179{
3180 ut_ad(!os_thread_count);
3181 fil_close_all_files();
3182 if (buf_dblwr) {
3183 buf_dblwr_free();
3184 }
3185}
3186
3187/***********************************************************************
3188Allocate and initialize the entry for databases and tables filtering
3189hash tables. If memory allocation is not successful, terminate program.
3190@return pointer to the created entry. */
3191static
3192xb_filter_entry_t *
3193xb_new_filter_entry(
3194/*================*/
3195 const char* name) /*!< in: name of table/database */
3196{
3197 xb_filter_entry_t *entry;
3198 ulint namelen = strlen(name);
3199
3200 ut_a(namelen <= NAME_LEN * 2 + 1);
3201
3202 entry = static_cast<xb_filter_entry_t *>
3203 (malloc(sizeof(xb_filter_entry_t) + namelen + 1));
3204 memset(entry, '\0', sizeof(xb_filter_entry_t) + namelen + 1);
3205 entry->name = ((char*)entry) + sizeof(xb_filter_entry_t);
3206 strcpy(entry->name, name);
3207 entry->has_tables = FALSE;
3208
3209 return entry;
3210}
3211
3212/***********************************************************************
3213Add entry to hash table. If hash table is NULL, allocate and initialize
3214new hash table */
3215static
3216xb_filter_entry_t*
3217xb_add_filter(
3218/*========================*/
3219 const char* name, /*!< in: name of table/database */
3220 hash_table_t** hash) /*!< in/out: hash to insert into */
3221{
3222 xb_filter_entry_t* entry;
3223
3224 entry = xb_new_filter_entry(name);
3225
3226 if (UNIV_UNLIKELY(*hash == NULL)) {
3227 *hash = hash_create(1000);
3228 }
3229 HASH_INSERT(xb_filter_entry_t,
3230 name_hash, *hash,
3231 ut_fold_string(entry->name),
3232 entry);
3233
3234 return entry;
3235}
3236
3237/***********************************************************************
3238Validate name of table or database. If name is invalid, program will
3239be finished with error code */
3240static
3241void
3242xb_validate_name(
3243/*=============*/
3244 const char* name, /*!< in: name */
3245 size_t len) /*!< in: length of name */
3246{
3247 const char* p;
3248
3249 /* perform only basic validation. validate length and
3250 path symbols */
3251 if (len > NAME_LEN) {
3252 msg("mariabackup: name `%s` is too long.\n", name);
3253 exit(EXIT_FAILURE);
3254 }
3255 p = strpbrk(name, "/\\~");
3256 if (p && (uint) (p - name) < NAME_LEN) {
3257 msg("mariabackup: name `%s` is not valid.\n", name);
3258 exit(EXIT_FAILURE);
3259 }
3260}
3261
3262/***********************************************************************
3263Register new filter entry which can be either database
3264or table name. */
3265static
3266void
3267xb_register_filter_entry(
3268/*=====================*/
3269 const char* name, /*!< in: name */
3270 hash_table_t** databases_hash,
3271 hash_table_t** tables_hash
3272 )
3273{
3274 const char* p;
3275 size_t namelen;
3276 xb_filter_entry_t* db_entry = NULL;
3277
3278 namelen = strlen(name);
3279 if ((p = strchr(name, '.')) != NULL) {
3280 char dbname[NAME_LEN + 1];
3281
3282 xb_validate_name(name, p - name);
3283 xb_validate_name(p + 1, namelen - (p - name));
3284
3285 strncpy(dbname, name, p - name);
3286 dbname[p - name] = 0;
3287
3288 if (*databases_hash) {
3289 HASH_SEARCH(name_hash, (*databases_hash),
3290 ut_fold_string(dbname),
3291 xb_filter_entry_t*,
3292 db_entry, (void) 0,
3293 !strcmp(db_entry->name, dbname));
3294 }
3295 if (!db_entry) {
3296 db_entry = xb_add_filter(dbname, databases_hash);
3297 }
3298 db_entry->has_tables = TRUE;
3299 xb_add_filter(name, tables_hash);
3300 } else {
3301 xb_validate_name(name, namelen);
3302
3303 xb_add_filter(name, databases_hash);
3304 }
3305}
3306
3307static
3308void
3309xb_register_include_filter_entry(
3310 const char* name
3311)
3312{
3313 xb_register_filter_entry(name, &databases_include_hash,
3314 &tables_include_hash);
3315}
3316
3317static
3318void
3319xb_register_exclude_filter_entry(
3320 const char* name
3321)
3322{
3323 xb_register_filter_entry(name, &databases_exclude_hash,
3324 &tables_exclude_hash);
3325}
3326
3327/***********************************************************************
3328Register new table for the filter. */
3329static
3330void
3331xb_register_table(
3332/*==============*/
3333 const char* name) /*!< in: name of table */
3334{
3335 if (strchr(name, '.') == NULL) {
3336 msg("mariabackup: `%s` is not fully qualified name.\n", name);
3337 exit(EXIT_FAILURE);
3338 }
3339
3340 xb_register_include_filter_entry(name);
3341}
3342
3343static
3344void
3345xb_add_regex_to_list(
3346 const char* regex, /*!< in: regex */
3347 const char* error_context, /*!< in: context to error message */
3348 regex_list_t* list) /*! in: list to put new regex to */
3349{
3350 char errbuf[100];
3351 int ret;
3352
3353 regex_t compiled_regex;
3354 ret = regcomp(&compiled_regex, regex, REG_EXTENDED);
3355
3356 if (ret != 0) {
3357 regerror(ret, &compiled_regex, errbuf, sizeof(errbuf));
3358 msg("mariabackup: error: %s regcomp(%s): %s\n",
3359 error_context, regex, errbuf);
3360 exit(EXIT_FAILURE);
3361 }
3362
3363 list->push_back(compiled_regex);
3364}
3365
3366/***********************************************************************
3367Register new regex for the include filter. */
3368static
3369void
3370xb_register_include_regex(
3371/*==============*/
3372 const char* regex) /*!< in: regex */
3373{
3374 xb_add_regex_to_list(regex, "tables", &regex_include_list);
3375}
3376
3377/***********************************************************************
3378Register new regex for the exclude filter. */
3379static
3380void
3381xb_register_exclude_regex(
3382/*==============*/
3383 const char* regex) /*!< in: regex */
3384{
3385 xb_add_regex_to_list(regex, "tables-exclude", &regex_exclude_list);
3386}
3387
3388typedef void (*insert_entry_func_t)(const char*);
3389
3390/***********************************************************************
3391Scan string and load filter entries from it. */
3392static
3393void
3394xb_load_list_string(
3395/*================*/
3396 char* list, /*!< in: string representing a list */
3397 const char* delimiters, /*!< in: delimiters of entries */
3398 insert_entry_func_t ins) /*!< in: callback to add entry */
3399{
3400 char* p;
3401 char* saveptr;
3402
3403 p = strtok_r(list, delimiters, &saveptr);
3404 while (p) {
3405
3406 ins(p);
3407
3408 p = strtok_r(NULL, delimiters, &saveptr);
3409 }
3410}
3411
3412/***********************************************************************
3413Scan file and load filter entries from it. */
3414static
3415void
3416xb_load_list_file(
3417/*==============*/
3418 const char* filename, /*!< in: name of file */
3419 insert_entry_func_t ins) /*!< in: callback to add entry */
3420{
3421 char name_buf[NAME_LEN*2+2];
3422 FILE* fp;
3423
3424 /* read and store the filenames */
3425 fp = fopen(filename, "r");
3426 if (!fp) {
3427 msg("mariabackup: cannot open %s\n",
3428 filename);
3429 exit(EXIT_FAILURE);
3430 }
3431 while (fgets(name_buf, sizeof(name_buf), fp) != NULL) {
3432 char* p = strchr(name_buf, '\n');
3433 if (p) {
3434 *p = '\0';
3435 } else {
3436 msg("mariabackup: `%s...` name is too long", name_buf);
3437 exit(EXIT_FAILURE);
3438 }
3439
3440 ins(name_buf);
3441 }
3442
3443 fclose(fp);
3444}
3445
3446
3447static
3448void
3449xb_filters_init()
3450{
3451 if (xtrabackup_databases) {
3452 xb_load_list_string(xtrabackup_databases, " \t",
3453 xb_register_include_filter_entry);
3454 }
3455
3456 if (xtrabackup_databases_file) {
3457 xb_load_list_file(xtrabackup_databases_file,
3458 xb_register_include_filter_entry);
3459 }
3460
3461 if (xtrabackup_databases_exclude) {
3462 xb_load_list_string(xtrabackup_databases_exclude, " \t",
3463 xb_register_exclude_filter_entry);
3464 }
3465
3466 if (xtrabackup_tables) {
3467 xb_load_list_string(xtrabackup_tables, ",",
3468 xb_register_include_regex);
3469 }
3470
3471 if (xtrabackup_tables_file) {
3472 xb_load_list_file(xtrabackup_tables_file, xb_register_table);
3473 }
3474
3475 if (xtrabackup_tables_exclude) {
3476 xb_load_list_string(xtrabackup_tables_exclude, ",",
3477 xb_register_exclude_regex);
3478 }
3479}
3480
3481static
3482void
3483xb_filter_hash_free(hash_table_t* hash)
3484{
3485 ulint i;
3486
3487 /* free the hash elements */
3488 for (i = 0; i < hash_get_n_cells(hash); i++) {
3489 xb_filter_entry_t* table;
3490
3491 table = static_cast<xb_filter_entry_t *>
3492 (HASH_GET_FIRST(hash, i));
3493
3494 while (table) {
3495 xb_filter_entry_t* prev_table = table;
3496
3497 table = static_cast<xb_filter_entry_t *>
3498 (HASH_GET_NEXT(name_hash, prev_table));
3499
3500 HASH_DELETE(xb_filter_entry_t, name_hash, hash,
3501 ut_fold_string(prev_table->name), prev_table);
3502 free(prev_table);
3503 }
3504 }
3505
3506 /* free hash */
3507 hash_table_free(hash);
3508}
3509
3510static void xb_regex_list_free(regex_list_t* list)
3511{
3512 while (list->size() > 0) {
3513 xb_regfree(&list->front());
3514 list->pop_front();
3515 }
3516}
3517
3518/************************************************************************
3519Destroy table filters for partial backup. */
3520static
3521void
3522xb_filters_free()
3523{
3524 xb_regex_list_free(&regex_include_list);
3525 xb_regex_list_free(&regex_exclude_list);
3526
3527 if (tables_include_hash) {
3528 xb_filter_hash_free(tables_include_hash);
3529 }
3530
3531 if (tables_exclude_hash) {
3532 xb_filter_hash_free(tables_exclude_hash);
3533 }
3534
3535 if (databases_include_hash) {
3536 xb_filter_hash_free(databases_include_hash);
3537 }
3538
3539 if (databases_exclude_hash) {
3540 xb_filter_hash_free(databases_exclude_hash);
3541 }
3542}
3543
3544/*********************************************************************//**
3545Creates or opens the log files and closes them.
3546@return DB_SUCCESS or error code */
3547static
3548ulint
3549open_or_create_log_file(
3550/*====================*/
3551 fil_space_t* space,
3552 ibool* log_file_created, /*!< out: TRUE if new log file
3553 created */
3554 ulint i) /*!< in: log file number in group */
3555{
3556 char name[10000];
3557 ulint dirnamelen;
3558
3559 *log_file_created = FALSE;
3560
3561 os_normalize_path(srv_log_group_home_dir);
3562
3563 dirnamelen = strlen(srv_log_group_home_dir);
3564 ut_a(dirnamelen < (sizeof name) - 10 - sizeof "ib_logfile");
3565 memcpy(name, srv_log_group_home_dir, dirnamelen);
3566
3567 /* Add a path separator if needed. */
3568 if (dirnamelen && name[dirnamelen - 1] != OS_PATH_SEPARATOR) {
3569 name[dirnamelen++] = OS_PATH_SEPARATOR;
3570 }
3571
3572 sprintf(name + dirnamelen, "%s%lu", "ib_logfile", (ulong) i);
3573
3574 ut_a(fil_validate());
3575
3576 ut_a(fil_node_create(name, ulint(srv_log_file_size >> srv_page_size_shift),
3577 space, false, false));
3578
3579 return(DB_SUCCESS);
3580}
3581
3582/***********************************************************************
3583Set the open files limit. Based on set_max_open_files().
3584
3585@return the resulting open files limit. May be less or more than the requested
3586value. */
3587static uint
3588xb_set_max_open_files(
3589/*==================*/
3590 uint max_file_limit) /*!<in: open files limit */
3591{
3592#if defined(RLIMIT_NOFILE)
3593 struct rlimit rlimit;
3594 uint old_cur;
3595
3596 if (getrlimit(RLIMIT_NOFILE, &rlimit)) {
3597
3598 goto end;
3599 }
3600
3601 old_cur = (uint) rlimit.rlim_cur;
3602
3603 if (rlimit.rlim_cur == RLIM_INFINITY) {
3604
3605 rlimit.rlim_cur = max_file_limit;
3606 }
3607
3608 if (rlimit.rlim_cur >= max_file_limit) {
3609
3610 max_file_limit = rlimit.rlim_cur;
3611 goto end;
3612 }
3613
3614 rlimit.rlim_cur = rlimit.rlim_max = max_file_limit;
3615
3616 if (setrlimit(RLIMIT_NOFILE, &rlimit)) {
3617
3618 max_file_limit = old_cur; /* Use original value */
3619 } else {
3620
3621 rlimit.rlim_cur = 0; /* Safety if next call fails */
3622
3623 (void) getrlimit(RLIMIT_NOFILE, &rlimit);
3624
3625 if (rlimit.rlim_cur) {
3626
3627 /* If call didn't fail */
3628 max_file_limit = (uint) rlimit.rlim_cur;
3629 }
3630 }
3631
3632end:
3633 return(max_file_limit);
3634#else
3635 return(0);
3636#endif
3637}
3638
3639static void stop_backup_threads()
3640{
3641 log_copying = false;
3642
3643 if (log_copying_stop) {
3644 os_event_set(log_copying_stop);
3645 msg("mariabackup: Stopping log copying thread.\n");
3646 while (log_copying_running) {
3647 msg(".");
3648 os_thread_sleep(200000); /*0.2 sec*/
3649 }
3650 msg("\n");
3651 os_event_destroy(log_copying_stop);
3652 }
3653
3654 if (wait_throttle) {
3655 /* wait for io_watching_thread completion */
3656 while (io_watching_thread_running) {
3657 os_thread_sleep(1000000);
3658 }
3659 os_event_destroy(wait_throttle);
3660 }
3661}
3662
3663/** Implement the core of --backup
3664@return whether the operation succeeded */
3665static
3666bool
3667xtrabackup_backup_low()
3668{
3669 /* read the latest checkpoint lsn */
3670 {
3671 ulint max_cp_field;
3672
3673 log_mutex_enter();
3674
3675 if (recv_find_max_checkpoint(&max_cp_field) == DB_SUCCESS
3676 && log_sys.log.format != 0) {
3677 metadata_to_lsn = mach_read_from_8(
3678 log_sys.checkpoint_buf + LOG_CHECKPOINT_LSN);
3679 msg("mariabackup: The latest check point"
3680 " (for incremental): '" LSN_PF "'\n",
3681 metadata_to_lsn);
3682 } else {
3683 metadata_to_lsn = 0;
3684 msg("mariabackup: Error: recv_find_max_checkpoint() failed.\n");
3685 }
3686 log_mutex_exit();
3687 }
3688
3689 stop_backup_threads();
3690
3691 if (!dst_log_file || xtrabackup_copy_logfile(COPY_LAST)) {
3692 return false;
3693 }
3694
3695 if (ds_close(dst_log_file)) {
3696 dst_log_file = NULL;
3697 return false;
3698 }
3699
3700 dst_log_file = NULL;
3701
3702 if(!xtrabackup_incremental) {
3703 strcpy(metadata_type, "full-backuped");
3704 metadata_from_lsn = 0;
3705 } else {
3706 strcpy(metadata_type, "incremental");
3707 metadata_from_lsn = incremental_lsn;
3708 }
3709 metadata_last_lsn = log_copy_scanned_lsn;
3710
3711 if (!xtrabackup_stream_metadata(ds_meta)) {
3712 msg("mariabackup: Error: failed to stream metadata.\n");
3713 return false;
3714 }
3715 if (xtrabackup_extra_lsndir) {
3716 char filename[FN_REFLEN];
3717
3718 sprintf(filename, "%s/%s", xtrabackup_extra_lsndir,
3719 XTRABACKUP_METADATA_FILENAME);
3720 if (!xtrabackup_write_metadata(filename)) {
3721 msg("mariabackup: Error: failed to write metadata "
3722 "to '%s'.\n", filename);
3723 return false;
3724 }
3725 sprintf(filename, "%s/%s", xtrabackup_extra_lsndir,
3726 XTRABACKUP_INFO);
3727 if (!write_xtrabackup_info(mysql_connection, filename, false)) {
3728 msg("mariabackup: Error: failed to write info "
3729 "to '%s'.\n", filename);
3730 return false;
3731 }
3732 }
3733
3734 return true;
3735}
3736
3737/** Implement --backup
3738@return whether the operation succeeded */
3739static
3740bool
3741xtrabackup_backup_func()
3742{
3743 MY_STAT stat_info;
3744 uint i;
3745 uint count;
3746 pthread_mutex_t count_mutex;
3747 data_thread_ctxt_t *data_threads;
3748
3749#ifdef USE_POSIX_FADVISE
3750 msg("mariabackup: uses posix_fadvise().\n");
3751#endif
3752
3753 /* cd to datadir */
3754
3755 if (my_setwd(mysql_real_data_home,MYF(MY_WME)))
3756 {
3757 msg("mariabackup: cannot my_setwd %s\n", mysql_real_data_home);
3758 return(false);
3759 }
3760 msg("mariabackup: cd to %s\n", mysql_real_data_home);
3761
3762 msg("mariabackup: open files limit requested %u, set to %u\n",
3763 (uint) xb_open_files_limit,
3764 xb_set_max_open_files(xb_open_files_limit));
3765
3766 mysql_data_home= mysql_data_home_buff;
3767 mysql_data_home[0]=FN_CURLIB; // all paths are relative from here
3768 mysql_data_home[1]=0;
3769
3770 srv_n_purge_threads = 1;
3771 srv_read_only_mode = TRUE;
3772
3773 srv_operation = SRV_OPERATION_BACKUP;
3774
3775 if (xb_close_files)
3776 msg("mariabackup: warning: close-files specified. Use it "
3777 "at your own risk. If there are DDL operations like table DROP TABLE "
3778 "or RENAME TABLE during the backup, inconsistent backup will be "
3779 "produced.\n");
3780
3781 /* initialize components */
3782 if(innodb_init_param()) {
3783fail:
3784 stop_backup_threads();
3785 if (fil_system.is_initialised()) {
3786 innodb_shutdown();
3787 }
3788 return(false);
3789 }
3790
3791 if (srv_buf_pool_size >= 1000 * 1024 * 1024) {
3792 /* Here we still have srv_pool_size counted
3793 in kilobytes (in 4.0 this was in bytes)
3794 srv_boot() converts the value to
3795 pages; if buffer pool is less than 1000 MB,
3796 assume fewer threads. */
3797 srv_max_n_threads = 50000;
3798
3799 } else if (srv_buf_pool_size >= 8 * 1024 * 1024) {
3800
3801 srv_max_n_threads = 10000;
3802 } else {
3803 srv_max_n_threads = 1000; /* saves several MB of memory,
3804 especially in 64-bit
3805 computers */
3806 }
3807
3808 sync_check_init();
3809 ut_d(sync_check_enable());
3810 /* Reset the system variables in the recovery module. */
3811 recv_sys_var_init();
3812 trx_pool_init();
3813
3814 ut_crc32_init();
3815 crc_init();
3816 recv_sys_init();
3817
3818#ifdef WITH_INNODB_DISALLOW_WRITES
3819 srv_allow_writes_event = os_event_create(0);
3820 os_event_set(srv_allow_writes_event);
3821#endif
3822
3823 xb_filters_init();
3824
3825 {
3826 ibool log_file_created;
3827 ibool log_created = FALSE;
3828 ibool log_opened = FALSE;
3829 ulint err;
3830 ulint i;
3831
3832 xb_fil_io_init();
3833 srv_n_file_io_threads = srv_n_read_io_threads;
3834
3835 os_aio_init(srv_n_read_io_threads, srv_n_write_io_threads,
3836 SRV_MAX_N_PENDING_SYNC_IOS);
3837
3838 log_sys.create();
3839 log_sys.log.create(srv_n_log_files);
3840 fil_space_t* space = fil_space_create(
3841 "innodb_redo_log", SRV_LOG_SPACE_FIRST_ID, 0,
3842 FIL_TYPE_LOG, NULL);
3843
3844 for (i = 0; i < srv_n_log_files; i++) {
3845 err = open_or_create_log_file(space, &log_file_created, i);
3846 if (err != DB_SUCCESS) {
3847 goto fail;
3848 }
3849
3850 if (log_file_created) {
3851 log_created = TRUE;
3852 } else {
3853 log_opened = TRUE;
3854 }
3855 if ((log_opened && log_created)) {
3856 msg(
3857 "mariabackup: Error: all log files must be created at the same time.\n"
3858 "mariabackup: All log files must be created also in database creation.\n"
3859 "mariabackup: If you want bigger or smaller log files, shut down the\n"
3860 "mariabackup: database and make sure there were no errors in shutdown.\n"
3861 "mariabackup: Then delete the existing log files. Edit the .cnf file\n"
3862 "mariabackup: and start the database again.\n");
3863
3864 goto fail;
3865 }
3866 }
3867
3868 /* log_file_created must not be TRUE, if online */
3869 if (log_file_created) {
3870 msg("mariabackup: Something wrong with source files...\n");
3871 goto fail;
3872 }
3873
3874 }
3875
3876 /* create extra LSN dir if it does not exist. */
3877 if (xtrabackup_extra_lsndir
3878 &&!my_stat(xtrabackup_extra_lsndir,&stat_info,MYF(0))
3879 && (my_mkdir(xtrabackup_extra_lsndir,0777,MYF(0)) < 0)) {
3880 msg("mariabackup: Error: cannot mkdir %d: %s\n",
3881 my_errno, xtrabackup_extra_lsndir);
3882 goto fail;
3883 }
3884
3885 /* create target dir if not exist */
3886 if (!xtrabackup_stream_str && !my_stat(xtrabackup_target_dir,&stat_info,MYF(0))
3887 && (my_mkdir(xtrabackup_target_dir,0777,MYF(0)) < 0)){
3888 msg("mariabackup: Error: cannot mkdir %d: %s\n",
3889 my_errno, xtrabackup_target_dir);
3890 goto fail;
3891 }
3892
3893 {
3894 /* definition from recv_recovery_from_checkpoint_start() */
3895 ulint max_cp_field;
3896
3897 /* start back ground thread to copy newer log */
3898 os_thread_id_t log_copying_thread_id;
3899 datafiles_iter_t *it;
3900
3901 /* get current checkpoint_lsn */
3902 /* Look for the latest checkpoint from any of the log groups */
3903
3904 log_mutex_enter();
3905
3906 dberr_t err = recv_find_max_checkpoint(&max_cp_field);
3907
3908 if (err != DB_SUCCESS) {
3909log_fail:
3910 log_mutex_exit();
3911 goto fail;
3912 }
3913
3914 if (log_sys.log.format == 0) {
3915old_format:
3916 msg("mariabackup: Error: cannot process redo log"
3917 " before MariaDB 10.2.2\n");
3918 log_mutex_exit();
3919 goto log_fail;
3920 }
3921
3922 ut_ad(!((log_sys.log.format ^ LOG_HEADER_FORMAT_CURRENT)
3923 & ~LOG_HEADER_FORMAT_ENCRYPTED));
3924
3925 const byte* buf = log_sys.checkpoint_buf;
3926
3927reread_log_header:
3928 checkpoint_lsn_start = log_sys.log.lsn;
3929 checkpoint_no_start = log_sys.next_checkpoint_no;
3930
3931 err = recv_find_max_checkpoint(&max_cp_field);
3932
3933 if (err != DB_SUCCESS) {
3934 goto log_fail;
3935 }
3936
3937 if (log_sys.log.format == 0) {
3938 goto old_format;
3939 }
3940
3941 ut_ad(!((log_sys.log.format ^ LOG_HEADER_FORMAT_CURRENT)
3942 & ~LOG_HEADER_FORMAT_ENCRYPTED));
3943
3944 log_header_read(max_cp_field);
3945
3946 if (checkpoint_no_start != mach_read_from_8(buf + LOG_CHECKPOINT_NO)) {
3947 goto reread_log_header;
3948 }
3949
3950 log_mutex_exit();
3951
3952 xtrabackup_init_datasinks();
3953
3954 if (!select_history()) {
3955 goto fail;
3956 }
3957
3958 /* open the log file */
3959 memset(&stat_info, 0, sizeof(MY_STAT));
3960 dst_log_file = ds_open(ds_redo, "ib_logfile0", &stat_info);
3961 if (dst_log_file == NULL) {
3962 msg("mariabackup: error: failed to open the target stream for "
3963 "'ib_logfile0'.\n");
3964 goto fail;
3965 }
3966
3967 /* label it */
3968 byte MY_ALIGNED(OS_FILE_LOG_BLOCK_SIZE) log_hdr[OS_FILE_LOG_BLOCK_SIZE];
3969 memset(log_hdr, 0, sizeof log_hdr);
3970 mach_write_to_4(LOG_HEADER_FORMAT + log_hdr, log_sys.log.format);
3971 mach_write_to_8(LOG_HEADER_START_LSN + log_hdr, checkpoint_lsn_start);
3972 strcpy(reinterpret_cast<char*>(LOG_HEADER_CREATOR + log_hdr),
3973 "Backup " MYSQL_SERVER_VERSION);
3974 log_block_set_checksum(log_hdr,
3975 log_block_calc_checksum_crc32(log_hdr));
3976
3977 /* Write the log header. */
3978 if (ds_write(dst_log_file, log_hdr, sizeof log_hdr)) {
3979 log_write_fail:
3980 msg("mariabackup: error: write to logfile failed\n");
3981 goto fail;
3982 }
3983 /* Adjust the checkpoint page. */
3984 memcpy(log_hdr, buf, OS_FILE_LOG_BLOCK_SIZE);
3985 mach_write_to_8(log_hdr + LOG_CHECKPOINT_OFFSET,
3986 (checkpoint_lsn_start & (OS_FILE_LOG_BLOCK_SIZE - 1))
3987 | LOG_FILE_HDR_SIZE);
3988 log_block_set_checksum(log_hdr,
3989 log_block_calc_checksum_crc32(log_hdr));
3990 /* Write checkpoint page 1 and two empty log pages before the
3991 payload. */
3992 if (ds_write(dst_log_file, log_hdr, OS_FILE_LOG_BLOCK_SIZE)
3993 || !memset(log_hdr, 0, sizeof log_hdr)
3994 || ds_write(dst_log_file, log_hdr, sizeof log_hdr)
3995 || ds_write(dst_log_file, log_hdr, sizeof log_hdr)) {
3996 goto log_write_fail;
3997 }
3998
3999 /* start flag */
4000 log_copying = TRUE;
4001
4002 /* start io throttle */
4003 if(xtrabackup_throttle) {
4004 os_thread_id_t io_watching_thread_id;
4005
4006 io_ticket = xtrabackup_throttle;
4007 wait_throttle = os_event_create(0);
4008 io_watching_thread_running = true;
4009
4010 os_thread_create(io_watching_thread, NULL,
4011 &io_watching_thread_id);
4012 }
4013
4014 /* Populate fil_system with tablespaces to copy */
4015 err = xb_load_tablespaces();
4016 if (err != DB_SUCCESS) {
4017 msg("mariabackup: error: xb_load_tablespaces() failed with"
4018 " error %s.\n", ut_strerr(err));
4019 goto fail;
4020 }
4021
4022 /* copy log file by current position */
4023 log_copy_scanned_lsn = checkpoint_lsn_start;
4024 recv_sys->recovered_lsn = log_copy_scanned_lsn;
4025
4026 if (xtrabackup_copy_logfile(COPY_FIRST))
4027 goto fail;
4028
4029 log_copying_stop = os_event_create(0);
4030 log_copying_running = true;
4031 os_thread_create(log_copying_thread, NULL, &log_copying_thread_id);
4032
4033 /* FLUSH CHANGED_PAGE_BITMAPS call */
4034 if (!flush_changed_page_bitmaps()) {
4035 goto fail;
4036 }
4037 debug_sync_point("xtrabackup_suspend_at_start");
4038
4039 if (xtrabackup_incremental) {
4040 if (!xtrabackup_incremental_force_scan) {
4041 changed_page_bitmap = xb_page_bitmap_init();
4042 }
4043 if (!changed_page_bitmap) {
4044 msg("mariabackup: using the full scan for incremental "
4045 "backup\n");
4046 } else if (incremental_lsn != checkpoint_lsn_start) {
4047 /* Do not print that bitmaps are used when dummy bitmap
4048 is build for an empty LSN range. */
4049 msg("mariabackup: using the changed page bitmap\n");
4050 }
4051 }
4052
4053 ut_a(xtrabackup_parallel > 0);
4054
4055 if (xtrabackup_parallel > 1) {
4056 msg("mariabackup: Starting %u threads for parallel data "
4057 "files transfer\n", xtrabackup_parallel);
4058 }
4059
4060 if (opt_lock_ddl_per_table) {
4061 mdl_lock_all();
4062 }
4063
4064 it = datafiles_iter_new();
4065 if (it == NULL) {
4066 msg("mariabackup: Error: datafiles_iter_new() failed.\n");
4067 goto fail;
4068 }
4069
4070 /* Create data copying threads */
4071 data_threads = (data_thread_ctxt_t *)
4072 malloc(sizeof(data_thread_ctxt_t) * xtrabackup_parallel);
4073 count = xtrabackup_parallel;
4074 pthread_mutex_init(&count_mutex, NULL);
4075
4076 for (i = 0; i < (uint) xtrabackup_parallel; i++) {
4077 data_threads[i].it = it;
4078 data_threads[i].num = i+1;
4079 data_threads[i].count = &count;
4080 data_threads[i].count_mutex = count_mutex;
4081 os_thread_create(data_copy_thread_func, data_threads + i,
4082 &data_threads[i].id);
4083 }
4084
4085 /* Wait for threads to exit */
4086 while (1) {
4087 os_thread_sleep(1000000);
4088 pthread_mutex_lock(&count_mutex);
4089 bool stop = count == 0;
4090 pthread_mutex_unlock(&count_mutex);
4091 if (stop) {
4092 break;
4093 }
4094 }
4095
4096 pthread_mutex_destroy(&count_mutex);
4097 free(data_threads);
4098 datafiles_iter_free(it);
4099
4100 if (changed_page_bitmap) {
4101 xb_page_bitmap_deinit(changed_page_bitmap);
4102 }
4103 }
4104
4105 bool ok = backup_start();
4106
4107 if (ok) {
4108 ok = xtrabackup_backup_low();
4109
4110 backup_release();
4111
4112 DBUG_EXECUTE_IF("check_mdl_lock_works",
4113 os_event_wait(dbug_alter_thread_done);
4114 os_event_destroy(dbug_alter_thread_done);
4115 );
4116
4117 if (ok) {
4118 backup_finish();
4119 }
4120 }
4121
4122 if (!ok) {
4123 goto fail;
4124 }
4125
4126 xtrabackup_destroy_datasinks();
4127
4128 msg("mariabackup: Redo log (from LSN " LSN_PF " to " LSN_PF
4129 ") was copied.\n", checkpoint_lsn_start, log_copy_scanned_lsn);
4130 xb_filters_free();
4131
4132 xb_data_files_close();
4133
4134 /* Make sure that the latest checkpoint was included */
4135 if (metadata_to_lsn > log_copy_scanned_lsn) {
4136 msg("mariabackup: error: failed to copy enough redo log ("
4137 "LSN=" LSN_PF "; checkpoint LSN=" LSN_PF ").\n",
4138 log_copy_scanned_lsn, metadata_to_lsn);
4139 goto fail;
4140 }
4141
4142 innodb_shutdown();
4143 return(true);
4144}
4145
4146/* ================= prepare ================= */
4147
4148/***********************************************************************
4149Generates path to the meta file path from a given path to an incremental .delta
4150by replacing trailing ".delta" with ".meta", or returns error if 'delta_path'
4151does not end with the ".delta" character sequence.
4152@return TRUE on success, FALSE on error. */
4153static
4154ibool
4155get_meta_path(
4156 const char *delta_path, /* in: path to a .delta file */
4157 char *meta_path) /* out: path to the corresponding .meta
4158 file */
4159{
4160 size_t len = strlen(delta_path);
4161
4162 if (len <= 6 || strcmp(delta_path + len - 6, ".delta")) {
4163 return FALSE;
4164 }
4165 memcpy(meta_path, delta_path, len - 6);
4166 strcpy(meta_path + len - 6, XB_DELTA_INFO_SUFFIX);
4167
4168 return TRUE;
4169}
4170
4171/****************************************************************//**
4172Create a new tablespace on disk and return the handle to its opened
4173file. Code adopted from fil_create_new_single_table_tablespace with
4174the main difference that only disk file is created without updating
4175the InnoDB in-memory dictionary data structures.
4176
4177@return true on success, false on error. */
4178static
4179bool
4180xb_space_create_file(
4181/*==================*/
4182 const char* path, /*!<in: path to tablespace */
4183 ulint space_id, /*!<in: space id */
4184 ulint flags, /*!<in: tablespace flags */
4185 pfs_os_file_t* file) /*!<out: file handle */
4186{
4187 bool ret;
4188 byte* buf;
4189 byte* page;
4190
4191 *file = os_file_create_simple_no_error_handling(
4192 0, path, OS_FILE_CREATE, OS_FILE_READ_WRITE, false, &ret);
4193 if (!ret) {
4194 msg("mariabackup: cannot create file %s\n", path);
4195 return ret;
4196 }
4197
4198 ret = os_file_set_size(path, *file,
4199 FIL_IBD_FILE_INITIAL_SIZE
4200 << srv_page_size_shift);
4201 if (!ret) {
4202 msg("mariabackup: cannot set size for file %s\n", path);
4203 os_file_close(*file);
4204 os_file_delete(0, path);
4205 return ret;
4206 }
4207
4208 buf = static_cast<byte *>(malloc(3U << srv_page_size_shift));
4209 /* Align the memory for file i/o if we might have O_DIRECT set */
4210 page = static_cast<byte *>(ut_align(buf, srv_page_size));
4211
4212 memset(page, '\0', srv_page_size);
4213
4214 fsp_header_init_fields(page, space_id, flags);
4215 mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id);
4216
4217 const page_size_t page_size(flags);
4218
4219 if (!page_size.is_compressed()) {
4220 buf_flush_init_for_writing(NULL, page, NULL, 0);
4221
4222 ret = os_file_write(IORequestWrite, path, *file, page, 0,
4223 srv_page_size);
4224 } else {
4225 page_zip_des_t page_zip;
4226 ulint zip_size = page_size.physical();
4227 page_zip_set_size(&page_zip, zip_size);
4228 page_zip.data = page + srv_page_size;
4229 fprintf(stderr, "zip_size = " ULINTPF "\n", zip_size);
4230
4231#ifdef UNIV_DEBUG
4232 page_zip.m_start =
4233#endif /* UNIV_DEBUG */
4234 page_zip.m_end = page_zip.m_nonempty =
4235 page_zip.n_blobs = 0;
4236
4237 buf_flush_init_for_writing(NULL, page, &page_zip, 0);
4238
4239 ret = os_file_write(IORequestWrite, path, *file,
4240 page_zip.data, 0, zip_size);
4241 }
4242
4243 free(buf);
4244
4245 if (!ret) {
4246 msg("mariabackup: could not write the first page to %s\n",
4247 path);
4248 os_file_close(*file);
4249 os_file_delete(0, path);
4250 return ret;
4251 }
4252
4253 return TRUE;
4254}
4255
4256static fil_space_t* fil_space_get_by_name(const char* name)
4257{
4258 ut_ad(mutex_own(&fil_system.mutex));
4259 for (fil_space_t* space = UT_LIST_GET_FIRST(fil_system.space_list);
4260 space != NULL;
4261 space = UT_LIST_GET_NEXT(space_list, space))
4262 if (!strcmp(space->name, name)) return space;
4263 return NULL;
4264}
4265
4266/***********************************************************************
4267Searches for matching tablespace file for given .delta file and space_id
4268in given directory. When matching tablespace found, renames it to match the
4269name of .delta file. If there was a tablespace with matching name and
4270mismatching ID, renames it to xtrabackup_tmp_#ID.ibd. If there was no
4271matching file, creates a new tablespace.
4272@return file handle of matched or created file */
4273static
4274pfs_os_file_t
4275xb_delta_open_matching_space(
4276 const char* dbname, /* in: path to destination database dir */
4277 const char* name, /* in: name of delta file (without .delta) */
4278 const xb_delta_info_t& info,
4279 char* real_name, /* out: full path of destination file */
4280 size_t real_name_len, /* out: buffer size for real_name */
4281 bool* success) /* out: indicates error. true = success */
4282{
4283 char dest_dir[FN_REFLEN];
4284 char dest_space_name[FN_REFLEN];
4285 fil_space_t* fil_space;
4286 pfs_os_file_t file;
4287 xb_filter_entry_t* table;
4288
4289 ut_a(dbname != NULL ||
4290 !fil_is_user_tablespace_id(info.space_id) ||
4291 info.space_id == ULINT_UNDEFINED);
4292
4293 *success = false;
4294
4295 if (dbname) {
4296 snprintf(dest_dir, FN_REFLEN, "%s/%s",
4297 xtrabackup_target_dir, dbname);
4298 os_normalize_path(dest_dir);
4299
4300 snprintf(dest_space_name, FN_REFLEN, "%s/%s", dbname, name);
4301 } else {
4302 snprintf(dest_dir, FN_REFLEN, "%s", xtrabackup_target_dir);
4303 os_normalize_path(dest_dir);
4304
4305 snprintf(dest_space_name, FN_REFLEN, "%s", name);
4306 }
4307
4308 snprintf(real_name, real_name_len,
4309 "%s/%s",
4310 xtrabackup_target_dir, dest_space_name);
4311 os_normalize_path(real_name);
4312 /* Truncate ".ibd" */
4313 dest_space_name[strlen(dest_space_name) - 4] = '\0';
4314
4315 /* Create the database directory if it doesn't exist yet */
4316 if (!os_file_create_directory(dest_dir, FALSE)) {
4317 msg("mariabackup: error: cannot create dir %s\n", dest_dir);
4318 return file;
4319 }
4320
4321 log_mutex_enter();
4322 if (!fil_is_user_tablespace_id(info.space_id)) {
4323found:
4324 /* open the file and return its handle */
4325
4326 file = os_file_create_simple_no_error_handling(
4327 0, real_name,
4328 OS_FILE_OPEN, OS_FILE_READ_WRITE, false, success);
4329
4330 if (!*success) {
4331 msg("mariabackup: Cannot open file %s\n", real_name);
4332 }
4333exit:
4334 log_mutex_exit();
4335 return file;
4336 }
4337
4338 /* remember space name for further reference */
4339 table = static_cast<xb_filter_entry_t *>
4340 (malloc(sizeof(xb_filter_entry_t) +
4341 strlen(dest_space_name) + 1));
4342
4343 table->name = ((char*)table) + sizeof(xb_filter_entry_t);
4344 strcpy(table->name, dest_space_name);
4345 HASH_INSERT(xb_filter_entry_t, name_hash, inc_dir_tables_hash,
4346 ut_fold_string(table->name), table);
4347
4348 mutex_enter(&fil_system.mutex);
4349 fil_space = fil_space_get_by_name(dest_space_name);
4350 mutex_exit(&fil_system.mutex);
4351
4352 if (fil_space != NULL) {
4353 if (fil_space->id == info.space_id
4354 || info.space_id == ULINT_UNDEFINED) {
4355 /* we found matching space */
4356 goto found;
4357 } else {
4358
4359 char tmpname[FN_REFLEN];
4360
4361 snprintf(tmpname, FN_REFLEN, "%s/xtrabackup_tmp_#" ULINTPF,
4362 dbname, fil_space->id);
4363
4364 msg("mariabackup: Renaming %s to %s.ibd\n",
4365 fil_space->name, tmpname);
4366
4367 if (fil_space->rename(tmpname, NULL, false)
4368 != DB_SUCCESS) {
4369 msg("mariabackup: Cannot rename %s to %s\n",
4370 fil_space->name, tmpname);
4371 goto exit;
4372 }
4373 }
4374 }
4375
4376 if (info.space_id == ULINT_UNDEFINED)
4377 {
4378 msg("mariabackup: Error: Cannot handle DDL operation on tablespace "
4379 "%s\n", dest_space_name);
4380 exit(EXIT_FAILURE);
4381 }
4382 mutex_enter(&fil_system.mutex);
4383 fil_space = fil_space_get_by_id(info.space_id);
4384 mutex_exit(&fil_system.mutex);
4385 if (fil_space != NULL) {
4386 char tmpname[FN_REFLEN];
4387
4388 strncpy(tmpname, dest_space_name, FN_REFLEN);
4389
4390 msg("mariabackup: Renaming %s to %s\n",
4391 fil_space->name, dest_space_name);
4392
4393 if (fil_space->rename(tmpname, NULL, false) != DB_SUCCESS)
4394 {
4395 msg("mariabackup: Cannot rename %s to %s\n",
4396 fil_space->name, dest_space_name);
4397 goto exit;
4398 }
4399
4400 goto found;
4401 }
4402
4403 /* No matching space found. create the new one. */
4404 const ulint flags = info.page_size.is_compressed()
4405 ? get_bit_shift(info.page_size.physical()
4406 >> (UNIV_ZIP_SIZE_SHIFT_MIN - 1))
4407 << FSP_FLAGS_POS_ZIP_SSIZE
4408 | FSP_FLAGS_MASK_POST_ANTELOPE
4409 | FSP_FLAGS_MASK_ATOMIC_BLOBS
4410 | (info.page_size.logical() == UNIV_PAGE_SIZE_ORIG
4411 ? 0
4412 : get_bit_shift(info.page_size.logical()
4413 >> (UNIV_ZIP_SIZE_SHIFT_MIN - 1))
4414 << FSP_FLAGS_POS_PAGE_SSIZE)
4415 : FSP_FLAGS_PAGE_SSIZE();
4416 ut_ad(page_size_t(flags).equals_to(info.page_size));
4417
4418 if (fil_space_create(dest_space_name, info.space_id, flags,
4419 FIL_TYPE_TABLESPACE, 0)) {
4420 *success = xb_space_create_file(real_name, info.space_id,
4421 flags, &file);
4422 } else {
4423 msg("mariabackup: Cannot create tablespace %s\n",
4424 dest_space_name);
4425 }
4426
4427 goto exit;
4428}
4429
4430/************************************************************************
4431Applies a given .delta file to the corresponding data file.
4432@return TRUE on success */
4433static
4434ibool
4435xtrabackup_apply_delta(
4436 const char* dirname, /* in: dir name of incremental */
4437 const char* dbname, /* in: database name (ibdata: NULL) */
4438 const char* filename, /* in: file name (not a path),
4439 including the .delta extension */
4440 void* /*data*/)
4441{
4442 pfs_os_file_t src_file;
4443 pfs_os_file_t dst_file;
4444 char src_path[FN_REFLEN];
4445 char dst_path[FN_REFLEN];
4446 char meta_path[FN_REFLEN];
4447 char space_name[FN_REFLEN];
4448 bool success;
4449
4450 ibool last_buffer = FALSE;
4451 ulint page_in_buffer;
4452 ulint incremental_buffers = 0;
4453
4454 xb_delta_info_t info(univ_page_size, SRV_TMP_SPACE_ID);
4455 ulint page_size;
4456 ulint page_size_shift;
4457 byte* incremental_buffer_base = NULL;
4458 byte* incremental_buffer;
4459
4460 size_t offset;
4461
4462 ut_a(xtrabackup_incremental);
4463
4464 if (dbname) {
4465 snprintf(src_path, sizeof(src_path), "%s/%s/%s",
4466 dirname, dbname, filename);
4467 snprintf(dst_path, sizeof(dst_path), "%s/%s/%s",
4468 xtrabackup_real_target_dir, dbname, filename);
4469 } else {
4470 snprintf(src_path, sizeof(src_path), "%s/%s",
4471 dirname, filename);
4472 snprintf(dst_path, sizeof(dst_path), "%s/%s",
4473 xtrabackup_real_target_dir, filename);
4474 }
4475 dst_path[strlen(dst_path) - 6] = '\0';
4476
4477 strncpy(space_name, filename, FN_REFLEN);
4478 space_name[strlen(space_name) - 6] = 0;
4479
4480 if (!get_meta_path(src_path, meta_path)) {
4481 goto error;
4482 }
4483
4484 os_normalize_path(dst_path);
4485 os_normalize_path(src_path);
4486 os_normalize_path(meta_path);
4487
4488 if (!xb_read_delta_metadata(meta_path, &info)) {
4489 goto error;
4490 }
4491
4492 page_size = info.page_size.physical();
4493 page_size_shift = get_bit_shift(page_size);
4494 msg("mariabackup: page size for %s is %zu bytes\n",
4495 src_path, page_size);
4496 if (page_size_shift < 10 ||
4497 page_size_shift > UNIV_PAGE_SIZE_SHIFT_MAX) {
4498 msg("mariabackup: error: invalid value of page_size "
4499 "(%zu bytes) read from %s\n", page_size, meta_path);
4500 goto error;
4501 }
4502
4503 src_file = os_file_create_simple_no_error_handling(
4504 0, src_path,
4505 OS_FILE_OPEN, OS_FILE_READ_WRITE, false, &success);
4506 if (!success) {
4507 os_file_get_last_error(TRUE);
4508 msg("mariabackup: error: cannot open %s\n", src_path);
4509 goto error;
4510 }
4511
4512 posix_fadvise(src_file, 0, 0, POSIX_FADV_SEQUENTIAL);
4513
4514 dst_file = xb_delta_open_matching_space(
4515 dbname, space_name, info,
4516 dst_path, sizeof(dst_path), &success);
4517 if (!success) {
4518 msg("mariabackup: error: cannot open %s\n", dst_path);
4519 goto error;
4520 }
4521
4522 posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED);
4523
4524 /* allocate buffer for incremental backup (4096 pages) */
4525 incremental_buffer_base = static_cast<byte *>
4526 (malloc((page_size / 4 + 1) * page_size));
4527 incremental_buffer = static_cast<byte *>
4528 (ut_align(incremental_buffer_base,
4529 page_size));
4530
4531 msg("Applying %s to %s...\n", src_path, dst_path);
4532
4533 while (!last_buffer) {
4534 ulint cluster_header;
4535
4536 /* read to buffer */
4537 /* first block of block cluster */
4538 offset = ((incremental_buffers * (page_size / 4))
4539 << page_size_shift);
4540 success = os_file_read(IORequestRead, src_file,
4541 incremental_buffer, offset, page_size);
4542 if (!success) {
4543 goto error;
4544 }
4545
4546 cluster_header = mach_read_from_4(incremental_buffer);
4547 switch(cluster_header) {
4548 case 0x78747261UL: /*"xtra"*/
4549 break;
4550 case 0x58545241UL: /*"XTRA"*/
4551 last_buffer = TRUE;
4552 break;
4553 default:
4554 msg("mariabackup: error: %s seems not "
4555 ".delta file.\n", src_path);
4556 goto error;
4557 }
4558
4559 /* FIXME: If the .delta modifies FSP_SIZE on page 0,
4560 extend the file to that size. */
4561
4562 for (page_in_buffer = 1; page_in_buffer < page_size / 4;
4563 page_in_buffer++) {
4564 if (mach_read_from_4(incremental_buffer + page_in_buffer * 4)
4565 == 0xFFFFFFFFUL)
4566 break;
4567 }
4568
4569 ut_a(last_buffer || page_in_buffer == page_size / 4);
4570
4571 /* read whole of the cluster */
4572 success = os_file_read(IORequestRead, src_file,
4573 incremental_buffer,
4574 offset, page_in_buffer * page_size);
4575 if (!success) {
4576 goto error;
4577 }
4578
4579 posix_fadvise(src_file, offset, page_in_buffer * page_size,
4580 POSIX_FADV_DONTNEED);
4581
4582 for (page_in_buffer = 1; page_in_buffer < page_size / 4;
4583 page_in_buffer++) {
4584 ulint offset_on_page;
4585
4586 offset_on_page = mach_read_from_4(incremental_buffer + page_in_buffer * 4);
4587
4588 if (offset_on_page == 0xFFFFFFFFUL)
4589 break;
4590
4591 uchar *buf = incremental_buffer + page_in_buffer * page_size;
4592 const os_offset_t off = os_offset_t(offset_on_page)*page_size;
4593
4594 if (off == 0) {
4595 /* Read tablespace size from page 0,
4596 and extend the file to specified size.*/
4597 os_offset_t n_pages = mach_read_from_4(
4598 buf + FSP_HEADER_OFFSET + FSP_SIZE);
4599 if (mach_read_from_4(buf
4600 + FIL_PAGE_SPACE_ID)) {
4601 if (!os_file_set_size(
4602 dst_path, dst_file,
4603 n_pages * page_size))
4604 goto error;
4605 } else if (fil_space_t* space
4606 = fil_system.sys_space) {
4607 /* The system tablespace can
4608 consist of multiple files. The
4609 first one has full tablespace
4610 size in page 0, but only the last
4611 file should be extended. */
4612 fil_node_t* n = UT_LIST_GET_FIRST(
4613 space->chain);
4614 bool fail = !strcmp(n->name, dst_path)
4615 && !fil_space_extend(
4616 space, (ulint)n_pages);
4617 if (fail) goto error;
4618 }
4619 }
4620
4621 success = os_file_write(IORequestWrite,
4622 dst_path, dst_file, buf, off, page_size);
4623 if (!success) {
4624 goto error;
4625 }
4626 }
4627
4628 /* Free file system buffer cache after the batch was written. */
4629#ifdef __linux__
4630 os_file_flush_func(dst_file);
4631#endif
4632 posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED);
4633
4634
4635 incremental_buffers++;
4636 }
4637
4638 free(incremental_buffer_base);
4639 if (src_file != OS_FILE_CLOSED) {
4640 os_file_close(src_file);
4641 os_file_delete(0,src_path);
4642 }
4643 if (dst_file != OS_FILE_CLOSED)
4644 os_file_close(dst_file);
4645 return TRUE;
4646
4647error:
4648 free(incremental_buffer_base);
4649 if (src_file != OS_FILE_CLOSED)
4650 os_file_close(src_file);
4651 if (dst_file != OS_FILE_CLOSED)
4652 os_file_close(dst_file);
4653 msg("mariabackup: Error: xtrabackup_apply_delta(): "
4654 "failed to apply %s to %s.\n", src_path, dst_path);
4655 return FALSE;
4656}
4657
4658/************************************************************************
4659Callback to handle datadir entry. Function of this type will be called
4660for each entry which matches the mask by xb_process_datadir.
4661@return should return TRUE on success */
4662typedef ibool (*handle_datadir_entry_func_t)(
4663/*=========================================*/
4664 const char* data_home_dir, /*!<in: path to datadir */
4665 const char* db_name, /*!<in: database name */
4666 const char* file_name, /*!<in: file name with suffix */
4667 void* arg); /*!<in: caller-provided data */
4668
4669/************************************************************************
4670Callback to handle datadir entry. Deletes entry if it has no matching
4671fil_space in fil_system directory.
4672@return FALSE if delete attempt was unsuccessful */
4673static
4674ibool
4675rm_if_not_found(
4676 const char* data_home_dir, /*!<in: path to datadir */
4677 const char* db_name, /*!<in: database name */
4678 const char* file_name, /*!<in: file name with suffix */
4679 void* arg __attribute__((unused)))
4680{
4681 char name[FN_REFLEN];
4682 xb_filter_entry_t* table;
4683
4684 snprintf(name, FN_REFLEN, "%s/%s", db_name, file_name);
4685 /* Truncate ".ibd" */
4686 name[strlen(name) - 4] = '\0';
4687
4688 HASH_SEARCH(name_hash, inc_dir_tables_hash, ut_fold_string(name),
4689 xb_filter_entry_t*,
4690 table, (void) 0,
4691 !strcmp(table->name, name));
4692
4693 if (!table) {
4694 snprintf(name, FN_REFLEN, "%s/%s/%s", data_home_dir,
4695 db_name, file_name);
4696 return os_file_delete(0, name);
4697 }
4698
4699 return(TRUE);
4700}
4701
4702/************************************************************************
4703Function enumerates files in datadir (provided by path) which are matched
4704by provided suffix. For each entry callback is called.
4705@return FALSE if callback for some entry returned FALSE */
4706static
4707ibool
4708xb_process_datadir(
4709 const char* path, /*!<in: datadir path */
4710 const char* suffix, /*!<in: suffix to match
4711 against */
4712 handle_datadir_entry_func_t func) /*!<in: callback */
4713{
4714 ulint ret;
4715 char dbpath[OS_FILE_MAX_PATH+1];
4716 os_file_dir_t dir;
4717 os_file_dir_t dbdir;
4718 os_file_stat_t dbinfo;
4719 os_file_stat_t fileinfo;
4720 ulint suffix_len;
4721 dberr_t err = DB_SUCCESS;
4722 static char current_dir[2];
4723
4724 current_dir[0] = FN_CURLIB;
4725 current_dir[1] = 0;
4726 srv_data_home = current_dir;
4727
4728 suffix_len = strlen(suffix);
4729
4730 /* datafile */
4731 dbdir = os_file_opendir(path, FALSE);
4732
4733 if (dbdir != NULL) {
4734 ret = fil_file_readdir_next_file(&err, path, dbdir,
4735 &fileinfo);
4736 while (ret == 0) {
4737 if (fileinfo.type == OS_FILE_TYPE_DIR) {
4738 goto next_file_item_1;
4739 }
4740
4741 if (strlen(fileinfo.name) > suffix_len
4742 && 0 == strcmp(fileinfo.name +
4743 strlen(fileinfo.name) - suffix_len,
4744 suffix)) {
4745 if (!func(
4746 path, NULL,
4747 fileinfo.name, NULL))
4748 {
4749 return(FALSE);
4750 }
4751 }
4752next_file_item_1:
4753 ret = fil_file_readdir_next_file(&err,
4754 path, dbdir,
4755 &fileinfo);
4756 }
4757
4758 os_file_closedir(dbdir);
4759 } else {
4760 msg("mariabackup: Cannot open dir %s\n",
4761 path);
4762 }
4763
4764 /* single table tablespaces */
4765 dir = os_file_opendir(path, FALSE);
4766
4767 if (dir == NULL) {
4768 msg("mariabackup: Cannot open dir %s\n",
4769 path);
4770 }
4771
4772 ret = fil_file_readdir_next_file(&err, path, dir,
4773 &dbinfo);
4774 while (ret == 0) {
4775 if (dbinfo.type == OS_FILE_TYPE_FILE
4776 || dbinfo.type == OS_FILE_TYPE_UNKNOWN) {
4777
4778 goto next_datadir_item;
4779 }
4780
4781 snprintf(dbpath, sizeof(dbpath)-1, "%s/%s", path, dbinfo.name);
4782
4783 os_normalize_path(dbpath);
4784
4785 dbdir = os_file_opendir(dbpath, FALSE);
4786
4787 if (dbdir != NULL) {
4788
4789 ret = fil_file_readdir_next_file(&err, dbpath, dbdir,
4790 &fileinfo);
4791 while (ret == 0) {
4792
4793 if (fileinfo.type == OS_FILE_TYPE_DIR) {
4794
4795 goto next_file_item_2;
4796 }
4797
4798 if (strlen(fileinfo.name) > suffix_len
4799 && 0 == strcmp(fileinfo.name +
4800 strlen(fileinfo.name) -
4801 suffix_len,
4802 suffix)) {
4803 /* The name ends in suffix; process
4804 the file */
4805 if (!func(
4806 path,
4807 dbinfo.name,
4808 fileinfo.name, NULL))
4809 {
4810 return(FALSE);
4811 }
4812 }
4813next_file_item_2:
4814 ret = fil_file_readdir_next_file(&err,
4815 dbpath, dbdir,
4816 &fileinfo);
4817 }
4818
4819 os_file_closedir(dbdir);
4820 }
4821next_datadir_item:
4822 ret = fil_file_readdir_next_file(&err,
4823 path,
4824 dir, &dbinfo);
4825 }
4826
4827 os_file_closedir(dir);
4828
4829 return(TRUE);
4830}
4831
4832/************************************************************************
4833Applies all .delta files from incremental_dir to the full backup.
4834@return TRUE on success. */
4835static
4836ibool
4837xtrabackup_apply_deltas()
4838{
4839 return xb_process_datadir(xtrabackup_incremental_dir, ".delta",
4840 xtrabackup_apply_delta);
4841}
4842
4843
4844static
4845void
4846innodb_free_param()
4847{
4848 srv_sys_space.shutdown();
4849 free_tmpdir(&mysql_tmpdir_list);
4850}
4851
4852
4853/** Store the current binary log coordinates in a specified file.
4854@param[in] filename file name
4855@param[in] name binary log file name
4856@param[in] pos binary log file position
4857@return whether the operation succeeded */
4858static bool
4859store_binlog_info(const char* filename, const char* name, ulonglong pos)
4860{
4861 FILE *fp = fopen(filename, "w");
4862
4863 if (!fp) {
4864 msg("mariabackup: failed to open '%s'\n", filename);
4865 return(false);
4866 }
4867
4868 fprintf(fp, "%s\t%llu\n", name, pos);
4869 fclose(fp);
4870
4871 return(true);
4872}
4873
4874/** Implement --prepare
4875@return whether the operation succeeded */
4876static bool
4877xtrabackup_prepare_func(char** argv)
4878{
4879 char metadata_path[FN_REFLEN];
4880
4881 /* cd to target-dir */
4882
4883 if (my_setwd(xtrabackup_real_target_dir,MYF(MY_WME)))
4884 {
4885 msg("mariabackup: cannot my_setwd %s\n",
4886 xtrabackup_real_target_dir);
4887 return(false);
4888 }
4889 msg("mariabackup: cd to %s\n", xtrabackup_real_target_dir);
4890
4891 int argc; for (argc = 0; argv[argc]; argc++) {}
4892 encryption_plugin_prepare_init(argc, argv);
4893
4894 xtrabackup_target_dir= mysql_data_home_buff;
4895 xtrabackup_target_dir[0]=FN_CURLIB; // all paths are relative from here
4896 xtrabackup_target_dir[1]=0;
4897 const lsn_t target_lsn = xtrabackup_incremental
4898 ? incremental_to_lsn : metadata_to_lsn;
4899
4900 /*
4901 read metadata of target
4902 */
4903 sprintf(metadata_path, "%s/%s", xtrabackup_target_dir,
4904 XTRABACKUP_METADATA_FILENAME);
4905
4906 if (!xtrabackup_read_metadata(metadata_path)) {
4907 msg("mariabackup: Error: failed to read metadata from '%s'\n",
4908 metadata_path);
4909 return(false);
4910 }
4911
4912 if (!strcmp(metadata_type, "full-backuped")) {
4913 if (xtrabackup_incremental) {
4914 msg("mariabackup: error: applying incremental backup "
4915 "needs a prepared target.\n");
4916 return(false);
4917 }
4918 msg("mariabackup: This target seems to be not prepared yet.\n");
4919 } else if (!strcmp(metadata_type, "log-applied")) {
4920 msg("mariabackup: This target seems to be already prepared.\n");
4921 } else {
4922 msg("mariabackup: This target does not have correct metadata.\n");
4923 return(false);
4924 }
4925
4926 bool ok = !xtrabackup_incremental
4927 || metadata_to_lsn == incremental_lsn;
4928 if (!ok) {
4929 msg("mariabackup: error: This incremental backup seems "
4930 "not to be proper for the target.\n"
4931 "mariabackup: Check 'to_lsn' of the target and "
4932 "'from_lsn' of the incremental.\n");
4933 return(false);
4934 }
4935
4936 srv_max_n_threads = 1000;
4937 srv_undo_logs = 1;
4938 srv_n_purge_threads = 1;
4939
4940 xb_filters_init();
4941
4942 srv_log_group_home_dir = NULL;
4943 srv_thread_concurrency = 1;
4944
4945 if (xtrabackup_incremental) {
4946 srv_operation = SRV_OPERATION_RESTORE_DELTA;
4947
4948 if (innodb_init_param()) {
4949 goto error_cleanup;
4950 }
4951
4952 sync_check_init();
4953 ut_d(sync_check_enable());
4954 ut_crc32_init();
4955 recv_sys_init();
4956 log_sys.create();
4957 recv_recovery_on = true;
4958
4959#ifdef WITH_INNODB_DISALLOW_WRITES
4960 srv_allow_writes_event = os_event_create(0);
4961 os_event_set(srv_allow_writes_event);
4962#endif
4963 dberr_t err = xb_data_files_init();
4964 if (err != DB_SUCCESS) {
4965 msg("mariabackup: error: xb_data_files_init() failed "
4966 "with error %s\n", ut_strerr(err));
4967 goto error_cleanup;
4968 }
4969
4970 inc_dir_tables_hash = hash_create(1000);
4971
4972 ok = xtrabackup_apply_deltas();
4973
4974 xb_data_files_close();
4975
4976 if (ok) {
4977 /* Cleanup datadir from tablespaces deleted
4978 between full and incremental backups */
4979
4980 xb_process_datadir("./", ".ibd", rm_if_not_found);
4981 }
4982
4983 xb_filter_hash_free(inc_dir_tables_hash);
4984
4985 fil_system.close();
4986#ifdef WITH_INNODB_DISALLOW_WRITES
4987 os_event_destroy(srv_allow_writes_event);
4988#endif
4989 innodb_free_param();
4990 log_sys.close();
4991 sync_check_close();
4992 if (!ok) goto error_cleanup;
4993 }
4994
4995 srv_operation = xtrabackup_export
4996 ? SRV_OPERATION_RESTORE_EXPORT : SRV_OPERATION_RESTORE;
4997
4998 if (innodb_init_param()) {
4999 goto error_cleanup;
5000 }
5001
5002 /* increase IO threads */
5003 if (srv_n_file_io_threads < 10) {
5004 srv_n_read_io_threads = 4;
5005 srv_n_write_io_threads = 4;
5006 }
5007
5008 msg("mariabackup: Starting InnoDB instance for recovery.\n"
5009 "mariabackup: Using %lld bytes for buffer pool "
5010 "(set by --use-memory parameter)\n", xtrabackup_use_memory);
5011
5012 srv_max_buf_pool_modified_pct = (double)max_buf_pool_modified_pct;
5013
5014 if (srv_max_dirty_pages_pct_lwm > srv_max_buf_pool_modified_pct) {
5015 srv_max_dirty_pages_pct_lwm = srv_max_buf_pool_modified_pct;
5016 }
5017
5018 if (innodb_init()) {
5019 goto error_cleanup;
5020 }
5021
5022 if (ok) {
5023 msg("Last binlog file %s, position %lld\n",
5024 trx_sys.recovered_binlog_filename,
5025 longlong(trx_sys.recovered_binlog_offset));
5026
5027 /* output to xtrabackup_binlog_pos_innodb and
5028 (if backup_safe_binlog_info was available on
5029 the server) to xtrabackup_binlog_info. In the
5030 latter case xtrabackup_binlog_pos_innodb
5031 becomes redundant and is created only for
5032 compatibility. */
5033 ok = store_binlog_info("xtrabackup_binlog_pos_innodb",
5034 trx_sys.recovered_binlog_filename,
5035 trx_sys.recovered_binlog_offset)
5036 && (!recover_binlog_info
5037 || store_binlog_info(XTRABACKUP_BINLOG_INFO,
5038 trx_sys.recovered_binlog_filename,
5039 trx_sys.recovered_binlog_offset));
5040 }
5041
5042 /* Check whether the log is applied enough or not. */
5043 if ((srv_start_lsn || fil_space_get(SRV_LOG_SPACE_FIRST_ID))
5044 && srv_start_lsn < target_lsn) {
5045 msg("mariabackup: error: "
5046 "The log was only applied up to LSN " LSN_PF
5047 ", instead of " LSN_PF "\n",
5048 srv_start_lsn, target_lsn);
5049 ok = false;
5050 }
5051#ifdef WITH_WSREP
5052 else if (ok) xb_write_galera_info(xtrabackup_incremental);
5053#endif
5054
5055 innodb_shutdown();
5056 innodb_free_param();
5057
5058 /* output to metadata file */
5059 if (ok) {
5060 char filename[FN_REFLEN];
5061
5062 strcpy(metadata_type, "log-applied");
5063
5064 if(xtrabackup_incremental
5065 && metadata_to_lsn < incremental_to_lsn)
5066 {
5067 metadata_to_lsn = incremental_to_lsn;
5068 metadata_last_lsn = incremental_last_lsn;
5069 }
5070
5071 sprintf(filename, "%s/%s", xtrabackup_target_dir, XTRABACKUP_METADATA_FILENAME);
5072 if (!xtrabackup_write_metadata(filename)) {
5073
5074 msg("mariabackup: Error: failed to write metadata "
5075 "to '%s'\n", filename);
5076 ok = false;
5077 } else if (xtrabackup_extra_lsndir) {
5078 sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_METADATA_FILENAME);
5079 if (!xtrabackup_write_metadata(filename)) {
5080 msg("mariabackup: Error: failed to write "
5081 "metadata to '%s'\n", filename);
5082 ok = false;
5083 }
5084 }
5085 }
5086
5087 if (ok) ok = apply_log_finish();
5088
5089 if (ok && xtrabackup_export)
5090 ok= (prepare_export() == 0);
5091
5092error_cleanup:
5093 xb_filters_free();
5094 return ok;
5095}
5096
5097/**************************************************************************
5098Append group name to xb_load_default_groups list. */
5099static
5100void
5101append_defaults_group(const char *group, const char *default_groups[],
5102 size_t default_groups_size)
5103{
5104 uint i;
5105 bool appended = false;
5106 for (i = 0; i < default_groups_size - 1; i++) {
5107 if (default_groups[i] == NULL) {
5108 default_groups[i] = group;
5109 appended = true;
5110 break;
5111 }
5112 }
5113 ut_a(appended);
5114}
5115
5116bool
5117xb_init()
5118{
5119 const char *mixed_options[4] = {NULL, NULL, NULL, NULL};
5120 int n_mixed_options;
5121
5122 /* sanity checks */
5123
5124 if (opt_slave_info
5125 && opt_no_lock
5126 && !opt_safe_slave_backup) {
5127 msg("Error: --slave-info is used with --no-lock but "
5128 "without --safe-slave-backup. The binlog position "
5129 "cannot be consistent with the backup data.\n");
5130 return(false);
5131 }
5132
5133 if (xtrabackup_backup && opt_rsync)
5134 {
5135 if (xtrabackup_stream_fmt)
5136 {
5137 msg("Error: --rsync doesn't work with --stream\n");
5138 return(false);
5139 }
5140 bool have_rsync = IF_WIN(false, (system("rsync --version > /dev/null 2>&1") == 0));
5141 if (!have_rsync)
5142 {
5143 msg("Error: rsync executable not found, cannot run backup with --rsync\n");
5144 return false;
5145 }
5146 }
5147
5148 n_mixed_options = 0;
5149
5150 if (opt_decompress) {
5151 mixed_options[n_mixed_options++] = "--decompress";
5152 }
5153
5154 if (xtrabackup_copy_back) {
5155 mixed_options[n_mixed_options++] = "--copy-back";
5156 }
5157
5158 if (xtrabackup_move_back) {
5159 mixed_options[n_mixed_options++] = "--move-back";
5160 }
5161
5162 if (xtrabackup_prepare) {
5163 mixed_options[n_mixed_options++] = "--apply-log";
5164 }
5165
5166 if (n_mixed_options > 1) {
5167 msg("Error: %s and %s are mutually exclusive\n",
5168 mixed_options[0], mixed_options[1]);
5169 return(false);
5170 }
5171
5172 if (xtrabackup_backup) {
5173 if ((mysql_connection = xb_mysql_connect()) == NULL) {
5174 return(false);
5175 }
5176
5177 if (!get_mysql_vars(mysql_connection)) {
5178 return(false);
5179 }
5180
5181 encryption_plugin_backup_init(mysql_connection);
5182 history_start_time = time(NULL);
5183
5184 }
5185
5186 return(true);
5187}
5188
5189
5190extern void init_signals(void);
5191
5192#include <sql_locale.h>
5193
5194/* Messages . Avoid loading errmsg.sys file */
5195void setup_error_messages()
5196{
5197 static const char *my_msgs[ERRORS_PER_RANGE];
5198 static const char **all_msgs[] = { my_msgs, my_msgs, my_msgs, my_msgs };
5199 my_default_lc_messages = &my_locale_en_US;
5200 my_default_lc_messages->errmsgs->errmsgs = all_msgs;
5201
5202 /* Populate the necessary error messages */
5203 struct {
5204 int id;
5205 const char *fmt;
5206 }
5207 xb_msgs[] =
5208 {
5209 { ER_DATABASE_NAME,"Database" },
5210 { ER_TABLE_NAME,"Table"},
5211 { ER_PARTITION_NAME, "Partition" },
5212 { ER_SUBPARTITION_NAME, "Subpartition" },
5213 { ER_TEMPORARY_NAME, "Temporary"},
5214 { ER_RENAMED_NAME, "Renamed"},
5215 { ER_CANT_FIND_DL_ENTRY, "Can't find symbol '%-.128s' in library"},
5216 { ER_CANT_OPEN_LIBRARY, "Can't open shared library '%-.192s' (errno: %d, %-.128s)" },
5217 { ER_OUTOFMEMORY, "Out of memory; restart server and try again (needed %d bytes)" },
5218 { ER_CANT_OPEN_LIBRARY, "Can't open shared library '%-.192s' (errno: %d, %-.128s)" },
5219 { ER_UDF_NO_PATHS, "No paths allowed for shared library" },
5220 { ER_CANT_INITIALIZE_UDF,"Can't initialize function '%-.192s'; %-.80s"},
5221 { ER_PLUGIN_IS_NOT_LOADED,"Plugin '%-.192s' is not loaded" }
5222 };
5223
5224 for (int i = 0; i < (int)array_elements(all_msgs); i++)
5225 all_msgs[0][i] = "Unknown error";
5226
5227 for (int i = 0; i < (int)array_elements(xb_msgs); i++)
5228 all_msgs[0][xb_msgs[i].id - ER_ERROR_FIRST] = xb_msgs[i].fmt;
5229}
5230
5231void
5232handle_options(int argc, char **argv, char ***argv_client, char ***argv_server)
5233{
5234 /* Setup some variables for Innodb.*/
5235
5236 srv_operation = SRV_OPERATION_RESTORE;
5237
5238 files_charset_info = &my_charset_utf8_general_ci;
5239 check_if_backup_includes = backup_includes;
5240
5241 setup_error_messages();
5242 sys_var_init();
5243 plugin_mutex_init();
5244 mysql_prlock_init(key_rwlock_LOCK_system_variables_hash, &LOCK_system_variables_hash);
5245 opt_stack_trace = 1;
5246 test_flags |= TEST_SIGINT;
5247 init_signals();
5248#ifndef _WIN32
5249 /* Exit process on SIGINT. */
5250 my_sigset(SIGINT, SIG_DFL);
5251#endif
5252
5253 sf_leaking_memory = 1; /* don't report memory leaks on early exist */
5254
5255 int i;
5256 int ho_error;
5257
5258 char* target_dir = NULL;
5259 bool prepare = false;
5260
5261 char conf_file[FN_REFLEN];
5262 int argc_client = argc;
5263 int argc_server = argc;
5264
5265 /* scan options for group and config file to load defaults from */
5266 for (i = 1; i < argc; i++) {
5267
5268 char *optend = strcend(argv[i], '=');
5269
5270 if (strncmp(argv[i], "--defaults-group",
5271 optend - argv[i]) == 0) {
5272 defaults_group = optend + 1;
5273 append_defaults_group(defaults_group,
5274 xb_server_default_groups,
5275 array_elements(xb_server_default_groups));
5276 }
5277
5278 if (strncmp(argv[i], "--login-path",
5279 optend - argv[i]) == 0) {
5280 append_defaults_group(optend + 1,
5281 xb_client_default_groups,
5282 array_elements(xb_client_default_groups));
5283 }
5284
5285 if (!strncmp(argv[i], "--prepare",
5286 optend - argv[i])) {
5287 prepare = true;
5288 }
5289
5290 if (!strncmp(argv[i], "--apply-log",
5291 optend - argv[i])) {
5292 prepare = true;
5293 }
5294
5295 if (!strncmp(argv[i], "--target-dir",
5296 optend - argv[i]) && *optend) {
5297 target_dir = optend + 1;
5298 }
5299
5300 if (!*optend && argv[i][0] != '-') {
5301 target_dir = argv[i];
5302 }
5303 }
5304
5305 snprintf(conf_file, sizeof(conf_file), "my");
5306
5307 if (prepare && target_dir) {
5308 snprintf(conf_file, sizeof(conf_file),
5309 "%s/backup-my.cnf", target_dir);
5310 if (!strncmp(argv[1], "--defaults-file=", 16)) {
5311 /* Remove defaults-file*/
5312 for (int i = 2; ; i++) {
5313 if ((argv[i-1]= argv[i]) == 0)
5314 break;
5315 }
5316 argc--;
5317 }
5318 }
5319
5320 *argv_client = argv;
5321 *argv_server = argv;
5322 load_defaults_or_exit(conf_file, xb_server_default_groups,
5323 &argc_server, argv_server);
5324
5325 int n;
5326 for (n = 0; (*argv_server)[n]; n++) {};
5327 argc_server = n;
5328
5329 print_param_str <<
5330 "# This MySQL options file was generated by XtraBackup.\n"
5331 "[" << defaults_group << "]\n";
5332
5333 /* We want xtrabackup to ignore unknown options, because it only
5334 recognizes a small subset of server variables */
5335 my_getopt_skip_unknown = TRUE;
5336
5337 /* Reset u_max_value for all options, as we don't want the
5338 --maximum-... modifier to set the actual option values */
5339 for (my_option *optp= xb_server_options; optp->name; optp++) {
5340 optp->u_max_value = (G_PTR *) &global_max_value;
5341 }
5342
5343
5344 /* Throw a descriptive error if --defaults-file or --defaults-extra-file
5345 is not the first command line argument */
5346 for (int i = 2 ; i < argc ; i++) {
5347 char *optend = strcend((argv)[i], '=');
5348
5349 if (optend - argv[i] == 15 &&
5350 !strncmp(argv[i], "--defaults-file", optend - argv[i])) {
5351
5352 msg("mariabackup: Error: --defaults-file "
5353 "must be specified first on the command "
5354 "line\n");
5355 exit(EXIT_FAILURE);
5356 }
5357 if (optend - argv[i] == 21 &&
5358 !strncmp(argv[i], "--defaults-extra-file",
5359 optend - argv[i])) {
5360
5361 msg("mariabackup: Error: --defaults-extra-file "
5362 "must be specified first on the command "
5363 "line\n");
5364 exit(EXIT_FAILURE);
5365 }
5366 }
5367
5368 if (argc_server > 0
5369 && (ho_error=handle_options(&argc_server, argv_server,
5370 xb_server_options, xb_get_one_option)))
5371 exit(ho_error);
5372
5373 load_defaults_or_exit(conf_file, xb_client_default_groups,
5374 &argc_client, argv_client);
5375
5376 for (n = 0; (*argv_client)[n]; n++) {};
5377 argc_client = n;
5378
5379 if (innobackupex_mode && argc_client > 0) {
5380 /* emulate innobackupex script */
5381 innobackupex_mode = true;
5382 if (!ibx_handle_options(&argc_client, argv_client)) {
5383 exit(EXIT_FAILURE);
5384 }
5385 }
5386
5387 if (argc_client > 0
5388 && (ho_error=handle_options(&argc_client, argv_client,
5389 xb_client_options, xb_get_one_option)))
5390 exit(ho_error);
5391
5392 /* Reject command line arguments that don't look like options, i.e. are
5393 not of the form '-X' (single-character options) or '--option' (long
5394 options) */
5395 for (int i = 0 ; i < argc_client ; i++) {
5396 const char * const opt = (*argv_client)[i];
5397
5398 if (strncmp(opt, "--", 2) &&
5399 !(strlen(opt) == 2 && opt[0] == '-')) {
5400 bool server_option = true;
5401
5402 for (int j = 0; j < argc_server; j++) {
5403 if (opt == (*argv_server)[j]) {
5404 server_option = false;
5405 break;
5406 }
5407 }
5408
5409 if (!server_option) {
5410 msg("mariabackup: Error:"
5411 " unknown argument: '%s'\n", opt);
5412 exit(EXIT_FAILURE);
5413 }
5414 }
5415 }
5416}
5417
5418static int main_low(char** argv);
5419static int get_exepath(char *buf, size_t size, const char *argv0);
5420
5421/* ================= main =================== */
5422int main(int argc, char **argv)
5423{
5424 char **client_defaults, **server_defaults;
5425
5426 if (get_exepath(mariabackup_exe,FN_REFLEN, argv[0]))
5427 strncpy(mariabackup_exe,argv[0], FN_REFLEN-1);
5428
5429
5430 if (argc > 1 )
5431 {
5432 /* In "prepare export", we need to start mysqld
5433 Since it is not always be installed on the machine,
5434 we start "mariabackup --mysqld", which acts as mysqld
5435 */
5436 if (strcmp(argv[1], "--mysqld") == 0)
5437 {
5438 extern int mysqld_main(int argc, char **argv);
5439 argc--;
5440 argv++;
5441 argv[0]+=2;
5442 return mysqld_main(argc, argv);
5443 }
5444 if(strcmp(argv[1], "--innobackupex") == 0)
5445 {
5446 argv++;
5447 argc--;
5448 innobackupex_mode = true;
5449 }
5450 }
5451
5452 if (argc > 1)
5453 strncpy(orig_argv1,argv[1],sizeof(orig_argv1) -1);
5454
5455 init_signals();
5456 MY_INIT(argv[0]);
5457
5458 pthread_key_create(&THR_THD, NULL);
5459 my_pthread_setspecific_ptr(THR_THD, NULL);
5460
5461 xb_regex_init();
5462
5463 capture_tool_command(argc, argv);
5464
5465 if (mysql_server_init(-1, NULL, NULL))
5466 {
5467 exit(EXIT_FAILURE);
5468 }
5469
5470 system_charset_info = &my_charset_utf8_general_ci;
5471 key_map_full.set_all();
5472
5473 logger.init_base();
5474 logger.set_handlers(LOG_FILE, LOG_NONE, LOG_NONE);
5475 mysql_mutex_init(key_LOCK_error_log, &LOCK_error_log,
5476 MY_MUTEX_INIT_FAST);
5477
5478 handle_options(argc, argv, &client_defaults, &server_defaults);
5479
5480#ifndef DBUG_OFF
5481 if (dbug_option) {
5482 DBUG_SET_INITIAL(dbug_option);
5483 DBUG_SET(dbug_option);
5484 }
5485#endif
5486
5487 int status = main_low(server_defaults);
5488
5489 backup_cleanup();
5490
5491 if (innobackupex_mode) {
5492 ibx_cleanup();
5493 }
5494
5495 free_defaults(client_defaults);
5496 free_defaults(server_defaults);
5497
5498#ifndef DBUG_OFF
5499 if (dbug_option) {
5500 DBUG_END();
5501 }
5502#endif
5503
5504 if (THR_THD)
5505 (void) pthread_key_delete(THR_THD);
5506
5507 logger.cleanup_base();
5508 mysql_mutex_destroy(&LOCK_error_log);
5509
5510 if (status == EXIT_SUCCESS) {
5511 msg_ts("completed OK!\n");
5512 }
5513
5514 return status;
5515}
5516
5517static int main_low(char** argv)
5518{
5519 if (innobackupex_mode) {
5520 if (!ibx_init()) {
5521 return(EXIT_FAILURE);
5522 }
5523 }
5524
5525 if (!xtrabackup_print_param && !xtrabackup_prepare
5526 && !strcmp(mysql_data_home, "./")) {
5527 if (!xtrabackup_print_param)
5528 usage();
5529 msg("\nmariabackup: Error: Please set parameter 'datadir'\n");
5530 return(EXIT_FAILURE);
5531 }
5532
5533 /* Expand target-dir, incremental-basedir, etc. */
5534
5535 char cwd[FN_REFLEN];
5536 my_getwd(cwd, sizeof(cwd), MYF(0));
5537
5538 my_load_path(xtrabackup_real_target_dir,
5539 xtrabackup_target_dir, cwd);
5540 unpack_dirname(xtrabackup_real_target_dir,
5541 xtrabackup_real_target_dir);
5542 xtrabackup_target_dir= xtrabackup_real_target_dir;
5543
5544 if (xtrabackup_incremental_basedir) {
5545 my_load_path(xtrabackup_real_incremental_basedir,
5546 xtrabackup_incremental_basedir, cwd);
5547 unpack_dirname(xtrabackup_real_incremental_basedir,
5548 xtrabackup_real_incremental_basedir);
5549 xtrabackup_incremental_basedir =
5550 xtrabackup_real_incremental_basedir;
5551 }
5552
5553 if (xtrabackup_incremental_dir) {
5554 my_load_path(xtrabackup_real_incremental_dir,
5555 xtrabackup_incremental_dir, cwd);
5556 unpack_dirname(xtrabackup_real_incremental_dir,
5557 xtrabackup_real_incremental_dir);
5558 xtrabackup_incremental_dir = xtrabackup_real_incremental_dir;
5559 }
5560
5561 if (xtrabackup_extra_lsndir) {
5562 my_load_path(xtrabackup_real_extra_lsndir,
5563 xtrabackup_extra_lsndir, cwd);
5564 unpack_dirname(xtrabackup_real_extra_lsndir,
5565 xtrabackup_real_extra_lsndir);
5566 xtrabackup_extra_lsndir = xtrabackup_real_extra_lsndir;
5567 }
5568
5569 /* get default temporary directory */
5570 if (!opt_mysql_tmpdir || !opt_mysql_tmpdir[0]) {
5571 opt_mysql_tmpdir = getenv("TMPDIR");
5572#if defined(__WIN__)
5573 if (!opt_mysql_tmpdir) {
5574 opt_mysql_tmpdir = getenv("TEMP");
5575 }
5576 if (!opt_mysql_tmpdir) {
5577 opt_mysql_tmpdir = getenv("TMP");
5578 }
5579#endif
5580 if (!opt_mysql_tmpdir || !opt_mysql_tmpdir[0]) {
5581 opt_mysql_tmpdir = const_cast<char*>(DEFAULT_TMPDIR);
5582 }
5583 }
5584
5585 /* temporary setting of enough size */
5586 srv_page_size_shift = UNIV_PAGE_SIZE_SHIFT_MAX;
5587 srv_page_size = UNIV_PAGE_SIZE_MAX;
5588 if (xtrabackup_backup && xtrabackup_incremental) {
5589 /* direct specification is only for --backup */
5590 /* and the lsn is prior to the other option */
5591
5592 char* endchar;
5593 int error = 0;
5594 incremental_lsn = strtoll(xtrabackup_incremental, &endchar, 10);
5595 if (*endchar != '\0')
5596 error = 1;
5597
5598 if (error) {
5599 msg("mariabackup: value '%s' may be wrong format for "
5600 "incremental option.\n", xtrabackup_incremental);
5601 return(EXIT_FAILURE);
5602 }
5603 } else if (xtrabackup_backup && xtrabackup_incremental_basedir) {
5604 char filename[FN_REFLEN];
5605
5606 sprintf(filename, "%s/%s", xtrabackup_incremental_basedir, XTRABACKUP_METADATA_FILENAME);
5607
5608 if (!xtrabackup_read_metadata(filename)) {
5609 msg("mariabackup: error: failed to read metadata from "
5610 "%s\n", filename);
5611 return(EXIT_FAILURE);
5612 }
5613
5614 incremental_lsn = metadata_to_lsn;
5615 xtrabackup_incremental = xtrabackup_incremental_basedir; //dummy
5616 } else if (xtrabackup_prepare && xtrabackup_incremental_dir) {
5617 char filename[FN_REFLEN];
5618
5619 sprintf(filename, "%s/%s", xtrabackup_incremental_dir, XTRABACKUP_METADATA_FILENAME);
5620
5621 if (!xtrabackup_read_metadata(filename)) {
5622 msg("mariabackup: error: failed to read metadata from "
5623 "%s\n", filename);
5624 return(EXIT_FAILURE);
5625 }
5626
5627 incremental_lsn = metadata_from_lsn;
5628 incremental_to_lsn = metadata_to_lsn;
5629 incremental_last_lsn = metadata_last_lsn;
5630 xtrabackup_incremental = xtrabackup_incremental_dir; //dummy
5631
5632 } else if (opt_incremental_history_name) {
5633 xtrabackup_incremental = opt_incremental_history_name;
5634 } else if (opt_incremental_history_uuid) {
5635 xtrabackup_incremental = opt_incremental_history_uuid;
5636 } else {
5637 xtrabackup_incremental = NULL;
5638 }
5639
5640 if (xtrabackup_stream && !xtrabackup_backup) {
5641 msg("Warning: --stream parameter is ignored, it only works together with --backup.\n");
5642 }
5643
5644 if (!xb_init()) {
5645 return(EXIT_FAILURE);
5646 }
5647
5648 /* --print-param */
5649 if (xtrabackup_print_param) {
5650 printf("%s", print_param_str.str().c_str());
5651 return(EXIT_SUCCESS);
5652 }
5653
5654 print_version();
5655 if (xtrabackup_incremental) {
5656 msg("incremental backup from " LSN_PF " is enabled.\n",
5657 incremental_lsn);
5658 }
5659
5660 if (xtrabackup_export && innobase_file_per_table == FALSE) {
5661 msg("mariabackup: auto-enabling --innodb-file-per-table due to "
5662 "the --export option\n");
5663 innobase_file_per_table = TRUE;
5664 }
5665
5666 /* cannot execute both for now */
5667 {
5668 int num = 0;
5669
5670 if (xtrabackup_backup) num++;
5671 if (xtrabackup_prepare) num++;
5672 if (xtrabackup_copy_back) num++;
5673 if (xtrabackup_move_back) num++;
5674 if (xtrabackup_decrypt_decompress) num++;
5675 if (num != 1) { /* !XOR (for now) */
5676 usage();
5677 return(EXIT_FAILURE);
5678 }
5679 }
5680
5681#ifndef __WIN__
5682 if (xtrabackup_debug_sync) {
5683 signal(SIGCONT, sigcont_handler);
5684 }
5685#endif
5686
5687 /* --backup */
5688 if (xtrabackup_backup && !xtrabackup_backup_func()) {
5689 return(EXIT_FAILURE);
5690 }
5691
5692 /* --prepare */
5693 if (xtrabackup_prepare
5694 && !xtrabackup_prepare_func(argv)) {
5695 return(EXIT_FAILURE);
5696 }
5697
5698 if (xtrabackup_copy_back || xtrabackup_move_back) {
5699 if (!check_if_param_set("datadir")) {
5700 msg("Error: datadir must be specified.\n");
5701 return(EXIT_FAILURE);
5702 }
5703 if (!copy_back())
5704 return(EXIT_FAILURE);
5705 }
5706
5707 if (xtrabackup_decrypt_decompress && !decrypt_decompress()) {
5708 return(EXIT_FAILURE);
5709 }
5710
5711 return(EXIT_SUCCESS);
5712}
5713
5714
5715static int get_exepath(char *buf, size_t size, const char *argv0)
5716{
5717#ifdef _WIN32
5718 DWORD ret = GetModuleFileNameA(NULL, buf, (DWORD)size);
5719 if (ret > 0)
5720 return 0;
5721#elif defined(__linux__)
5722 ssize_t ret = readlink("/proc/self/exe", buf, size-1);
5723 if(ret > 0)
5724 return 0;
5725#endif
5726
5727 return my_realpath(buf, argv0, 0);
5728}
5729