1/*
2 Copyright (c) 2005, 2016, Oracle and/or its affiliates. All rights reserved.
3 Copyright (c) 2014, 2017, MariaDB Corporation.
4
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; version 2 of the License.
8
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17*/
18
19/*
20 InnoDB offline file checksum utility. 85% of the code in this utility
21 is included from the InnoDB codebase.
22
23 The final 15% was originally written by Mark Smith of Danga
24 Interactive, Inc. <junior@danga.com>
25
26 Published with a permission.
27*/
28
29#include <my_global.h>
30#include <stdio.h>
31#include <stdlib.h>
32#include <time.h>
33#include <sys/types.h>
34#include <sys/stat.h>
35#ifndef __WIN__
36# include <unistd.h>
37#endif
38#include <my_getopt.h>
39#include <m_string.h>
40#include <welcome_copyright_notice.h> /* ORACLE_WELCOME_COPYRIGHT_NOTICE */
41
42/* Only parts of these files are included from the InnoDB codebase.
43The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */
44
45typedef void fil_space_t;
46
47#include "univ.i" /* include all of this */
48#include "page0size.h"
49
50#define FLST_BASE_NODE_SIZE (4 + 2 * FIL_ADDR_SIZE)
51#define FLST_NODE_SIZE (2 * FIL_ADDR_SIZE)
52#define FSEG_PAGE_DATA FIL_PAGE_DATA
53#define FSEG_HEADER_SIZE 10
54#define UT_BITS_IN_BYTES(b) (((b) + 7) / 8)
55
56#include "ut0ut.h"
57#include "ut0byte.h"
58#include "mtr0types.h"
59#include "mach0data.h"
60#include "fsp0types.h"
61#include "rem0rec.h"
62#include "buf0checksum.h" /* buf_calc_page_*() */
63#include "buf0buf.h" /* buf_page_is_corrupted */
64#include "fil0fil.h" /* FIL_* */
65#include "page0page.h" /* PAGE_* */
66#include "page0zip.h" /* page_zip_*() */
67#include "trx0undo.h" /* TRX_* */
68#include "fsp0fsp.h" /* fsp_flags_get_page_size() &
69 fsp_flags_get_zip_size() */
70#include "ut0crc32.h" /* ut_crc32_init() */
71#include "fsp0pagecompress.h" /* fil_get_compression_alg_name */
72#include "fil0crypt.h" /* fil_space_verify_crypt_checksum */
73
74#include <string.h>
75
76#ifdef UNIV_NONINL
77# include "fsp0fsp.ic"
78# include "mach0data.ic"
79# include "ut0rnd.ic"
80#endif
81
82#ifndef PRIuMAX
83#define PRIuMAX "llu"
84#endif
85
86/* Global variables */
87static bool verbose;
88static bool just_count;
89static unsigned long long start_page;
90static unsigned long long end_page;
91static unsigned long long do_page;
92static bool use_end_page;
93static bool do_one_page;
94static my_bool do_leaf;
95static my_bool per_page_details;
96static ulint n_merge;
97extern ulong srv_checksum_algorithm;
98static ulint physical_page_size; /* Page size in bytes on disk. */
99static ulint logical_page_size; /* Page size when uncompressed. */
100ulong srv_page_size;
101ulong srv_page_size_shift;
102page_size_t univ_page_size(0, 0, false);
103/* Current page number (0 based). */
104unsigned long long cur_page_num;
105/* Skip the checksum verification. */
106static bool no_check;
107/* Enabled for strict checksum verification. */
108bool strict_verify = 0;
109/* Enabled for rewrite checksum. */
110static bool do_write;
111/* Mismatches count allowed (0 by default). */
112static unsigned long long allow_mismatches=0;
113static bool page_type_summary;
114static bool page_type_dump;
115/* Store filename for page-type-dump option. */
116char* page_dump_filename = 0;
117/* skip the checksum verification & rewrite if page is doublewrite buffer. */
118static bool skip_page = 0;
119const char *dbug_setting = "FALSE";
120char* log_filename = NULL;
121/* User defined filename for logging. */
122FILE* log_file = NULL;
123/* Enabled for log write option. */
124static bool is_log_enabled = false;
125
126#ifndef _WIN32
127/* advisory lock for non-window system. */
128struct flock lk;
129#endif /* _WIN32 */
130
131/* Strict check algorithm name. */
132static ulong strict_check;
133/* Rewrite checksum algorithm name. */
134static ulong write_check;
135
136/* Innodb page type. */
137struct innodb_page_type {
138 int n_undo_state_active;
139 int n_undo_state_cached;
140 int n_undo_state_to_free;
141 int n_undo_state_to_purge;
142 int n_undo_state_prepared;
143 int n_undo_state_other;
144 int n_undo_insert;
145 int n_undo_update;
146 int n_undo_other;
147 int n_fil_page_index;
148 int n_fil_page_undo_log;
149 int n_fil_page_inode;
150 int n_fil_page_ibuf_free_list;
151 int n_fil_page_ibuf_bitmap;
152 int n_fil_page_type_sys;
153 int n_fil_page_type_trx_sys;
154 int n_fil_page_type_fsp_hdr;
155 int n_fil_page_type_allocated;
156 int n_fil_page_type_xdes;
157 int n_fil_page_type_blob;
158 int n_fil_page_type_zblob;
159 int n_fil_page_type_other;
160 int n_fil_page_type_zblob2;
161 int n_fil_page_type_page_compressed;
162 int n_fil_page_type_page_compressed_encrypted;
163} page_type;
164
165/* Possible values for "--strict-check" for strictly verify checksum
166and "--write" for rewrite checksum. */
167static const char *innochecksum_algorithms[] = {
168 "crc32",
169 "crc32",
170 "innodb",
171 "innodb",
172 "none",
173 "none",
174 NullS
175};
176
177/* Used to define an enumerate type of the "innochecksum algorithm". */
178static TYPELIB innochecksum_algorithms_typelib = {
179 array_elements(innochecksum_algorithms)-1,"",
180 innochecksum_algorithms, NULL
181};
182
183#define SIZE_RANGES_FOR_PAGE 10
184#define NUM_RETRIES 3
185#define DEFAULT_RETRY_DELAY 1000000
186
187struct per_page_stats {
188 ulint n_recs;
189 ulint data_size;
190 ulint left_page_no;
191 ulint right_page_no;
192 per_page_stats(ulint n, ulint data, ulint left, ulint right) :
193 n_recs(n), data_size(data), left_page_no(left), right_page_no(right) {}
194 per_page_stats() : n_recs(0), data_size(0), left_page_no(0), right_page_no(0) {}
195};
196
197struct per_index_stats {
198 unsigned long long pages;
199 unsigned long long leaf_pages;
200 ulint first_leaf_page;
201 ulint count;
202 ulint free_pages;
203 ulint max_data_size;
204 unsigned long long total_n_recs;
205 unsigned long long total_data_bytes;
206
207 /*!< first element for empty pages,
208 last element for pages with more than logical_page_size */
209 unsigned long long pages_in_size_range[SIZE_RANGES_FOR_PAGE+2];
210
211 std::map<unsigned long long, per_page_stats> leaves;
212
213 per_index_stats():pages(0), leaf_pages(0), first_leaf_page(0),
214 count(0), free_pages(0), max_data_size(0), total_n_recs(0),
215 total_data_bytes(0)
216 {
217 memset(pages_in_size_range, 0, sizeof(pages_in_size_range));
218 }
219};
220
221std::map<unsigned long long, per_index_stats> index_ids;
222
223void print_index_leaf_stats(
224 unsigned long long id,
225 const per_index_stats& index,
226 FILE* fil_out)
227
228{
229 ulint page_no = index.first_leaf_page;
230 std::map<unsigned long long, per_page_stats>::const_iterator it_page = index.leaves.find(page_no);
231 fprintf(fil_out, "\nindex: %llu leaf page stats: n_pages = %llu\n",
232 id, index.leaf_pages);
233 fprintf(fil_out, "page_no\tdata_size\tn_recs\n");
234 while (it_page != index.leaves.end()) {
235 const per_page_stats& stat = it_page->second;
236 fprintf(fil_out, "%llu\t" ULINTPF "\t" ULINTPF "\n", it_page->first, stat.data_size, stat.n_recs);
237 page_no = stat.right_page_no;
238 it_page = index.leaves.find(page_no);
239 }
240}
241
242void defrag_analysis(
243 unsigned long long id,
244 const per_index_stats& index,
245 FILE* fil_out)
246{
247 // TODO: make it work for compressed pages too
248 std::map<unsigned long long, per_page_stats>::const_iterator it = index.leaves.find(index.first_leaf_page);
249 ulint n_pages = 0;
250 ulint n_leaf_pages = 0;
251 while (it != index.leaves.end()) {
252 ulint data_size_total = 0;
253 for (ulong i = 0; i < n_merge; i++) {
254 const per_page_stats& stat = it->second;
255 n_leaf_pages ++;
256 data_size_total += stat.data_size;
257 it = index.leaves.find(stat.right_page_no);
258 if (it == index.leaves.end()) {
259 break;
260 }
261 }
262
263 if (index.max_data_size) {
264 n_pages += data_size_total / index.max_data_size;
265 if (data_size_total % index.max_data_size != 0) {
266 n_pages += 1;
267 }
268 }
269 }
270
271 if (index.leaf_pages) {
272 fprintf(fil_out, "count = " ULINTPF " free = " ULINTPF "\n", index.count, index.free_pages);
273 }
274
275 if (!n_leaf_pages) {
276 n_leaf_pages = 1;
277 }
278
279 fprintf(fil_out, "%llu\t\t%llu\t\t" ULINTPF "\t\t" ULINTPF "\t\t" ULINTPF "\t\t%.2f\t" ULINTPF "\n",
280 id, index.leaf_pages, n_leaf_pages, n_merge, n_pages,
281 1.0 - (double)n_pages / (double)n_leaf_pages, index.max_data_size);
282}
283
284void print_leaf_stats(
285 FILE* fil_out)
286{
287 fprintf(fil_out, "\n**************************************************\n");
288 fprintf(fil_out, "index_id\t#leaf_pages\t#actual_leaf_pages\tn_merge\t"
289 "#leaf_after_merge\tdefrag\n");
290 for (std::map<unsigned long long, per_index_stats>::const_iterator it = index_ids.begin();
291 it != index_ids.end(); it++) {
292 const per_index_stats& index = it->second;
293
294 if (verbose) {
295 print_index_leaf_stats(it->first, index, fil_out);
296 }
297
298 if (n_merge) {
299 defrag_analysis(it->first, index, fil_out);
300 }
301 }
302}
303
304/** Get the page size of the filespace from the filespace header.
305@param[in] buf buffer used to read the page.
306@return page size */
307static
308const page_size_t
309get_page_size(
310 byte* buf)
311{
312 const unsigned flags = mach_read_from_4(buf + FIL_PAGE_DATA
313 + FSP_SPACE_FLAGS);
314
315 const ulong ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags);
316
317 srv_page_size_shift = ssize
318 ? UNIV_ZIP_SIZE_SHIFT_MIN - 1 + ssize
319 : UNIV_PAGE_SIZE_SHIFT_ORIG;
320
321 srv_page_size = 1U << srv_page_size_shift;
322
323 univ_page_size.copy_from(
324 page_size_t(srv_page_size, srv_page_size, false));
325
326 return(page_size_t(flags));
327}
328
329#ifdef _WIN32
330/***********************************************//*
331 @param [in] error error no. from the getLastError().
332
333 @retval error message corresponding to error no.
334*/
335static
336char*
337error_message(
338 int error)
339{
340 static char err_msg[1024] = {'\0'};
341 FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM,
342 NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
343 (LPTSTR)err_msg, sizeof(err_msg), NULL );
344
345 return (err_msg);
346}
347#endif /* _WIN32 */
348
349/***********************************************//*
350 @param>>_______[in] name>_____name of file.
351 @retval file pointer; file pointer is NULL when error occured.
352*/
353
354FILE*
355open_file(
356 const char* name)
357{
358 int fd; /* file descriptor. */
359 FILE* fil_in;
360#ifdef _WIN32
361 HANDLE hFile; /* handle to open file. */
362 DWORD access; /* define access control */
363 int flags = 0; /* define the mode for file
364 descriptor */
365
366 if (do_write) {
367 access = GENERIC_READ | GENERIC_WRITE;
368 flags = _O_RDWR | _O_BINARY;
369 } else {
370 access = GENERIC_READ;
371 flags = _O_RDONLY | _O_BINARY;
372 }
373
374 /* CreateFile() also provide advisory lock with the usage of
375 access and share mode of the file.*/
376 hFile = CreateFile(
377 (LPCTSTR) name, access, 0L, NULL,
378 OPEN_EXISTING, NULL, NULL);
379
380 if (hFile == INVALID_HANDLE_VALUE) {
381 /* print the error message. */
382 fprintf(stderr, "Filename::%s %s\n", name,
383 error_message(GetLastError()));
384
385 return (NULL);
386 }
387
388 /* get the file descriptor. */
389 fd= _open_osfhandle((intptr_t)hFile, flags);
390#else /* _WIN32 */
391
392 int create_flag;
393 /* define the advisory lock and open file mode. */
394 if (do_write) {
395 create_flag = O_RDWR;
396 lk.l_type = F_WRLCK;
397 } else {
398 create_flag = O_RDONLY;
399 lk.l_type = F_RDLCK;
400 }
401
402 fd = open(name, create_flag);
403
404 lk.l_whence = SEEK_SET;
405 lk.l_start = lk.l_len = 0;
406
407 if (fcntl(fd, F_SETLK, &lk) == -1) {
408 fprintf(stderr, "Error: Unable to lock file::"
409 " %s\n", name);
410 perror("fcntl");
411 return (NULL);
412 }
413#endif /* _WIN32 */
414
415 if (do_write) {
416 fil_in = fdopen(fd, "rb+");
417 } else {
418 fil_in = fdopen(fd, "rb");
419 }
420
421 return (fil_in);
422}
423
424/************************************************************//*
425 Read the content of file
426
427 @param [in,out] buf read the file in buffer
428 @param [in] partial_page_read enable when to read the
429 remaining buffer for first page.
430 @param [in] physical_page_size Physical/Commpressed page size.
431 @param [in,out] fil_in file pointer created for the
432 tablespace.
433 @retval no. of bytes read.
434*/
435ulint read_file(
436 byte* buf,
437 bool partial_page_read,
438 ulint physical_page_size,
439 FILE* fil_in)
440{
441 ulint bytes = 0;
442
443 DBUG_ASSERT(physical_page_size >= UNIV_ZIP_SIZE_MIN);
444
445 if (partial_page_read) {
446 buf += UNIV_ZIP_SIZE_MIN;
447 physical_page_size -= UNIV_ZIP_SIZE_MIN;
448 bytes = UNIV_ZIP_SIZE_MIN;
449 }
450
451 bytes += ulint(fread(buf, 1, physical_page_size, fil_in));
452
453 return bytes;
454}
455
456/** Check if page is corrupted or not.
457@param[in] buf page frame
458@param[in] page_size page size
459@param[in] is_encrypted true if page0 contained cryp_data
460 with crypt_scheme encrypted
461@param[in] is_compressed true if page0 fsp_flags contained
462 page compression flag
463@retval true if page is corrupted otherwise false. */
464static
465bool
466is_page_corrupted(
467 byte* buf,
468 const page_size_t& page_size,
469 bool is_encrypted,
470 bool is_compressed)
471{
472
473 /* enable if page is corrupted. */
474 bool is_corrupted;
475 /* use to store LSN values. */
476 ulint logseq;
477 ulint logseqfield;
478 ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE);
479 uint key_version = mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
480 ulint space_id = mach_read_from_4(
481 buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
482
483 /* We can't trust only a page type, thus we take account
484 also fsp_flags or crypt_data on page 0 */
485 if ((page_type == FIL_PAGE_PAGE_COMPRESSED && is_compressed) ||
486 (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED &&
487 is_compressed && is_encrypted)) {
488 /* Page compressed tables do not contain post compression
489 checksum. */
490 return (false);
491 }
492
493 if (page_size.is_compressed()) {
494 /* check the stored log sequence numbers
495 for uncompressed tablespace. */
496 logseq = mach_read_from_4(buf + FIL_PAGE_LSN + 4);
497 logseqfield = mach_read_from_4(
498 buf + page_size.logical() -
499 FIL_PAGE_END_LSN_OLD_CHKSUM + 4);
500
501 if (is_log_enabled) {
502 fprintf(log_file,
503 "space::" ULINTPF " page::%llu"
504 "; log sequence number:first = " ULINTPF
505 "; second = " ULINTPF "\n",
506 space_id, cur_page_num, logseq, logseqfield);
507 if (logseq != logseqfield) {
508 fprintf(log_file,
509 "Fail; space::" ULINTPF " page::%llu"
510 " invalid (fails log "
511 "sequence number check)\n",
512 space_id, cur_page_num);
513 }
514 }
515 }
516
517 /* Again we can't trust only FIL_PAGE_FILE_FLUSH_LSN field
518 now repurposed as FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION,
519 we need to check also crypt_data contents.
520
521 If page is encrypted, use different checksum calculation
522 as innochecksum can't decrypt pages. Note that some old InnoDB
523 versions did not initialize FIL_PAGE_FILE_FLUSH_LSN field
524 so if crypt checksum does not match we verify checksum using
525 normal method. */
526 if (is_encrypted && key_version != 0) {
527 is_corrupted = !fil_space_verify_crypt_checksum(buf,
528 page_size, space_id, (ulint)cur_page_num);
529 } else {
530 is_corrupted = true;
531 }
532
533 if (is_corrupted) {
534 is_corrupted = buf_page_is_corrupted(
535 true, buf, page_size, NULL);
536 }
537
538 return(is_corrupted);
539}
540
541/********************************************//*
542 Check if page is doublewrite buffer or not.
543 @param [in] page buffer page
544
545 @retval true if page is doublewrite buffer otherwise false.
546*/
547static
548bool
549is_page_doublewritebuffer(
550 const byte* page)
551{
552 if ((cur_page_num >= FSP_EXTENT_SIZE)
553 && (cur_page_num < FSP_EXTENT_SIZE * 3)) {
554 /* page is doublewrite buffer. */
555 return (true);
556 }
557
558 return (false);
559}
560
561/*******************************************************//*
562Check if page is empty or not.
563 @param [in] page page to checked for empty.
564 @param [in] len size of page.
565
566 @retval true if page is empty.
567 @retval false if page is not empty.
568*/
569static
570bool
571is_page_empty(
572 const byte* page,
573 size_t len)
574{
575 while (len--) {
576 if (*page++) {
577 return (false);
578 }
579 }
580
581 return (true);
582}
583
584/********************************************************************//**
585Rewrite the checksum for the page.
586@param [in/out] page page buffer
587@param [in] physical_page_size page size in bytes on disk.
588@param [in] iscompressed Is compressed/Uncompressed Page.
589
590@retval true : do rewrite
591@retval false : skip the rewrite as checksum stored match with
592 calculated or page is doublwrite buffer.
593*/
594
595bool
596update_checksum(
597 byte* page,
598 ulong physical_page_size,
599 bool iscompressed)
600{
601 ib_uint32_t checksum = 0;
602 byte stored1[4]; /* get FIL_PAGE_SPACE_OR_CHKSUM field checksum */
603 byte stored2[4]; /* get FIL_PAGE_END_LSN_OLD_CHKSUM field checksum */
604
605 ut_ad(page);
606 /* If page is doublewrite buffer, skip the rewrite of checksum. */
607 if (skip_page) {
608 return (false);
609 }
610
611 memcpy(stored1, page + FIL_PAGE_SPACE_OR_CHKSUM, 4);
612 memcpy(stored2, page + physical_page_size -
613 FIL_PAGE_END_LSN_OLD_CHKSUM, 4);
614
615 /* Check if page is empty, exclude the checksum field */
616 if (is_page_empty(page + 4, physical_page_size - 12)
617 && is_page_empty(page + physical_page_size - 4, 4)) {
618
619 memset(page + FIL_PAGE_SPACE_OR_CHKSUM, 0, 4);
620 memset(page + physical_page_size -
621 FIL_PAGE_END_LSN_OLD_CHKSUM, 0, 4);
622
623 goto func_exit;
624 }
625
626 if (iscompressed) {
627 /* page is compressed */
628 checksum = page_zip_calc_checksum(
629 page, physical_page_size,
630 static_cast<srv_checksum_algorithm_t>(write_check));
631
632 mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum);
633 if (is_log_enabled) {
634 fprintf(log_file, "page::%llu; Updated checksum ="
635 " %u\n", cur_page_num, checksum);
636 }
637
638 } else {
639 /* page is uncompressed. */
640
641 /* Store the new formula checksum */
642 switch ((srv_checksum_algorithm_t) write_check) {
643
644 case SRV_CHECKSUM_ALGORITHM_CRC32:
645 case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
646 checksum = buf_calc_page_crc32(page);
647 break;
648
649 case SRV_CHECKSUM_ALGORITHM_INNODB:
650 case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
651 checksum = (ib_uint32_t)
652 buf_calc_page_new_checksum(page);
653 break;
654
655 case SRV_CHECKSUM_ALGORITHM_NONE:
656 case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
657 checksum = BUF_NO_CHECKSUM_MAGIC;
658 break;
659 /* no default so the compiler will emit a warning if new
660 enum is added and not handled here */
661 }
662
663 mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum);
664 if (is_log_enabled) {
665 fprintf(log_file, "page::%llu; Updated checksum field1"
666 " = %u\n", cur_page_num, checksum);
667 }
668
669 if (write_check == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB
670 || write_check == SRV_CHECKSUM_ALGORITHM_INNODB) {
671 checksum = (ib_uint32_t)
672 buf_calc_page_old_checksum(page);
673 }
674
675 mach_write_to_4(page + physical_page_size -
676 FIL_PAGE_END_LSN_OLD_CHKSUM,checksum);
677
678 if (is_log_enabled) {
679 fprintf(log_file, "page::%llu; Updated checksum "
680 "field2 = %u\n", cur_page_num, checksum);
681 }
682
683 }
684
685func_exit:
686 /* The following code is to check the stored checksum with the
687 calculated checksum. If it matches, then return FALSE to skip
688 the rewrite of checksum, otherwise return TRUE. */
689 if (iscompressed) {
690 if (!memcmp(stored1, page + FIL_PAGE_SPACE_OR_CHKSUM, 4)) {
691 return (false);
692 }
693 return (true);
694 }
695
696 if (!memcmp(stored1, page + FIL_PAGE_SPACE_OR_CHKSUM, 4)
697 && !memcmp(stored2, page + physical_page_size -
698 FIL_PAGE_END_LSN_OLD_CHKSUM, 4)) {
699 return (false);
700
701 }
702
703 return (true);
704}
705
706/**
707 Write the content to the file
708@param[in] filename name of the file.
709@param[in,out] file file pointer where content
710 have to be written
711@param[in] buf file buffer read
712@param[in] compressed Enabled if tablespace is
713 compressed.
714@param[in,out] pos current file position.
715@param[in] page_size page size in bytes on disk.
716
717@retval true if successfully written
718@retval false if a non-recoverable error occurred
719*/
720static
721bool
722write_file(
723 const char* filename,
724 FILE* file,
725 byte* buf,
726 bool compressed,
727 fpos_t* pos,
728 ulong page_size)
729{
730 bool do_update;
731
732 do_update = update_checksum(buf, page_size, compressed);
733
734 if (file != stdin) {
735 if (do_update) {
736 /* Set the previous file pointer position
737 saved in pos to current file position. */
738 if (0 != fsetpos(file, pos)) {
739 perror("fsetpos");
740 return(false);
741 }
742 } else {
743 /* Store the current file position in pos */
744 if (0 != fgetpos(file, pos)) {
745 perror("fgetpos");
746 return(false);
747 }
748 return(true);
749 }
750 }
751
752 if (page_size
753 != fwrite(buf, 1, page_size, file == stdin ? stdout : file)) {
754 fprintf(stderr, "Failed to write page::%llu to %s: %s\n",
755 cur_page_num, filename, strerror(errno));
756
757 return(false);
758 }
759 if (file != stdin) {
760 fflush(file);
761 /* Store the current file position in pos */
762 if (0 != fgetpos(file, pos)) {
763 perror("fgetpos");
764 return(false);
765 }
766 }
767
768 return(true);
769}
770
771/*
772Parse the page and collect/dump the information about page type
773@param [in] page buffer page
774@param [out] xdes extend descriptor page
775@param [in] file file for diagnosis.
776@param [in] page_size page_size
777@param [in] is_encrypted tablespace is encrypted
778*/
779void
780parse_page(
781 const byte* page,
782 byte* xdes,
783 FILE* file,
784 const page_size_t& page_size,
785 bool is_encrypted)
786{
787 unsigned long long id;
788 ulint undo_page_type;
789 char str[20]={'\0'};
790 ulint n_recs;
791 ulint page_no;
792 ulint left_page_no;
793 ulint right_page_no;
794 ulint data_bytes;
795 bool is_leaf;
796 ulint size_range_id;
797
798 /* Check whether page is doublewrite buffer. */
799 if(skip_page) {
800 strcpy(str, "Double_write_buffer");
801 } else {
802 strcpy(str, "-");
803 }
804
805 switch (mach_read_from_2(page + FIL_PAGE_TYPE)) {
806
807 case FIL_PAGE_INDEX: {
808 uint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
809 page_type.n_fil_page_index++;
810
811 /* If page is encrypted we can't read index header */
812 if (!is_encrypted) {
813 id = mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID);
814 n_recs = mach_read_from_2(page + PAGE_HEADER + PAGE_N_RECS);
815 page_no = mach_read_from_4(page + FIL_PAGE_OFFSET);
816 left_page_no = mach_read_from_4(page + FIL_PAGE_PREV);
817 right_page_no = mach_read_from_4(page + FIL_PAGE_NEXT);
818 ulint is_comp = mach_read_from_2(page + PAGE_HEADER + PAGE_N_HEAP) & 0x8000;
819 ulint level = mach_read_from_2(page + PAGE_HEADER + PAGE_LEVEL);
820 ulint garbage = mach_read_from_2(page + PAGE_HEADER + PAGE_GARBAGE);
821
822
823 data_bytes = (ulint)(mach_read_from_2(page + PAGE_HEADER + PAGE_HEAP_TOP)
824 - (is_comp
825 ? PAGE_NEW_SUPREMUM_END
826 : PAGE_OLD_SUPREMUM_END)
827 - garbage);
828
829 is_leaf = (!*(const uint16*) (page + (PAGE_HEADER + PAGE_LEVEL)));
830
831 if (page_type_dump) {
832 fprintf(file, "#::%llu\t\t|\t\tIndex page\t\t\t|"
833 "\tindex id=%llu,", cur_page_num, id);
834
835 fprintf(file,
836 " page level=" ULINTPF
837 ", No. of records=" ULINTPF
838 ", garbage=" ULINTPF ", %s\n",
839 level, n_recs, garbage, str);
840 }
841
842 size_range_id = (data_bytes * SIZE_RANGES_FOR_PAGE
843 + page_size.logical() - 1) /
844 page_size.logical();
845
846 if (size_range_id > SIZE_RANGES_FOR_PAGE + 1) {
847 /* data_bytes is bigger than logical_page_size */
848 size_range_id = SIZE_RANGES_FOR_PAGE + 1;
849 }
850 if (per_page_details) {
851 printf("index id=%llu page " ULINTPF " leaf %d n_recs " ULINTPF " data_bytes " ULINTPF
852 "\n", id, page_no, is_leaf, n_recs, data_bytes);
853 }
854 /* update per-index statistics */
855 {
856 if (index_ids.count(id) == 0) {
857 index_ids[id] = per_index_stats();
858 }
859 std::map<unsigned long long, per_index_stats>::iterator it;
860 it = index_ids.find(id);
861 per_index_stats &index = (it->second);
862 const byte* des = xdes + XDES_ARR_OFFSET
863 + XDES_SIZE * ((page_no & (page_size.physical() - 1))
864 / FSP_EXTENT_SIZE);
865 if (xdes_get_bit(des, XDES_FREE_BIT,
866 page_no % FSP_EXTENT_SIZE)) {
867 index.free_pages++;
868 return;
869 }
870
871 index.pages++;
872
873 if (is_leaf) {
874 index.leaf_pages++;
875 if (data_bytes > index.max_data_size) {
876 index.max_data_size = data_bytes;
877 }
878 struct per_page_stats pp(n_recs, data_bytes,
879 left_page_no, right_page_no);
880
881 index.leaves[page_no] = pp;
882
883 if (left_page_no == ULINT32_UNDEFINED) {
884 index.first_leaf_page = page_no;
885 index.count++;
886 }
887 }
888
889 index.total_n_recs += n_recs;
890 index.total_data_bytes += data_bytes;
891 index.pages_in_size_range[size_range_id] ++;
892 }
893 } else {
894 fprintf(file, "#::%llu\t\t|\t\tEncrypted Index page\t\t\t|"
895 "\tkey_version %u,%s\n", cur_page_num, key_version, str);
896 }
897
898 break;
899 }
900 case FIL_PAGE_UNDO_LOG:
901 page_type.n_fil_page_undo_log++;
902 undo_page_type = mach_read_from_2(page +
903 TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_TYPE);
904 if (page_type_dump) {
905 fprintf(file, "#::%llu\t\t|\t\tUndo log page\t\t\t|",
906 cur_page_num);
907 }
908 if (undo_page_type == TRX_UNDO_INSERT) {
909 page_type.n_undo_insert++;
910 if (page_type_dump) {
911 fprintf(file, "\t%s",
912 "Insert Undo log page");
913 }
914
915 } else if (undo_page_type == TRX_UNDO_UPDATE) {
916 page_type.n_undo_update++;
917 if (page_type_dump) {
918 fprintf(file, "\t%s",
919 "Update undo log page");
920 }
921 }
922
923 undo_page_type = mach_read_from_2(page + TRX_UNDO_SEG_HDR +
924 TRX_UNDO_STATE);
925 switch (undo_page_type) {
926 case TRX_UNDO_ACTIVE:
927 page_type.n_undo_state_active++;
928 if (page_type_dump) {
929 fprintf(file, ", %s", "Undo log of "
930 "an active transaction");
931 }
932 break;
933
934 case TRX_UNDO_CACHED:
935 page_type.n_undo_state_cached++;
936 if (page_type_dump) {
937 fprintf(file, ", %s", "Page is "
938 "cached for quick reuse");
939 }
940 break;
941
942 case TRX_UNDO_TO_FREE:
943 page_type.n_undo_state_to_free++;
944 if (page_type_dump) {
945 fprintf(file, ", %s", "Insert undo "
946 "segment that can be freed");
947 }
948 break;
949
950 case TRX_UNDO_TO_PURGE:
951 page_type.n_undo_state_to_purge++;
952 if (page_type_dump) {
953 fprintf(file, ", %s", "Will be "
954 "freed in purge when all undo"
955 "data in it is removed");
956 }
957 break;
958
959 case TRX_UNDO_PREPARED:
960 page_type.n_undo_state_prepared++;
961 if (page_type_dump) {
962 fprintf(file, ", %s", "Undo log of "
963 "an prepared transaction");
964 }
965 break;
966
967 default:
968 page_type.n_undo_state_other++;
969 break;
970 }
971 if(page_type_dump) {
972 fprintf(file, ", %s\n", str);
973 }
974 break;
975
976 case FIL_PAGE_INODE:
977 page_type.n_fil_page_inode++;
978 if (page_type_dump) {
979 fprintf(file, "#::%llu\t\t|\t\tInode page\t\t\t|"
980 "\t%s\n",cur_page_num, str);
981 }
982 break;
983
984 case FIL_PAGE_IBUF_FREE_LIST:
985 page_type.n_fil_page_ibuf_free_list++;
986 if (page_type_dump) {
987 fprintf(file, "#::%llu\t\t|\t\tInsert buffer free list"
988 " page\t|\t%s\n", cur_page_num, str);
989 }
990 break;
991
992 case FIL_PAGE_TYPE_ALLOCATED:
993 page_type.n_fil_page_type_allocated++;
994 if (page_type_dump) {
995 fprintf(file, "#::%llu\t\t|\t\tFreshly allocated "
996 "page\t\t|\t%s\n", cur_page_num, str);
997 }
998 break;
999
1000 case FIL_PAGE_IBUF_BITMAP:
1001 page_type.n_fil_page_ibuf_bitmap++;
1002 if (page_type_dump) {
1003 fprintf(file, "#::%llu\t\t|\t\tInsert Buffer "
1004 "Bitmap\t\t|\t%s\n", cur_page_num, str);
1005 }
1006 break;
1007
1008 case FIL_PAGE_TYPE_SYS:
1009 page_type.n_fil_page_type_sys++;
1010 if (page_type_dump) {
1011 fprintf(file, "#::%llu\t\t|\t\tSystem page\t\t\t|"
1012 "\t%s\n",cur_page_num, str);
1013 }
1014 break;
1015
1016 case FIL_PAGE_TYPE_TRX_SYS:
1017 page_type.n_fil_page_type_trx_sys++;
1018 if (page_type_dump) {
1019 fprintf(file, "#::%llu\t\t|\t\tTransaction system "
1020 "page\t\t|\t%s\n", cur_page_num, str);
1021 }
1022 break;
1023
1024 case FIL_PAGE_TYPE_FSP_HDR:
1025 page_type.n_fil_page_type_fsp_hdr++;
1026 memcpy(xdes, page, page_size.physical());
1027 if (page_type_dump) {
1028 fprintf(file, "#::%llu\t\t|\t\tFile Space "
1029 "Header\t\t|\t%s\n", cur_page_num, str);
1030 }
1031 break;
1032
1033 case FIL_PAGE_TYPE_XDES:
1034 page_type.n_fil_page_type_xdes++;
1035 memcpy(xdes, page, page_size.physical());
1036 if (page_type_dump) {
1037 fprintf(file, "#::%llu\t\t|\t\tExtent descriptor "
1038 "page\t\t|\t%s\n", cur_page_num, str);
1039 }
1040 break;
1041
1042 case FIL_PAGE_TYPE_BLOB:
1043 page_type.n_fil_page_type_blob++;
1044 if (page_type_dump) {
1045 fprintf(file, "#::%llu\t\t|\t\tBLOB page\t\t\t|\t%s\n",
1046 cur_page_num, str);
1047 }
1048 break;
1049
1050 case FIL_PAGE_TYPE_ZBLOB:
1051 page_type.n_fil_page_type_zblob++;
1052 if (page_type_dump) {
1053 fprintf(file, "#::%llu\t\t|\t\tCompressed BLOB "
1054 "page\t\t|\t%s\n", cur_page_num, str);
1055 }
1056 break;
1057
1058 case FIL_PAGE_TYPE_ZBLOB2:
1059 page_type.n_fil_page_type_zblob2++;
1060 if (page_type_dump) {
1061 fprintf(file, "#::%llu\t\t|\t\tSubsequent Compressed "
1062 "BLOB page\t|\t%s\n", cur_page_num, str);
1063 }
1064 break;
1065
1066 case FIL_PAGE_PAGE_COMPRESSED:
1067 page_type.n_fil_page_type_page_compressed++;
1068 if (page_type_dump) {
1069 fprintf(file, "#::%llu\t\t|\t\tPage compressed "
1070 "page\t|\t%s\n", cur_page_num, str);
1071 }
1072 break;
1073
1074 case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED:
1075 page_type.n_fil_page_type_page_compressed_encrypted++;
1076 if (page_type_dump) {
1077 fprintf(file, "#::%llu\t\t|\t\tPage compressed encrypted "
1078 "page\t|\t%s\n", cur_page_num, str);
1079 }
1080 break;
1081 default:
1082 page_type.n_fil_page_type_other++;
1083 break;
1084 }
1085}
1086/**
1087@param [in/out] file_name name of the filename
1088
1089@retval FILE pointer if successfully created else NULL when error occured.
1090*/
1091FILE*
1092create_file(
1093 char* file_name)
1094{
1095 FILE* file = NULL;
1096
1097#ifndef _WIN32
1098 file = fopen(file_name, "wb");
1099 if (file == NULL) {
1100 fprintf(stderr, "Failed to create file: %s: %s\n",
1101 file_name, strerror(errno));
1102 return(NULL);
1103 }
1104#else
1105 HANDLE hFile; /* handle to open file. */
1106 int fd = 0;
1107 hFile = CreateFile((LPCTSTR) file_name,
1108 GENERIC_READ | GENERIC_WRITE,
1109 FILE_SHARE_READ | FILE_SHARE_DELETE,
1110 NULL, CREATE_NEW, NULL, NULL);
1111
1112 if (hFile == INVALID_HANDLE_VALUE) {
1113 /* print the error message. */
1114 fprintf(stderr, "Filename::%s %s\n",
1115 file_name,
1116 error_message(GetLastError()));
1117
1118 return(NULL);
1119 }
1120
1121 /* get the file descriptor. */
1122 fd= _open_osfhandle((intptr_t)hFile, _O_RDWR | _O_BINARY);
1123 file = fdopen(fd, "wb");
1124#endif /* _WIN32 */
1125
1126 return(file);
1127}
1128
1129/*
1130 Print the page type count of a tablespace.
1131 @param [in] fil_out stream where the output goes.
1132*/
1133void
1134print_summary(
1135 FILE* fil_out)
1136{
1137 fprintf(fil_out, "\n================PAGE TYPE SUMMARY==============\n");
1138 fprintf(fil_out, "#PAGE_COUNT\tPAGE_TYPE");
1139 fprintf(fil_out, "\n===============================================\n");
1140 fprintf(fil_out, "%8d\tIndex page\n",
1141 page_type.n_fil_page_index);
1142 fprintf(fil_out, "%8d\tUndo log page\n",
1143 page_type.n_fil_page_undo_log);
1144 fprintf(fil_out, "%8d\tInode page\n",
1145 page_type.n_fil_page_inode);
1146 fprintf(fil_out, "%8d\tInsert buffer free list page\n",
1147 page_type.n_fil_page_ibuf_free_list);
1148 fprintf(fil_out, "%8d\tFreshly allocated page\n",
1149 page_type.n_fil_page_type_allocated);
1150 fprintf(fil_out, "%8d\tInsert buffer bitmap\n",
1151 page_type.n_fil_page_ibuf_bitmap);
1152 fprintf(fil_out, "%8d\tSystem page\n",
1153 page_type.n_fil_page_type_sys);
1154 fprintf(fil_out, "%8d\tTransaction system page\n",
1155 page_type.n_fil_page_type_trx_sys);
1156 fprintf(fil_out, "%8d\tFile Space Header\n",
1157 page_type.n_fil_page_type_fsp_hdr);
1158 fprintf(fil_out, "%8d\tExtent descriptor page\n",
1159 page_type.n_fil_page_type_xdes);
1160 fprintf(fil_out, "%8d\tBLOB page\n",
1161 page_type.n_fil_page_type_blob);
1162 fprintf(fil_out, "%8d\tCompressed BLOB page\n",
1163 page_type.n_fil_page_type_zblob);
1164 fprintf(fil_out, "%8d\tPage compressed page\n",
1165 page_type.n_fil_page_type_page_compressed);
1166 fprintf(fil_out, "%8d\tPage compressed encrypted page\n",
1167 page_type.n_fil_page_type_page_compressed_encrypted);
1168 fprintf(fil_out, "%8d\tOther type of page\n",
1169 page_type.n_fil_page_type_other);
1170
1171 fprintf(fil_out, "\n===============================================\n");
1172 fprintf(fil_out, "Additional information:\n");
1173 fprintf(fil_out, "Undo page type: %d insert, %d update, %d other\n",
1174 page_type.n_undo_insert,
1175 page_type.n_undo_update,
1176 page_type.n_undo_other);
1177 fprintf(fil_out, "Undo page state: %d active, %d cached, %d to_free, %d"
1178 " to_purge, %d prepared, %d other\n",
1179 page_type.n_undo_state_active,
1180 page_type.n_undo_state_cached,
1181 page_type.n_undo_state_to_free,
1182 page_type.n_undo_state_to_purge,
1183 page_type.n_undo_state_prepared,
1184 page_type.n_undo_state_other);
1185
1186 fprintf(fil_out, "index_id\t#pages\t\t#leaf_pages\t#recs_per_page"
1187 "\t#bytes_per_page\n");
1188
1189 for (std::map<unsigned long long, per_index_stats>::const_iterator it = index_ids.begin();
1190 it != index_ids.end(); it++) {
1191 const per_index_stats& index = it->second;
1192 fprintf(fil_out, "%lld\t\t%lld\t\t%lld\t\t%lld\t\t%lld\n",
1193 it->first, index.pages, index.leaf_pages,
1194 index.total_n_recs / index.pages,
1195 index.total_data_bytes / index.pages);
1196 }
1197
1198 fprintf(fil_out, "\n");
1199 fprintf(fil_out, "index_id\tpage_data_bytes_histgram(empty,...,oversized)\n");
1200
1201 for (std::map<unsigned long long, per_index_stats>::const_iterator it = index_ids.begin();
1202 it != index_ids.end(); it++) {
1203 fprintf(fil_out, "%lld\t", it->first);
1204 const per_index_stats& index = it->second;
1205 for (ulint i = 0; i < SIZE_RANGES_FOR_PAGE+2; i++) {
1206 fprintf(fil_out, "\t%lld", index.pages_in_size_range[i]);
1207 }
1208 fprintf(fil_out, "\n");
1209 }
1210
1211 if (do_leaf) {
1212 print_leaf_stats(fil_out);
1213 }
1214}
1215
1216/* command line argument for innochecksum tool. */
1217static struct my_option innochecksum_options[] = {
1218 {"help", '?', "Displays this help and exits.",
1219 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
1220 {"info", 'I', "Synonym for --help.",
1221 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
1222 {"version", 'V', "Displays version information and exits.",
1223 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
1224 {"verbose", 'v', "Verbose (prints progress every 5 seconds).",
1225 &verbose, &verbose, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1226#ifndef DBUG_OFF
1227 {"debug", '#', "Output debug log. See " REFMAN "dbug-package.html",
1228 &dbug_setting, &dbug_setting, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
1229#endif /* !DBUG_OFF */
1230 {"count", 'c', "Print the count of pages in the file and exits.",
1231 &just_count, &just_count, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1232 {"start_page", 's', "Start on this page number (0 based).",
1233 &start_page, &start_page, 0, GET_ULL, REQUIRED_ARG,
1234 0, 0, ULLONG_MAX, 0, 1, 0},
1235 {"end_page", 'e', "End at this page number (0 based).",
1236 &end_page, &end_page, 0, GET_ULL, REQUIRED_ARG,
1237 0, 0, ULLONG_MAX, 0, 1, 0},
1238 {"page", 'p', "Check only this page (0 based).",
1239 &do_page, &do_page, 0, GET_ULL, REQUIRED_ARG,
1240 0, 0, ULLONG_MAX, 0, 1, 0},
1241 {"strict-check", 'C', "Specify the strict checksum algorithm by the user.",
1242 &strict_check, &strict_check, &innochecksum_algorithms_typelib,
1243 GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1244 {"no-check", 'n', "Ignore the checksum verification.",
1245 &no_check, &no_check, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1246 {"allow-mismatches", 'a', "Maximum checksum mismatch allowed.",
1247 &allow_mismatches, &allow_mismatches, 0,
1248 GET_ULL, REQUIRED_ARG, 0, 0, ULLONG_MAX, 0, 1, 0},
1249 {"write", 'w', "Rewrite the checksum algorithm by the user.",
1250 &write_check, &write_check, &innochecksum_algorithms_typelib,
1251 GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1252 {"page-type-summary", 'S', "Display a count of each page type "
1253 "in a tablespace.", &page_type_summary, &page_type_summary, 0,
1254 GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1255 {"page-type-dump", 'D', "Dump the page type info for each page in a "
1256 "tablespace.", &page_dump_filename, &page_dump_filename, 0,
1257 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1258 {"per-page-details", 'i', "Print out per-page detail information.",
1259 &per_page_details, &per_page_details, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1260 {"log", 'l', "log output.",
1261 &log_filename, &log_filename, 0,
1262 GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
1263 {"leaf", 'f', "Examine leaf index pages",
1264 &do_leaf, &do_leaf, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
1265 {"merge", 'm', "leaf page count if merge given number of consecutive pages",
1266 &n_merge, &n_merge, 0, GET_ULONG, REQUIRED_ARG, 0, 0, (longlong)10L, 0, 1, 0},
1267
1268 {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
1269};
1270
1271/* Print out the Innodb version and machine information. */
1272static void print_version(void)
1273{
1274#ifdef DBUG_OFF
1275 printf("%s Ver %s, for %s (%s)\n",
1276 my_progname, INNODB_VERSION_STR,
1277 SYSTEM_TYPE, MACHINE_TYPE);
1278#else
1279 printf("%s-debug Ver %s, for %s (%s)\n",
1280 my_progname, INNODB_VERSION_STR,
1281 SYSTEM_TYPE, MACHINE_TYPE);
1282#endif /* DBUG_OFF */
1283}
1284
1285static void usage(void)
1286{
1287 print_version();
1288 puts(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000"));
1289 printf("InnoDB offline file checksum utility.\n");
1290 printf("Usage: %s [-c] [-s <start page>] [-e <end page>] "
1291 "[-p <page>] [-i] [-v] [-a <allow mismatches>] [-n] "
1292 "[-C <strict-check>] [-w <write>] [-S] [-D <page type dump>] "
1293 "[-l <log>] [-l] [-m <merge pages>] <filename or [-]>\n", my_progname);
1294 printf("See " REFMAN "innochecksum.html for usage hints.\n");
1295 my_print_help(innochecksum_options);
1296 my_print_variables(innochecksum_options);
1297}
1298
1299extern "C" my_bool
1300innochecksum_get_one_option(
1301 int optid,
1302 const struct my_option *opt MY_ATTRIBUTE((unused)),
1303 char *argument MY_ATTRIBUTE((unused)))
1304{
1305 switch (optid) {
1306#ifndef DBUG_OFF
1307 case '#':
1308 dbug_setting = argument
1309 ? argument
1310 : IF_WIN("d:O,innochecksum.trace",
1311 "d:o,/tmp/innochecksum.trace");
1312 DBUG_PUSH(dbug_setting);
1313 break;
1314#endif /* !DBUG_OFF */
1315 case 'e':
1316 use_end_page = true;
1317 break;
1318 case 'p':
1319 end_page = start_page = do_page;
1320 use_end_page = true;
1321 do_one_page = true;
1322 break;
1323 case 'V':
1324 print_version();
1325 my_end(0);
1326 exit(EXIT_SUCCESS);
1327 break;
1328 case 'C':
1329 strict_verify = true;
1330 switch ((srv_checksum_algorithm_t) strict_check) {
1331
1332 case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
1333 case SRV_CHECKSUM_ALGORITHM_CRC32:
1334 srv_checksum_algorithm =
1335 SRV_CHECKSUM_ALGORITHM_STRICT_CRC32;
1336 break;
1337
1338 case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
1339 case SRV_CHECKSUM_ALGORITHM_INNODB:
1340 srv_checksum_algorithm =
1341 SRV_CHECKSUM_ALGORITHM_STRICT_INNODB;
1342 break;
1343
1344 case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
1345 case SRV_CHECKSUM_ALGORITHM_NONE:
1346 srv_checksum_algorithm =
1347 SRV_CHECKSUM_ALGORITHM_STRICT_NONE;
1348 break;
1349 default:
1350 return(true);
1351 }
1352 break;
1353 case 'n':
1354 no_check = true;
1355 break;
1356 case 'a':
1357 case 'S':
1358 break;
1359 case 'w':
1360 do_write = true;
1361 break;
1362 case 'D':
1363 page_type_dump = true;
1364 break;
1365 case 'l':
1366 is_log_enabled = true;
1367 break;
1368 case 'I':
1369 case '?':
1370 usage();
1371 my_end(0);
1372 exit(EXIT_SUCCESS);
1373 break;
1374 }
1375
1376 return(false);
1377}
1378
1379static
1380bool
1381get_options(
1382 int *argc,
1383 char ***argv)
1384{
1385 if (handle_options(argc, argv, innochecksum_options,
1386 innochecksum_get_one_option)) {
1387 my_end(0);
1388 exit(true);
1389 }
1390
1391 /* The next arg must be the filename */
1392 if (!*argc) {
1393 usage();
1394 my_end(0);
1395 return (true);
1396 }
1397
1398 return (false);
1399}
1400
1401/** Check from page 0 if table is encrypted.
1402@param[in] filename Filename
1403@param[in] page_size page size
1404@param[in] page Page 0
1405@retval true if tablespace is encrypted, false if not
1406*/
1407static
1408bool check_encryption(
1409 const char* filename,
1410 const page_size_t& page_size,
1411 byte * page)
1412{
1413 ulint offset = (FSP_HEADER_OFFSET + (XDES_ARR_OFFSET + XDES_SIZE *
1414 (page_size.physical()) / FSP_EXTENT_SIZE));
1415
1416 if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) {
1417 return false;
1418 }
1419
1420 ulint type = mach_read_from_1(page + offset + MAGIC_SZ + 0);
1421
1422 if (! (type == CRYPT_SCHEME_UNENCRYPTED ||
1423 type == CRYPT_SCHEME_1)) {
1424 return false;
1425 }
1426
1427 ulint iv_length = mach_read_from_1(page + offset + MAGIC_SZ + 1);
1428
1429 if (iv_length != CRYPT_SCHEME_1_IV_LEN) {
1430 return false;
1431 }
1432
1433 uint min_key_version = mach_read_from_4
1434 (page + offset + MAGIC_SZ + 2 + iv_length);
1435
1436 uint key_id = mach_read_from_4
1437 (page + offset + MAGIC_SZ + 2 + iv_length + 4);
1438
1439 if (type == CRYPT_SCHEME_1 && is_log_enabled) {
1440 fprintf(log_file,"Tablespace %s encrypted key_version %u key_id %u\n",
1441 filename, min_key_version, key_id);
1442 }
1443
1444 return (type == CRYPT_SCHEME_1);
1445}
1446
1447/**
1448Verify page checksum.
1449@param[in] buf page to verify
1450@param[in] page_size page size
1451@param[in] is_encrypted true if tablespace is encrypted
1452@param[in] is_compressed true if tablespace is page compressed
1453@param[in,out] mismatch_count Number of pages failed in checksum verify
1454@retval 0 if page checksum matches or 1 if it does not match
1455*/
1456static
1457int verify_checksum(
1458 byte* buf,
1459 const page_size_t& page_size,
1460 bool is_encrypted,
1461 bool is_compressed,
1462 unsigned long long* mismatch_count)
1463{
1464 int exit_status = 0;
1465 bool is_corrupted = false;
1466
1467 is_corrupted = is_page_corrupted(
1468 buf, page_size, is_encrypted, is_compressed);
1469
1470 if (is_corrupted) {
1471 fprintf(stderr, "Fail: page::%llu invalid\n",
1472 cur_page_num);
1473
1474 (*mismatch_count)++;
1475
1476 if (*mismatch_count > allow_mismatches) {
1477 fprintf(stderr,
1478 "Exceeded the "
1479 "maximum allowed "
1480 "checksum mismatch "
1481 "count::%llu current::%llu\n",
1482 *mismatch_count,
1483 allow_mismatches);
1484
1485 exit_status = 1;
1486 }
1487 }
1488
1489 return (exit_status);
1490}
1491
1492/** Rewrite page checksum if needed.
1493@param[in] filename File name
1494@param[in] fil_in File pointer
1495@param[in] buf page
1496@param[in] page_size page size
1497@param[in] pos File position
1498@param[in] is_encrypted true if tablespace is encrypted
1499@param[in] is_compressed true if tablespace is page compressed
1500@retval 0 if checksum rewrite was successful, 1 if error was detected */
1501static
1502int
1503rewrite_checksum(
1504 const char* filename,
1505 FILE* fil_in,
1506 byte* buf,
1507 const page_size_t& page_size,
1508 fpos_t* pos,
1509 bool is_encrypted,
1510 bool is_compressed)
1511{
1512 int exit_status = 0;
1513 /* Rewrite checksum. Note that for encrypted and
1514 page compressed tables this is not currently supported. */
1515 if (do_write &&
1516 !is_encrypted &&
1517 !is_compressed
1518 && !write_file(filename, fil_in, buf,
1519 page_size.is_compressed(), pos,
1520 static_cast<ulong>(page_size.physical()))) {
1521
1522 exit_status = 1;
1523 }
1524
1525 return (exit_status);
1526}
1527
1528int main(
1529 int argc,
1530 char **argv)
1531{
1532 /* our input file. */
1533 FILE* fil_in = NULL;
1534 /* our input filename. */
1535 char* filename;
1536 /* Buffer to store pages read. */
1537 byte* buf_ptr = NULL;
1538 byte* xdes_ptr = NULL;
1539 byte* buf = NULL;
1540 byte* xdes = NULL;
1541 /* bytes read count */
1542 ulint bytes;
1543 /* current time */
1544 time_t now;
1545 /* last time */
1546 time_t lastt;
1547 /* stat, to get file size. */
1548#ifdef _WIN32
1549 struct _stat64 st;
1550#else
1551 struct stat st;
1552#endif /* _WIN32 */
1553
1554 int exit_status = 0;
1555
1556 /* size of file (has to be 64 bits) */
1557 unsigned long long int size = 0;
1558 /* number of pages in file */
1559 ulint pages;
1560
1561 off_t offset = 0;
1562 /* count the no. of page corrupted. */
1563 unsigned long long mismatch_count = 0;
1564
1565 bool partial_page_read = false;
1566 /* Enabled when read from stdin is done. */
1567 bool read_from_stdin = false;
1568 FILE* fil_page_type = NULL;
1569 fpos_t pos;
1570
1571 /* Use to check the space id of given file. If space_id is zero,
1572 then check whether page is doublewrite buffer.*/
1573 ulint space_id = 0UL;
1574 /* enable when space_id of given file is zero. */
1575 bool is_system_tablespace = false;
1576
1577 ut_crc32_init();
1578 MY_INIT(argv[0]);
1579 DBUG_ENTER("main");
1580 DBUG_PROCESS(argv[0]);
1581
1582 if (get_options(&argc,&argv)) {
1583 exit_status = 1;
1584 goto my_exit;
1585 }
1586
1587 if (strict_verify && no_check) {
1588 fprintf(stderr, "Error: --strict-check option cannot be used "
1589 "together with --no-check option.\n");
1590 exit_status = 1;
1591 goto my_exit;
1592 }
1593
1594 if (no_check && !do_write) {
1595 fprintf(stderr, "Error: --no-check must be associated with "
1596 "--write option.\n");
1597 exit_status = 1;
1598 goto my_exit;
1599 }
1600
1601 if (page_type_dump) {
1602 fil_page_type = create_file(page_dump_filename);
1603 if (!fil_page_type) {
1604 exit_status = 1;
1605 goto my_exit;
1606 }
1607 }
1608
1609 if (is_log_enabled) {
1610 log_file = create_file(log_filename);
1611 if (!log_file) {
1612 exit_status = 1;
1613 goto my_exit;
1614 }
1615 fprintf(log_file, "InnoDB File Checksum Utility.\n");
1616 }
1617
1618 if (verbose) {
1619 my_print_variables(innochecksum_options);
1620 }
1621
1622
1623 buf_ptr = (byte*) malloc(UNIV_PAGE_SIZE_MAX * 2);
1624 xdes_ptr = (byte*)malloc(UNIV_PAGE_SIZE_MAX * 2);
1625 buf = (byte *) ut_align(buf_ptr, UNIV_PAGE_SIZE_MAX);
1626 xdes = (byte *) ut_align(xdes_ptr, UNIV_PAGE_SIZE_MAX);
1627
1628 /* The file name is not optional. */
1629 for (int i = 0; i < argc; ++i) {
1630
1631 /* Reset parameters for each file. */
1632 filename = argv[i];
1633 memset(&page_type, 0, sizeof(innodb_page_type));
1634 partial_page_read = false;
1635 skip_page = false;
1636
1637 if (is_log_enabled) {
1638 fprintf(log_file, "Filename = %s\n", filename);
1639 }
1640
1641 if (*filename == '-') {
1642 /* read from stdin. */
1643 fil_in = stdin;
1644 read_from_stdin = true;
1645
1646 }
1647
1648 /* stat the file to get size and page count. */
1649 if (!read_from_stdin &&
1650#ifdef _WIN32
1651 _stat64(filename, &st)) {
1652#else
1653 stat(filename, &st)) {
1654#endif /* _WIN32 */
1655 fprintf(stderr, "Error: %s cannot be found\n",
1656 filename);
1657
1658 exit_status = 1;
1659 goto my_exit;
1660 }
1661
1662 if (!read_from_stdin) {
1663 size = st.st_size;
1664 fil_in = open_file(filename);
1665 /*If fil_in is NULL, terminate as some error encountered */
1666 if(fil_in == NULL) {
1667 exit_status = 1;
1668 goto my_exit;
1669 }
1670 /* Save the current file pointer in pos variable.*/
1671 if (0 != fgetpos(fil_in, &pos)) {
1672 perror("fgetpos");
1673 exit_status = 1;
1674 goto my_exit;
1675 }
1676 }
1677
1678 /* Read the minimum page size. */
1679 bytes = fread(buf, 1, UNIV_ZIP_SIZE_MIN, fil_in);
1680 partial_page_read = true;
1681
1682 if (bytes != UNIV_ZIP_SIZE_MIN) {
1683 fprintf(stderr, "Error: Was not able to read the "
1684 "minimum page size ");
1685 fprintf(stderr, "of %d bytes. Bytes read was " ULINTPF "\n",
1686 UNIV_ZIP_SIZE_MIN, bytes);
1687
1688 exit_status = 1;
1689 goto my_exit;
1690 }
1691
1692 /* enable variable is_system_tablespace when space_id of given
1693 file is zero. Use to skip the checksum verification and rewrite
1694 for doublewrite pages. */
1695 is_system_tablespace = (!memcmp(&space_id, buf +
1696 FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, 4))
1697 ? true : false;
1698
1699 /* Determine page size, zip_size and page compression
1700 from fsp_flags and encryption metadata from page 0 */
1701 const page_size_t& page_size = get_page_size(buf);
1702
1703 ulint flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf);
1704 ulint zip_size = page_size.is_compressed() ? page_size.logical() : 0;
1705 logical_page_size = page_size.is_compressed() ? zip_size : 0;
1706 physical_page_size = page_size.physical();
1707 bool is_compressed = FSP_FLAGS_HAS_PAGE_COMPRESSION(flags);
1708
1709 if (page_size.physical() > UNIV_ZIP_SIZE_MIN) {
1710 /* Read rest of the page 0 to determine crypt_data */
1711 bytes = read_file(buf, partial_page_read, page_size.physical(), fil_in);
1712 if (bytes != page_size.physical()) {
1713 fprintf(stderr, "Error: Was not able to read the "
1714 "rest of the page ");
1715 fprintf(stderr, "of " ULINTPF " bytes. Bytes read was " ULINTPF "\n",
1716 page_size.physical() - UNIV_ZIP_SIZE_MIN, bytes);
1717
1718 exit_status = 1;
1719 goto my_exit;
1720 }
1721 partial_page_read = false;
1722 }
1723
1724 /* Now that we have full page 0 in buffer, check encryption */
1725 bool is_encrypted = check_encryption(filename, page_size, buf);
1726
1727 /* Verify page 0 contents. Note that we can't allow
1728 checksum mismatch on page 0, because that would mean we
1729 could not trust it content. */
1730 if (!no_check) {
1731 unsigned long long tmp_allow_mismatches = allow_mismatches;
1732 allow_mismatches = 0;
1733
1734 exit_status = verify_checksum(buf, page_size, is_encrypted, is_compressed, &mismatch_count);
1735
1736 if (exit_status) {
1737 fprintf(stderr, "Error: Page 0 checksum mismatch, can't continue. \n");
1738 goto my_exit;
1739 }
1740 allow_mismatches = tmp_allow_mismatches;
1741 }
1742
1743 if ((exit_status = rewrite_checksum(filename, fil_in, buf,
1744 page_size, &pos, is_encrypted, is_compressed))) {
1745 goto my_exit;
1746 }
1747
1748 if (page_type_dump) {
1749 fprintf(fil_page_type,
1750 "\n\nFilename::%s\n", filename);
1751 fprintf(fil_page_type,
1752 "========================================"
1753 "======================================\n");
1754 fprintf(fil_page_type,
1755 "\tPAGE_NO\t\t|\t\tPAGE_TYPE\t\t"
1756 "\t|\tEXTRA INFO\n");
1757 fprintf(fil_page_type,
1758 "========================================"
1759 "======================================\n");
1760 }
1761
1762 if (per_page_details) {
1763 printf("page %llu ", cur_page_num);
1764 }
1765
1766 if (page_type_summary || page_type_dump) {
1767 parse_page(buf, xdes, fil_page_type, page_size, is_encrypted);
1768 }
1769
1770 pages = (ulint) (size / page_size.physical());
1771
1772 if (just_count) {
1773 if (read_from_stdin) {
1774 fprintf(stderr, "Number of pages:" ULINTPF "\n", pages);
1775 } else {
1776 printf("Number of pages:" ULINTPF "\n", pages);
1777 }
1778 continue;
1779 } else if (verbose && !read_from_stdin) {
1780 if (is_log_enabled) {
1781 fprintf(log_file, "file %s = %llu bytes "
1782 "(" ULINTPF " pages)\n", filename, size, pages);
1783 if (do_one_page) {
1784 fprintf(log_file, "Innochecksum: "
1785 "checking page::%llu;\n",
1786 do_page);
1787 }
1788 }
1789 } else {
1790 if (is_log_enabled) {
1791 fprintf(log_file, "Innochecksum: checking "
1792 "pages in range::%llu to %llu\n",
1793 start_page, use_end_page ?
1794 end_page : (pages - 1));
1795 }
1796 }
1797
1798 /* seek to the necessary position */
1799 if (start_page) {
1800 if (!read_from_stdin) {
1801 /* If read is not from stdin, we can use
1802 fseeko() to position the file pointer to
1803 the desired page. */
1804 partial_page_read = false;
1805
1806 offset = (off_t) start_page
1807 * (off_t) page_size.physical();
1808#ifdef _WIN32
1809 if (_fseeki64(fil_in, offset, SEEK_SET)) {
1810#else
1811 if (fseeko(fil_in, offset, SEEK_SET)) {
1812#endif /* _WIN32 */
1813 perror("Error: Unable to seek to "
1814 "necessary offset");
1815
1816 exit_status = 1;
1817 goto my_exit;
1818 }
1819 /* Save the current file pointer in
1820 pos variable. */
1821 if (0 != fgetpos(fil_in, &pos)) {
1822 perror("fgetpos");
1823
1824 exit_status = 1;
1825 goto my_exit;
1826 }
1827 } else {
1828
1829 ulong count = 0;
1830
1831 while (!feof(fil_in)) {
1832 if (start_page == count) {
1833 break;
1834 }
1835 /* We read a part of page to find the
1836 minimum page size. We cannot reset
1837 the file pointer to the beginning of
1838 the page if we are reading from stdin
1839 (fseeko() on stdin doesn't work). So
1840 read only the remaining part of page,
1841 if partial_page_read is enable. */
1842 bytes = read_file(buf,
1843 partial_page_read,
1844 static_cast<ulong>(
1845 page_size.physical()),
1846 fil_in);
1847
1848 partial_page_read = false;
1849 count++;
1850
1851 if (!bytes || feof(fil_in)) {
1852 fprintf(stderr, "Error: Unable "
1853 "to seek to necessary "
1854 "offset");
1855
1856 exit_status = 1;
1857 goto my_exit;
1858 }
1859 }
1860 }
1861 }
1862
1863 /* main checksumming loop */
1864 cur_page_num = start_page ? start_page : cur_page_num + 1;
1865
1866 lastt = 0;
1867 while (!feof(fil_in)) {
1868
1869 bytes = read_file(buf, partial_page_read,
1870 static_cast<ulong>(
1871 page_size.physical()), fil_in);
1872 partial_page_read = false;
1873
1874 if (!bytes && feof(fil_in)) {
1875 break;
1876 }
1877
1878 if (ferror(fil_in)) {
1879 fprintf(stderr, "Error reading " ULINTPF " bytes",
1880 page_size.physical());
1881 perror(" ");
1882
1883 exit_status = 1;
1884 goto my_exit;
1885 }
1886
1887 if (bytes != page_size.physical()) {
1888 fprintf(stderr, "Error: bytes read (" ULINTPF ") "
1889 "doesn't match page size (" ULINTPF ")\n",
1890 bytes, page_size.physical());
1891 exit_status = 1;
1892 goto my_exit;
1893 }
1894
1895 if (is_system_tablespace) {
1896 /* enable when page is double write buffer.*/
1897 skip_page = is_page_doublewritebuffer(buf);
1898 } else {
1899 skip_page = false;
1900 }
1901
1902 ulint cur_page_type = mach_read_from_2(buf+FIL_PAGE_TYPE);
1903
1904 /* FIXME: Page compressed or Page compressed and encrypted
1905 pages do not contain checksum. */
1906 if (cur_page_type == FIL_PAGE_PAGE_COMPRESSED ||
1907 cur_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
1908 skip_page = true;
1909 }
1910
1911 /* If no-check is enabled, skip the
1912 checksum verification.*/
1913 if (!no_check
1914 && !skip_page
1915 && (exit_status = verify_checksum(buf, page_size,
1916 is_encrypted, is_compressed, &mismatch_count))) {
1917 goto my_exit;
1918 }
1919
1920 if ((exit_status = rewrite_checksum(filename, fil_in, buf,
1921 page_size, &pos, is_encrypted, is_compressed))) {
1922 goto my_exit;
1923 }
1924
1925 /* end if this was the last page we were supposed to check */
1926 if (use_end_page && (cur_page_num >= end_page)) {
1927 break;
1928 }
1929
1930 if (per_page_details) {
1931 printf("page %llu ", cur_page_num);
1932 }
1933
1934 if (page_type_summary || page_type_dump) {
1935 parse_page(buf, xdes, fil_page_type, page_size, is_encrypted);
1936 }
1937
1938 /* do counter increase and progress printing */
1939 cur_page_num++;
1940
1941 if (verbose && !read_from_stdin) {
1942 if ((cur_page_num % 64) == 0) {
1943 now = time(0);
1944 if (!lastt) {
1945 lastt= now;
1946 }
1947 if (now - lastt >= 1
1948 && is_log_enabled) {
1949 fprintf(log_file, "page::%llu "
1950 "okay: %.3f%% done\n",
1951 (cur_page_num - 1),
1952 (float) cur_page_num / pages * 100);
1953 lastt = now;
1954 }
1955 }
1956 }
1957 }
1958
1959 if (!read_from_stdin) {
1960 /* flcose() will flush the data and release the lock if
1961 any acquired. */
1962 fclose(fil_in);
1963 }
1964
1965 /* Enabled for page type summary. */
1966 if (page_type_summary) {
1967 if (!read_from_stdin) {
1968 fprintf(stdout, "\nFile::%s",filename);
1969 print_summary(stdout);
1970 } else {
1971 print_summary(stderr);
1972 }
1973 }
1974 }
1975
1976 if (is_log_enabled) {
1977 fclose(log_file);
1978 }
1979
1980 free(buf_ptr);
1981 free(xdes_ptr);
1982
1983 my_end(exit_status);
1984 DBUG_RETURN(exit_status);
1985
1986my_exit:
1987 if (buf_ptr) {
1988 free(buf_ptr);
1989 }
1990
1991 if (xdes_ptr) {
1992 free(xdes_ptr);
1993 }
1994
1995 if (!read_from_stdin && fil_in) {
1996 fclose(fil_in);
1997 }
1998
1999 if (log_file) {
2000 fclose(log_file);
2001 }
2002
2003 my_end(exit_status);
2004 DBUG_RETURN(exit_status);
2005}
2006