1/* Copyright (c) 2000, 2013, Oracle and/or its affiliates.
2
3 This program is free software; you can redistribute it and/or modify
4 it under the terms of the GNU General Public License as published by
5 the Free Software Foundation; version 2 of the License.
6
7 This program is distributed in the hope that it will be useful,
8 but WITHOUT ANY WARRANTY; without even the implied warranty of
9 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 GNU General Public License for more details.
11
12 You should have received a copy of the GNU General Public License
13 along with this program; if not, write to the Free Software
14 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
15
16/* Describe, check and repair of MyISAM tables */
17
18/*
19 About checksum calculation.
20
21 There are two types of checksums. Table checksum and row checksum.
22
23 Row checksum is an additional byte at the end of dynamic length
24 records. It must be calculated if the table is configured for them.
25 Otherwise they must not be used. The variable
26 MYISAM_SHARE::calc_checksum determines if row checksums are used.
27 MI_INFO::checksum is used as temporary storage during row handling.
28 For parallel repair we must assure that only one thread can use this
29 variable. There is no problem on the write side as this is done by one
30 thread only. But when checking a record after read this could go
31 wrong. But since all threads read through a common read buffer, it is
32 sufficient if only one thread checks it.
33
34 Table checksum is an eight byte value in the header of the index file.
35 It can be calculated even if row checksums are not used. The variable
36 MI_CHECK::glob_crc is calculated over all records.
37 MI_SORT_PARAM::calc_checksum determines if this should be done. This
38 variable is not part of MI_CHECK because it must be set per thread for
39 parallel repair. The global glob_crc must be changed by one thread
40 only. And it is sufficient to calculate the checksum once only.
41*/
42
43#include "ftdefs.h"
44#include <m_ctype.h>
45#include <my_getopt.h>
46#ifdef HAVE_SYS_VADVISE_H
47#include <sys/vadvise.h>
48#endif
49#ifdef HAVE_SYS_MMAN_H
50#include <sys/mman.h>
51#endif
52#include "rt_index.h"
53#include <mysqld_error.h>
54
55 /* Functions defined in this file */
56
57static int check_k_link(HA_CHECK *param, MI_INFO *info,uint nr);
58static int chk_index(HA_CHECK *, MI_INFO *, MI_KEYDEF *, my_off_t, uchar *,
59 ha_rows *, ha_checksum *, uint);
60static uint isam_key_length(MI_INFO *info,MI_KEYDEF *keyinfo);
61static ha_checksum calc_checksum(ha_rows count);
62static int writekeys(MI_SORT_PARAM *sort_param);
63static int sort_one_index(HA_CHECK *, MI_INFO *, MI_KEYDEF *, my_off_t, File);
64static int sort_key_read(MI_SORT_PARAM *sort_param,void *key);
65static int sort_ft_key_read(MI_SORT_PARAM *sort_param,void *key);
66static int sort_get_next_record(MI_SORT_PARAM *sort_param);
67static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,const void *b);
68static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a);
69static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a);
70static my_off_t get_record_for_key(MI_INFO *, MI_KEYDEF *, uchar *);
71static int sort_insert_key(MI_SORT_PARAM *, SORT_KEY_BLOCKS *, uchar *, my_off_t);
72static int sort_delete_record(MI_SORT_PARAM *sort_param);
73/*static int flush_pending_blocks(HA_CHECK *param);*/
74static SORT_KEY_BLOCKS *alloc_key_blocks(HA_CHECK *, uint, uint);
75static ha_checksum mi_byte_checksum(const uchar *buf, uint length);
76static void set_data_file_type(MI_SORT_INFO *sort_info, MYISAM_SHARE *share);
77static int replace_data_file(HA_CHECK *param, MI_INFO *info, File new_file);
78
79void myisamchk_init(HA_CHECK *param)
80{
81 bzero((uchar*) param,sizeof(*param));
82 /* Set all params that are not 0 */
83 param->opt_follow_links=1;
84 param->keys_in_use= ~(ulonglong) 0;
85 param->search_after_block=HA_OFFSET_ERROR;
86 param->use_buffers= KEY_BUFFER_INIT;
87 param->read_buffer_length=READ_BUFFER_INIT;
88 param->write_buffer_length=READ_BUFFER_INIT;
89 param->sort_buffer_length=SORT_BUFFER_INIT;
90 param->sort_key_blocks=BUFFERS_WHEN_SORTING;
91 param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL;
92 param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL);
93 param->max_record_length= LONGLONG_MAX;
94 param->key_cache_block_size= KEY_CACHE_BLOCK_SIZE;
95 param->stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL;
96 param->need_print_msg_lock= 0;
97}
98
99 /* Check the status flags for the table */
100
101int chk_status(HA_CHECK *param, register MI_INFO *info)
102{
103 MYISAM_SHARE *share=info->s;
104
105 if (mi_is_crashed_on_repair(info))
106 mi_check_print_warning(param,
107 "Table is marked as crashed and last repair failed");
108 else if (mi_is_crashed(info))
109 mi_check_print_warning(param,
110 "Table is marked as crashed");
111 if (share->state.open_count != (uint) (info->s->global_changed ? 1 : 0))
112 {
113 /* Don't count this as a real warning, as check can correct this ! */
114 uint save=param->warning_printed;
115 mi_check_print_warning(param,
116 share->state.open_count==1 ?
117 "%d client is using or hasn't closed the table properly" :
118 "%d clients are using or haven't closed the table properly",
119 share->state.open_count);
120 /* If this will be fixed by the check, forget the warning */
121 if (param->testflag & T_UPDATE_STATE)
122 param->warning_printed=save;
123 }
124 return 0;
125}
126
127 /* Check delete links */
128
129int chk_del(HA_CHECK *param, register MI_INFO *info, ulonglong test_flag)
130{
131 reg2 ha_rows i;
132 uint delete_link_length;
133 my_off_t empty,next_link,UNINIT_VAR(old_link);
134 char buff[22],buff2[22];
135 DBUG_ENTER("chk_del");
136
137 param->record_checksum=0;
138 delete_link_length=((info->s->options & HA_OPTION_PACK_RECORD) ? 20 :
139 info->s->rec_reflength+1);
140
141 if (!(test_flag & T_SILENT))
142 puts("- check record delete-chain");
143
144 next_link=info->s->state.dellink;
145 if (info->state->del == 0)
146 {
147 if (test_flag & T_VERBOSE)
148 {
149 puts("No recordlinks");
150 }
151 }
152 else
153 {
154 if (test_flag & T_VERBOSE)
155 printf("Recordlinks: ");
156 empty=0;
157 for (i= info->state->del ; i > 0L && next_link != HA_OFFSET_ERROR ; i--)
158 {
159 if (killed_ptr(param))
160 DBUG_RETURN(1);
161 if (test_flag & T_VERBOSE)
162 printf(" %9s",llstr(next_link,buff));
163 if (next_link >= info->state->data_file_length)
164 goto wrong;
165 if (mysql_file_pread(info->dfile, (uchar*) buff, delete_link_length,
166 next_link, MYF(MY_NABP)))
167 {
168 if (test_flag & T_VERBOSE) puts("");
169 mi_check_print_error(param,"Can't read delete-link at filepos: %s",
170 llstr(next_link,buff));
171 DBUG_RETURN(1);
172 }
173 if (*buff != '\0')
174 {
175 if (test_flag & T_VERBOSE) puts("");
176 mi_check_print_error(param,"Record at pos: %s is not remove-marked",
177 llstr(next_link,buff));
178 goto wrong;
179 }
180 if (info->s->options & HA_OPTION_PACK_RECORD)
181 {
182 my_off_t prev_link=mi_sizekorr(buff+12);
183 if (empty && prev_link != old_link)
184 {
185 if (test_flag & T_VERBOSE) puts("");
186 mi_check_print_error(param,"Deleted block at %s doesn't point back at previous delete link",llstr(next_link,buff2));
187 goto wrong;
188 }
189 old_link=next_link;
190 next_link=mi_sizekorr(buff+4);
191 empty+=mi_uint3korr(buff+1);
192 }
193 else
194 {
195 param->record_checksum+=(ha_checksum) next_link;
196 next_link=_mi_rec_pos(info->s,(uchar*) buff+1);
197 empty+=info->s->base.pack_reclength;
198 }
199 }
200 if (test_flag & T_VERBOSE)
201 puts("\n");
202 if (empty != info->state->empty)
203 {
204 mi_check_print_warning(param,
205 "Found %s deleted space in delete link chain. Should be %s",
206 llstr(empty,buff2),
207 llstr(info->state->empty,buff));
208 }
209 if (next_link != HA_OFFSET_ERROR)
210 {
211 mi_check_print_error(param,
212 "Found more than the expected %s deleted rows in delete link chain",
213 llstr(info->state->del, buff));
214 goto wrong;
215 }
216 if (i != 0)
217 {
218 mi_check_print_error(param,
219 "Found %s deleted rows in delete link chain. Should be %s",
220 llstr(info->state->del - i, buff2),
221 llstr(info->state->del, buff));
222 goto wrong;
223 }
224 }
225 DBUG_RETURN(0);
226
227wrong:
228 param->testflag|=T_RETRY_WITHOUT_QUICK;
229 if (test_flag & T_VERBOSE) puts("");
230 mi_check_print_error(param,"record delete-link-chain corrupted");
231 DBUG_RETURN(1);
232} /* chk_del */
233
234
235 /* Check delete links in index file */
236
237static int check_k_link(HA_CHECK *param, register MI_INFO *info, uint nr)
238{
239 my_off_t next_link;
240 uint block_size=(nr+1)*MI_MIN_KEY_BLOCK_LENGTH;
241 ha_rows records;
242 char llbuff[21], llbuff2[21];
243 uchar *buff;
244 DBUG_ENTER("check_k_link");
245 DBUG_PRINT("enter", ("block_size: %u", block_size));
246
247 if (param->testflag & T_VERBOSE)
248 printf("block_size %4u:", block_size); /* purecov: tested */
249
250 next_link=info->s->state.key_del[nr];
251 records= (ha_rows) (info->state->key_file_length / block_size);
252 while (next_link != HA_OFFSET_ERROR && records > 0)
253 {
254 if (killed_ptr(param))
255 DBUG_RETURN(1);
256 if (param->testflag & T_VERBOSE)
257 printf("%16s",llstr(next_link,llbuff));
258
259 /* Key blocks must lay within the key file length entirely. */
260 if (next_link + block_size > info->state->key_file_length)
261 {
262 /* purecov: begin tested */
263 mi_check_print_error(param, "Invalid key block position: %s "
264 "key block size: %u file_length: %s",
265 llstr(next_link, llbuff), block_size,
266 llstr(info->state->key_file_length, llbuff2));
267 DBUG_RETURN(1);
268 /* purecov: end */
269 }
270
271 /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
272 if (next_link & (MI_MIN_KEY_BLOCK_LENGTH - 1))
273 {
274 /* purecov: begin tested */
275 mi_check_print_error(param, "Mis-aligned key block: %s "
276 "minimum key block length: %u",
277 llstr(next_link, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
278 DBUG_RETURN(1);
279 /* purecov: end */
280 }
281
282 /*
283 Read the key block with MI_MIN_KEY_BLOCK_LENGTH to find next link.
284 If the key cache block size is smaller than block_size, we can so
285 avoid unecessary eviction of cache block.
286 */
287 if (!(buff=key_cache_read(info->s->key_cache,
288 info->s->kfile, next_link, DFLT_INIT_HITS,
289 (uchar*) info->buff, MI_MIN_KEY_BLOCK_LENGTH,
290 MI_MIN_KEY_BLOCK_LENGTH, 1)))
291 {
292 /* purecov: begin tested */
293 mi_check_print_error(param, "key cache read error for block: %s",
294 llstr(next_link,llbuff));
295 DBUG_RETURN(1);
296 /* purecov: end */
297 }
298 next_link=mi_sizekorr(buff);
299 records--;
300 param->key_file_blocks+=block_size;
301 }
302 if (param->testflag & T_VERBOSE)
303 {
304 if (next_link != HA_OFFSET_ERROR)
305 printf("%16s\n",llstr(next_link,llbuff));
306 else
307 puts("");
308 }
309 DBUG_RETURN (next_link != HA_OFFSET_ERROR);
310} /* check_k_link */
311
312
313 /* Check sizes of files */
314
315int chk_size(HA_CHECK *param, register MI_INFO *info)
316{
317 int error=0;
318 register my_off_t skr,size;
319 char buff[22],buff2[22];
320 DBUG_ENTER("chk_size");
321
322 if (!(param->testflag & T_SILENT)) puts("- check file-size");
323
324 /* The following is needed if called externally (not from myisamchk) */
325 flush_key_blocks(info->s->key_cache,
326 info->s->kfile, &info->s->dirty_part_map,
327 FLUSH_FORCE_WRITE);
328
329 size= mysql_file_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(MY_THREADSAFE));
330 if ((skr=(my_off_t) info->state->key_file_length) != size)
331 {
332 /* Don't give error if file generated by myisampack */
333 if (skr > size && mi_is_any_key_active(info->s->state.key_map))
334 {
335 error=1;
336 mi_check_print_error(param,
337 "Size of indexfile is: %-8s Should be: %s",
338 llstr(size,buff), llstr(skr,buff2));
339 }
340 else
341 mi_check_print_warning(param,
342 "Size of indexfile is: %-8s Should be: %s",
343 llstr(size,buff), llstr(skr,buff2));
344 }
345 if (!(param->testflag & T_VERY_SILENT) &&
346 ! (info->s->options & HA_OPTION_COMPRESS_RECORD) &&
347 ulonglong2double(info->state->key_file_length) >
348 ulonglong2double(info->s->base.margin_key_file_length)*0.9)
349 mi_check_print_warning(param,"Keyfile is almost full, %10s of %10s used",
350 llstr(info->state->key_file_length,buff),
351 llstr(info->s->base.max_key_file_length-1,buff));
352
353 size= mysql_file_seek(info->dfile, 0L, MY_SEEK_END, MYF(0));
354 skr=(my_off_t) info->state->data_file_length;
355 if (info->s->options & HA_OPTION_COMPRESS_RECORD)
356 skr+= MEMMAP_EXTRA_MARGIN;
357#ifdef USE_RELOC
358 if (info->data_file_type == STATIC_RECORD &&
359 skr < (my_off_t) info->s->base.reloc*info->s->base.min_pack_length)
360 skr=(my_off_t) info->s->base.reloc*info->s->base.min_pack_length;
361#endif
362 if (skr != size)
363 {
364 info->state->data_file_length=size; /* Skip other errors */
365 if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN)
366 {
367 error=1;
368 mi_check_print_error(param,"Size of datafile is: %-9s Should be: %s",
369 llstr(size,buff), llstr(skr,buff2));
370 param->testflag|=T_RETRY_WITHOUT_QUICK;
371 }
372 else
373 {
374 mi_check_print_warning(param,
375 "Size of datafile is: %-9s Should be: %s",
376 llstr(size,buff), llstr(skr,buff2));
377 }
378 }
379 if (!(param->testflag & T_VERY_SILENT) &&
380 !(info->s->options & HA_OPTION_COMPRESS_RECORD) &&
381 ulonglong2double(info->state->data_file_length) >
382 (ulonglong2double(info->s->base.max_data_file_length)*0.9))
383 mi_check_print_warning(param, "Datafile is almost full, %10s of %10s used",
384 llstr(info->state->data_file_length,buff),
385 llstr(info->s->base.max_data_file_length-1,buff2));
386 DBUG_RETURN(error);
387} /* chk_size */
388
389
390 /* Check keys */
391
392int chk_key(HA_CHECK *param, register MI_INFO *info)
393{
394 uint key,found_keys=0,full_text_keys=0,result=0;
395 ha_rows keys;
396 ha_checksum old_record_checksum,init_checksum;
397 my_off_t all_keydata,all_totaldata,key_totlength,length;
398 ulong *rec_per_key_part;
399 MYISAM_SHARE *share=info->s;
400 MI_KEYDEF *keyinfo;
401 char buff[22],buff2[22];
402 DBUG_ENTER("chk_key");
403
404 if (!(param->testflag & T_SILENT))
405 puts("- check key delete-chain");
406
407 param->key_file_blocks=info->s->base.keystart;
408 for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
409 if (check_k_link(param,info,key))
410 {
411 if (param->testflag & T_VERBOSE) puts("");
412 mi_check_print_error(param,"key delete-link-chain corrupted");
413 DBUG_RETURN(-1);
414 }
415
416 if (!(param->testflag & T_SILENT)) puts("- check index reference");
417
418 all_keydata=all_totaldata=key_totlength=0;
419 old_record_checksum=0;
420 init_checksum=param->record_checksum;
421 if (!(share->options &
422 (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
423 old_record_checksum=calc_checksum(info->state->records+info->state->del-1)*
424 share->base.pack_reclength;
425 rec_per_key_part= param->rec_per_key_part;
426 for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
427 rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++)
428 {
429 param->key_crc[key]=0;
430 if (! mi_is_key_active(share->state.key_map, key))
431 {
432 /* Remember old statistics for key */
433 memcpy((char*) rec_per_key_part,
434 (char*) (share->state.rec_per_key_part +
435 (uint) (rec_per_key_part - param->rec_per_key_part)),
436 keyinfo->keysegs*sizeof(*rec_per_key_part));
437 continue;
438 }
439 found_keys++;
440
441 param->record_checksum=init_checksum;
442
443 bzero((char*) &param->unique_count,sizeof(param->unique_count));
444 bzero((char*) &param->notnull_count,sizeof(param->notnull_count));
445
446 if ((!(param->testflag & T_SILENT)))
447 printf ("- check data record references index: %d\n",key+1);
448 if (keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL))
449 full_text_keys++;
450 if (share->state.key_root[key] == HA_OFFSET_ERROR &&
451 (info->state->records == 0 || keyinfo->flag & HA_FULLTEXT))
452 goto do_stat;
453 if (!_mi_fetch_keypage(info,keyinfo,share->state.key_root[key],
454 DFLT_INIT_HITS,info->buff,0))
455 {
456 mi_check_print_error(param,"Can't read indexpage from filepos: %s",
457 llstr(share->state.key_root[key],buff));
458 if (!(param->testflag & T_INFO))
459 DBUG_RETURN(-1);
460 result= -1;
461 continue;
462 }
463 param->key_file_blocks+=keyinfo->block_length;
464 keys=0;
465 param->keydata=param->totaldata=0;
466 param->key_blocks=0;
467 param->max_level=0;
468 if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff,
469 &keys, param->key_crc+key,1))
470 DBUG_RETURN(-1);
471 if(!(keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL)))
472 {
473 if (keys != info->state->records)
474 {
475 mi_check_print_error(param,"Found %s keys of %s",llstr(keys,buff),
476 llstr(info->state->records,buff2));
477 if (!(param->testflag & T_INFO))
478 DBUG_RETURN(-1);
479 result= -1;
480 continue;
481 }
482 if (found_keys - full_text_keys == 1 &&
483 ((share->options &
484 (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ||
485 (param->testflag & T_DONT_CHECK_CHECKSUM)))
486 old_record_checksum=param->record_checksum;
487 else if (old_record_checksum != param->record_checksum)
488 {
489 if (key)
490 mi_check_print_error(param,"Key %u doesn't point at same records that key 1",
491 key+1);
492 else
493 mi_check_print_error(param,"Key 1 doesn't point at all records");
494 if (!(param->testflag & T_INFO))
495 DBUG_RETURN(-1);
496 result= -1;
497 continue;
498 }
499 }
500 if ((uint) share->base.auto_key -1 == key)
501 {
502 /* Check that auto_increment key is bigger than max key value */
503 ulonglong auto_increment;
504 info->lastinx=key;
505 _mi_read_key_record(info, 0L, info->rec_buff);
506 auto_increment= retrieve_auto_increment(info, info->rec_buff);
507 if (auto_increment > info->s->state.auto_increment)
508 {
509 mi_check_print_warning(param, "Auto-increment value: %s is smaller "
510 "than max used value: %s",
511 llstr(info->s->state.auto_increment,buff2),
512 llstr(auto_increment, buff));
513 }
514 if (param->testflag & T_AUTO_INC)
515 {
516 set_if_bigger(info->s->state.auto_increment,
517 auto_increment);
518 set_if_bigger(info->s->state.auto_increment,
519 param->auto_increment_value);
520 }
521
522 /* Check that there isn't a row with auto_increment = 0 in the table */
523 mi_extra(info,HA_EXTRA_KEYREAD,0);
524 bzero(info->lastkey,keyinfo->seg->length);
525 if (!mi_rkey(info, info->rec_buff, key, (const uchar*) info->lastkey,
526 (key_part_map)1, HA_READ_KEY_EXACT))
527 {
528 /* Don't count this as a real warning, as myisamchk can't correct it */
529 uint save=param->warning_printed;
530 mi_check_print_warning(param, "Found row where the auto_increment "
531 "column has the value 0");
532 param->warning_printed=save;
533 }
534 mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
535 }
536
537 length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2;
538 if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L)
539 printf("Key: %2d: Keyblocks used: %3d%% Packed: %4d%% Max levels: %2d\n",
540 key+1,
541 (int) (my_off_t2double(param->keydata)*100.0/my_off_t2double(param->totaldata)),
542 (int) ((my_off_t2double(length) - my_off_t2double(param->keydata))*100.0/
543 my_off_t2double(length)),
544 param->max_level);
545 all_keydata+=param->keydata; all_totaldata+=param->totaldata; key_totlength+=length;
546
547do_stat:
548 if (param->testflag & T_STATISTICS)
549 update_key_parts(keyinfo, rec_per_key_part, param->unique_count,
550 param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
551 param->notnull_count: NULL,
552 (ulonglong)info->state->records);
553 }
554 if (param->testflag & T_INFO)
555 {
556 if (all_totaldata != 0L && found_keys > 0)
557 printf("Total: Keyblocks used: %3d%% Packed: %4d%%\n\n",
558 (int) (my_off_t2double(all_keydata)*100.0/
559 my_off_t2double(all_totaldata)),
560 (int) ((my_off_t2double(key_totlength) -
561 my_off_t2double(all_keydata))*100.0/
562 my_off_t2double(key_totlength)));
563 else if (all_totaldata != 0L && mi_is_any_key_active(share->state.key_map))
564 puts("");
565 }
566 if (param->key_file_blocks != info->state->key_file_length &&
567 param->keys_in_use != ~(ulonglong) 0)
568 mi_check_print_warning(param, "Some data are unreferenced in keyfile");
569 if (found_keys != full_text_keys)
570 param->record_checksum=old_record_checksum-init_checksum; /* Remove delete links */
571 else
572 param->record_checksum=0;
573 DBUG_RETURN(result);
574} /* chk_key */
575
576
577static int chk_index_down(HA_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
578 my_off_t page, uchar *buff, ha_rows *keys,
579 ha_checksum *key_checksum, uint level)
580{
581 char llbuff[22],llbuff2[22];
582 DBUG_ENTER("chk_index_down");
583
584 /* Key blocks must lay within the key file length entirely. */
585 if (page + keyinfo->block_length > info->state->key_file_length)
586 {
587 /* purecov: begin tested */
588 /* Give it a chance to fit in the real file size. */
589 my_off_t max_length= mysql_file_seek(info->s->kfile, 0L, MY_SEEK_END,
590 MYF(MY_THREADSAFE));
591 mi_check_print_error(param, "Invalid key block position: %s "
592 "key block size: %u file_length: %s",
593 llstr(page, llbuff), keyinfo->block_length,
594 llstr(info->state->key_file_length, llbuff2));
595 if (page + keyinfo->block_length > max_length)
596 goto err;
597 /* Fix the remebered key file length. */
598 info->state->key_file_length= (max_length &
599 ~ (my_off_t) (keyinfo->block_length - 1));
600 /* purecov: end */
601 }
602
603 /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
604 if (page & (MI_MIN_KEY_BLOCK_LENGTH - 1))
605 {
606 /* purecov: begin tested */
607 mi_check_print_error(param, "Mis-aligned key block: %s "
608 "minimum key block length: %u",
609 llstr(page, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
610 goto err;
611 /* purecov: end */
612 }
613
614 if (!_mi_fetch_keypage(info,keyinfo,page, DFLT_INIT_HITS,buff,0))
615 {
616 mi_check_print_error(param,"Can't read key from filepos: %s",
617 llstr(page,llbuff));
618 goto err;
619 }
620 param->key_file_blocks+=keyinfo->block_length;
621 if (chk_index(param,info,keyinfo,page,buff,keys,key_checksum,level))
622 goto err;
623
624 DBUG_RETURN(0);
625
626 /* purecov: begin tested */
627err:
628 DBUG_RETURN(1);
629 /* purecov: end */
630}
631
632
633/*
634 "Ignore NULLs" statistics collection method: process first index tuple.
635
636 SYNOPSIS
637 mi_collect_stats_nonulls_first()
638 keyseg IN Array of key part descriptions
639 notnull INOUT Array, notnull[i] = (number of {keypart1...keypart_i}
640 tuples that don't contain NULLs)
641 key IN Key values tuple
642
643 DESCRIPTION
644 Process the first index tuple - find out which prefix tuples don't
645 contain NULLs, and update the array of notnull counters accordingly.
646*/
647
648static
649void mi_collect_stats_nonulls_first(HA_KEYSEG *keyseg, ulonglong *notnull,
650 uchar *key)
651{
652 uint first_null, kp;
653 first_null= (uint) (ha_find_null(keyseg, key) - keyseg);
654 /*
655 All prefix tuples that don't include keypart_{first_null} are not-null
656 tuples (and all others aren't), increment counters for them.
657 */
658 for (kp= 0; kp < first_null; kp++)
659 notnull[kp]++;
660}
661
662
663/*
664 "Ignore NULLs" statistics collection method: process next index tuple.
665
666 SYNOPSIS
667 mi_collect_stats_nonulls_next()
668 keyseg IN Array of key part descriptions
669 notnull INOUT Array, notnull[i] = (number of {keypart1...keypart_i}
670 tuples that don't contain NULLs)
671 prev_key IN Previous key values tuple
672 last_key IN Next key values tuple
673
674 DESCRIPTION
675 Process the next index tuple:
676 1. Find out which prefix tuples of last_key don't contain NULLs, and
677 update the array of notnull counters accordingly.
678 2. Find the first keypart number where the prev_key and last_key tuples
679 are different(A), or last_key has NULL value(B), and return it, so the
680 caller can count number of unique tuples for each key prefix. We don't
681 need (B) to be counted, and that is compensated back in
682 update_key_parts().
683
684 RETURN
685 1 + number of first keypart where values differ or last_key tuple has NULL
686*/
687
688static
689int mi_collect_stats_nonulls_next(HA_KEYSEG *keyseg, ulonglong *notnull,
690 uchar *prev_key, uchar *last_key)
691{
692 uint diffs[2];
693 uint first_null_seg, kp;
694 HA_KEYSEG *seg;
695
696 /*
697 Find the first keypart where values are different or either of them is
698 NULL. We get results in diffs array:
699 diffs[0]= 1 + number of first different keypart
700 diffs[1]=offset: (last_key + diffs[1]) points to first value in
701 last_key that is NULL or different from corresponding
702 value in prev_key.
703 */
704 ha_key_cmp(keyseg, prev_key, last_key, USE_WHOLE_KEY,
705 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diffs);
706 seg= keyseg + diffs[0] - 1;
707
708 /* Find first NULL in last_key */
709 first_null_seg= (uint) (ha_find_null(seg, last_key + diffs[1]) - keyseg);
710 for (kp= 0; kp < first_null_seg; kp++)
711 notnull[kp]++;
712
713 /*
714 Return 1+ number of first key part where values differ. Don't care if
715 these were NULLs and not .... We compensate for that in
716 update_key_parts.
717 */
718 return diffs[0];
719}
720
721
722 /* Check if index is ok */
723
724static int chk_index(HA_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
725 my_off_t page, uchar *buff, ha_rows *keys,
726 ha_checksum *key_checksum, uint level)
727{
728 int flag;
729 uint used_length,comp_flag,nod_flag,key_length=0;
730 uchar key[HA_MAX_POSSIBLE_KEY_BUFF],*temp_buff,*keypos,*old_keypos,*endpos;
731 my_off_t next_page,record;
732 char llbuff[22];
733 uint diff_pos[2];
734 DBUG_ENTER("chk_index");
735 DBUG_DUMP("buff",(uchar*) buff,mi_getint(buff));
736
737 /* TODO: implement appropriate check for RTree keys */
738 if (keyinfo->flag & HA_SPATIAL)
739 DBUG_RETURN(0);
740
741 if (!(temp_buff=(uchar*) my_alloca((uint) keyinfo->block_length)))
742 {
743 mi_check_print_error(param,"Not enough memory for keyblock");
744 DBUG_RETURN(-1);
745 }
746
747 if (keyinfo->flag & HA_NOSAME)
748 {
749 /* Not real duplicates */
750 comp_flag= SEARCH_FIND | SEARCH_UPDATE | SEARCH_INSERT;
751 }
752 else
753 comp_flag=SEARCH_SAME; /* Keys in positionorder */
754 nod_flag=mi_test_if_nod(buff);
755 used_length=mi_getint(buff);
756 keypos=buff+2+nod_flag;
757 endpos=buff+used_length;
758
759 param->keydata+=used_length; param->totaldata+=keyinfo->block_length; /* INFO */
760 param->key_blocks++;
761 if (level > param->max_level)
762 param->max_level=level;
763
764 if (used_length > keyinfo->block_length)
765 {
766 mi_check_print_error(param,"Wrong pageinfo at page: %s",
767 llstr(page,llbuff));
768 goto err;
769 }
770 for ( ;; )
771 {
772 if (killed_ptr(param))
773 goto err;
774 memcpy((char*) info->lastkey,(char*) key,key_length);
775 info->lastkey_length=key_length;
776 if (nod_flag)
777 {
778 next_page=_mi_kpos(nod_flag,keypos);
779 if (chk_index_down(param,info,keyinfo,next_page,
780 temp_buff,keys,key_checksum,level+1))
781 goto err;
782 }
783 old_keypos=keypos;
784 if (keypos >= endpos ||
785 (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
786 break;
787 if (keypos > endpos)
788 {
789 mi_check_print_error(param,"Wrong key block length at page: %s",llstr(page,llbuff));
790 goto err;
791 }
792 if ((*keys)++ &&
793 (flag=ha_key_cmp(keyinfo->seg,info->lastkey,key,key_length,
794 comp_flag, diff_pos)) >=0)
795 {
796 DBUG_DUMP("old",info->lastkey, info->lastkey_length);
797 DBUG_DUMP("new",key, key_length);
798 DBUG_DUMP("new_in_page",old_keypos,(uint) (keypos-old_keypos));
799
800 if (comp_flag & SEARCH_FIND && flag == 0)
801 mi_check_print_error(param,"Found duplicated key at page %s",llstr(page,llbuff));
802 else
803 mi_check_print_error(param,"Key in wrong position at page %s",llstr(page,llbuff));
804 goto err;
805 }
806 if (param->testflag & T_STATISTICS)
807 {
808 if (*keys != 1L) /* not first_key */
809 {
810 if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
811 ha_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY,
812 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL,
813 diff_pos);
814 else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
815 {
816 diff_pos[0]= mi_collect_stats_nonulls_next(keyinfo->seg,
817 param->notnull_count,
818 info->lastkey, key);
819 }
820 param->unique_count[diff_pos[0]-1]++;
821 }
822 else
823 {
824 if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
825 mi_collect_stats_nonulls_first(keyinfo->seg, param->notnull_count,
826 key);
827 }
828 }
829 (*key_checksum)+= mi_byte_checksum((uchar*) key,
830 key_length- info->s->rec_reflength);
831 record= _mi_dpos(info,0,key+key_length);
832 if (keyinfo->flag & HA_FULLTEXT) /* special handling for ft2 */
833 {
834 uint off;
835 int subkeys;
836 get_key_full_length_rdonly(off, key);
837 subkeys=ft_sintXkorr(key+off);
838 if (subkeys < 0)
839 {
840 ha_rows tmp_keys=0;
841 if (chk_index_down(param,info,&info->s->ft2_keyinfo,record,
842 temp_buff,&tmp_keys,key_checksum,1))
843 goto err;
844 if (tmp_keys + subkeys)
845 {
846 mi_check_print_error(param,
847 "Number of words in the 2nd level tree "
848 "does not match the number in the header. "
849 "Parent word in on the page %s, offset %u",
850 llstr(page,llbuff), (uint) (old_keypos-buff));
851 goto err;
852 }
853 (*keys)+=tmp_keys-1;
854 continue;
855 }
856 /* fall through */
857 }
858 if (record >= info->state->data_file_length)
859 {
860#ifndef DBUG_OFF
861 char llbuff2[22], llbuff3[22];
862#endif
863 mi_check_print_error(param,"Found key at page %s that points to record outside datafile",llstr(page,llbuff));
864 DBUG_PRINT("test",("page: %s record: %s filelength: %s",
865 llstr(page,llbuff),llstr(record,llbuff2),
866 llstr(info->state->data_file_length,llbuff3)));
867 DBUG_DUMP("key",key,key_length);
868 DBUG_DUMP("new_in_page",old_keypos,(uint) (keypos-old_keypos));
869 goto err;
870 }
871 param->record_checksum+=(ha_checksum) record;
872 }
873 if (keypos != endpos)
874 {
875 mi_check_print_error(param,"Keyblock size at page %s is not correct. Block length: %d key length: %d",
876 llstr(page,llbuff), used_length, (keypos - buff));
877 goto err;
878 }
879 my_afree((uchar*) temp_buff);
880 DBUG_RETURN(0);
881 err:
882 my_afree((uchar*) temp_buff);
883 DBUG_RETURN(1);
884} /* chk_index */
885
886
887 /* Calculate a checksum of 1+2+3+4...N = N*(N+1)/2 without overflow */
888
889static ha_checksum calc_checksum(ha_rows count)
890{
891 ulonglong sum,a,b;
892 DBUG_ENTER("calc_checksum");
893
894 sum=0;
895 a=count; b=count+1;
896 if (a & 1)
897 b>>=1;
898 else
899 a>>=1;
900 while (b)
901 {
902 if (b & 1)
903 sum+=a;
904 a<<=1; b>>=1;
905 }
906 DBUG_PRINT("exit",("sum: %lx",(ulong) sum));
907 DBUG_RETURN((ha_checksum) sum);
908} /* calc_checksum */
909
910
911 /* Calc length of key in normal isam */
912
913static uint isam_key_length(MI_INFO *info, register MI_KEYDEF *keyinfo)
914{
915 uint length;
916 HA_KEYSEG *keyseg;
917 DBUG_ENTER("isam_key_length");
918
919 length= info->s->rec_reflength;
920 for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++)
921 length+= keyseg->length;
922
923 DBUG_PRINT("exit",("length: %d",length));
924 DBUG_RETURN(length);
925} /* key_length */
926
927
928 /* Check that record-link is ok */
929
930int chk_data_link(HA_CHECK *param, MI_INFO *info, my_bool extend)
931{
932 int error,got_error,flag;
933 uint key, UNINIT_VAR(left_length), b_type;
934 ha_rows records,del_blocks;
935 my_off_t used,empty,pos,splits,UNINIT_VAR(start_recpos),
936 del_length,link_used,start_block;
937 uchar *record= 0, *UNINIT_VAR(to);
938 char llbuff[22],llbuff2[22],llbuff3[22];
939 ha_checksum intern_record_checksum;
940 ha_checksum key_checksum[HA_MAX_POSSIBLE_KEY];
941 MI_KEYDEF *keyinfo;
942 MI_BLOCK_INFO block_info;
943 DBUG_ENTER("chk_data_link");
944
945 if (!(param->testflag & T_SILENT))
946 {
947 if (extend)
948 puts("- check records and index references");
949 else
950 puts("- check record links");
951 }
952
953 if (!mi_alloc_rec_buff(info, -1, &record))
954 {
955 mi_check_print_error(param,"Not enough memory for record");
956 DBUG_RETURN(-1);
957 }
958 records=del_blocks=0;
959 used=link_used=splits=del_length=0;
960 intern_record_checksum=param->glob_crc=0;
961 got_error=error=0;
962 empty=info->s->pack.header_length;
963
964 pos=my_b_tell(&param->read_cache);
965 bzero((char*) key_checksum, info->s->base.keys * sizeof(key_checksum[0]));
966 while (pos < info->state->data_file_length)
967 {
968 if (killed_ptr(param))
969 goto err2;
970 switch (info->s->data_file_type) {
971 case STATIC_RECORD:
972 if (my_b_read(&param->read_cache,(uchar*) record,
973 info->s->base.pack_reclength))
974 goto err;
975 start_recpos=pos;
976 pos+=info->s->base.pack_reclength;
977 splits++;
978 if (*record == '\0')
979 {
980 del_blocks++;
981 del_length+=info->s->base.pack_reclength;
982 continue; /* Record removed */
983 }
984 param->glob_crc+= (*info->s->calc_check_checksum)(info,record);
985 used+=info->s->base.pack_reclength;
986 break;
987 case DYNAMIC_RECORD:
988 flag=block_info.second_read=0;
989 block_info.next_filepos=pos;
990 do
991 {
992 if (_mi_read_cache(&param->read_cache,(uchar*) block_info.header,
993 (start_block=block_info.next_filepos),
994 sizeof(block_info.header),
995 (flag ? 0 : READING_NEXT) | READING_HEADER))
996 goto err;
997 if (start_block & (MI_DYN_ALIGN_SIZE-1))
998 {
999 mi_check_print_error(param,"Wrong aligned block at %s",
1000 llstr(start_block,llbuff));
1001 goto err2;
1002 }
1003 b_type=_mi_get_block_info(&block_info,-1,start_block);
1004 if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
1005 BLOCK_FATAL_ERROR))
1006 {
1007 if (b_type & BLOCK_SYNC_ERROR)
1008 {
1009 if (flag)
1010 {
1011 mi_check_print_error(param,"Unexpected byte: %d at link: %s",
1012 (int) block_info.header[0],
1013 llstr(start_block,llbuff));
1014 goto err2;
1015 }
1016 pos=block_info.filepos+block_info.block_len;
1017 goto next;
1018 }
1019 if (b_type & BLOCK_DELETED)
1020 {
1021 if (block_info.block_len < info->s->base.min_block_length)
1022 {
1023 mi_check_print_error(param,
1024 "Deleted block with impossible length %lu at %s",
1025 block_info.block_len,llstr(pos,llbuff));
1026 goto err2;
1027 }
1028 if ((block_info.next_filepos != HA_OFFSET_ERROR &&
1029 block_info.next_filepos >= info->state->data_file_length) ||
1030 (block_info.prev_filepos != HA_OFFSET_ERROR &&
1031 block_info.prev_filepos >= info->state->data_file_length))
1032 {
1033 mi_check_print_error(param,"Delete link points outside datafile at %s",
1034 llstr(pos,llbuff));
1035 goto err2;
1036 }
1037 del_blocks++;
1038 del_length+=block_info.block_len;
1039 pos=block_info.filepos+block_info.block_len;
1040 splits++;
1041 goto next;
1042 }
1043 mi_check_print_error(param,"Wrong bytesec: %d-%d-%d at linkstart: %s",
1044 block_info.header[0],block_info.header[1],
1045 block_info.header[2],
1046 llstr(start_block,llbuff));
1047 goto err2;
1048 }
1049 if (info->state->data_file_length < block_info.filepos+
1050 block_info.block_len)
1051 {
1052 mi_check_print_error(param,
1053 "Recordlink that points outside datafile at %s",
1054 llstr(pos,llbuff));
1055 got_error=1;
1056 break;
1057 }
1058 splits++;
1059 if (!flag++) /* First block */
1060 {
1061 start_recpos=pos;
1062 pos=block_info.filepos+block_info.block_len;
1063 if (block_info.rec_len > (uint) info->s->base.max_pack_length)
1064 {
1065 mi_check_print_error(param,"Found too long record (%lu) at %s",
1066 (ulong) block_info.rec_len,
1067 llstr(start_recpos,llbuff));
1068 got_error=1;
1069 break;
1070 }
1071 if (info->s->base.blobs)
1072 {
1073 if (!(to= mi_alloc_rec_buff(info, block_info.rec_len,
1074 &info->rec_buff)))
1075 {
1076 mi_check_print_error(param,
1077 "Not enough memory (%lu) for blob at %s",
1078 (ulong) block_info.rec_len,
1079 llstr(start_recpos,llbuff));
1080 got_error=1;
1081 break;
1082 }
1083 }
1084 else
1085 to= info->rec_buff;
1086 left_length=block_info.rec_len;
1087 }
1088 if (left_length < block_info.data_len)
1089 {
1090 mi_check_print_error(param,"Found too long record (%lu) at %s",
1091 (ulong) block_info.data_len,
1092 llstr(start_recpos,llbuff));
1093 got_error=1;
1094 break;
1095 }
1096 if (_mi_read_cache(&param->read_cache,(uchar*) to,block_info.filepos,
1097 (uint) block_info.data_len,
1098 flag == 1 ? READING_NEXT : 0))
1099 goto err;
1100 to+=block_info.data_len;
1101 link_used+= block_info.filepos-start_block;
1102 used+= block_info.filepos - start_block + block_info.data_len;
1103 empty+=block_info.block_len-block_info.data_len;
1104 left_length-=block_info.data_len;
1105 if (left_length)
1106 {
1107 if (b_type & BLOCK_LAST)
1108 {
1109 mi_check_print_error(param,
1110 "Wrong record length %s of %s at %s",
1111 llstr(block_info.rec_len-left_length,llbuff),
1112 llstr(block_info.rec_len, llbuff2),
1113 llstr(start_recpos,llbuff3));
1114 got_error=1;
1115 break;
1116 }
1117 if (info->state->data_file_length < block_info.next_filepos)
1118 {
1119 mi_check_print_error(param,
1120 "Found next-recordlink that points outside datafile at %s",
1121 llstr(block_info.filepos,llbuff));
1122 got_error=1;
1123 break;
1124 }
1125 }
1126 } while (left_length);
1127 if (! got_error)
1128 {
1129 if (_mi_rec_unpack(info,record,info->rec_buff,block_info.rec_len) ==
1130 MY_FILE_ERROR)
1131 {
1132 mi_check_print_error(param,"Found wrong record at %s",
1133 llstr(start_recpos,llbuff));
1134 got_error=1;
1135 }
1136 else
1137 {
1138 info->checksum= (*info->s->calc_check_checksum)(info,record);
1139 if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE))
1140 {
1141 if (_mi_rec_check(info,record, info->rec_buff,block_info.rec_len,
1142 MY_TEST(info->s->calc_checksum)))
1143 {
1144 mi_check_print_error(param,"Found wrong packed record at %s",
1145 llstr(start_recpos,llbuff));
1146 got_error=1;
1147 }
1148 }
1149 if (!got_error)
1150 param->glob_crc+= info->checksum;
1151 }
1152 }
1153 else if (!flag)
1154 pos=block_info.filepos+block_info.block_len;
1155 break;
1156 case COMPRESSED_RECORD:
1157 if (_mi_read_cache(&param->read_cache,(uchar*) block_info.header, pos,
1158 info->s->pack.ref_length, READING_NEXT))
1159 goto err;
1160 start_recpos=pos;
1161 splits++;
1162 (void) _mi_pack_get_block_info(info, &info->bit_buff, &block_info,
1163 &info->rec_buff, -1, start_recpos);
1164 pos=block_info.filepos+block_info.rec_len;
1165 if (block_info.rec_len < (uint) info->s->min_pack_length ||
1166 block_info.rec_len > (uint) info->s->max_pack_length)
1167 {
1168 mi_check_print_error(param,
1169 "Found block with wrong recordlength: %d at %s",
1170 block_info.rec_len, llstr(start_recpos,llbuff));
1171 got_error=1;
1172 break;
1173 }
1174 if (_mi_read_cache(&param->read_cache,(uchar*) info->rec_buff,
1175 block_info.filepos, block_info.rec_len, READING_NEXT))
1176 goto err;
1177 if (_mi_pack_rec_unpack(info, &info->bit_buff, record,
1178 info->rec_buff, block_info.rec_len))
1179 {
1180 mi_check_print_error(param,"Found wrong record at %s",
1181 llstr(start_recpos,llbuff));
1182 got_error=1;
1183 }
1184 param->glob_crc+= (*info->s->calc_check_checksum)(info,record);
1185 link_used+= (block_info.filepos - start_recpos);
1186 used+= (pos-start_recpos);
1187 break;
1188 default:
1189 DBUG_ASSERT(0); /* Impossible */
1190 break;
1191 } /* switch */
1192 if (param->fix_record)
1193 param->fix_record(info, record, -1);
1194 if (! got_error)
1195 {
1196 intern_record_checksum+=(ha_checksum) start_recpos;
1197 records++;
1198 if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0)
1199 {
1200 printf("%s\r", llstr(records,llbuff)); (void) fflush(stdout);
1201 }
1202
1203 /* Check if keys match the record */
1204
1205 for (key=0,keyinfo= info->s->keyinfo; key < info->s->base.keys;
1206 key++,keyinfo++)
1207 {
1208 if (mi_is_key_active(info->s->state.key_map, key))
1209 {
1210 if(!(keyinfo->flag & HA_FULLTEXT))
1211 {
1212 uint key_length=_mi_make_key(info,key,info->lastkey,record,
1213 start_recpos);
1214 if (extend)
1215 {
1216 /* We don't need to lock the key tree here as we don't allow
1217 concurrent threads when running myisamchk
1218 */
1219 int search_result=
1220#ifdef HAVE_RTREE_KEYS
1221 (keyinfo->flag & HA_SPATIAL) ?
1222 rtree_find_first(info, key, info->lastkey, key_length,
1223 MBR_EQUAL | MBR_DATA) :
1224#endif
1225 _mi_search(info,keyinfo,info->lastkey,key_length,
1226 SEARCH_SAME, info->s->state.key_root[key]);
1227 if (search_result)
1228 {
1229 mi_check_print_error(param,"Record at: %10s "
1230 "Can't find key for index: %2d",
1231 llstr(start_recpos,llbuff),key+1);
1232 if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1233 goto err2;
1234 }
1235 }
1236 else
1237 key_checksum[key]+=mi_byte_checksum((uchar*) info->lastkey,
1238 key_length);
1239 }
1240 }
1241 }
1242 }
1243 else
1244 {
1245 got_error=0;
1246 if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1247 goto err2;
1248 }
1249 next:; /* Next record */
1250 }
1251 if (param->testflag & T_WRITE_LOOP)
1252 {
1253 (void) fputs(" \r",stdout); (void) fflush(stdout);
1254 }
1255 if (records != info->state->records)
1256 {
1257 mi_check_print_error(param,"Record-count is not ok; is %-10s Should be: %s",
1258 llstr(records,llbuff), llstr(info->state->records,llbuff2));
1259 error=1;
1260 }
1261 else if (param->record_checksum &&
1262 param->record_checksum != intern_record_checksum)
1263 {
1264 mi_check_print_error(param,
1265 "Keypointers and record positions doesn't match");
1266 error=1;
1267 }
1268 else if (param->glob_crc != info->state->checksum &&
1269 (info->s->options &
1270 (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD)))
1271 {
1272 mi_check_print_warning(param,
1273 "Record checksum is not the same as checksum stored in the index file\n");
1274 error=1;
1275 }
1276 else if (!extend)
1277 {
1278 for (key=0 ; key < info->s->base.keys; key++)
1279 {
1280 if (key_checksum[key] != param->key_crc[key] &&
1281 !(info->s->keyinfo[key].flag & (HA_FULLTEXT | HA_SPATIAL)))
1282 {
1283 mi_check_print_error(param,"Checksum for key: %2d doesn't match checksum for records",
1284 key+1);
1285 error=1;
1286 }
1287 }
1288 }
1289
1290 if (del_length != info->state->empty)
1291 {
1292 mi_check_print_warning(param,
1293 "Found %s deleted space. Should be %s",
1294 llstr(del_length,llbuff2),
1295 llstr(info->state->empty,llbuff));
1296 }
1297 if (used+empty+del_length != info->state->data_file_length)
1298 {
1299 mi_check_print_warning(param,
1300 "Found %s record-data and %s unused data and %s deleted-data",
1301 llstr(used,llbuff),llstr(empty,llbuff2),
1302 llstr(del_length,llbuff3));
1303 mi_check_print_warning(param,
1304 "Total %s, Should be: %s",
1305 llstr((used+empty+del_length),llbuff),
1306 llstr(info->state->data_file_length,llbuff2));
1307 }
1308 if (del_blocks != info->state->del)
1309 {
1310 mi_check_print_warning(param,
1311 "Found %10s deleted blocks Should be: %s",
1312 llstr(del_blocks,llbuff),
1313 llstr(info->state->del,llbuff2));
1314 }
1315 if (splits != info->s->state.split)
1316 {
1317 mi_check_print_warning(param,
1318 "Found %10s key parts. Should be: %s",
1319 llstr(splits,llbuff),
1320 llstr(info->s->state.split,llbuff2));
1321 }
1322 if (param->testflag & T_INFO)
1323 {
1324 if (param->warning_printed || param->error_printed || param->note_printed)
1325 puts("");
1326 if (used != 0 && ! param->error_printed)
1327 {
1328 printf("Records:%18s M.recordlength:%9lu Packed:%14.0f%%\n",
1329 llstr(records,llbuff), (long)((used-link_used)/records),
1330 (info->s->base.blobs ? 0.0 :
1331 (ulonglong2double((ulonglong) info->s->base.reclength*records)-
1332 my_off_t2double(used))/
1333 ulonglong2double((ulonglong) info->s->base.reclength*records)*100.0));
1334 printf("Recordspace used:%9.0f%% Empty space:%12d%% Blocks/Record: %6.2f\n",
1335 (ulonglong2double(used-link_used)/ulonglong2double(used-link_used+empty)*100.0),
1336 (!records ? 100 : (int) (ulonglong2double(del_length+empty)/
1337 my_off_t2double(used)*100.0)),
1338 ulonglong2double(splits - del_blocks) / records);
1339 }
1340 printf("Record blocks:%12s Delete blocks:%10s\n",
1341 llstr(splits-del_blocks,llbuff),llstr(del_blocks,llbuff2));
1342 printf("Record data: %12s Deleted data: %10s\n",
1343 llstr(used-link_used,llbuff),llstr(del_length,llbuff2));
1344 printf("Lost space: %12s Linkdata: %10s\n",
1345 llstr(empty,llbuff),llstr(link_used,llbuff2));
1346 }
1347 my_free(mi_get_rec_buff_ptr(info, record));
1348 DBUG_RETURN (error);
1349 err:
1350 mi_check_print_error(param,"got error: %d when reading datafile at record: %s",my_errno, llstr(records,llbuff));
1351 err2:
1352 my_free(mi_get_rec_buff_ptr(info, record));
1353 param->testflag|=T_RETRY_WITHOUT_QUICK;
1354 DBUG_RETURN(1);
1355} /* chk_data_link */
1356
1357
1358/**
1359 @brief Drop all indexes
1360
1361 @param[in] param check parameters
1362 @param[in] info MI_INFO handle
1363 @param[in] force if to force drop all indexes
1364
1365 @return status
1366 @retval 0 OK
1367 @retval != 0 Error
1368
1369 @note
1370 Once allocated, index blocks remain part of the key file forever.
1371 When indexes are disabled, no block is freed. When enabling indexes,
1372 no block is freed either. The new indexes are create from new
1373 blocks. (Bug #4692)
1374
1375 Before recreating formerly disabled indexes, the unused blocks
1376 must be freed. There are two options to do this:
1377 - Follow the tree of disabled indexes, add all blocks to the
1378 deleted blocks chain. Would require a lot of random I/O.
1379 - Drop all blocks by clearing all index root pointers and all
1380 delete chain pointers and resetting key_file_length to the end
1381 of the index file header. This requires to recreate all indexes,
1382 even those that may still be intact.
1383 The second method is probably faster in most cases.
1384
1385 When disabling indexes, MySQL disables either all indexes or all
1386 non-unique indexes. When MySQL [re-]enables disabled indexes
1387 (T_CREATE_MISSING_KEYS), then we either have "lost" blocks in the
1388 index file, or there are no non-unique indexes. In the latter case,
1389 mi_repair*() would not be called as there would be no disabled
1390 indexes.
1391
1392 If there would be more unique indexes than disabled (non-unique)
1393 indexes, we could do the first method. But this is not implemented
1394 yet. By now we drop and recreate all indexes when repair is called.
1395
1396 However, there is an exception. Sometimes MySQL disables non-unique
1397 indexes when the table is empty (e.g. when copying a table in
1398 mysql_alter_table()). When enabling the non-unique indexes, they
1399 are still empty. So there is no index block that can be lost. This
1400 optimization is implemented in this function.
1401
1402 Note that in normal repair (T_CREATE_MISSING_KEYS not set) we
1403 recreate all enabled indexes unconditonally. We do not change the
1404 key_map. Otherwise we invert the key map temporarily (outside of
1405 this function) and recreate the then "seemingly" enabled indexes.
1406 When we cannot use the optimization, and drop all indexes, we
1407 pretend that all indexes were disabled. By the inversion, we will
1408 then recrate all indexes.
1409*/
1410
1411static int mi_drop_all_indexes(HA_CHECK *param, MI_INFO *info, my_bool force)
1412{
1413 MYISAM_SHARE *share= info->s;
1414 MI_STATE_INFO *state= &share->state;
1415 uint i;
1416 int error;
1417 DBUG_ENTER("mi_drop_all_indexes");
1418
1419 /*
1420 If any of the disabled indexes has a key block assigned, we must
1421 drop and recreate all indexes to avoid losing index blocks.
1422
1423 If we want to recreate disabled indexes only _and_ all of these
1424 indexes are empty, we don't need to recreate the existing indexes.
1425 */
1426 if (!force && (param->testflag & T_CREATE_MISSING_KEYS))
1427 {
1428 DBUG_PRINT("repair", ("creating missing indexes"));
1429 for (i= 0; i < share->base.keys; i++)
1430 {
1431 DBUG_PRINT("repair", ("index #: %u key_root: 0x%lx active: %d",
1432 i, (long) state->key_root[i],
1433 mi_is_key_active(state->key_map, i)));
1434 if ((state->key_root[i] != HA_OFFSET_ERROR) &&
1435 !mi_is_key_active(state->key_map, i))
1436 {
1437 /*
1438 This index has at least one key block and it is disabled.
1439 We would lose its block(s) if would just recreate it.
1440 So we need to drop and recreate all indexes.
1441 */
1442 DBUG_PRINT("repair", ("nonempty and disabled: recreate all"));
1443 break;
1444 }
1445 }
1446 if (i >= share->base.keys)
1447 {
1448 /*
1449 All of the disabled indexes are empty. We can just recreate them.
1450 Flush dirty blocks of this index file from key cache and remove
1451 all blocks of this index file from key cache.
1452 */
1453 DBUG_PRINT("repair", ("all disabled are empty: create missing"));
1454 error= flush_key_blocks(share->key_cache, share->kfile,
1455 &share->dirty_part_map,
1456 FLUSH_FORCE_WRITE);
1457 goto end;
1458 }
1459 /*
1460 We do now drop all indexes and declare them disabled. With the
1461 T_CREATE_MISSING_KEYS flag, mi_repair*() will recreate all
1462 disabled indexes and enable them.
1463 */
1464 mi_clear_all_keys_active(state->key_map);
1465 DBUG_PRINT("repair", ("declared all indexes disabled"));
1466 }
1467
1468 /* Remove all key blocks of this index file from key cache. */
1469 if ((error= flush_key_blocks(share->key_cache, share->kfile,
1470 &share->dirty_part_map,
1471 FLUSH_IGNORE_CHANGED)))
1472 goto end; /* purecov: inspected */
1473
1474 /* Clear index root block pointers. */
1475 for (i= 0; i < share->base.keys; i++)
1476 state->key_root[i]= HA_OFFSET_ERROR;
1477
1478 /* Clear the delete chains. */
1479 for (i= 0; i < state->header.max_block_size_index; i++)
1480 state->key_del[i]= HA_OFFSET_ERROR;
1481
1482 /* Reset index file length to end of index file header. */
1483 info->state->key_file_length= share->base.keystart;
1484
1485 DBUG_PRINT("repair", ("dropped all indexes"));
1486 /* error= 0; set by last (error= flush_key_bocks()). */
1487
1488 end:
1489 DBUG_RETURN(error);
1490}
1491
1492
1493 /* Recover old table by reading each record and writing all keys */
1494 /* Save new datafile-name in temp_filename */
1495
1496int mi_repair(HA_CHECK *param, register MI_INFO *info,
1497 char * name, int rep_quick)
1498{
1499 int error,got_error;
1500 ha_rows start_records,new_header_length;
1501 my_off_t del;
1502 File new_file;
1503 MYISAM_SHARE *share=info->s;
1504 char llbuff[22],llbuff2[22];
1505 MI_SORT_INFO sort_info;
1506 MI_SORT_PARAM sort_param;
1507 DBUG_ENTER("mi_repair");
1508
1509 bzero((char *)&sort_info, sizeof(sort_info));
1510 bzero((char *)&sort_param, sizeof(sort_param));
1511 start_records=info->state->records;
1512 new_header_length=(param->testflag & T_UNPACK) ? 0L :
1513 share->pack.header_length;
1514 got_error=1;
1515 new_file= -1;
1516 sort_param.sort_info=&sort_info;
1517 param->retry_repair= 0;
1518 param->warning_printed= param->error_printed= param->note_printed= 0;
1519
1520 if (!(param->testflag & T_SILENT))
1521 {
1522 printf("- recovering (with keycache) MyISAM-table '%s'\n",name);
1523 printf("Data records: %s\n", llstr(info->state->records,llbuff));
1524 }
1525 param->testflag|=T_REP; /* for easy checking */
1526
1527 if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
1528 param->testflag|=T_CALC_CHECKSUM;
1529
1530 DBUG_ASSERT(param->use_buffers < SIZE_T_MAX);
1531
1532 if (!param->using_global_keycache)
1533 (void) init_key_cache(dflt_key_cache, param->key_cache_block_size,
1534 (size_t) param->use_buffers, 0, 0, 0, 0);
1535
1536 if (init_io_cache(&param->read_cache,info->dfile,
1537 (uint) param->read_buffer_length,
1538 READ_CACHE,share->pack.header_length,1,MYF(MY_WME)))
1539 {
1540 bzero(&info->rec_cache,sizeof(info->rec_cache));
1541 goto err;
1542 }
1543 info->opt_flag|=WRITE_CACHE_USED;
1544 if (!mi_alloc_rec_buff(info, -1, &sort_param.record) ||
1545 !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff))
1546 {
1547 mi_check_print_error(param, "Not enough memory for extra record");
1548 goto err;
1549 }
1550
1551 if (!rep_quick)
1552 {
1553 /* Get real path for data file */
1554 if ((new_file= mysql_file_create(mi_key_file_datatmp,
1555 fn_format(param->temp_filename,
1556 share->data_file_name, "",
1557 DATA_TMP_EXT, 2+4),
1558 0, param->tmpfile_createflag,
1559 MYF(0))) < 0)
1560 {
1561 mi_check_print_error(param,"Can't create new tempfile: '%s'",
1562 param->temp_filename);
1563 goto err;
1564 }
1565 if (new_header_length &&
1566 filecopy(param,new_file,info->dfile,0L,new_header_length,
1567 "datafile-header"))
1568 goto err;
1569 info->s->state.dellink= HA_OFFSET_ERROR;
1570 if (param->testflag & T_UNPACK)
1571 {
1572 share->options&= ~HA_OPTION_COMPRESS_RECORD;
1573 mi_int2store(share->state.header.options,share->options);
1574 }
1575 if (init_io_cache(&info->rec_cache, new_file,
1576 (uint) param->write_buffer_length,
1577 WRITE_CACHE, new_header_length, 1,
1578 MYF(MY_WME | MY_WAIT_IF_FULL)))
1579 goto err;
1580 }
1581 sort_info.info=info;
1582 sort_info.param = param;
1583 sort_param.read_cache=param->read_cache;
1584 sort_param.pos=sort_param.max_pos=share->pack.header_length;
1585 sort_param.filepos=new_header_length;
1586 param->read_cache.end_of_file=sort_info.filelength=
1587 mysql_file_seek(info->dfile, 0L, MY_SEEK_END, MYF(0));
1588 sort_info.dupp=0;
1589 sort_param.fix_datafile= (my_bool) (! rep_quick);
1590 sort_param.master=1;
1591 sort_info.max_records= ~(ha_rows) 0;
1592
1593 set_data_file_type(&sort_info, share);
1594 del=info->state->del;
1595 info->state->records=info->state->del=share->state.split=0;
1596 info->state->empty=0;
1597 param->glob_crc=0;
1598 if (param->testflag & T_CALC_CHECKSUM)
1599 sort_param.calc_checksum= 1;
1600
1601 info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1602
1603 /* This function always recreates all enabled indexes. */
1604 if (param->testflag & T_CREATE_MISSING_KEYS)
1605 mi_set_all_keys_active(share->state.key_map, share->base.keys);
1606 mi_drop_all_indexes(param, info, TRUE);
1607
1608 lock_memory(param); /* Everything is alloced */
1609
1610 /* Re-create all keys, which are set in key_map. */
1611 while (!(error=sort_get_next_record(&sort_param)))
1612 {
1613 if (writekeys(&sort_param))
1614 {
1615 if (my_errno != HA_ERR_FOUND_DUPP_KEY)
1616 goto err;
1617 DBUG_DUMP("record",(uchar*) sort_param.record,share->base.pack_reclength);
1618 mi_check_print_info(param,"Duplicate key %2d for record at %10s against new record at %10s",
1619 info->errkey+1,
1620 llstr(sort_param.start_recpos,llbuff),
1621 llstr(info->dupp_key_pos,llbuff2));
1622 if (param->testflag & T_VERBOSE)
1623 {
1624 (void) _mi_make_key(info,(uint) info->errkey,info->lastkey,
1625 sort_param.record,0L);
1626 _mi_print_key(stdout,share->keyinfo[info->errkey].seg,info->lastkey,
1627 USE_WHOLE_KEY);
1628 }
1629 sort_info.dupp++;
1630 if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
1631 {
1632 param->testflag|=T_RETRY_WITHOUT_QUICK;
1633 param->error_printed=1;
1634 goto err;
1635 }
1636 continue;
1637 }
1638 if (sort_write_record(&sort_param))
1639 goto err;
1640 }
1641 if (error > 0 || write_data_suffix(&sort_info, (my_bool)!rep_quick) ||
1642 flush_io_cache(&info->rec_cache) || param->read_cache.error < 0)
1643 goto err;
1644
1645 if (param->testflag & T_WRITE_LOOP)
1646 {
1647 (void) fputs(" \r",stdout); (void) fflush(stdout);
1648 }
1649 if (mysql_file_chsize(share->kfile, info->state->key_file_length, 0, MYF(0)))
1650 {
1651 mi_check_print_warning(param,
1652 "Can't change size of indexfile, error: %d",
1653 my_errno);
1654 goto err;
1655 }
1656
1657 if (rep_quick && del+sort_info.dupp != info->state->del)
1658 {
1659 mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
1660 mi_check_print_error(param,"Run recovery again without --quick");
1661 got_error=1;
1662 param->retry_repair=1;
1663 param->testflag|=T_RETRY_WITHOUT_QUICK;
1664 goto err;
1665 }
1666 if (param->testflag & T_SAFE_REPAIR)
1667 {
1668 /* Don't repair if we loosed more than one row */
1669 if (info->state->records+1 < start_records)
1670 {
1671 info->state->records=start_records;
1672 got_error=1;
1673 goto err;
1674 }
1675 }
1676
1677 if (!rep_quick)
1678 {
1679 mysql_file_close(info->dfile, MYF(0));
1680 info->dfile=new_file;
1681 info->state->data_file_length=sort_param.filepos;
1682 share->state.version=(ulong) time((time_t*) 0); /* Force reopen */
1683 }
1684 else
1685 {
1686 info->state->data_file_length=sort_param.max_pos;
1687 }
1688 if (param->testflag & T_CALC_CHECKSUM)
1689 info->state->checksum=param->glob_crc;
1690
1691 if (!(param->testflag & T_SILENT))
1692 {
1693 if (start_records != info->state->records)
1694 printf("Data records: %s\n", llstr(info->state->records,llbuff));
1695 if (sort_info.dupp)
1696 mi_check_print_warning(param,
1697 "%s records have been removed",
1698 llstr(sort_info.dupp,llbuff));
1699 }
1700
1701 got_error=0;
1702 /* If invoked by external program that uses thr_lock */
1703 if (&share->state.state != info->state)
1704 memcpy( &share->state.state, info->state, sizeof(*info->state));
1705
1706err:
1707 if (!got_error)
1708 {
1709 /* Replace the actual file with the temporary file */
1710 if (new_file >= 0)
1711 {
1712 got_error= replace_data_file(param, info, new_file);
1713 new_file= -1;
1714 param->retry_repair= 0;
1715 }
1716 }
1717 if (got_error)
1718 {
1719 if (! param->error_printed)
1720 mi_check_print_error(param,"%d for record at pos %s",my_errno,
1721 llstr(sort_param.start_recpos,llbuff));
1722 if (new_file >= 0)
1723 {
1724 (void) mysql_file_close(new_file, MYF(0));
1725 (void) mysql_file_delete(mi_key_file_datatmp,
1726 param->temp_filename, MYF(MY_WME));
1727 info->rec_cache.file=-1; /* don't flush data to new_file, it's closed */
1728 }
1729 mi_mark_crashed_on_repair(info);
1730 }
1731 my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff));
1732 my_free(mi_get_rec_buff_ptr(info, sort_param.record));
1733 my_free(sort_info.buff);
1734 (void) end_io_cache(&param->read_cache);
1735 info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
1736 (void) end_io_cache(&info->rec_cache);
1737 got_error|=flush_blocks(param, share->key_cache, share->kfile,
1738 &share->dirty_part_map);
1739 if (!got_error && param->testflag & T_UNPACK)
1740 {
1741 share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
1742 share->pack.header_length=0;
1743 share->data_file_type=sort_info.new_data_file_type;
1744 }
1745 share->state.changed|= (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES |
1746 STATE_NOT_ANALYZED);
1747 DBUG_RETURN(got_error);
1748}
1749
1750
1751/* Uppate keyfile when doing repair */
1752
1753static int writekeys(MI_SORT_PARAM *sort_param)
1754{
1755 register uint i;
1756 uchar *key;
1757 MI_INFO *info= sort_param->sort_info->info;
1758 uchar *buff= sort_param->record;
1759 my_off_t filepos= sort_param->filepos;
1760 DBUG_ENTER("writekeys");
1761
1762 key=info->lastkey+info->s->base.max_key_length;
1763 for (i=0 ; i < info->s->base.keys ; i++)
1764 {
1765 if (mi_is_key_active(info->s->state.key_map, i))
1766 {
1767 if (info->s->keyinfo[i].flag & HA_FULLTEXT )
1768 {
1769 if (_mi_ft_add(info, i, key, buff, filepos))
1770 goto err;
1771 }
1772#ifdef HAVE_SPATIAL
1773 else if (info->s->keyinfo[i].flag & HA_SPATIAL)
1774 {
1775 uint key_length=_mi_make_key(info,i,key,buff,filepos);
1776 if (rtree_insert(info, i, key, key_length))
1777 goto err;
1778 }
1779#endif /*HAVE_SPATIAL*/
1780 else
1781 {
1782 uint key_length=_mi_make_key(info,i,key,buff,filepos);
1783 if (_mi_ck_write(info,i,key,key_length))
1784 goto err;
1785 }
1786 }
1787 }
1788 DBUG_RETURN(0);
1789
1790 err:
1791 if (my_errno == HA_ERR_FOUND_DUPP_KEY)
1792 {
1793 info->errkey=(int) i; /* This key was found */
1794 while ( i-- > 0 )
1795 {
1796 if (mi_is_key_active(info->s->state.key_map, i))
1797 {
1798 if (info->s->keyinfo[i].flag & HA_FULLTEXT)
1799 {
1800 if (_mi_ft_del(info,i, key,buff,filepos))
1801 break;
1802 }
1803 else
1804 {
1805 uint key_length=_mi_make_key(info,i,key,buff,filepos);
1806 if (_mi_ck_delete(info,i,key,key_length))
1807 break;
1808 }
1809 }
1810 }
1811 }
1812 /* Remove checksum that was added to glob_crc in sort_get_next_record */
1813 if (sort_param->calc_checksum)
1814 sort_param->sort_info->param->glob_crc-= info->checksum;
1815 DBUG_PRINT("error",("errno: %d",my_errno));
1816 DBUG_RETURN(-1);
1817} /* writekeys */
1818
1819
1820 /* Change all key-pointers that points to a records */
1821
1822int movepoint(register MI_INFO *info, uchar *record, my_off_t oldpos,
1823 my_off_t newpos, uint prot_key)
1824{
1825 register uint i;
1826 uchar *key;
1827 uint key_length;
1828 DBUG_ENTER("movepoint");
1829
1830 key=info->lastkey+info->s->base.max_key_length;
1831 for (i=0 ; i < info->s->base.keys; i++)
1832 {
1833 if (i != prot_key && mi_is_key_active(info->s->state.key_map, i))
1834 {
1835 key_length=_mi_make_key(info,i,key,record,oldpos);
1836 if (info->s->keyinfo[i].flag & HA_NOSAME)
1837 { /* Change pointer direct */
1838 uint nod_flag;
1839 MI_KEYDEF *keyinfo;
1840 keyinfo=info->s->keyinfo+i;
1841 if (_mi_search(info,keyinfo,key,USE_WHOLE_KEY,
1842 (uint) (SEARCH_SAME | SEARCH_SAVE_BUFF),
1843 info->s->state.key_root[i]))
1844 DBUG_RETURN(-1);
1845 nod_flag=mi_test_if_nod(info->buff);
1846 _mi_dpointer(info,info->int_keypos-nod_flag-
1847 info->s->rec_reflength,newpos);
1848 if (_mi_write_keypage(info,keyinfo,info->last_keypage,
1849 DFLT_INIT_HITS,info->buff))
1850 DBUG_RETURN(-1);
1851 }
1852 else
1853 { /* Change old key to new */
1854 if (_mi_ck_delete(info,i,key,key_length))
1855 DBUG_RETURN(-1);
1856 key_length=_mi_make_key(info,i,key,record,newpos);
1857 if (_mi_ck_write(info,i,key,key_length))
1858 DBUG_RETURN(-1);
1859 }
1860 }
1861 }
1862 DBUG_RETURN(0);
1863} /* movepoint */
1864
1865
1866 /* Tell system that we want all memory for our cache */
1867
1868void lock_memory(HA_CHECK *param __attribute__((unused)))
1869{
1870#ifdef SUN_OS /* Key-cacheing thrases on sun 4.1 */
1871 if (param->opt_lock_memory)
1872 {
1873 int success = mlockall(MCL_CURRENT); /* or plock(DATLOCK); */
1874 if (geteuid() == 0 && success != 0)
1875 mi_check_print_warning(param,
1876 "Failed to lock memory. errno %d",my_errno);
1877 }
1878#endif
1879} /* lock_memory */
1880
1881
1882 /* Flush all changed blocks to disk */
1883
1884int flush_blocks(HA_CHECK *param, KEY_CACHE *key_cache, File file,
1885 ulonglong *dirty_part_map)
1886{
1887 if (flush_key_blocks(key_cache, file, dirty_part_map, FLUSH_RELEASE))
1888 {
1889 mi_check_print_error(param,"%d when trying to write buffers",my_errno);
1890 return(1);
1891 }
1892 if (!param->using_global_keycache)
1893 end_key_cache(key_cache,1);
1894 return 0;
1895} /* flush_blocks */
1896
1897
1898 /* Sort index for more efficent reads */
1899
1900int mi_sort_index(HA_CHECK *param, register MI_INFO *info, char * name)
1901{
1902 reg2 uint key;
1903 reg1 MI_KEYDEF *keyinfo;
1904 File new_file;
1905 my_off_t index_pos[HA_MAX_POSSIBLE_KEY];
1906 uint r_locks,w_locks;
1907 int old_lock;
1908 MYISAM_SHARE *share=info->s;
1909 MI_STATE_INFO old_state;
1910 DBUG_ENTER("mi_sort_index");
1911
1912 /* cannot sort index files with R-tree indexes */
1913 for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
1914 key++,keyinfo++)
1915 if (keyinfo->key_alg == HA_KEY_ALG_RTREE)
1916 DBUG_RETURN(0);
1917
1918 if (!(param->testflag & T_SILENT))
1919 printf("- Sorting index for MyISAM-table '%s'\n",name);
1920
1921 /* Get real path for index file */
1922 fn_format(param->temp_filename,name,"", MI_NAME_IEXT,2+4+32);
1923 if ((new_file= mysql_file_create(mi_key_file_datatmp,
1924 fn_format(param->temp_filename,
1925 param->temp_filename,
1926 "", INDEX_TMP_EXT, 2+4),
1927 0, param->tmpfile_createflag, MYF(0))) < 0)
1928 {
1929 mi_check_print_error(param,"Can't create new tempfile: '%s'",
1930 param->temp_filename);
1931 DBUG_RETURN(-1);
1932 }
1933 if (filecopy(param, new_file,share->kfile,0L,
1934 (ulong) share->base.keystart, "headerblock"))
1935 goto err;
1936
1937 param->new_file_pos=share->base.keystart;
1938 for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
1939 key++,keyinfo++)
1940 {
1941 if (mi_is_key_active(info->s->state.key_map, key) &&
1942 share->state.key_root[key] != HA_OFFSET_ERROR)
1943 {
1944 index_pos[key]=param->new_file_pos; /* Write first block here */
1945 if (sort_one_index(param,info,keyinfo,share->state.key_root[key],
1946 new_file))
1947 goto err;
1948 }
1949 else
1950 index_pos[key]= HA_OFFSET_ERROR; /* No blocks */
1951 }
1952
1953 /* Flush key cache for this file if we are calling this outside myisamchk */
1954 flush_key_blocks(share->key_cache, share->kfile, &share->dirty_part_map,
1955 FLUSH_IGNORE_CHANGED);
1956
1957 share->state.version=(ulong) time((time_t*) 0);
1958 old_state= share->state; /* save state if not stored */
1959 r_locks= share->r_locks;
1960 w_locks= share->w_locks;
1961 old_lock= info->lock_type;
1962
1963 /* Put same locks as old file */
1964 share->r_locks= share->w_locks= share->tot_locks= 0;
1965 (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE);
1966 (void) mysql_file_close(share->kfile, MYF(MY_WME));
1967 share->kfile = -1;
1968 (void) mysql_file_close(new_file, MYF(MY_WME));
1969 if (change_to_newfile(share->index_file_name,MI_NAME_IEXT,INDEX_TMP_EXT,
1970 0, MYF(0)) ||
1971 mi_open_keyfile(share))
1972 goto err2;
1973 info->lock_type= F_UNLCK; /* Force mi_readinfo to lock */
1974 _mi_readinfo(info,F_WRLCK,0); /* Will lock the table */
1975 info->lock_type= old_lock;
1976 share->r_locks= r_locks;
1977 share->w_locks= w_locks;
1978 share->tot_locks= r_locks+w_locks;
1979 share->state= old_state; /* Restore old state */
1980
1981 info->state->key_file_length=param->new_file_pos;
1982 info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1983 for (key=0 ; key < info->s->base.keys ; key++)
1984 info->s->state.key_root[key]=index_pos[key];
1985 for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
1986 info->s->state.key_del[key]= HA_OFFSET_ERROR;
1987
1988 info->s->state.changed&= ~STATE_NOT_SORTED_PAGES;
1989 DBUG_RETURN(0);
1990
1991err:
1992 (void) mysql_file_close(new_file, MYF(MY_WME));
1993err2:
1994 (void) mysql_file_delete(mi_key_file_datatmp,
1995 param->temp_filename, MYF(MY_WME));
1996 DBUG_RETURN(-1);
1997} /* mi_sort_index */
1998
1999
2000 /* Sort records recursive using one index */
2001
2002static int sort_one_index(HA_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
2003 my_off_t pagepos, File new_file)
2004{
2005 uint length,nod_flag,used_length, key_length;
2006 uchar *buff,*keypos,*endpos;
2007 uchar key[HA_MAX_POSSIBLE_KEY_BUFF];
2008 my_off_t new_page_pos,next_page;
2009 char llbuff[22];
2010 DBUG_ENTER("sort_one_index");
2011
2012 /* cannot walk over R-tree indices */
2013 DBUG_ASSERT(keyinfo->key_alg != HA_KEY_ALG_RTREE);
2014 new_page_pos=param->new_file_pos;
2015 param->new_file_pos+=keyinfo->block_length;
2016
2017 if (!(buff=(uchar*) my_alloca((uint) keyinfo->block_length)))
2018 {
2019 mi_check_print_error(param,"Not enough memory for key block");
2020 DBUG_RETURN(-1);
2021 }
2022 if (!_mi_fetch_keypage(info,keyinfo,pagepos,DFLT_INIT_HITS,buff,0))
2023 {
2024 mi_check_print_error(param,"Can't read key block from filepos: %s",
2025 llstr(pagepos,llbuff));
2026 goto err;
2027 }
2028 if ((nod_flag=mi_test_if_nod(buff)) || keyinfo->flag & HA_FULLTEXT)
2029 {
2030 used_length=mi_getint(buff);
2031 keypos=buff+2+nod_flag;
2032 endpos=buff+used_length;
2033 for ( ;; )
2034 {
2035 if (nod_flag)
2036 {
2037 next_page=_mi_kpos(nod_flag,keypos);
2038 _mi_kpointer(info,keypos-nod_flag,param->new_file_pos); /* Save new pos */
2039 if (sort_one_index(param,info,keyinfo,next_page,new_file))
2040 {
2041 DBUG_PRINT("error",
2042 ("From page: %ld, keyoffset: %lu used_length: %d",
2043 (ulong) pagepos, (ulong) (keypos - buff),
2044 (int) used_length));
2045 DBUG_DUMP("buff",(uchar*) buff,used_length);
2046 goto err;
2047 }
2048 }
2049 if (keypos >= endpos ||
2050 (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
2051 break;
2052 DBUG_ASSERT(keypos <= endpos);
2053 if (keyinfo->flag & HA_FULLTEXT)
2054 {
2055 uint off;
2056 int subkeys;
2057 get_key_full_length_rdonly(off, key);
2058 subkeys=ft_sintXkorr(key+off);
2059 if (subkeys < 0)
2060 {
2061 next_page= _mi_dpos(info,0,key+key_length);
2062 _mi_dpointer(info,keypos-nod_flag-info->s->rec_reflength,
2063 param->new_file_pos); /* Save new pos */
2064 if (sort_one_index(param,info,&info->s->ft2_keyinfo,
2065 next_page,new_file))
2066 goto err;
2067 }
2068 }
2069 }
2070 }
2071
2072 /* Fill block with zero and write it to the new index file */
2073 length=mi_getint(buff);
2074 bzero((uchar*) buff+length,keyinfo->block_length-length);
2075 if (mysql_file_pwrite(new_file, (uchar*) buff, (uint) keyinfo->block_length,
2076 new_page_pos, MYF(MY_NABP | MY_WAIT_IF_FULL)))
2077 {
2078 mi_check_print_error(param,"Can't write indexblock, error: %d",my_errno);
2079 goto err;
2080 }
2081 my_afree((uchar*) buff);
2082 DBUG_RETURN(0);
2083err:
2084 my_afree((uchar*) buff);
2085 DBUG_RETURN(1);
2086} /* sort_one_index */
2087
2088
2089 /*
2090 Let temporary file replace old file.
2091 This assumes that the new file was created in the same
2092 directory as given by realpath(filename).
2093 This will ensure that any symlinks that are used will still work.
2094 Copy stats from old file to new file, deletes orignal and
2095 changes new file name to old file name
2096 */
2097
2098int change_to_newfile(const char * filename, const char * old_ext,
2099 const char * new_ext,
2100 time_t backup_time,
2101 myf MyFlags)
2102{
2103 char old_filename[FN_REFLEN],new_filename[FN_REFLEN];
2104 /* Get real path to filename */
2105 (void) fn_format(old_filename,filename,"",old_ext,2+4+32);
2106 return my_redel(old_filename,
2107 fn_format(new_filename,old_filename,"",new_ext,2+4),
2108 backup_time, MYF(MY_WME | MY_LINK_WARNING | MyFlags));
2109} /* change_to_newfile */
2110
2111
2112 /* Locks a whole file */
2113 /* Gives an error-message if file can't be locked */
2114
2115int lock_file(HA_CHECK *param, File file, my_off_t start, int lock_type,
2116 const char *filetype, const char *filename)
2117{
2118 if (my_lock(file,lock_type,start,F_TO_EOF,
2119 param->testflag & T_WAIT_FOREVER ? MYF(MY_SEEK_NOT_DONE) :
2120 MYF(MY_SEEK_NOT_DONE | MY_SHORT_WAIT)))
2121 {
2122 mi_check_print_error(param," %d when locking %s '%s'",my_errno,filetype,filename);
2123 param->error_printed=2; /* Don't give that data is crashed */
2124 return 1;
2125 }
2126 return 0;
2127} /* lock_file */
2128
2129
2130 /* Copy a block between two files */
2131
2132int filecopy(HA_CHECK *param, File to,File from,my_off_t start,
2133 my_off_t length, const char *type)
2134{
2135 char tmp_buff[IO_SIZE],*buff;
2136 ulong buff_length;
2137 DBUG_ENTER("filecopy");
2138
2139 buff_length=(ulong) MY_MIN(param->write_buffer_length,length);
2140 if (!(buff=my_malloc(buff_length,MYF(0))))
2141 {
2142 buff=tmp_buff; buff_length=IO_SIZE;
2143 }
2144
2145 mysql_file_seek(from, start, MY_SEEK_SET, MYF(0));
2146 while (length > buff_length)
2147 {
2148 if (mysql_file_read(from, (uchar*) buff, buff_length, MYF(MY_NABP)) ||
2149 mysql_file_write(to, (uchar*) buff, buff_length, param->myf_rw))
2150 goto err;
2151 length-= buff_length;
2152 }
2153 if (mysql_file_read(from, (uchar*) buff, (uint) length, MYF(MY_NABP)) ||
2154 mysql_file_write(to, (uchar*) buff, (uint) length, param->myf_rw))
2155 goto err;
2156 if (buff != tmp_buff)
2157 my_free(buff);
2158 DBUG_RETURN(0);
2159err:
2160 if (buff != tmp_buff)
2161 my_free(buff);
2162 mi_check_print_error(param,"Can't copy %s to tempfile, error %d",
2163 type,my_errno);
2164 DBUG_RETURN(1);
2165}
2166
2167
2168/*
2169 Repair table or given index using sorting
2170
2171 SYNOPSIS
2172 mi_repair_by_sort()
2173 param Repair parameters
2174 info MyISAM handler to repair
2175 name Name of table (for warnings)
2176 rep_quick set to <> 0 if we should not change data file
2177
2178 RESULT
2179 0 ok
2180 <>0 Error
2181*/
2182
2183int mi_repair_by_sort(HA_CHECK *param, register MI_INFO *info,
2184 const char * name, int rep_quick)
2185{
2186 int got_error;
2187 uint i;
2188 ulong length;
2189 ha_rows start_records;
2190 my_off_t new_header_length,del;
2191 File new_file;
2192 MI_SORT_PARAM sort_param;
2193 MYISAM_SHARE *share=info->s;
2194 HA_KEYSEG *keyseg;
2195 ulong *rec_per_key_part;
2196 char llbuff[22], llbuff2[22];
2197 MI_SORT_INFO sort_info;
2198 ulonglong UNINIT_VAR(key_map);
2199 DBUG_ENTER("mi_repair_by_sort");
2200
2201 start_records=info->state->records;
2202 got_error=1;
2203 new_file= -1;
2204 new_header_length=(param->testflag & T_UNPACK) ? 0 :
2205 share->pack.header_length;
2206 if (!(param->testflag & T_SILENT))
2207 {
2208 printf("- recovering (with sort) MyISAM-table '%s'\n",name);
2209 printf("Data records: %s\n", llstr(start_records,llbuff));
2210 }
2211 param->testflag|=T_REP_BY_SORT; /* for easy checking */
2212 param->retry_repair= 0;
2213 param->warning_printed= param->error_printed= param->note_printed= 0;
2214
2215 if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
2216 param->testflag|=T_CALC_CHECKSUM;
2217
2218 bzero((char*)&sort_info,sizeof(sort_info));
2219 bzero((char *)&sort_param, sizeof(sort_param));
2220
2221 if (!(sort_info.key_block=
2222 alloc_key_blocks(param,
2223 (uint) param->sort_key_blocks,
2224 share->base.max_key_block_length)))
2225 goto err;
2226
2227 if (init_io_cache(&param->read_cache,info->dfile,
2228 (uint) param->read_buffer_length,
2229 READ_CACHE,share->pack.header_length,1,MYF(MY_WME)))
2230 goto err;
2231
2232 sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
2233 info->opt_flag|=WRITE_CACHE_USED;
2234
2235 if (!mi_alloc_rec_buff(info, -1, &sort_param.record) ||
2236 !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff))
2237 {
2238 mi_check_print_error(param, "Not enough memory for extra record");
2239 goto err;
2240 }
2241 if (!rep_quick)
2242 {
2243 /* Get real path for data file */
2244 if ((new_file= mysql_file_create(mi_key_file_datatmp,
2245 fn_format(param->temp_filename,
2246 share->data_file_name, "",
2247 DATA_TMP_EXT, 2+4),
2248 0, param->tmpfile_createflag,
2249 MYF(0))) < 0)
2250 {
2251 mi_check_print_error(param,"Can't create new tempfile: '%s'",
2252 param->temp_filename);
2253 goto err;
2254 }
2255 if (new_header_length &&
2256 filecopy(param, new_file,info->dfile,0L,new_header_length,
2257 "datafile-header"))
2258 goto err;
2259 if (param->testflag & T_UNPACK)
2260 {
2261 share->options&= ~HA_OPTION_COMPRESS_RECORD;
2262 mi_int2store(share->state.header.options,share->options);
2263 }
2264 share->state.dellink= HA_OFFSET_ERROR;
2265 if (init_io_cache(&info->rec_cache, new_file,
2266 (uint) param->write_buffer_length,
2267 WRITE_CACHE, new_header_length, 1,
2268 MYF((param->myf_rw & MY_WAIT_IF_FULL) | MY_WME)))
2269 goto err;
2270 }
2271
2272 info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2273
2274 /* Optionally drop indexes and optionally modify the key_map. */
2275 mi_drop_all_indexes(param, info, FALSE);
2276 key_map= share->state.key_map;
2277 if (param->testflag & T_CREATE_MISSING_KEYS)
2278 {
2279 /* Invert the copied key_map to recreate all disabled indexes. */
2280 key_map= ~key_map;
2281 }
2282
2283 sort_info.info=info;
2284 sort_info.param = param;
2285
2286 set_data_file_type(&sort_info, share);
2287 sort_param.filepos=new_header_length;
2288 sort_info.dupp=0;
2289 sort_info.buff=0;
2290 param->read_cache.end_of_file=sort_info.filelength=
2291 mysql_file_seek(param->read_cache.file, 0L, MY_SEEK_END, MYF(0));
2292
2293 sort_param.wordlist=NULL;
2294 init_alloc_root(&sort_param.wordroot, "sort", FTPARSER_MEMROOT_ALLOC_SIZE, 0,
2295 MYF(param->malloc_flags));
2296
2297 if (share->data_file_type == DYNAMIC_RECORD)
2298 length=MY_MAX(share->base.min_pack_length+1,share->base.min_block_length);
2299 else if (share->data_file_type == COMPRESSED_RECORD)
2300 length=share->base.min_block_length;
2301 else
2302 length=share->base.pack_reclength;
2303 sort_info.max_records=
2304 ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records :
2305 (ha_rows) (sort_info.filelength/length+1));
2306 sort_param.key_cmp=sort_key_cmp;
2307 sort_param.lock_in_memory=lock_memory;
2308 sort_param.tmpdir=param->tmpdir;
2309 sort_param.sort_info=&sort_info;
2310 sort_param.fix_datafile= (my_bool) (! rep_quick);
2311 sort_param.master =1;
2312
2313 del=info->state->del;
2314 param->glob_crc=0;
2315 if (param->testflag & T_CALC_CHECKSUM)
2316 sort_param.calc_checksum= 1;
2317
2318 rec_per_key_part= param->rec_per_key_part;
2319 for (sort_param.key=0 ; sort_param.key < share->base.keys ;
2320 rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++)
2321 {
2322 sort_param.read_cache=param->read_cache;
2323 sort_param.keyinfo=share->keyinfo+sort_param.key;
2324 sort_param.seg=sort_param.keyinfo->seg;
2325 /*
2326 Skip this index if it is marked disabled in the copied
2327 (and possibly inverted) key_map.
2328 */
2329 if (! mi_is_key_active(key_map, sort_param.key))
2330 {
2331 /* Remember old statistics for key */
2332 memcpy((char*) rec_per_key_part,
2333 (char*) (share->state.rec_per_key_part +
2334 (uint) (rec_per_key_part - param->rec_per_key_part)),
2335 sort_param.keyinfo->keysegs*sizeof(*rec_per_key_part));
2336 DBUG_PRINT("repair", ("skipping seemingly disabled index #: %u",
2337 sort_param.key));
2338 continue;
2339 }
2340
2341 if ((!(param->testflag & T_SILENT)))
2342 printf ("- Fixing index %d\n",sort_param.key+1);
2343 sort_param.max_pos=sort_param.pos=share->pack.header_length;
2344 keyseg=sort_param.seg;
2345 bzero((char*) sort_param.unique,sizeof(sort_param.unique));
2346 sort_param.key_length=share->rec_reflength;
2347 for (i=0 ; keyseg[i].type != HA_KEYTYPE_END; i++)
2348 {
2349 sort_param.key_length+=keyseg[i].length;
2350 if (keyseg[i].flag & HA_SPACE_PACK)
2351 sort_param.key_length+=get_pack_length(keyseg[i].length);
2352 if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
2353 sort_param.key_length+= 2 + MY_TEST(keyseg[i].length >= 127);
2354 if (keyseg[i].flag & HA_NULL_PART)
2355 sort_param.key_length++;
2356 }
2357 info->state->records=info->state->del=share->state.split=0;
2358 info->state->empty=0;
2359
2360 if (sort_param.keyinfo->flag & HA_FULLTEXT)
2361 {
2362 uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
2363 sort_param.keyinfo->seg->charset->mbmaxlen;
2364 sort_param.key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
2365 /*
2366 fulltext indexes may have much more entries than the
2367 number of rows in the table. We estimate the number here.
2368 */
2369 if (sort_param.keyinfo->parser == &ft_default_parser)
2370 {
2371 /*
2372 for built-in parser the number of generated index entries
2373 cannot be larger than the size of the data file divided
2374 by the minimal word's length
2375 */
2376 sort_info.max_records=
2377 (ha_rows) (sort_info.filelength/ft_min_word_len+1);
2378 }
2379 else
2380 {
2381 /*
2382 for external plugin parser we cannot tell anything at all :(
2383 so, we'll use all the sort memory and start from ~10 buffpeks.
2384 (see _create_index_by_sort)
2385 */
2386 sort_info.max_records= 10 *
2387 MY_MAX(param->sort_buffer_length, MIN_SORT_BUFFER) /
2388 sort_param.key_length;
2389 }
2390
2391 sort_param.key_read=sort_ft_key_read;
2392 sort_param.key_write=sort_ft_key_write;
2393 }
2394 else
2395 {
2396 sort_param.key_read=sort_key_read;
2397 sort_param.key_write=sort_key_write;
2398 }
2399
2400 if (_create_index_by_sort(&sort_param,
2401 (my_bool) (!(param->testflag & T_VERBOSE)),
2402 param->sort_buffer_length))
2403 {
2404 if ((param->testflag & T_CREATE_UNIQUE_BY_SORT) && sort_param.sort_info->dupp)
2405 share->state.dupp_key= sort_param.key;
2406 else
2407 param->retry_repair= 1;
2408 if (! param->error_printed)
2409 mi_check_print_error(param, "Couldn't fix table with create_index_by_sort(). Error: %d",
2410 my_errno);
2411 goto err;
2412 }
2413 /* No need to calculate checksum again. */
2414 sort_param.calc_checksum= 0;
2415 free_root(&sort_param.wordroot, MYF(0));
2416
2417 /* Set for next loop */
2418 sort_info.max_records= (ha_rows) info->state->records;
2419
2420 if (param->testflag & T_STATISTICS)
2421 update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique,
2422 param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
2423 sort_param.notnull: NULL,
2424 (ulonglong) info->state->records);
2425 /* Enable this index in the permanent (not the copied) key_map. */
2426 mi_set_key_active(share->state.key_map, sort_param.key);
2427 DBUG_PRINT("repair", ("set enabled index #: %u", sort_param.key));
2428
2429 if (sort_param.fix_datafile)
2430 {
2431 param->read_cache.end_of_file=sort_param.filepos;
2432 if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache))
2433 goto err;
2434 if (param->testflag & T_SAFE_REPAIR)
2435 {
2436 /* Don't repair if we loosed more than one row */
2437 if (info->state->records+1 < start_records)
2438 {
2439 mi_check_print_error(param,
2440 "Couldn't fix table as SAFE_REPAIR was requested and we would loose too many rows. %s -> %s",
2441 llstr(start_records, llbuff), llstr(info->state->records, llbuff2));
2442 info->state->records= start_records;
2443 goto err;
2444 }
2445 }
2446 share->state.state.data_file_length = info->state->data_file_length=
2447 sort_param.filepos;
2448 /* Only whole records */
2449 share->state.version=(ulong) time((time_t*) 0);
2450 mysql_file_close(info->dfile, MYF(0));
2451 info->dfile=new_file;
2452 share->data_file_type=sort_info.new_data_file_type;
2453 share->pack.header_length=(ulong) new_header_length;
2454 sort_param.fix_datafile=0;
2455 }
2456 else
2457 info->state->data_file_length=sort_param.max_pos;
2458
2459 param->read_cache.file=info->dfile; /* re-init read cache */
2460 reinit_io_cache(&param->read_cache,READ_CACHE,share->pack.header_length,
2461 1,1);
2462 }
2463
2464 if (param->testflag & T_WRITE_LOOP)
2465 {
2466 (void) fputs(" \r",stdout); (void) fflush(stdout);
2467 }
2468
2469 if (rep_quick && del+sort_info.dupp != info->state->del)
2470 {
2471 mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
2472 mi_check_print_error(param,"Run recovery again without --quick");
2473 got_error=1;
2474 param->retry_repair=1;
2475 param->testflag|=T_RETRY_WITHOUT_QUICK;
2476 goto err;
2477 }
2478
2479 if (rep_quick && (param->testflag & T_FORCE_UNIQUENESS))
2480 {
2481 my_off_t skr=info->state->data_file_length+
2482 (share->options & HA_OPTION_COMPRESS_RECORD ?
2483 MEMMAP_EXTRA_MARGIN : 0);
2484#ifdef USE_RELOC
2485 if (share->data_file_type == STATIC_RECORD &&
2486 skr < share->base.reloc*share->base.min_pack_length)
2487 skr=share->base.reloc*share->base.min_pack_length;
2488#endif
2489 if (skr != sort_info.filelength)
2490 if (mysql_file_chsize(info->dfile, skr, 0, MYF(0)))
2491 mi_check_print_warning(param,
2492 "Can't change size of datafile, error: %d",
2493 my_errno);
2494 }
2495 if (param->testflag & T_CALC_CHECKSUM)
2496 info->state->checksum=param->glob_crc;
2497
2498 if (mysql_file_chsize(share->kfile, info->state->key_file_length, 0, MYF(0)))
2499 mi_check_print_warning(param,
2500 "Can't change size of indexfile, error: %d",
2501 my_errno);
2502
2503 if (!(param->testflag & T_SILENT))
2504 {
2505 if (start_records != info->state->records)
2506 printf("Data records: %s\n", llstr(info->state->records,llbuff));
2507 if (sort_info.dupp)
2508 mi_check_print_warning(param,
2509 "%s records have been removed",
2510 llstr(sort_info.dupp,llbuff));
2511 }
2512 got_error=0;
2513
2514 if (&share->state.state != info->state)
2515 memcpy( &share->state.state, info->state, sizeof(*info->state));
2516
2517err:
2518 got_error|= flush_blocks(param, share->key_cache, share->kfile,
2519 &share->dirty_part_map);
2520 (void) end_io_cache(&info->rec_cache);
2521 if (!got_error)
2522 {
2523 /* Replace the actual file with the temporary file */
2524 if (new_file >= 0)
2525 {
2526 got_error= replace_data_file(param, info, new_file);
2527 new_file= -1;
2528 }
2529 }
2530 if (got_error)
2531 {
2532 if (! param->error_printed)
2533 mi_check_print_error(param,"%d when fixing table",my_errno);
2534 if (new_file >= 0)
2535 {
2536 (void) mysql_file_close(new_file, MYF(0));
2537 (void) mysql_file_delete(mi_key_file_datatmp,
2538 param->temp_filename, MYF(MY_WME));
2539 if (info->dfile == new_file) /* Retry with key cache */
2540 if (unlikely(mi_open_datafile(info, share)))
2541 param->retry_repair= 0; /* Safety */
2542 }
2543 mi_mark_crashed_on_repair(info);
2544 if (killed_ptr(param))
2545 param->retry_repair= 0; /* No use to retry repair */
2546 }
2547 else if (key_map == share->state.key_map)
2548 share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
2549 share->state.changed|=STATE_NOT_SORTED_PAGES;
2550
2551 my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff));
2552 my_free(mi_get_rec_buff_ptr(info, sort_param.record));
2553 my_free(sort_info.key_block);
2554 my_free(sort_info.ft_buf);
2555 my_free(sort_info.buff);
2556 (void) end_io_cache(&param->read_cache);
2557 info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
2558 if (!got_error && (param->testflag & T_UNPACK))
2559 {
2560 share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
2561 share->pack.header_length=0;
2562 }
2563 DBUG_RETURN(got_error);
2564}
2565
2566/*
2567 Threaded repair of table using sorting
2568
2569 SYNOPSIS
2570 mi_repair_parallel()
2571 param Repair parameters
2572 info MyISAM handler to repair
2573 name Name of table (for warnings)
2574 rep_quick set to <> 0 if we should not change data file
2575
2576 DESCRIPTION
2577 Same as mi_repair_by_sort but do it multithreaded
2578 Each key is handled by a separate thread.
2579 TODO: make a number of threads a parameter
2580
2581 In parallel repair we use one thread per index. There are two modes:
2582
2583 Quick
2584
2585 Only the indexes are rebuilt. All threads share a read buffer.
2586 Every thread that needs fresh data in the buffer enters the shared
2587 cache lock. The last thread joining the lock reads the buffer from
2588 the data file and wakes all other threads.
2589
2590 Non-quick
2591
2592 The data file is rebuilt and all indexes are rebuilt to point to
2593 the new record positions. One thread is the master thread. It
2594 reads from the old data file and writes to the new data file. It
2595 also creates one of the indexes. The other threads read from a
2596 buffer which is filled by the master. If they need fresh data,
2597 they enter the shared cache lock. If the masters write buffer is
2598 full, it flushes it to the new data file and enters the shared
2599 cache lock too. When all threads joined in the lock, the master
2600 copies its write buffer to the read buffer for the other threads
2601 and wakes them.
2602
2603 RESULT
2604 0 ok
2605 <>0 Error
2606*/
2607
2608int mi_repair_parallel(HA_CHECK *param, register MI_INFO *info,
2609 const char * name, int rep_quick)
2610{
2611 int got_error;
2612 uint i,key, total_key_length, istep;
2613 ulong rec_length;
2614 ha_rows start_records;
2615 my_off_t new_header_length,del;
2616 File new_file;
2617 MI_SORT_PARAM *sort_param=0;
2618 MYISAM_SHARE *share=info->s;
2619 ulong *rec_per_key_part;
2620 HA_KEYSEG *keyseg;
2621 char llbuff[22];
2622 IO_CACHE new_data_cache; /* For non-quick repair. */
2623 IO_CACHE_SHARE io_share;
2624 MI_SORT_INFO sort_info;
2625 ulonglong UNINIT_VAR(key_map);
2626 pthread_attr_t thr_attr;
2627 ulong max_pack_reclength;
2628 int error;
2629 DBUG_ENTER("mi_repair_parallel");
2630
2631 start_records=info->state->records;
2632 got_error=1;
2633 new_file= -1;
2634 new_header_length=(param->testflag & T_UNPACK) ? 0 :
2635 share->pack.header_length;
2636 if (!(param->testflag & T_SILENT))
2637 {
2638 printf("- parallel recovering (with sort) MyISAM-table '%s'\n",name);
2639 printf("Data records: %s\n", llstr(start_records,llbuff));
2640 }
2641 param->testflag|=T_REP_PARALLEL; /* for easy checking */
2642 param->retry_repair= 0;
2643 param->warning_printed= 0;
2644 param->error_printed= 0;
2645
2646 if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
2647 param->testflag|=T_CALC_CHECKSUM;
2648
2649 /*
2650 Quick repair (not touching data file, rebuilding indexes):
2651 {
2652 Read cache is (HA_CHECK *param)->read_cache using info->dfile.
2653 }
2654
2655 Non-quick repair (rebuilding data file and indexes):
2656 {
2657 Master thread:
2658
2659 Read cache is (HA_CHECK *param)->read_cache using info->dfile.
2660 Write cache is (MI_INFO *info)->rec_cache using new_file.
2661
2662 Slave threads:
2663
2664 Read cache is new_data_cache synced to master rec_cache.
2665
2666 The final assignment of the filedescriptor for rec_cache is done
2667 after the cache creation.
2668
2669 Don't check file size on new_data_cache, as the resulting file size
2670 is not known yet.
2671
2672 As rec_cache and new_data_cache are synced, write_buffer_length is
2673 used for the read cache 'new_data_cache'. Both start at the same
2674 position 'new_header_length'.
2675 }
2676 */
2677 DBUG_PRINT("info", ("is quick repair: %d", rep_quick));
2678 bzero((char*)&sort_info,sizeof(sort_info));
2679 if (!rep_quick)
2680 my_b_clear(&new_data_cache);
2681 /* Initialize pthread structures before goto err. */
2682 mysql_mutex_init(mi_key_mutex_MI_SORT_INFO_mutex,
2683 &sort_info.mutex, MY_MUTEX_INIT_FAST);
2684 mysql_cond_init(mi_key_cond_MI_SORT_INFO_cond, &sort_info.cond, 0);
2685 mysql_mutex_init(mi_key_mutex_MI_CHECK_print_msg,
2686 &param->print_msg_mutex, MY_MUTEX_INIT_FAST);
2687 param->need_print_msg_lock= 1;
2688
2689 if (!(sort_info.key_block=
2690 alloc_key_blocks(param, (uint) param->sort_key_blocks,
2691 share->base.max_key_block_length)))
2692 goto err;
2693
2694 if (init_io_cache(&param->read_cache, info->dfile,
2695 (uint) param->read_buffer_length,
2696 READ_CACHE, share->pack.header_length, 1, MYF(MY_WME)))
2697 goto err;
2698
2699 sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
2700 info->opt_flag|=WRITE_CACHE_USED;
2701
2702 if (!rep_quick)
2703 {
2704 /* Get real path for data file */
2705 if ((new_file= mysql_file_create(mi_key_file_datatmp,
2706 fn_format(param->temp_filename,
2707 share->data_file_name, "",
2708 DATA_TMP_EXT, 2+4),
2709 0, param->tmpfile_createflag,
2710 MYF(0))) < 0)
2711 {
2712 mi_check_print_error(param,"Can't create new tempfile: '%s'",
2713 param->temp_filename);
2714 goto err;
2715 }
2716 if (new_header_length &&
2717 filecopy(param, new_file,info->dfile,0L,new_header_length,
2718 "datafile-header"))
2719 goto err;
2720 if (param->testflag & T_UNPACK)
2721 {
2722 share->options&= ~HA_OPTION_COMPRESS_RECORD;
2723 mi_int2store(share->state.header.options,share->options);
2724 }
2725 share->state.dellink= HA_OFFSET_ERROR;
2726
2727 if (init_io_cache(&info->rec_cache, new_file,
2728 (uint) param->write_buffer_length,
2729 WRITE_CACHE, new_header_length, 1,
2730 MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw))
2731 goto err;
2732
2733 if (init_io_cache(&new_data_cache, -1,
2734 (uint) param->write_buffer_length,
2735 READ_CACHE, new_header_length, 1,
2736 MYF(MY_WME | MY_DONT_CHECK_FILESIZE)))
2737 goto err;
2738 }
2739
2740 info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2741
2742 /* Optionally drop indexes and optionally modify the key_map. */
2743 mi_drop_all_indexes(param, info, FALSE);
2744 key_map= share->state.key_map;
2745 if (param->testflag & T_CREATE_MISSING_KEYS)
2746 {
2747 /* Invert the copied key_map to recreate all disabled indexes. */
2748 key_map= ~key_map;
2749 }
2750
2751 sort_info.info=info;
2752 sort_info.param = param;
2753
2754 set_data_file_type(&sort_info, share);
2755 sort_info.dupp=0;
2756 sort_info.buff=0;
2757 param->read_cache.end_of_file=sort_info.filelength=
2758 mysql_file_seek(param->read_cache.file, 0L, MY_SEEK_END, MYF(0));
2759
2760 if (share->data_file_type == DYNAMIC_RECORD)
2761 rec_length=MY_MAX(share->base.min_pack_length+1,share->base.min_block_length);
2762 else if (share->data_file_type == COMPRESSED_RECORD)
2763 rec_length=share->base.min_block_length;
2764 else
2765 rec_length=share->base.pack_reclength;
2766 /*
2767 +1 below is required hack for parallel repair mode.
2768 The info->state->records value, that is compared later
2769 to sort_info.max_records and cannot exceed it, is
2770 increased in sort_key_write. In mi_repair_by_sort, sort_key_write
2771 is called after sort_key_read, where the comparison is performed,
2772 but in parallel mode master thread can call sort_key_write
2773 before some other repair thread calls sort_key_read.
2774 Furthermore I'm not even sure +1 would be enough.
2775 May be sort_info.max_records shold be always set to max value in
2776 parallel mode.
2777 */
2778 sort_info.max_records=
2779 ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records + 1:
2780 (ha_rows) (sort_info.filelength/rec_length+1));
2781
2782 del=info->state->del;
2783 param->glob_crc=0;
2784 /* for compressed tables */
2785 max_pack_reclength= MY_MAX(share->base.pack_reclength, share->vreclength);
2786 if (share->options & HA_OPTION_COMPRESS_RECORD)
2787 set_if_bigger(max_pack_reclength, share->max_pack_length);
2788 if (!(sort_param=(MI_SORT_PARAM *)
2789 my_malloc((uint) share->base.keys *
2790 (sizeof(MI_SORT_PARAM) + max_pack_reclength),
2791 MYF(MY_ZEROFILL))))
2792 {
2793 mi_check_print_error(param,"Not enough memory for key!");
2794 goto err;
2795 }
2796 total_key_length=0;
2797 rec_per_key_part= param->rec_per_key_part;
2798 info->state->records=info->state->del=share->state.split=0;
2799 info->state->empty=0;
2800
2801 for (i=key=0, istep=1 ; key < share->base.keys ;
2802 rec_per_key_part+=sort_param[i].keyinfo->keysegs, i+=istep, key++)
2803 {
2804 sort_param[i].key=key;
2805 sort_param[i].keyinfo=share->keyinfo+key;
2806 sort_param[i].seg=sort_param[i].keyinfo->seg;
2807 /*
2808 Skip this index if it is marked disabled in the copied
2809 (and possibly inverted) key_map.
2810 */
2811 if (! mi_is_key_active(key_map, key))
2812 {
2813 /* Remember old statistics for key */
2814 memcpy((char*) rec_per_key_part,
2815 (char*) (share->state.rec_per_key_part+
2816 (uint) (rec_per_key_part - param->rec_per_key_part)),
2817 sort_param[i].keyinfo->keysegs*sizeof(*rec_per_key_part));
2818 istep=0;
2819 continue;
2820 }
2821 istep=1;
2822 if ((!(param->testflag & T_SILENT)))
2823 printf ("- Fixing index %d\n",key+1);
2824 if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
2825 {
2826 sort_param[i].key_read=sort_ft_key_read;
2827 sort_param[i].key_write=sort_ft_key_write;
2828 }
2829 else
2830 {
2831 sort_param[i].key_read=sort_key_read;
2832 sort_param[i].key_write=sort_key_write;
2833 }
2834 sort_param[i].key_cmp=sort_key_cmp;
2835 sort_param[i].lock_in_memory=lock_memory;
2836 sort_param[i].tmpdir=param->tmpdir;
2837 sort_param[i].sort_info=&sort_info;
2838 sort_param[i].master=0;
2839 sort_param[i].fix_datafile=0;
2840 sort_param[i].calc_checksum= 0;
2841
2842 sort_param[i].filepos=new_header_length;
2843 sort_param[i].max_pos=sort_param[i].pos=share->pack.header_length;
2844
2845 sort_param[i].record= (((uchar *)(sort_param+share->base.keys))+
2846 (max_pack_reclength * i));
2847 if (!mi_alloc_rec_buff(info, -1, &sort_param[i].rec_buff))
2848 {
2849 mi_check_print_error(param,"Not enough memory!");
2850 goto err;
2851 }
2852
2853 sort_param[i].key_length=share->rec_reflength;
2854 for (keyseg=sort_param[i].seg; keyseg->type != HA_KEYTYPE_END;
2855 keyseg++)
2856 {
2857 sort_param[i].key_length+=keyseg->length;
2858 if (keyseg->flag & HA_SPACE_PACK)
2859 sort_param[i].key_length+=get_pack_length(keyseg->length);
2860 if (keyseg->flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
2861 sort_param[i].key_length+= 2 + MY_TEST(keyseg->length >= 127);
2862 if (keyseg->flag & HA_NULL_PART)
2863 sort_param[i].key_length++;
2864 }
2865 total_key_length+=sort_param[i].key_length;
2866
2867 if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
2868 {
2869 uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
2870 sort_param[i].keyinfo->seg->charset->mbmaxlen;
2871 sort_param[i].key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
2872 init_alloc_root(&sort_param[i].wordroot, "sort",
2873 FTPARSER_MEMROOT_ALLOC_SIZE, 0,
2874 MYF(param->malloc_flags));
2875 }
2876 }
2877 sort_info.total_keys=i;
2878 sort_param[0].master= 1;
2879 sort_param[0].fix_datafile= (my_bool)(! rep_quick);
2880 sort_param[0].calc_checksum= MY_TEST(param->testflag & T_CALC_CHECKSUM);
2881
2882 if (!ftparser_alloc_param(info))
2883 goto err;
2884
2885 sort_info.got_error=0;
2886 mysql_mutex_lock(&sort_info.mutex);
2887
2888 /*
2889 Initialize the I/O cache share for use with the read caches and, in
2890 case of non-quick repair, the write cache. When all threads join on
2891 the cache lock, the writer copies the write cache contents to the
2892 read caches.
2893 */
2894 if (i > 1)
2895 {
2896 if (rep_quick)
2897 init_io_cache_share(&param->read_cache, &io_share, NULL, i);
2898 else
2899 init_io_cache_share(&new_data_cache, &io_share, &info->rec_cache, i);
2900 }
2901 else
2902 io_share.total_threads= 0; /* share not used */
2903
2904 (void) pthread_attr_init(&thr_attr);
2905 (void) pthread_attr_setdetachstate(&thr_attr,PTHREAD_CREATE_DETACHED);
2906
2907 for (i=0 ; i < sort_info.total_keys ; i++)
2908 {
2909 /*
2910 Copy the properly initialized IO_CACHE structure so that every
2911 thread has its own copy. In quick mode param->read_cache is shared
2912 for use by all threads. In non-quick mode all threads but the
2913 first copy the shared new_data_cache, which is synchronized to the
2914 write cache of the first thread. The first thread copies
2915 param->read_cache, which is not shared.
2916 */
2917 sort_param[i].read_cache= ((rep_quick || !i) ? param->read_cache :
2918 new_data_cache);
2919 DBUG_PRINT("io_cache_share", ("thread: %u read_cache: %p",
2920 i, &sort_param[i].read_cache));
2921
2922 /*
2923 two approaches: the same amount of memory for each thread
2924 or the memory for the same number of keys for each thread...
2925 In the second one all the threads will fill their sort_buffers
2926 (and call write_keys) at the same time, putting more stress on i/o.
2927 */
2928 sort_param[i].sortbuff_size=
2929#ifndef USING_SECOND_APPROACH
2930 param->sort_buffer_length/sort_info.total_keys;
2931#else
2932 param->sort_buffer_length*sort_param[i].key_length/total_key_length;
2933#endif
2934 if ((error= mysql_thread_create(mi_key_thread_find_all_keys,
2935 &sort_param[i].thr, &thr_attr,
2936 thr_find_all_keys,
2937 (void *) (sort_param+i))))
2938 {
2939 mi_check_print_error(param,"Cannot start a repair thread (errno= %d)",
2940 error);
2941 /* Cleanup: Detach from the share. Avoid others to be blocked. */
2942 if (io_share.total_threads)
2943 remove_io_thread(&sort_param[i].read_cache);
2944 DBUG_PRINT("error", ("Cannot start a repair thread"));
2945 sort_info.got_error=1;
2946 }
2947 else
2948 sort_info.threads_running++;
2949 }
2950 (void) pthread_attr_destroy(&thr_attr);
2951
2952 /* waiting for all threads to finish */
2953 while (sort_info.threads_running)
2954 mysql_cond_wait(&sort_info.cond, &sort_info.mutex);
2955 mysql_mutex_unlock(&sort_info.mutex);
2956
2957 if ((got_error= thr_write_keys(sort_param)))
2958 {
2959 param->retry_repair=1;
2960 goto err;
2961 }
2962 got_error=1; /* Assume the following may go wrong */
2963
2964 if (sort_param[0].fix_datafile)
2965 {
2966 /*
2967 Append some nuls to the end of a memory mapped file. Destroy the
2968 write cache. The master thread did already detach from the share
2969 by remove_io_thread() in sort.c:thr_find_all_keys().
2970 */
2971 if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache))
2972 goto err;
2973 if (param->testflag & T_SAFE_REPAIR)
2974 {
2975 /* Don't repair if we loosed more than one row */
2976 if (info->state->records+1 < start_records)
2977 {
2978 info->state->records=start_records;
2979 goto err;
2980 }
2981 }
2982 share->state.state.data_file_length= info->state->data_file_length=
2983 sort_param->filepos;
2984 /* Only whole records */
2985 share->state.version=(ulong) time((time_t*) 0);
2986
2987 /*
2988 Exchange the data file descriptor of the table, so that we use the
2989 new file from now on.
2990 */
2991 mysql_file_close(info->dfile, MYF(0));
2992 info->dfile=new_file;
2993
2994 share->data_file_type=sort_info.new_data_file_type;
2995 share->pack.header_length=(ulong) new_header_length;
2996 }
2997 else
2998 info->state->data_file_length=sort_param->max_pos;
2999
3000 if (rep_quick && del+sort_info.dupp != info->state->del)
3001 {
3002 mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
3003 mi_check_print_error(param,"Run recovery again without -q");
3004 param->retry_repair=1;
3005 param->testflag|=T_RETRY_WITHOUT_QUICK;
3006 goto err;
3007 }
3008
3009 if (rep_quick && (param->testflag & T_FORCE_UNIQUENESS))
3010 {
3011 my_off_t skr=info->state->data_file_length+
3012 (share->options & HA_OPTION_COMPRESS_RECORD ?
3013 MEMMAP_EXTRA_MARGIN : 0);
3014#ifdef USE_RELOC
3015 if (share->data_file_type == STATIC_RECORD &&
3016 skr < share->base.reloc*share->base.min_pack_length)
3017 skr=share->base.reloc*share->base.min_pack_length;
3018#endif
3019 if (skr != sort_info.filelength)
3020 if (mysql_file_chsize(info->dfile, skr, 0, MYF(0)))
3021 mi_check_print_warning(param,
3022 "Can't change size of datafile, error: %d",
3023 my_errno);
3024 }
3025 if (param->testflag & T_CALC_CHECKSUM)
3026 info->state->checksum=param->glob_crc;
3027
3028 if (mysql_file_chsize(share->kfile, info->state->key_file_length, 0, MYF(0)))
3029 mi_check_print_warning(param,
3030 "Can't change size of indexfile, error: %d", my_errno);
3031
3032 if (!(param->testflag & T_SILENT))
3033 {
3034 if (start_records != info->state->records)
3035 printf("Data records: %s\n", llstr(info->state->records,llbuff));
3036 if (sort_info.dupp)
3037 mi_check_print_warning(param,
3038 "%s records have been removed",
3039 llstr(sort_info.dupp,llbuff));
3040 }
3041 got_error=0;
3042
3043 if (&share->state.state != info->state)
3044 memcpy(&share->state.state, info->state, sizeof(*info->state));
3045
3046err:
3047 got_error|= flush_blocks(param, share->key_cache, share->kfile,
3048 &share->dirty_part_map);
3049 /*
3050 Destroy the write cache. The master thread did already detach from
3051 the share by remove_io_thread() or it was not yet started (if the
3052 error happend before creating the thread).
3053 */
3054 (void) end_io_cache(&info->rec_cache);
3055 /*
3056 Destroy the new data cache in case of non-quick repair. All slave
3057 threads did either detach from the share by remove_io_thread()
3058 already or they were not yet started (if the error happend before
3059 creating the threads).
3060 */
3061 if (!rep_quick && my_b_inited(&new_data_cache))
3062 (void) end_io_cache(&new_data_cache);
3063 if (!got_error)
3064 {
3065 /* Replace the actual file with the temporary file */
3066 if (new_file >= 0)
3067 {
3068 got_error= replace_data_file(param, info, new_file);
3069 new_file= -1;
3070 }
3071 }
3072 if (got_error)
3073 {
3074 if (! param->error_printed)
3075 mi_check_print_error(param,"%d when fixing table",my_errno);
3076 if (new_file >= 0)
3077 {
3078 (void) mysql_file_close(new_file, MYF(0));
3079 (void) mysql_file_delete(mi_key_file_datatmp,
3080 param->temp_filename, MYF(MY_WME));
3081 if (info->dfile == new_file) /* Retry with key cache */
3082 if (unlikely(mi_open_datafile(info, share)))
3083 param->retry_repair= 0; /* Safety */
3084 }
3085 mi_mark_crashed_on_repair(info);
3086 if (killed_ptr(param))
3087 param->retry_repair= 0;
3088 }
3089 else if (key_map == share->state.key_map)
3090 share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
3091 share->state.changed|=STATE_NOT_SORTED_PAGES;
3092
3093 mysql_cond_destroy(&sort_info.cond);
3094 mysql_mutex_destroy(&sort_info.mutex);
3095 mysql_mutex_destroy(&param->print_msg_mutex);
3096 param->need_print_msg_lock= 0;
3097
3098 my_free(sort_info.ft_buf);
3099 my_free(sort_info.key_block);
3100 my_free(sort_param);
3101 my_free(sort_info.buff);
3102 (void) end_io_cache(&param->read_cache);
3103 info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
3104 if (!got_error && (param->testflag & T_UNPACK))
3105 {
3106 share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
3107 share->pack.header_length=0;
3108 }
3109 DBUG_RETURN(got_error);
3110}
3111
3112 /* Read next record and return next key */
3113
3114static int sort_key_read(MI_SORT_PARAM *sort_param, void *key)
3115{
3116 int error;
3117 MI_SORT_INFO *sort_info=sort_param->sort_info;
3118 MI_INFO *info=sort_info->info;
3119 DBUG_ENTER("sort_key_read");
3120
3121 if ((error=sort_get_next_record(sort_param)))
3122 {
3123 DBUG_ASSERT(error < 0 ||
3124 sort_info->param->error_printed ||
3125 sort_info->param->warning_printed ||
3126 sort_info->param->note_printed);
3127 DBUG_RETURN(error);
3128 }
3129 if (info->state->records == sort_info->max_records)
3130 {
3131 my_errno= HA_ERR_WRONG_IN_RECORD;
3132 mi_check_print_error(sort_info->param,
3133 "Key %d - Found too many records; Can't continue",
3134 sort_param->key+1);
3135 DBUG_RETURN(1);
3136 }
3137 sort_param->real_key_length=
3138 (info->s->rec_reflength+
3139 _mi_make_key(info, sort_param->key, (uchar*) key,
3140 sort_param->record, sort_param->filepos));
3141#ifdef HAVE_valgrind
3142 bzero(key+sort_param->real_key_length,
3143 (sort_param->key_length-sort_param->real_key_length));
3144#endif
3145 DBUG_RETURN(sort_write_record(sort_param));
3146} /* sort_key_read */
3147
3148static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key)
3149{
3150 int error;
3151 MI_SORT_INFO *sort_info=sort_param->sort_info;
3152 MI_INFO *info=sort_info->info;
3153 FT_WORD *wptr=0;
3154 DBUG_ENTER("sort_ft_key_read");
3155
3156 if (!sort_param->wordlist)
3157 {
3158 for (;;)
3159 {
3160 free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
3161 if ((error=sort_get_next_record(sort_param)))
3162 DBUG_RETURN(error);
3163 if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record,
3164 &sort_param->wordroot)))
3165 DBUG_RETURN(1);
3166 if (wptr->pos)
3167 break;
3168 error=sort_write_record(sort_param);
3169 }
3170 sort_param->wordptr=sort_param->wordlist=wptr;
3171 }
3172 else
3173 {
3174 error=0;
3175 wptr=(FT_WORD*)(sort_param->wordptr);
3176 }
3177
3178 sort_param->real_key_length=(info->s->rec_reflength+
3179 _ft_make_key(info, sort_param->key,
3180 key, wptr++, sort_param->filepos));
3181#ifdef HAVE_valgrind
3182 if (sort_param->key_length > sort_param->real_key_length)
3183 bzero(key+sort_param->real_key_length,
3184 (sort_param->key_length-sort_param->real_key_length));
3185#endif
3186 if (!wptr->pos)
3187 {
3188 free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
3189 sort_param->wordlist=0;
3190 error=sort_write_record(sort_param);
3191 }
3192 else
3193 sort_param->wordptr=(void*)wptr;
3194
3195 DBUG_RETURN(error);
3196} /* sort_ft_key_read */
3197
3198
3199/*
3200 Read next record from file using parameters in sort_info.
3201
3202 SYNOPSIS
3203 sort_get_next_record()
3204 sort_param Information about and for the sort process
3205
3206 NOTE
3207
3208 Dynamic Records With Non-Quick Parallel Repair
3209
3210 For non-quick parallel repair we use a synchronized read/write
3211 cache. This means that one thread is the master who fixes the data
3212 file by reading each record from the old data file and writing it
3213 to the new data file. By doing this the records in the new data
3214 file are written contiguously. Whenever the write buffer is full,
3215 it is copied to the read buffer. The slaves read from the read
3216 buffer, which is not associated with a file. Thus read_cache.file
3217 is -1. When using _mi_read_cache(), the slaves must always set
3218 flag to READING_NEXT so that the function never tries to read from
3219 file. This is safe because the records are contiguous. There is no
3220 need to read outside the cache. This condition is evaluated in the
3221 variable 'parallel_flag' for quick reference. read_cache.file must
3222 be >= 0 in every other case.
3223
3224 RETURN
3225 -1 end of file
3226 0 ok
3227 > 0 error
3228*/
3229
3230static int sort_get_next_record(MI_SORT_PARAM *sort_param)
3231{
3232 int searching;
3233 int parallel_flag;
3234 uint found_record,b_type,left_length;
3235 my_off_t pos;
3236 uchar *UNINIT_VAR(to);
3237 MI_BLOCK_INFO block_info;
3238 MI_SORT_INFO *sort_info=sort_param->sort_info;
3239 HA_CHECK *param=sort_info->param;
3240 MI_INFO *info=sort_info->info;
3241 MYISAM_SHARE *share=info->s;
3242 char llbuff[22],llbuff2[22];
3243 DBUG_ENTER("sort_get_next_record");
3244
3245 if (killed_ptr(param))
3246 {
3247 mi_check_print_error(param, "Repair killed by user with cause: %d",
3248 (int) killed_ptr(param));
3249 param->retry_repair= 0;
3250 DBUG_RETURN(1);
3251 }
3252
3253 switch (share->data_file_type) {
3254 case STATIC_RECORD:
3255 for (;;)
3256 {
3257 if (my_b_read(&sort_param->read_cache,sort_param->record,
3258 share->base.pack_reclength))
3259 {
3260 if (sort_param->read_cache.error)
3261 param->out_flag |= O_DATA_LOST;
3262 param->retry_repair=1;
3263 param->testflag|=T_RETRY_WITHOUT_QUICK;
3264 DBUG_RETURN(-1);
3265 }
3266 sort_param->start_recpos=sort_param->pos;
3267 if (!sort_param->fix_datafile)
3268 {
3269 sort_param->filepos=sort_param->pos;
3270 if (sort_param->master)
3271 share->state.split++;
3272 }
3273 sort_param->max_pos=(sort_param->pos+=share->base.pack_reclength);
3274 if (*sort_param->record)
3275 {
3276 if (sort_param->calc_checksum)
3277 info->checksum= (*info->s->calc_check_checksum)(info, sort_param->record);
3278 goto finish;
3279 }
3280 if (!sort_param->fix_datafile && sort_param->master)
3281 {
3282 info->state->del++;
3283 info->state->empty+=share->base.pack_reclength;
3284 }
3285 }
3286 case DYNAMIC_RECORD:
3287 pos=sort_param->pos;
3288 searching=(sort_param->fix_datafile && (param->testflag & T_EXTEND));
3289 parallel_flag= (sort_param->read_cache.file < 0) ? READING_NEXT : 0;
3290 for (;;)
3291 {
3292 found_record=block_info.second_read= 0;
3293 left_length=1;
3294 if (searching)
3295 {
3296 pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE);
3297 param->testflag|=T_RETRY_WITHOUT_QUICK;
3298 sort_param->start_recpos=pos;
3299 }
3300 do
3301 {
3302 if (pos > sort_param->max_pos)
3303 sort_param->max_pos=pos;
3304 if (pos & (MI_DYN_ALIGN_SIZE-1))
3305 {
3306 if ((param->testflag & T_VERBOSE) || searching == 0)
3307 mi_check_print_info(param,"Wrong aligned block at %s",
3308 llstr(pos,llbuff));
3309 if (searching)
3310 goto try_next;
3311 }
3312 if (found_record && pos == param->search_after_block)
3313 mi_check_print_info(param,"Block: %s used by record at %s",
3314 llstr(param->search_after_block,llbuff),
3315 llstr(sort_param->start_recpos,llbuff2));
3316 if (_mi_read_cache(&sort_param->read_cache,
3317 (uchar*) block_info.header,pos,
3318 MI_BLOCK_INFO_HEADER_LENGTH,
3319 (! found_record ? READING_NEXT : 0) |
3320 parallel_flag | READING_HEADER))
3321 {
3322 if (found_record)
3323 {
3324 mi_check_print_info(param,
3325 "Can't read whole record at %s (errno: %d)",
3326 llstr(sort_param->start_recpos,llbuff),errno);
3327 goto try_next;
3328 }
3329 DBUG_RETURN(-1);
3330 }
3331 if (searching && ! sort_param->fix_datafile)
3332 {
3333 mi_check_print_info(param,
3334 "Datafile is corrupted; Restart repair with option to copy datafile");
3335 param->error_printed=1;
3336 param->retry_repair=1;
3337 param->testflag|=T_RETRY_WITHOUT_QUICK;
3338 my_errno= HA_ERR_WRONG_IN_RECORD;
3339 DBUG_RETURN(1); /* Something wrong with data */
3340 }
3341 b_type=_mi_get_block_info(&block_info,-1,pos);
3342 if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) ||
3343 ((b_type & BLOCK_FIRST) &&
3344 (block_info.rec_len < (uint) share->base.min_pack_length ||
3345 block_info.rec_len > (uint) share->base.max_pack_length)))
3346 {
3347 uint i;
3348 if (param->testflag & T_VERBOSE || searching == 0)
3349 mi_check_print_info(param,
3350 "Wrong bytesec: %3d-%3d-%3d at %10s; Skipped",
3351 block_info.header[0],block_info.header[1],
3352 block_info.header[2],llstr(pos,llbuff));
3353 if (found_record)
3354 goto try_next;
3355 block_info.second_read=0;
3356 searching=1;
3357 /* Search after block in read header string */
3358 for (i=MI_DYN_ALIGN_SIZE ;
3359 i < MI_BLOCK_INFO_HEADER_LENGTH ;
3360 i+= MI_DYN_ALIGN_SIZE)
3361 if (block_info.header[i] >= 1 &&
3362 block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE)
3363 break;
3364 pos+=(ulong) i;
3365 sort_param->start_recpos=pos;
3366 continue;
3367 }
3368 if (b_type & BLOCK_DELETED)
3369 {
3370 my_bool error=0;
3371 if (block_info.block_len+ (uint) (block_info.filepos-pos) <
3372 share->base.min_block_length)
3373 {
3374 if (!searching)
3375 mi_check_print_info(param,
3376 "Deleted block with impossible length %u at %s",
3377 block_info.block_len,llstr(pos,llbuff));
3378 error=1;
3379 }
3380 else
3381 {
3382 if ((block_info.next_filepos != HA_OFFSET_ERROR &&
3383 block_info.next_filepos >=
3384 info->state->data_file_length) ||
3385 (block_info.prev_filepos != HA_OFFSET_ERROR &&
3386 block_info.prev_filepos >= info->state->data_file_length))
3387 {
3388 if (!searching)
3389 mi_check_print_info(param,
3390 "Delete link points outside datafile at %s",
3391 llstr(pos,llbuff));
3392 error=1;
3393 }
3394 }
3395 if (error)
3396 {
3397 DBUG_ASSERT(param->note_printed);
3398 if (found_record)
3399 goto try_next;
3400 searching=1;
3401 pos+= MI_DYN_ALIGN_SIZE;
3402 sort_param->start_recpos=pos;
3403 block_info.second_read=0;
3404 continue;
3405 }
3406 }
3407 else
3408 {
3409 if (block_info.block_len+ (uint) (block_info.filepos-pos) <
3410 share->base.min_block_length ||
3411 block_info.block_len > (uint) share->base.max_pack_length+
3412 MI_SPLIT_LENGTH)
3413 {
3414 if (!searching)
3415 mi_check_print_info(param,
3416 "Found block with impossible length %u at %s; Skipped",
3417 block_info.block_len+ (uint) (block_info.filepos-pos),
3418 llstr(pos,llbuff));
3419 if (found_record)
3420 goto try_next;
3421 searching=1;
3422 pos+= MI_DYN_ALIGN_SIZE;
3423 sort_param->start_recpos=pos;
3424 block_info.second_read=0;
3425 continue;
3426 }
3427 }
3428 if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR))
3429 {
3430 if (!sort_param->fix_datafile && sort_param->master &&
3431 (b_type & BLOCK_DELETED))
3432 {
3433 info->state->empty+=block_info.block_len;
3434 info->state->del++;
3435 share->state.split++;
3436 }
3437 if (found_record)
3438 {
3439 mi_check_print_info(param,
3440 "Found row block followed by deleted block");
3441 goto try_next;
3442 }
3443 if (searching)
3444 {
3445 pos+=MI_DYN_ALIGN_SIZE;
3446 sort_param->start_recpos=pos;
3447 }
3448 else
3449 pos=block_info.filepos+block_info.block_len;
3450 block_info.second_read=0;
3451 continue;
3452 }
3453
3454 if (!sort_param->fix_datafile && sort_param->master)
3455 share->state.split++;
3456 if (! found_record++)
3457 {
3458 sort_param->find_length=left_length=block_info.rec_len;
3459 sort_param->start_recpos=pos;
3460 if (!sort_param->fix_datafile)
3461 sort_param->filepos=sort_param->start_recpos;
3462 if (sort_param->fix_datafile && (param->testflag & T_EXTEND))
3463 sort_param->pos=block_info.filepos+1;
3464 else
3465 sort_param->pos=block_info.filepos+block_info.block_len;
3466 if (share->base.blobs)
3467 {
3468 if (!(to=mi_alloc_rec_buff(info,block_info.rec_len,
3469 &(sort_param->rec_buff))))
3470 {
3471 if (param->max_record_length >= block_info.rec_len)
3472 {
3473 mi_check_print_error(param,"Not enough memory for blob at %s (need %lu)",
3474 llstr(sort_param->start_recpos,llbuff),
3475 (ulong) block_info.rec_len);
3476 DBUG_ASSERT(param->error_printed);
3477 DBUG_RETURN(1);
3478 }
3479 else
3480 {
3481 mi_check_print_info(param,"Not enough memory for blob at %s (need %lu); Row skipped",
3482 llstr(sort_param->start_recpos,llbuff),
3483 (ulong) block_info.rec_len);
3484 goto try_next;
3485 }
3486 }
3487 }
3488 else
3489 to= sort_param->rec_buff;
3490 }
3491 if (left_length < block_info.data_len || ! block_info.data_len)
3492 {
3493 mi_check_print_info(param,
3494 "Found block with too small length at %s; Skipped",
3495 llstr(sort_param->start_recpos,llbuff));
3496 goto try_next;
3497 }
3498 if (block_info.filepos + block_info.data_len >
3499 sort_param->read_cache.end_of_file)
3500 {
3501 mi_check_print_info(param,
3502 "Found block that points outside data file at %s",
3503 llstr(sort_param->start_recpos,llbuff));
3504 goto try_next;
3505 }
3506 /*
3507 Copy information that is already read. Avoid accessing data
3508 below the cache start. This could happen if the header
3509 streched over the end of the previous buffer contents.
3510 */
3511 {
3512 uint header_len= (uint) (block_info.filepos - pos);
3513 uint prefetch_len= (MI_BLOCK_INFO_HEADER_LENGTH - header_len);
3514
3515 if (prefetch_len > block_info.data_len)
3516 prefetch_len= block_info.data_len;
3517 if (prefetch_len)
3518 {
3519 memcpy(to, block_info.header + header_len, prefetch_len);
3520 block_info.filepos+= prefetch_len;
3521 block_info.data_len-= prefetch_len;
3522 left_length-= prefetch_len;
3523 to+= prefetch_len;
3524 }
3525 }
3526 if (block_info.data_len &&
3527 _mi_read_cache(&sort_param->read_cache,to,block_info.filepos,
3528 block_info.data_len,
3529 (found_record == 1 ? READING_NEXT : 0) |
3530 parallel_flag))
3531 {
3532 mi_check_print_info(param,
3533 "Read error for block at: %s (error: %d); Skipped",
3534 llstr(block_info.filepos,llbuff),my_errno);
3535 goto try_next;
3536 }
3537 left_length-=block_info.data_len;
3538 to+=block_info.data_len;
3539 pos=block_info.next_filepos;
3540 if (pos == HA_OFFSET_ERROR && left_length)
3541 {
3542 mi_check_print_info(param,"Wrong block with wrong total length starting at %s",
3543 llstr(sort_param->start_recpos,llbuff));
3544 goto try_next;
3545 }
3546 if (pos + MI_BLOCK_INFO_HEADER_LENGTH > sort_param->read_cache.end_of_file)
3547 {
3548 mi_check_print_info(param,"Found link that points at %s (outside data file) at %s",
3549 llstr(pos,llbuff2),
3550 llstr(sort_param->start_recpos,llbuff));
3551 goto try_next;
3552 }
3553 } while (left_length);
3554
3555 if (_mi_rec_unpack(info,sort_param->record,sort_param->rec_buff,
3556 sort_param->find_length) != MY_FILE_ERROR)
3557 {
3558 if (sort_param->calc_checksum)
3559 info->checksum= (*info->s->calc_check_checksum)(info,
3560 sort_param->record);
3561 if ((param->testflag & (T_EXTEND | T_REP_ANY)) || searching)
3562 {
3563 if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff,
3564 sort_param->find_length,
3565 (param->testflag & T_QUICK) &&
3566 sort_param->calc_checksum &&
3567 MY_TEST(info->s->calc_checksum)))
3568 {
3569 mi_check_print_info(param,"Found wrong packed record at %s",
3570 llstr(sort_param->start_recpos,llbuff));
3571 goto try_next;
3572 }
3573 }
3574 goto finish;
3575 }
3576 if (!searching)
3577 mi_check_print_info(param,"Key %d - Found wrong stored record at %s",
3578 sort_param->key+1,
3579 llstr(sort_param->start_recpos,llbuff));
3580 try_next:
3581 DBUG_ASSERT(param->error_printed || param->note_printed);
3582 pos=(sort_param->start_recpos+=MI_DYN_ALIGN_SIZE);
3583 searching=1;
3584 }
3585 case COMPRESSED_RECORD:
3586 for (searching=0 ;; searching=1, sort_param->pos++)
3587 {
3588 if (_mi_read_cache(&sort_param->read_cache,(uchar*) block_info.header,
3589 sort_param->pos,
3590 share->pack.ref_length,READING_NEXT))
3591 DBUG_RETURN(-1);
3592 if (searching && ! sort_param->fix_datafile)
3593 {
3594 param->error_printed=1;
3595 param->retry_repair=1;
3596 param->testflag|=T_RETRY_WITHOUT_QUICK;
3597 my_errno= HA_ERR_WRONG_IN_RECORD;
3598 DBUG_RETURN(1); /* Something wrong with data */
3599 }
3600 sort_param->start_recpos=sort_param->pos;
3601 if (_mi_pack_get_block_info(info, &sort_param->bit_buff, &block_info,
3602 &sort_param->rec_buff, -1, sort_param->pos))
3603 DBUG_RETURN(-1);
3604 if (!block_info.rec_len &&
3605 sort_param->pos + MEMMAP_EXTRA_MARGIN ==
3606 sort_param->read_cache.end_of_file)
3607 DBUG_RETURN(-1);
3608 if (block_info.rec_len < (uint) share->min_pack_length ||
3609 block_info.rec_len > (uint) share->max_pack_length)
3610 {
3611 if (! searching)
3612 mi_check_print_info(param,"Found block with wrong recordlength: %d at %s\n",
3613 block_info.rec_len,
3614 llstr(sort_param->pos,llbuff));
3615 continue;
3616 }
3617 if (_mi_read_cache(&sort_param->read_cache,(uchar*) sort_param->rec_buff,
3618 block_info.filepos, block_info.rec_len,
3619 READING_NEXT))
3620 {
3621 if (! searching)
3622 mi_check_print_info(param,"Couldn't read whole record from %s",
3623 llstr(sort_param->pos,llbuff));
3624 continue;
3625 }
3626 if (_mi_pack_rec_unpack(info, &sort_param->bit_buff, sort_param->record,
3627 sort_param->rec_buff, block_info.rec_len))
3628 {
3629 if (! searching)
3630 mi_check_print_info(param,"Found wrong record at %s",
3631 llstr(sort_param->pos,llbuff));
3632 continue;
3633 }
3634 if (!sort_param->fix_datafile)
3635 {
3636 sort_param->filepos=sort_param->pos;
3637 if (sort_param->master)
3638 share->state.split++;
3639 }
3640 sort_param->max_pos=(sort_param->pos=block_info.filepos+
3641 block_info.rec_len);
3642 info->packed_length=block_info.rec_len;
3643 if (sort_param->calc_checksum)
3644 info->checksum= (*info->s->calc_check_checksum)(info, sort_param->record);
3645 goto finish;
3646 }
3647 default:
3648 DBUG_ASSERT(0); /* Impossible */
3649 break;
3650 }
3651 DBUG_ASSERT(0); /* Impossible */
3652 DBUG_RETURN(1); /* Impossible */
3653finish:
3654 if (sort_param->calc_checksum)
3655 param->glob_crc+= info->checksum;
3656 if (param->fix_record)
3657 param->fix_record(info, sort_param->record,
3658 param->testflag & T_REP_BY_SORT ? (int)sort_param->key
3659 : -1);
3660 DBUG_RETURN(0);
3661}
3662
3663
3664/*
3665 Write record to new file.
3666
3667 SYNOPSIS
3668 sort_write_record()
3669 sort_param Sort parameters.
3670
3671 NOTE
3672 This is only called by a master thread if parallel repair is used.
3673
3674 RETURN
3675 0 OK
3676 1 Error
3677*/
3678
3679int sort_write_record(MI_SORT_PARAM *sort_param)
3680{
3681 int flag;
3682 uint length;
3683 ulong block_length,reclength;
3684 uchar *from;
3685 uchar block_buff[8];
3686 MI_SORT_INFO *sort_info=sort_param->sort_info;
3687 HA_CHECK *param=sort_info->param;
3688 MI_INFO *info=sort_info->info;
3689 MYISAM_SHARE *share=info->s;
3690 DBUG_ENTER("sort_write_record");
3691
3692 if (sort_param->fix_datafile)
3693 {
3694 switch (sort_info->new_data_file_type) {
3695 case STATIC_RECORD:
3696 if (my_b_write(&info->rec_cache,sort_param->record,
3697 share->base.pack_reclength))
3698 {
3699 mi_check_print_error(param,"%d when writing to datafile",my_errno);
3700 DBUG_RETURN(1);
3701 }
3702 sort_param->filepos+=share->base.pack_reclength;
3703 info->s->state.split++;
3704 break;
3705 case DYNAMIC_RECORD:
3706 if (! info->blobs)
3707 from=sort_param->rec_buff;
3708 else
3709 {
3710 /* must be sure that local buffer is big enough */
3711 reclength=info->s->base.pack_reclength+
3712 _mi_calc_total_blob_length(info,sort_param->record)+
3713 ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+
3714 MI_DYN_DELETE_BLOCK_HEADER;
3715 if (sort_info->buff_length < reclength)
3716 {
3717 if (!(sort_info->buff=my_realloc(sort_info->buff, (uint) reclength,
3718 MYF(MY_FREE_ON_ERROR | MY_WME |
3719 MY_ALLOW_ZERO_PTR))))
3720 DBUG_RETURN(1);
3721 sort_info->buff_length=reclength;
3722 }
3723 from= sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER);
3724 }
3725 /* We can use info->checksum here as only one thread calls this. */
3726 info->checksum= (*info->s->calc_check_checksum)(info,sort_param->record);
3727 reclength=_mi_rec_pack(info,from,sort_param->record);
3728 flag=0;
3729
3730 do
3731 {
3732 block_length= reclength + 3 + MY_TEST(reclength >= (65520 - 3));
3733 if (block_length < share->base.min_block_length)
3734 block_length=share->base.min_block_length;
3735 info->update|=HA_STATE_WRITE_AT_END;
3736 block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE);
3737 if (block_length > MI_MAX_BLOCK_LENGTH)
3738 block_length=MI_MAX_BLOCK_LENGTH;
3739 if (_mi_write_part_record(info,0L,block_length,
3740 sort_param->filepos+block_length,
3741 &from,&reclength,&flag))
3742 {
3743 mi_check_print_error(param,"%d when writing to datafile",my_errno);
3744 DBUG_RETURN(1);
3745 }
3746 sort_param->filepos+=block_length;
3747 info->s->state.split++;
3748 } while (reclength);
3749 /* sort_info->param->glob_crc+=info->checksum; */
3750 break;
3751 case COMPRESSED_RECORD:
3752 reclength=info->packed_length;
3753 length= save_pack_length((uint) share->pack.version, block_buff,
3754 reclength);
3755 if (info->s->base.blobs)
3756 length+= save_pack_length((uint) share->pack.version,
3757 block_buff + length, info->blob_length);
3758 if (my_b_write(&info->rec_cache,block_buff,length) ||
3759 my_b_write(&info->rec_cache,(uchar*) sort_param->rec_buff,reclength))
3760 {
3761 mi_check_print_error(param,"%d when writing to datafile",my_errno);
3762 DBUG_RETURN(1);
3763 }
3764 /* sort_info->param->glob_crc+=info->checksum; */
3765 sort_param->filepos+=reclength+length;
3766 info->s->state.split++;
3767 break;
3768 default:
3769 DBUG_ASSERT(0); /* Impossible */
3770 break;
3771 }
3772 }
3773 if (sort_param->master)
3774 {
3775 info->state->records++;
3776 if ((param->testflag & T_WRITE_LOOP) &&
3777 (info->state->records % WRITE_COUNT) == 0)
3778 {
3779 char llbuff[22];
3780 printf("%s\r", llstr(info->state->records,llbuff));
3781 (void) fflush(stdout);
3782 }
3783 }
3784 DBUG_RETURN(0);
3785} /* sort_write_record */
3786
3787
3788 /* Compare two keys from _create_index_by_sort */
3789
3790static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,
3791 const void *b)
3792{
3793 uint not_used[2];
3794 return (ha_key_cmp(sort_param->seg, *((uchar**) a), *((uchar**) b),
3795 USE_WHOLE_KEY, SEARCH_SAME, not_used));
3796} /* sort_key_cmp */
3797
3798
3799static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a)
3800{
3801 uint diff_pos[2];
3802 char llbuff[22],llbuff2[22];
3803 MI_SORT_INFO *sort_info=sort_param->sort_info;
3804 HA_CHECK *param= sort_info->param;
3805 int cmp;
3806
3807 if (sort_info->key_block->inited)
3808 {
3809 cmp=ha_key_cmp(sort_param->seg, (uchar*) sort_info->key_block->lastkey,
3810 (uchar*) a, USE_WHOLE_KEY,
3811 SEARCH_FIND | SEARCH_UPDATE | SEARCH_INSERT,
3812 diff_pos);
3813 if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
3814 ha_key_cmp(sort_param->seg, (uchar*) sort_info->key_block->lastkey,
3815 (uchar*) a, USE_WHOLE_KEY,
3816 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos);
3817 else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
3818 {
3819 diff_pos[0]= mi_collect_stats_nonulls_next(sort_param->seg,
3820 sort_param->notnull,
3821 (uchar*) sort_info->
3822 key_block->lastkey,
3823 (uchar*)a);
3824 }
3825 sort_param->unique[diff_pos[0]-1]++;
3826 }
3827 else
3828 {
3829 cmp= -1;
3830 if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
3831 mi_collect_stats_nonulls_first(sort_param->seg, sort_param->notnull,
3832 (uchar*)a);
3833 }
3834 if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0)
3835 {
3836 sort_info->dupp++;
3837 sort_info->info->lastpos=get_record_for_key(sort_info->info,
3838 sort_param->keyinfo,
3839 (uchar*) a);
3840 if ((param->testflag & (T_CREATE_UNIQUE_BY_SORT | T_SUPPRESS_ERR_HANDLING))
3841 == T_CREATE_UNIQUE_BY_SORT)
3842 param->testflag|= T_SUPPRESS_ERR_HANDLING;
3843 mi_check_print_warning(param,
3844 "Duplicate key for record at %10s against record at %10s",
3845 llstr(sort_info->info->lastpos,llbuff),
3846 llstr(get_record_for_key(sort_info->info,
3847 sort_param->keyinfo,
3848 (uchar*) sort_info->
3849 key_block->lastkey),
3850 llbuff2));
3851 param->testflag|=T_RETRY_WITHOUT_QUICK;
3852 if (sort_info->param->testflag & T_VERBOSE)
3853 _mi_print_key(stdout,sort_param->seg,(uchar*) a, USE_WHOLE_KEY);
3854 return (sort_delete_record(sort_param));
3855 }
3856#ifndef DBUG_OFF
3857 if (cmp > 0)
3858 {
3859 mi_check_print_error(param,
3860 "Internal error: Keys are not in order from sort");
3861 return(1);
3862 }
3863#endif
3864 return (sort_insert_key(sort_param,sort_info->key_block,
3865 (uchar*) a, HA_OFFSET_ERROR));
3866} /* sort_key_write */
3867
3868int sort_ft_buf_flush(MI_SORT_PARAM *sort_param)
3869{
3870 MI_SORT_INFO *sort_info=sort_param->sort_info;
3871 SORT_KEY_BLOCKS *key_block=sort_info->key_block;
3872 MYISAM_SHARE *share=sort_info->info->s;
3873 uint val_off, val_len;
3874 int error;
3875 SORT_FT_BUF *ft_buf=sort_info->ft_buf;
3876 uchar *from, *to;
3877
3878 val_len=share->ft2_keyinfo.keylength;
3879 get_key_full_length_rdonly(val_off, ft_buf->lastkey);
3880 to= (uchar*) ft_buf->lastkey+val_off;
3881
3882 if (ft_buf->buf)
3883 {
3884 /* flushing first-level tree */
3885 error=sort_insert_key(sort_param,key_block, (uchar*) ft_buf->lastkey,
3886 HA_OFFSET_ERROR);
3887 for (from=to+val_len;
3888 !error && from < (uchar*) ft_buf->buf;
3889 from+= val_len)
3890 {
3891 memcpy(to, from, val_len);
3892 error=sort_insert_key(sort_param,key_block, (uchar*) ft_buf->lastkey,
3893 HA_OFFSET_ERROR);
3894 }
3895 return error;
3896 }
3897 /* flushing second-level tree keyblocks */
3898 error=flush_pending_blocks(sort_param);
3899 /* updating lastkey with second-level tree info */
3900 ft_intXstore(ft_buf->lastkey+val_off, -ft_buf->count);
3901 _mi_dpointer(sort_info->info, (uchar*) ft_buf->lastkey+val_off+HA_FT_WLEN,
3902 share->state.key_root[sort_param->key]);
3903 /* restoring first level tree data in sort_info/sort_param */
3904 sort_info->key_block=sort_info->key_block_end- sort_info->param->sort_key_blocks;
3905 sort_param->keyinfo=share->keyinfo+sort_param->key;
3906 share->state.key_root[sort_param->key]=HA_OFFSET_ERROR;
3907 /* writing lastkey in first-level tree */
3908 return error ? error :
3909 sort_insert_key(sort_param,sort_info->key_block,
3910 (uchar*) ft_buf->lastkey,HA_OFFSET_ERROR);
3911}
3912
3913static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a)
3914{
3915 uint a_len, val_off, val_len, error;
3916 uchar *p;
3917 MI_SORT_INFO *sort_info=sort_param->sort_info;
3918 SORT_FT_BUF *ft_buf=sort_info->ft_buf;
3919 SORT_KEY_BLOCKS *key_block=sort_info->key_block;
3920
3921 val_len= HA_FT_WLEN + sort_info->info->s->rec_reflength;
3922 get_key_full_length_rdonly(a_len, (uchar *)a);
3923
3924 if (!ft_buf)
3925 {
3926 /*
3927 use two-level tree only if key_reflength fits in rec_reflength place
3928 and row format is NOT static - for _mi_dpointer not to garble offsets
3929 */
3930 if ((sort_info->info->s->base.key_reflength <=
3931 sort_info->info->s->rec_reflength) &&
3932 (sort_info->info->s->options &
3933 (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
3934 ft_buf=(SORT_FT_BUF *)my_malloc(sort_param->keyinfo->block_length +
3935 sizeof(SORT_FT_BUF), MYF(MY_WME));
3936
3937 if (!ft_buf)
3938 {
3939 sort_param->key_write=sort_key_write;
3940 return sort_key_write(sort_param, a);
3941 }
3942 sort_info->ft_buf=ft_buf;
3943 goto word_init_ft_buf; /* no need to duplicate the code */
3944 }
3945 get_key_full_length_rdonly(val_off, ft_buf->lastkey);
3946
3947 if (ha_compare_text(sort_param->seg->charset,
3948 ((uchar *)a)+1,a_len-1,
3949 (uchar*) ft_buf->lastkey+1,val_off-1, 0)==0)
3950 {
3951 if (!ft_buf->buf) /* store in second-level tree */
3952 {
3953 ft_buf->count++;
3954 return sort_insert_key(sort_param,key_block,
3955 ((uchar *)a)+a_len, HA_OFFSET_ERROR);
3956 }
3957
3958 /* storing the key in the buffer. */
3959 memcpy (ft_buf->buf, (char *)a+a_len, val_len);
3960 ft_buf->buf+=val_len;
3961 if (ft_buf->buf < ft_buf->end)
3962 return 0;
3963
3964 /* converting to two-level tree */
3965 p= (uchar*) ft_buf->lastkey+val_off;
3966
3967 while (key_block->inited)
3968 key_block++;
3969 sort_info->key_block=key_block;
3970 sort_param->keyinfo=& sort_info->info->s->ft2_keyinfo;
3971 ft_buf->count=(int)((uchar*) ft_buf->buf - p)/val_len;
3972
3973 /* flushing buffer to second-level tree */
3974 for (error=0; !error && p < (uchar*) ft_buf->buf; p+= val_len)
3975 error=sort_insert_key(sort_param,key_block,p,HA_OFFSET_ERROR);
3976 ft_buf->buf=0;
3977 return error;
3978 }
3979
3980 /* flushing buffer */
3981 if ((error=sort_ft_buf_flush(sort_param)))
3982 return error;
3983
3984word_init_ft_buf:
3985 a_len+=val_len;
3986 memcpy(ft_buf->lastkey, a, a_len);
3987 ft_buf->buf=ft_buf->lastkey+a_len;
3988 /*
3989 32 is just a safety margin here
3990 (at least MY_MAX(val_len, sizeof(nod_flag)) should be there).
3991 May be better performance could be achieved if we'd put
3992 (sort_info->keyinfo->block_length-32)/XXX
3993 instead.
3994 TODO: benchmark the best value for XXX.
3995 */
3996 ft_buf->end=ft_buf->lastkey+ (sort_param->keyinfo->block_length-32);
3997 return 0;
3998} /* sort_ft_key_write */
3999
4000
4001 /* get pointer to record from a key */
4002
4003static my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo,
4004 uchar *key)
4005{
4006 return _mi_dpos(info,0,key+_mi_keylength(keyinfo,key));
4007} /* get_record_for_key */
4008
4009
4010 /* Insert a key in sort-key-blocks */
4011
4012static int sort_insert_key(MI_SORT_PARAM *sort_param,
4013 register SORT_KEY_BLOCKS *key_block, uchar *key,
4014 my_off_t prev_block)
4015{
4016 uint a_length,t_length,nod_flag;
4017 my_off_t filepos,key_file_length;
4018 uchar *anc_buff,*lastkey;
4019 MI_KEY_PARAM s_temp;
4020 MI_INFO *info;
4021 MI_KEYDEF *keyinfo=sort_param->keyinfo;
4022 MI_SORT_INFO *sort_info= sort_param->sort_info;
4023 HA_CHECK *param=sort_info->param;
4024 DBUG_ENTER("sort_insert_key");
4025
4026 anc_buff= (uchar*) key_block->buff;
4027 info=sort_info->info;
4028 lastkey= (uchar*) key_block->lastkey;
4029 nod_flag= (key_block == sort_info->key_block ? 0 :
4030 info->s->base.key_reflength);
4031
4032 if (!key_block->inited)
4033 {
4034 key_block->inited=1;
4035 if (key_block == sort_info->key_block_end)
4036 {
4037 mi_check_print_error(param,"To many key-block-levels; Try increasing sort_key_blocks");
4038 DBUG_RETURN(1);
4039 }
4040 a_length=2+nod_flag;
4041 key_block->end_pos= anc_buff+2;
4042 lastkey=0; /* No previous key in block */
4043 }
4044 else
4045 a_length=mi_getint(anc_buff);
4046
4047 /* Save pointer to previous block */
4048 if (nod_flag)
4049 _mi_kpointer(info,(uchar*) key_block->end_pos,prev_block);
4050
4051 t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,
4052 (uchar*) 0,lastkey,lastkey,key,
4053 &s_temp);
4054 (*keyinfo->store_key)(keyinfo, (uchar*) key_block->end_pos+nod_flag,&s_temp);
4055 a_length+=t_length;
4056 mi_putint(anc_buff,a_length,nod_flag);
4057 key_block->end_pos+=t_length;
4058 if (a_length <= keyinfo->block_length)
4059 {
4060 (void) _mi_move_key(keyinfo,key_block->lastkey,key);
4061 key_block->last_length=a_length-t_length;
4062 DBUG_RETURN(0);
4063 }
4064
4065 /* Fill block with end-zero and write filled block */
4066 mi_putint(anc_buff,key_block->last_length,nod_flag);
4067 bzero((uchar*) anc_buff+key_block->last_length,
4068 keyinfo->block_length- key_block->last_length);
4069 key_file_length=info->state->key_file_length;
4070 if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
4071 DBUG_RETURN(1);
4072
4073 /* If we read the page from the key cache, we have to write it back to it */
4074 if (key_file_length == info->state->key_file_length)
4075 {
4076 if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS, anc_buff))
4077 DBUG_RETURN(1);
4078 }
4079 else if (mysql_file_pwrite(info->s->kfile, (uchar*) anc_buff,
4080 (uint) keyinfo->block_length, filepos,
4081 param->myf_rw))
4082 DBUG_RETURN(1);
4083 DBUG_DUMP("buff",(uchar*) anc_buff,mi_getint(anc_buff));
4084
4085 /* Write separator-key to block in next level */
4086 if (sort_insert_key(sort_param,key_block+1,(uchar*) key_block->lastkey,
4087 filepos))
4088 DBUG_RETURN(1);
4089
4090 /* clear old block and write new key in it */
4091 key_block->inited=0;
4092 DBUG_RETURN(sort_insert_key(sort_param, key_block,key,prev_block));
4093} /* sort_insert_key */
4094
4095
4096 /* Delete record when we found a duplicated key */
4097
4098static int sort_delete_record(MI_SORT_PARAM *sort_param)
4099{
4100 uint i;
4101 int old_file,error;
4102 uchar *key;
4103 MI_SORT_INFO *sort_info=sort_param->sort_info;
4104 HA_CHECK *param=sort_info->param;
4105 MI_INFO *info=sort_info->info;
4106 DBUG_ENTER("sort_delete_record");
4107
4108 if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
4109 {
4110 mi_check_print_error(param,
4111 "Quick-recover aborted; Run recovery without switch -q or with switch -qq");
4112 DBUG_RETURN(1);
4113 }
4114 if (info->s->options & HA_OPTION_COMPRESS_RECORD)
4115 {
4116 mi_check_print_error(param,
4117 "Recover aborted; Can't run standard recovery on compressed tables with errors in data-file. Use switch 'myisamchk --safe-recover' to fix it\n",stderr);;
4118 DBUG_RETURN(1);
4119 }
4120
4121 old_file=info->dfile;
4122 info->dfile=info->rec_cache.file;
4123 if (sort_info->current_key)
4124 {
4125 key=info->lastkey+info->s->base.max_key_length;
4126 if ((error=(*info->s->read_rnd)(info,sort_param->record,info->lastpos,0)) &&
4127 error != HA_ERR_RECORD_DELETED)
4128 {
4129 mi_check_print_error(param,"Can't read record to be removed");
4130 info->dfile=old_file;
4131 DBUG_RETURN(1);
4132 }
4133
4134 for (i=0 ; i < sort_info->current_key ; i++)
4135 {
4136 uint key_length=_mi_make_key(info,i,key,sort_param->record,info->lastpos);
4137 if (_mi_ck_delete(info,i,key,key_length))
4138 {
4139 mi_check_print_error(param,"Can't delete key %d from record to be removed",i+1);
4140 info->dfile=old_file;
4141 DBUG_RETURN(1);
4142 }
4143 }
4144 if (sort_param->calc_checksum)
4145 param->glob_crc-=(*info->s->calc_checksum)(info, sort_param->record);
4146 }
4147 error=flush_io_cache(&info->rec_cache) || (*info->s->delete_record)(info);
4148 info->dfile=old_file; /* restore actual value */
4149 info->state->records--;
4150 DBUG_RETURN(error);
4151} /* sort_delete_record */
4152
4153 /* Fix all pending blocks and flush everything to disk */
4154
4155int flush_pending_blocks(MI_SORT_PARAM *sort_param)
4156{
4157 uint nod_flag,length;
4158 my_off_t filepos,key_file_length;
4159 SORT_KEY_BLOCKS *key_block;
4160 MI_SORT_INFO *sort_info= sort_param->sort_info;
4161 myf myf_rw=sort_info->param->myf_rw;
4162 MI_INFO *info=sort_info->info;
4163 MI_KEYDEF *keyinfo=sort_param->keyinfo;
4164 DBUG_ENTER("flush_pending_blocks");
4165
4166 filepos= HA_OFFSET_ERROR; /* if empty file */
4167 nod_flag=0;
4168 for (key_block=sort_info->key_block ; key_block->inited ; key_block++)
4169 {
4170 key_block->inited=0;
4171 length=mi_getint(key_block->buff);
4172 if (nod_flag)
4173 _mi_kpointer(info,(uchar*) key_block->end_pos,filepos);
4174 key_file_length=info->state->key_file_length;
4175 bzero((uchar*) key_block->buff+length, keyinfo->block_length-length);
4176 if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
4177 DBUG_RETURN(1);
4178
4179 /* If we read the page from the key cache, we have to write it back */
4180 if (key_file_length == info->state->key_file_length)
4181 {
4182 if (_mi_write_keypage(info, keyinfo, filepos,
4183 DFLT_INIT_HITS, (uchar*) key_block->buff))
4184 DBUG_RETURN(1);
4185 }
4186 else if (mysql_file_pwrite(info->s->kfile, (uchar*) key_block->buff,
4187 (uint) keyinfo->block_length, filepos, myf_rw))
4188 DBUG_RETURN(1);
4189 DBUG_DUMP("buff",(uchar*) key_block->buff,length);
4190 nod_flag=1;
4191 }
4192 info->s->state.key_root[sort_param->key]=filepos; /* Last is root for tree */
4193 DBUG_RETURN(0);
4194} /* flush_pending_blocks */
4195
4196 /* alloc space and pointers for key_blocks */
4197
4198static SORT_KEY_BLOCKS *alloc_key_blocks(HA_CHECK *param, uint blocks,
4199 uint buffer_length)
4200{
4201 reg1 uint i;
4202 SORT_KEY_BLOCKS *block;
4203 DBUG_ENTER("alloc_key_blocks");
4204
4205 if (!(block=(SORT_KEY_BLOCKS*) my_malloc((sizeof(SORT_KEY_BLOCKS)+
4206 buffer_length+IO_SIZE)*blocks,
4207 MYF(0))))
4208 {
4209 mi_check_print_error(param,"Not enough memory for sort-key-blocks");
4210 return(0);
4211 }
4212 for (i=0 ; i < blocks ; i++)
4213 {
4214 block[i].inited=0;
4215 block[i].buff=(uchar*) (block+blocks)+(buffer_length+IO_SIZE)*i;
4216 }
4217 DBUG_RETURN(block);
4218} /* alloc_key_blocks */
4219
4220
4221 /* Check if file is almost full */
4222
4223int test_if_almost_full(MI_INFO *info)
4224{
4225 if (info->s->options & HA_OPTION_COMPRESS_RECORD)
4226 return 0;
4227 return mysql_file_seek(info->s->kfile, 0L, MY_SEEK_END,
4228 MYF(MY_THREADSAFE)) / 10 * 9 >
4229 (my_off_t) info->s->base.max_key_file_length ||
4230 mysql_file_seek(info->dfile, 0L, MY_SEEK_END,
4231 MYF(0)) / 10 * 9 >
4232 (my_off_t) info->s->base.max_data_file_length;
4233}
4234
4235 /* Recreate table with bigger more alloced record-data */
4236
4237int recreate_table(HA_CHECK *param, MI_INFO **org_info, char *filename)
4238{
4239 int error;
4240 MI_INFO info;
4241 MYISAM_SHARE share;
4242 MI_KEYDEF *keyinfo,*key,*key_end;
4243 HA_KEYSEG *keysegs,*keyseg;
4244 MI_COLUMNDEF *recdef,*rec,*end;
4245 MI_UNIQUEDEF *uniquedef,*u_ptr,*u_end;
4246 MI_STATUS_INFO status_info;
4247 uint unpack,key_parts;
4248 ha_rows max_records;
4249 ulonglong file_length,tmp_length;
4250 MI_CREATE_INFO create_info;
4251 DBUG_ENTER("recreate_table");
4252
4253 error=1; /* Default error */
4254 info= **org_info;
4255 status_info= (*org_info)->state[0];
4256 info.state= &status_info;
4257 share= *(*org_info)->s;
4258 unpack= (share.options & HA_OPTION_COMPRESS_RECORD) &&
4259 (param->testflag & T_UNPACK);
4260 if (!(keyinfo=(MI_KEYDEF*) my_alloca(sizeof(MI_KEYDEF)*share.base.keys)))
4261 DBUG_RETURN(0);
4262 memcpy((uchar*) keyinfo,(uchar*) share.keyinfo,
4263 (size_t) (sizeof(MI_KEYDEF)*share.base.keys));
4264
4265 key_parts= share.base.all_key_parts;
4266 if (!(keysegs=(HA_KEYSEG*) my_alloca(sizeof(HA_KEYSEG)*
4267 (key_parts+share.base.keys))))
4268 {
4269 my_afree((uchar*) keyinfo);
4270 DBUG_RETURN(1);
4271 }
4272 if (!(recdef=(MI_COLUMNDEF*)
4273 my_alloca(sizeof(MI_COLUMNDEF)*(share.base.fields+1))))
4274 {
4275 my_afree((uchar*) keyinfo);
4276 my_afree((uchar*) keysegs);
4277 DBUG_RETURN(1);
4278 }
4279 if (!(uniquedef=(MI_UNIQUEDEF*)
4280 my_alloca(sizeof(MI_UNIQUEDEF)*(share.state.header.uniques+1))))
4281 {
4282 my_afree((uchar*) recdef);
4283 my_afree((uchar*) keyinfo);
4284 my_afree((uchar*) keysegs);
4285 DBUG_RETURN(1);
4286 }
4287
4288 /* Copy the column definitions */
4289 memcpy((uchar*) recdef,(uchar*) share.rec,
4290 (size_t) (sizeof(MI_COLUMNDEF)*(share.base.fields+1)));
4291 for (rec=recdef,end=recdef+share.base.fields; rec != end ; rec++)
4292 {
4293 if (unpack && !(share.options & HA_OPTION_PACK_RECORD) &&
4294 rec->type != FIELD_BLOB &&
4295 rec->type != FIELD_VARCHAR &&
4296 rec->type != FIELD_CHECK)
4297 rec->type=(int) FIELD_NORMAL;
4298 }
4299
4300 /* Change the new key to point at the saved key segments */
4301 memcpy((uchar*) keysegs,(uchar*) share.keyparts,
4302 (size_t) (sizeof(HA_KEYSEG)*(key_parts+share.base.keys+
4303 share.state.header.uniques)));
4304 keyseg=keysegs;
4305 for (key=keyinfo,key_end=keyinfo+share.base.keys; key != key_end ; key++)
4306 {
4307 key->seg=keyseg;
4308 for (; keyseg->type ; keyseg++)
4309 {
4310 if (param->language)
4311 keyseg->language=param->language; /* change language */
4312 }
4313 keyseg++; /* Skip end pointer */
4314 }
4315
4316 /* Copy the unique definitions and change them to point at the new key
4317 segments*/
4318 memcpy((uchar*) uniquedef,(uchar*) share.uniqueinfo,
4319 (size_t) (sizeof(MI_UNIQUEDEF)*(share.state.header.uniques)));
4320 for (u_ptr=uniquedef,u_end=uniquedef+share.state.header.uniques;
4321 u_ptr != u_end ; u_ptr++)
4322 {
4323 u_ptr->seg=keyseg;
4324 keyseg+=u_ptr->keysegs+1;
4325 }
4326 unpack= (share.options & HA_OPTION_COMPRESS_RECORD) &&
4327 (param->testflag & T_UNPACK);
4328 share.options&= ~HA_OPTION_TEMP_COMPRESS_RECORD;
4329
4330 file_length=(ulonglong) mysql_file_seek(info.dfile, 0L, MY_SEEK_END, MYF(0));
4331 tmp_length= file_length+file_length/10;
4332 set_if_bigger(file_length,param->max_data_file_length);
4333 set_if_bigger(file_length,tmp_length);
4334 set_if_bigger(file_length,(ulonglong) share.base.max_data_file_length);
4335
4336 if (share.options & HA_OPTION_COMPRESS_RECORD)
4337 share.base.records= max_records= info.state->records;
4338 else if (!(share.options & HA_OPTION_PACK_RECORD))
4339 max_records= (ha_rows) (file_length / share.base.pack_reclength);
4340 else
4341 max_records= 0;
4342
4343 (void) mi_close(*org_info);
4344 bzero((char*) &create_info,sizeof(create_info));
4345 create_info.max_rows= max_records;
4346 create_info.reloc_rows=share.base.reloc;
4347 create_info.old_options=(share.options |
4348 (unpack ? HA_OPTION_TEMP_COMPRESS_RECORD : 0));
4349
4350 create_info.data_file_length=file_length;
4351 create_info.auto_increment=share.state.auto_increment;
4352 create_info.language = (param->language ? param->language :
4353 share.state.header.language);
4354 create_info.key_file_length= status_info.key_file_length;
4355 /*
4356 Allow for creating an auto_increment key. This has an effect only if
4357 an auto_increment key exists in the original table.
4358 */
4359 create_info.with_auto_increment= TRUE;
4360 /* We don't have to handle symlinks here because we are using
4361 HA_DONT_TOUCH_DATA */
4362 if (mi_create(filename,
4363 share.base.keys - share.state.header.uniques,
4364 keyinfo, share.base.fields, recdef,
4365 share.state.header.uniques, uniquedef,
4366 &create_info,
4367 HA_DONT_TOUCH_DATA))
4368 {
4369 mi_check_print_error(param,"Got error %d when trying to recreate indexfile",my_errno);
4370 goto end;
4371 }
4372 *org_info=mi_open(filename,O_RDWR,
4373 (param->testflag & T_WAIT_FOREVER) ? HA_OPEN_WAIT_IF_LOCKED :
4374 (param->testflag & T_DESCRIPT) ? HA_OPEN_IGNORE_IF_LOCKED :
4375 HA_OPEN_ABORT_IF_LOCKED);
4376 if (!*org_info)
4377 {
4378 mi_check_print_error(param,"Got error %d when trying to open re-created indexfile",
4379 my_errno);
4380 goto end;
4381 }
4382 /* We are modifing */
4383 (*org_info)->s->options&= ~HA_OPTION_READ_ONLY_DATA;
4384 (void) _mi_readinfo(*org_info,F_WRLCK,0);
4385 (*org_info)->state->records=info.state->records;
4386 if (share.state.create_time)
4387 (*org_info)->s->state.create_time=share.state.create_time;
4388 (*org_info)->s->state.unique=(*org_info)->this_unique=
4389 share.state.unique;
4390 (*org_info)->state->checksum=info.state->checksum;
4391 (*org_info)->state->del=info.state->del;
4392 (*org_info)->s->state.dellink=share.state.dellink;
4393 (*org_info)->state->empty=info.state->empty;
4394 (*org_info)->state->data_file_length=info.state->data_file_length;
4395 if (update_state_info(param,*org_info,UPDATE_TIME | UPDATE_STAT |
4396 UPDATE_OPEN_COUNT))
4397 goto end;
4398 error=0;
4399end:
4400 my_afree((uchar*) uniquedef);
4401 my_afree((uchar*) keyinfo);
4402 my_afree((uchar*) recdef);
4403 my_afree((uchar*) keysegs);
4404 DBUG_RETURN(error);
4405}
4406
4407
4408 /* write suffix to data file if neaded */
4409
4410int write_data_suffix(MI_SORT_INFO *sort_info, my_bool fix_datafile)
4411{
4412 MI_INFO *info=sort_info->info;
4413
4414 if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile)
4415 {
4416 uchar buff[MEMMAP_EXTRA_MARGIN];
4417 bzero(buff,sizeof(buff));
4418 if (my_b_write(&info->rec_cache,buff,sizeof(buff)))
4419 {
4420 mi_check_print_error(sort_info->param,
4421 "%d when writing to datafile",my_errno);
4422 return 1;
4423 }
4424 sort_info->param->read_cache.end_of_file+=sizeof(buff);
4425 }
4426 return 0;
4427}
4428
4429 /* Update state and myisamchk_time of indexfile */
4430
4431int update_state_info(HA_CHECK *param, MI_INFO *info,uint update)
4432{
4433 MYISAM_SHARE *share=info->s;
4434
4435 if (update & UPDATE_OPEN_COUNT)
4436 {
4437 share->state.open_count=0;
4438 share->global_changed=0;
4439 }
4440 if (update & UPDATE_STAT)
4441 {
4442 uint i, key_parts= mi_uint2korr(share->state.header.key_parts);
4443 share->state.rec_per_key_rows=info->state->records;
4444 share->state.changed&= ~STATE_NOT_ANALYZED;
4445 if (info->state->records)
4446 {
4447 for (i=0; i<key_parts; i++)
4448 {
4449 if (!(share->state.rec_per_key_part[i]=param->rec_per_key_part[i]))
4450 share->state.changed|= STATE_NOT_ANALYZED;
4451 }
4452 }
4453 }
4454 if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC))
4455 {
4456 if (update & UPDATE_TIME)
4457 {
4458 share->state.check_time= time((time_t*) 0);
4459 if (!share->state.create_time)
4460 share->state.create_time=share->state.check_time;
4461 }
4462 /*
4463 When tables are locked we haven't synched the share state and the
4464 real state for a while so we better do it here before synching
4465 the share state to disk. Only when table is write locked is it
4466 necessary to perform this synch.
4467 */
4468 if (info->lock_type == F_WRLCK)
4469 share->state.state= *info->state;
4470 if (mi_state_info_write(share->kfile,&share->state,1+2))
4471 goto err;
4472 share->changed=0;
4473 }
4474 { /* Force update of status */
4475 int error;
4476 uint r_locks=share->r_locks,w_locks=share->w_locks;
4477 share->r_locks= share->w_locks= share->tot_locks= 0;
4478 error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK);
4479 share->r_locks=r_locks;
4480 share->w_locks=w_locks;
4481 share->tot_locks=r_locks+w_locks;
4482 if (!error)
4483 return 0;
4484 }
4485err:
4486 mi_check_print_error(param,"%d when updating keyfile",my_errno);
4487 return 1;
4488}
4489
4490 /*
4491 Update auto increment value for a table
4492 When setting the 'repair_only' flag we only want to change the
4493 old auto_increment value if its wrong (smaller than some given key).
4494 The reason is that we shouldn't change the auto_increment value
4495 for a table without good reason when only doing a repair; If the
4496 user have inserted and deleted rows, the auto_increment value
4497 may be bigger than the biggest current row and this is ok.
4498
4499 If repair_only is not set, we will update the flag to the value in
4500 param->auto_increment is bigger than the biggest key.
4501 */
4502
4503void update_auto_increment_key(HA_CHECK *param, MI_INFO *info,
4504 my_bool repair_only)
4505{
4506 uchar *record= 0;
4507 DBUG_ENTER("update_auto_increment_key");
4508
4509 if (!info->s->base.auto_key ||
4510 ! mi_is_key_active(info->s->state.key_map, info->s->base.auto_key - 1))
4511 {
4512 if (!(param->testflag & T_VERY_SILENT))
4513 mi_check_print_info(param,
4514 "Table: %s doesn't have an auto increment key\n",
4515 param->isam_file_name);
4516 DBUG_VOID_RETURN;
4517 }
4518 if (!(param->testflag & T_SILENT) &&
4519 !(param->testflag & T_REP_ANY))
4520 printf("Updating MyISAM file: %s\n", param->isam_file_name);
4521 /*
4522 We have to use an allocated buffer instead of info->rec_buff as
4523 _mi_put_key_in_record() may use info->rec_buff
4524 */
4525 if (!mi_alloc_rec_buff(info, -1, &record))
4526 {
4527 mi_check_print_error(param,"Not enough memory for extra record");
4528 DBUG_VOID_RETURN;
4529 }
4530
4531 mi_extra(info,HA_EXTRA_KEYREAD,0);
4532 if (mi_rlast(info, record, info->s->base.auto_key-1))
4533 {
4534 if (my_errno != HA_ERR_END_OF_FILE)
4535 {
4536 mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
4537 my_free(mi_get_rec_buff_ptr(info, record));
4538 mi_check_print_error(param,"%d when reading last record",my_errno);
4539 DBUG_VOID_RETURN;
4540 }
4541 if (!repair_only)
4542 info->s->state.auto_increment=param->auto_increment_value;
4543 }
4544 else
4545 {
4546 ulonglong auto_increment= retrieve_auto_increment(info, record);
4547 set_if_bigger(info->s->state.auto_increment,auto_increment);
4548 if (!repair_only)
4549 set_if_bigger(info->s->state.auto_increment, param->auto_increment_value);
4550 }
4551 mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
4552 my_free(mi_get_rec_buff_ptr(info, record));
4553 update_state_info(param, info, UPDATE_AUTO_INC);
4554 DBUG_VOID_RETURN;
4555}
4556
4557
4558/*
4559 Update statistics for each part of an index
4560
4561 SYNOPSIS
4562 update_key_parts()
4563 keyinfo IN Index information (only key->keysegs used)
4564 rec_per_key_part OUT Store statistics here
4565 unique IN Array of (#distinct tuples)
4566 notnull_tuples IN Array of (#tuples), or NULL
4567 records Number of records in the table
4568
4569 DESCRIPTION
4570 This function is called produce index statistics values from unique and
4571 notnull_tuples arrays after these arrays were produced with sequential
4572 index scan (the scan is done in two places: chk_index() and
4573 sort_key_write()).
4574
4575 This function handles all 3 index statistics collection methods.
4576
4577 Unique is an array:
4578 unique[0]= (#different values of {keypart1}) - 1
4579 unique[1]= (#different values of {keypart1,keypart2} tuple)-unique[0]-1
4580 ...
4581
4582 For MI_STATS_METHOD_IGNORE_NULLS method, notnull_tuples is an array too:
4583 notnull_tuples[0]= (#of {keypart1} tuples such that keypart1 is not NULL)
4584 notnull_tuples[1]= (#of {keypart1,keypart2} tuples such that all
4585 keypart{i} are not NULL)
4586 ...
4587 For all other statistics collection methods notnull_tuples==NULL.
4588
4589 Output is an array:
4590 rec_per_key_part[k] =
4591 = E(#records in the table such that keypart_1=c_1 AND ... AND
4592 keypart_k=c_k for arbitrary constants c_1 ... c_k)
4593
4594 = {assuming that values have uniform distribution and index contains all
4595 tuples from the domain (or that {c_1, ..., c_k} tuple is choosen from
4596 index tuples}
4597
4598 = #tuples-in-the-index / #distinct-tuples-in-the-index.
4599
4600 The #tuples-in-the-index and #distinct-tuples-in-the-index have different
4601 meaning depending on which statistics collection method is used:
4602
4603 MI_STATS_METHOD_* how are nulls compared? which tuples are counted?
4604 NULLS_EQUAL NULL == NULL all tuples in table
4605 NULLS_NOT_EQUAL NULL != NULL all tuples in table
4606 IGNORE_NULLS n/a tuples that don't have NULLs
4607*/
4608
4609void update_key_parts(MI_KEYDEF *keyinfo, ulong *rec_per_key_part,
4610 ulonglong *unique, ulonglong *notnull,
4611 ulonglong records)
4612{
4613 ulonglong count=0,tmp, unique_tuples;
4614 ulonglong tuples= records;
4615 uint parts;
4616 for (parts=0 ; parts < keyinfo->keysegs ; parts++)
4617 {
4618 count+=unique[parts];
4619 unique_tuples= count + 1;
4620 if (notnull)
4621 {
4622 tuples= notnull[parts];
4623 /*
4624 #(unique_tuples not counting tuples with NULLs) =
4625 #(unique_tuples counting tuples with NULLs as different) -
4626 #(tuples with NULLs)
4627 */
4628 unique_tuples -= (records - notnull[parts]);
4629 }
4630
4631 if (unique_tuples == 0)
4632 tmp= 1;
4633 else if (count == 0)
4634 tmp= tuples; /* 1 unique tuple */
4635 else
4636 tmp= (tuples + unique_tuples/2) / unique_tuples;
4637
4638 /*
4639 for some weird keys (e.g. FULLTEXT) tmp can be <1 here.
4640 let's ensure it is not
4641 */
4642 set_if_bigger(tmp,1);
4643 /* Keys are stored as 32 byte int's; Ensure we don't get an overflow */
4644 if (tmp >= (ulonglong) ~(uint32) 0)
4645 tmp=(ulonglong) ~(uint32) 0;
4646
4647 *rec_per_key_part=(ulong) tmp;
4648 rec_per_key_part++;
4649 }
4650}
4651
4652
4653static ha_checksum mi_byte_checksum(const uchar *buf, uint length)
4654{
4655 ha_checksum crc;
4656 const uchar *end=buf+length;
4657 for (crc=0; buf != end; buf++)
4658 crc=((crc << 1) + *((uchar*) buf)) +
4659 MY_TEST(crc & (((ha_checksum) 1) << (8 * sizeof(ha_checksum) - 1)));
4660 return crc;
4661}
4662
4663static my_bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows)
4664{
4665 uint key_maxlength=key->maxlength;
4666 if (key->flag & HA_FULLTEXT)
4667 {
4668 uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
4669 key->seg->charset->mbmaxlen;
4670 key_maxlength+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
4671 }
4672 return (key->flag & HA_SPATIAL) ||
4673 (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY | HA_FULLTEXT) &&
4674 ((ulonglong) rows * key_maxlength > myisam_max_temp_length));
4675}
4676
4677/*
4678 Deactivate all indexes that can be recreated fast.
4679 These include packed keys on which sorting will use more temporary
4680 space than the max allowed file length or for which the unpacked keys
4681 will take much more space than packed keys.
4682 Note that 'rows' may be zero for the case when we don't know how many
4683 rows we will put into the file.
4684 */
4685
4686void mi_disable_indexes_for_rebuild(MI_INFO *info, ha_rows rows,
4687 my_bool all_keys)
4688{
4689 MYISAM_SHARE *share=info->s;
4690 MI_KEYDEF *key=share->keyinfo;
4691 uint i;
4692
4693 DBUG_ASSERT(info->state->records == 0 &&
4694 (!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES));
4695 for (i=0 ; i < share->base.keys ; i++,key++)
4696 {
4697 if (!(key->flag & (HA_SPATIAL | HA_AUTO_KEY)) &&
4698 ! mi_too_big_key_for_sort(key,rows) && info->s->base.auto_key != i+1 &&
4699 (all_keys || !(key->flag & HA_NOSAME)))
4700 {
4701 mi_clear_key_active(share->state.key_map, i);
4702 info->update|= HA_STATE_CHANGED;
4703 info->create_unique_index_by_sort= all_keys;
4704 }
4705 }
4706}
4707
4708
4709/*
4710 Return TRUE if we can use repair by sorting
4711 One can set the force argument to force to use sorting
4712 even if the temporary file would be quite big!
4713*/
4714
4715my_bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows,
4716 ulonglong key_map, my_bool force)
4717{
4718 MYISAM_SHARE *share=info->s;
4719 MI_KEYDEF *key=share->keyinfo;
4720 uint i;
4721
4722 /*
4723 mi_repair_by_sort only works if we have at least one key. If we don't
4724 have any keys, we should use the normal repair.
4725 */
4726 if (! mi_is_any_key_active(key_map))
4727 return FALSE; /* Can't use sort */
4728 for (i=0 ; i < share->base.keys ; i++,key++)
4729 {
4730 if (!force && mi_too_big_key_for_sort(key,rows))
4731 return FALSE;
4732 }
4733 return TRUE;
4734}
4735
4736
4737static void
4738set_data_file_type(MI_SORT_INFO *sort_info, MYISAM_SHARE *share)
4739{
4740 if ((sort_info->new_data_file_type=share->data_file_type) ==
4741 COMPRESSED_RECORD && sort_info->param->testflag & T_UNPACK)
4742 {
4743 MYISAM_SHARE tmp;
4744
4745 if (share->options & HA_OPTION_PACK_RECORD)
4746 sort_info->new_data_file_type = DYNAMIC_RECORD;
4747 else
4748 sort_info->new_data_file_type = STATIC_RECORD;
4749
4750 /* Set delete_function for sort_delete_record() */
4751 memcpy((char*) &tmp, share, sizeof(*share));
4752 tmp.options= ~HA_OPTION_COMPRESS_RECORD;
4753 mi_setup_functions(&tmp);
4754 share->delete_record=tmp.delete_record;
4755 }
4756}
4757
4758
4759int mi_make_backup_of_index(MI_INFO *info, time_t backup_time, myf flags)
4760{
4761 char backup_name[FN_REFLEN + MY_BACKUP_NAME_EXTRA_LENGTH];
4762 my_create_backup_name(backup_name, info->s->index_file_name, backup_time);
4763 return my_copy(info->s->index_file_name, backup_name, flags);
4764}
4765
4766
4767static int replace_data_file(HA_CHECK *param, MI_INFO *info, File new_file)
4768{
4769 MYISAM_SHARE *share=info->s;
4770
4771 mysql_file_close(new_file,MYF(0));
4772 info->dfile= -1;
4773 if (param->testflag & T_BACKUP_DATA)
4774 {
4775 char buff[MY_BACKUP_NAME_EXTRA_LENGTH+1];
4776 my_create_backup_name(buff, "", param->backup_time);
4777 my_printf_error(ER_GET_ERRMSG,
4778 "Making backup of data file %s with extension '%s'",
4779 MYF(ME_JUST_INFO | ME_NOREFRESH), share->data_file_name,
4780 buff);
4781 }
4782
4783 /*
4784 On Windows, the old data file cannot be deleted if it is either
4785 open, or memory mapped. Closing the file won't remove the memory
4786 map implicilty on Windows. We closed the data file, but we keep
4787 the MyISAM table open. A memory map will be closed on the final
4788 mi_close() only. So we need to unmap explicitly here. After
4789 renaming the new file under the hook, we couldn't use the map of
4790 the old file any more anyway.
4791 */
4792 if (info->s->file_map)
4793 {
4794 (void) my_munmap((char*) info->s->file_map, info->s->mmaped_length);
4795 info->s->file_map= NULL;
4796 }
4797
4798 if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
4799 DATA_TMP_EXT, param->backup_time,
4800 (param->testflag & T_BACKUP_DATA ?
4801 MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
4802 mi_open_datafile(info, share))
4803 return 1;
4804 return 0;
4805}
4806