1/* -*- c-basic-offset: 2 -*- */
2/*
3 Copyright(C) 2011-2017 Brazil
4
5 This library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License version 2.1 as published by the Free Software Foundation.
8
9 This library is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 Lesser General Public License for more details.
13
14 You should have received a copy of the GNU Lesser General Public
15 License along with this library; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17*/
18#include "grn.h"
19#include <sys/types.h>
20#include <sys/stat.h>
21#include <cstring>
22#include <new>
23#include "grn_str.h"
24#include "grn_io.h"
25#include "grn_dat.h"
26#include "grn_util.h"
27#include "grn_normalizer.h"
28
29#include "dat/trie.hpp"
30#include "dat/cursor-factory.hpp"
31
32namespace {
33
34const uint32_t FILE_ID_LENGTH = 3;
35
36class CriticalSection {
37 public:
38 CriticalSection() : lock_(NULL) {}
39 explicit CriticalSection(grn_critical_section *lock) : lock_(lock) {
40 CRITICAL_SECTION_ENTER(*lock_);
41 }
42 ~CriticalSection() {
43 leave();
44 }
45
46 void enter(grn_critical_section *lock) {
47 leave();
48 lock_ = lock;
49 }
50 void leave() {
51 if (lock_ != NULL) {
52 CRITICAL_SECTION_LEAVE(*lock_);
53 lock_ = NULL;
54 }
55 }
56
57 private:
58 grn_critical_section *lock_;
59
60 // Disallows copy and assignment.
61 CriticalSection(const CriticalSection &);
62 CriticalSection &operator=(const CriticalSection &);
63};
64
65/*
66 grn_dat_remove_file() removes a file specified by `path' and then returns
67 true on success, false on failure. Note that grn_dat_remove_file() does not
68 change `ctx->rc'.
69 */
70bool
71grn_dat_remove_file(grn_ctx *ctx, const char *path)
72{
73 struct stat stat;
74
75 if (::stat(path, &stat) == -1) {
76 return false;
77 }
78
79 if (grn_unlink(path) == -1) {
80 const char *system_message = grn_strerror(errno);
81 GRN_LOG(ctx, GRN_LOG_WARNING,
82 "[dat][remove-file] failed to remove path: %s: <%s>",
83 system_message, path);
84 return false;
85 }
86
87 GRN_LOG(ctx, GRN_LOG_INFO,
88 "[dat][remove-file] removed: <%s>", path);
89 return true;
90}
91
92grn_rc
93grn_dat_translate_error_code(grn::dat::ErrorCode error_code) {
94 switch (error_code) {
95 case grn::dat::PARAM_ERROR: {
96 return GRN_INVALID_ARGUMENT;
97 }
98 case grn::dat::IO_ERROR: {
99 return GRN_INPUT_OUTPUT_ERROR;
100 }
101 case grn::dat::FORMAT_ERROR: {
102 return GRN_INVALID_FORMAT;
103 }
104 case grn::dat::MEMORY_ERROR: {
105 return GRN_NO_MEMORY_AVAILABLE;
106 }
107 case grn::dat::SIZE_ERROR:
108 case grn::dat::UNEXPECTED_ERROR: {
109 return GRN_UNKNOWN_ERROR;
110 }
111 case grn::dat::STATUS_ERROR: {
112 return GRN_FILE_CORRUPT;
113 }
114 default: {
115 return GRN_UNKNOWN_ERROR;
116 }
117 }
118}
119
120void
121grn_dat_init(grn_ctx *, grn_dat *dat)
122{
123 GRN_DB_OBJ_SET_TYPE(dat, GRN_TABLE_DAT_KEY);
124 dat->io = NULL;
125 dat->header = NULL;
126 dat->file_id = 0;
127 dat->encoding = GRN_ENC_DEFAULT;
128 dat->trie = NULL;
129 dat->old_trie = NULL;
130 dat->tokenizer = NULL;
131 dat->normalizer = NULL;
132 GRN_PTR_INIT(&(dat->token_filters), GRN_OBJ_VECTOR, GRN_ID_NIL);
133 CRITICAL_SECTION_INIT(dat->lock);
134 dat->is_dirty = GRN_FALSE;
135}
136
137void
138grn_dat_fin(grn_ctx *ctx, grn_dat *dat)
139{
140 CRITICAL_SECTION_FIN(dat->lock);
141 delete static_cast<grn::dat::Trie *>(dat->old_trie);
142 delete static_cast<grn::dat::Trie *>(dat->trie);
143 dat->old_trie = NULL;
144 dat->trie = NULL;
145 if (dat->io) {
146 if (dat->is_dirty) {
147 uint32_t n_dirty_opens;
148 GRN_ATOMIC_ADD_EX(&(dat->header->n_dirty_opens), -1, n_dirty_opens);
149 }
150 grn_io_close(ctx, dat->io);
151 dat->io = NULL;
152 }
153 GRN_OBJ_FIN(ctx, &(dat->token_filters));
154}
155
156/*
157 grn_dat_generate_trie_path() generates the path from `base_path' and
158 `file_id'. The generated path is stored in `trie_path'.
159 */
160void
161grn_dat_generate_trie_path(const char *base_path, char *trie_path, uint32_t file_id)
162{
163 if (!base_path || !base_path[0]) {
164 trie_path[0] = '\0';
165 return;
166 }
167 const size_t len = std::strlen(base_path);
168 grn_memcpy(trie_path, base_path, len);
169 trie_path[len] = '.';
170 grn_itoh(file_id % (1U << (4 * FILE_ID_LENGTH)),
171 trie_path + len + 1, FILE_ID_LENGTH);
172 trie_path[len + 1 + FILE_ID_LENGTH] = '\0';
173}
174
175bool
176grn_dat_open_trie_if_needed(grn_ctx *ctx, grn_dat *dat)
177{
178 if (!dat) {
179 ERR(GRN_INVALID_ARGUMENT, "dat is null");
180 return false;
181 }
182
183 const uint32_t file_id = dat->header->file_id;
184 if (!file_id || (dat->trie && (file_id <= dat->file_id))) {
185 /*
186 There is no need to open file when no trie file is available or the
187 current trie file is the latest one.
188 */
189 return true;
190 }
191
192 CriticalSection critical_section(&dat->lock);
193
194 if (dat->trie && (file_id <= dat->file_id)) {
195 /*
196 There is no need to open file if the latest file has been opened by
197 another thread.
198 */
199 return true;
200 }
201
202 char trie_path[PATH_MAX];
203 grn_dat_generate_trie_path(grn_io_path(dat->io), trie_path, file_id);
204 grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
205 grn::dat::Trie * const old_trie = static_cast<grn::dat::Trie *>(dat->old_trie);
206 grn::dat::Trie * const new_trie = new (std::nothrow) grn::dat::Trie;
207 if (!new_trie) {
208 MERR("new grn::dat::Trie failed");
209 return false;
210 }
211
212 if (trie_path[0] == '\0') {
213 try {
214 new_trie->create(trie_path);
215 } catch (const grn::dat::Exception &ex) {
216 ERR(grn_dat_translate_error_code(ex.code()),
217 "grn::dat::Trie::create failed: %s",
218 ex.what());
219 delete new_trie;
220 return false;
221 }
222 } else {
223 try {
224 new_trie->open(trie_path);
225 } catch (const grn::dat::Exception &ex) {
226 ERR(grn_dat_translate_error_code(ex.code()),
227 "grn::dat::Trie::open failed: %s",
228 ex.what());
229 delete new_trie;
230 return false;
231 }
232 }
233
234 dat->old_trie = trie;
235 dat->trie = new_trie;
236 dat->file_id = file_id;
237
238 critical_section.leave();
239
240 delete old_trie;
241 if (file_id >= 3) {
242 grn_dat_generate_trie_path(grn_io_path(dat->io), trie_path, file_id - 2);
243 grn_dat_remove_file(ctx, trie_path);
244 }
245 return true;
246}
247
248bool grn_dat_rebuild_trie(grn_ctx *ctx, grn_dat *dat) {
249 const grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
250 grn::dat::Trie * const new_trie = new (std::nothrow) grn::dat::Trie;
251 if (!new_trie) {
252 MERR("new grn::dat::Trie failed");
253 return false;
254 }
255
256 const uint32_t file_id = dat->header->file_id;
257 char trie_path[PATH_MAX];
258 grn_dat_generate_trie_path(grn_io_path(dat->io), trie_path, file_id + 1);
259
260 for (uint64_t file_size = trie->file_size() * 2;; file_size *= 2) {
261 try {
262 new_trie->create(*trie, trie_path, file_size);
263 } catch (const grn::dat::SizeError &) {
264 continue;
265 } catch (const grn::dat::Exception &ex) {
266 ERR(grn_dat_translate_error_code(ex.code()),
267 "grn::dat::Trie::open failed: %s",
268 ex.what());
269 delete new_trie;
270 return false;
271 }
272 break;
273 }
274
275 grn::dat::Trie * const old_trie = static_cast<grn::dat::Trie *>(dat->old_trie);
276 dat->old_trie = dat->trie;
277 dat->trie = new_trie;
278 dat->header->file_id = dat->file_id = file_id + 1;
279
280 delete old_trie;
281 if (file_id >= 2) {
282 char trie_path[PATH_MAX];
283 grn_dat_generate_trie_path(grn_io_path(dat->io), trie_path, file_id - 1);
284 grn_dat_remove_file(ctx, trie_path);
285 }
286 return true;
287}
288
289void grn_dat_cursor_init(grn_ctx *, grn_dat_cursor *cursor) {
290 GRN_DB_OBJ_SET_TYPE(cursor, GRN_CURSOR_TABLE_DAT_KEY);
291 cursor->dat = NULL;
292 cursor->cursor = NULL;
293 cursor->key = &grn::dat::Key::invalid_key();
294 cursor->curr_rec = GRN_ID_NIL;
295}
296
297void grn_dat_cursor_fin(grn_ctx *, grn_dat_cursor *cursor) {
298 delete static_cast<grn::dat::Cursor *>(cursor->cursor);
299 cursor->dat = NULL;
300 cursor->cursor = NULL;
301 cursor->key = &grn::dat::Key::invalid_key();
302 cursor->curr_rec = GRN_ID_NIL;
303}
304
305} // namespace
306
307extern "C" {
308
309grn_dat *
310grn_dat_create(grn_ctx *ctx, const char *path, uint32_t,
311 uint32_t, uint32_t flags)
312{
313 if (path) {
314 if (path[0] == '\0') {
315 path = NULL;
316 } else if (std::strlen(path) >= (PATH_MAX - (FILE_ID_LENGTH + 1))) {
317 ERR(GRN_FILENAME_TOO_LONG, "too long path");
318 return NULL;
319 }
320 }
321
322 grn_dat * const dat = static_cast<grn_dat *>(GRN_CALLOC(sizeof(grn_dat)));
323 if (!dat) {
324 return NULL;
325 }
326 grn_dat_init(ctx, dat);
327
328 dat->io = grn_io_create(ctx, path, sizeof(struct grn_dat_header),
329 4096, 0, grn_io_auto, GRN_IO_EXPIRE_SEGMENT);
330 if (!dat->io) {
331 GRN_FREE(dat);
332 return NULL;
333 }
334 grn_io_set_type(dat->io, GRN_TABLE_DAT_KEY);
335
336 dat->header = static_cast<struct grn_dat_header *>(grn_io_header(dat->io));
337 if (!dat->header) {
338 grn_io_close(ctx, dat->io);
339 grn_dat_remove_file(ctx, path);
340 GRN_FREE(dat);
341 return NULL;
342 }
343 const grn_encoding encoding = (ctx->encoding != GRN_ENC_DEFAULT) ?
344 ctx->encoding : grn_gctx.encoding;
345 dat->header->flags = flags;
346 dat->header->encoding = encoding;
347 dat->header->tokenizer = GRN_ID_NIL;
348 dat->header->file_id = 0;
349 if (dat->header->flags & GRN_OBJ_KEY_NORMALIZE) {
350 dat->header->flags &= ~GRN_OBJ_KEY_NORMALIZE;
351 dat->normalizer = grn_ctx_get(ctx, GRN_NORMALIZER_AUTO_NAME, -1);
352 dat->header->normalizer = grn_obj_id(ctx, dat->normalizer);
353 } else {
354 dat->normalizer = NULL;
355 dat->header->normalizer = GRN_ID_NIL;
356 }
357 dat->encoding = encoding;
358 dat->tokenizer = NULL;
359 GRN_PTR_INIT(&(dat->token_filters), GRN_OBJ_VECTOR, GRN_ID_NIL);
360
361 dat->obj.header.flags = dat->header->flags;
362
363 return dat;
364}
365
366grn_dat *
367grn_dat_open(grn_ctx *ctx, const char *path)
368{
369 if (path && (std::strlen(path) >= (PATH_MAX - (FILE_ID_LENGTH + 1)))) {
370 ERR(GRN_FILENAME_TOO_LONG, "too long path");
371 return NULL;
372 }
373
374 grn_dat * const dat = static_cast<grn_dat *>(GRN_MALLOC(sizeof(grn_dat)));
375 if (!dat) {
376 return NULL;
377 }
378
379 grn_dat_init(ctx, dat);
380 dat->io = grn_io_open(ctx, path, grn_io_auto);
381 if (!dat->io) {
382 GRN_FREE(dat);
383 return NULL;
384 }
385
386 dat->header = (struct grn_dat_header *)grn_io_header(dat->io);
387 if (!dat->header) {
388 grn_io_close(ctx, dat->io);
389 GRN_FREE(dat);
390 return NULL;
391 }
392 dat->file_id = dat->header->file_id;
393 dat->encoding = dat->header->encoding;
394 dat->tokenizer = grn_ctx_at(ctx, dat->header->tokenizer);
395 if (dat->header->flags & GRN_OBJ_KEY_NORMALIZE) {
396 dat->header->flags &= ~GRN_OBJ_KEY_NORMALIZE;
397 dat->normalizer = grn_ctx_get(ctx, GRN_NORMALIZER_AUTO_NAME, -1);
398 dat->header->normalizer = grn_obj_id(ctx, dat->normalizer);
399 } else {
400 dat->normalizer = grn_ctx_at(ctx, dat->header->normalizer);
401 }
402 GRN_PTR_INIT(&(dat->token_filters), GRN_OBJ_VECTOR, GRN_ID_NIL);
403 dat->obj.header.flags = dat->header->flags;
404 return dat;
405}
406
407grn_rc
408grn_dat_close(grn_ctx *ctx, grn_dat *dat)
409{
410 if (dat) {
411 grn_dat_fin(ctx, dat);
412 GRN_FREE(dat);
413 }
414 return GRN_SUCCESS;
415}
416
417grn_rc
418grn_dat_remove(grn_ctx *ctx, const char *path)
419{
420 if (!path) {
421 ERR(GRN_INVALID_ARGUMENT, "path is null");
422 return GRN_INVALID_ARGUMENT;
423 }
424
425 grn_dat * const dat = grn_dat_open(ctx, path);
426 if (!dat) {
427 return ctx->rc;
428 }
429 const uint32_t file_id = dat->header->file_id;
430 grn_dat_close(ctx, dat);
431
432 /*
433 grn_dat_remove() tries to remove (file_id + 1)th trie file because
434 grn::dat::Trie::create() might leave an incomplete file on failure.
435 */
436 char trie_path[PATH_MAX];
437 grn_dat_generate_trie_path(path, trie_path, file_id + 1);
438 grn_dat_remove_file(ctx, trie_path);
439 for (uint32_t i = file_id; i > 0; --i) {
440 grn_dat_generate_trie_path(path, trie_path, i);
441 if (!grn_dat_remove_file(ctx, trie_path)) {
442 break;
443 }
444 }
445
446 /*
447 grn_io_remove() reports an error when it fails to remove `path'.
448 */
449 return grn_io_remove(ctx, path);
450}
451
452grn_id
453grn_dat_get(grn_ctx *ctx, grn_dat *dat, const void *key,
454 unsigned int key_size, void **)
455{
456 if (!grn_dat_open_trie_if_needed(ctx, dat)) {
457 return GRN_ID_NIL;
458 }
459 const grn::dat::Trie * const trie = static_cast<const grn::dat::Trie *>(dat->trie);
460 if (!trie) {
461 return GRN_ID_NIL;
462 }
463 grn::dat::UInt32 key_pos;
464 try {
465 if (trie->search(key, key_size, &key_pos)) {
466 return trie->get_key(key_pos).id();
467 }
468 } catch (const grn::dat::Exception &ex) {
469 ERR(grn_dat_translate_error_code(ex.code()),
470 "grn::dat::Trie::search failed: %s",
471 ex.what());
472 }
473 return GRN_ID_NIL;
474}
475
476grn_id
477grn_dat_add(grn_ctx *ctx, grn_dat *dat, const void *key,
478 unsigned int key_size, void **, int *added)
479{
480 if (!key_size) {
481 return GRN_ID_NIL;
482 } else if (!grn_dat_open_trie_if_needed(ctx, dat)) {
483 return GRN_ID_NIL;
484 }
485
486 if (!dat->trie) {
487 char trie_path[PATH_MAX];
488 grn_dat_generate_trie_path(grn_io_path(dat->io), trie_path, 1);
489 grn::dat::Trie * const new_trie = new (std::nothrow) grn::dat::Trie;
490 if (!new_trie) {
491 MERR("new grn::dat::Trie failed");
492 return GRN_ID_NIL;
493 }
494 try {
495 new_trie->create(trie_path);
496 } catch (const grn::dat::Exception &ex) {
497 ERR(grn_dat_translate_error_code(ex.code()),
498 "grn::dat::Trie::create failed: %s",
499 ex.what());
500 delete new_trie;
501 return GRN_ID_NIL;
502 }
503 dat->trie = new_trie;
504 dat->file_id = dat->header->file_id = 1;
505 }
506
507 grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
508 try {
509 grn::dat::UInt32 key_pos;
510 const bool res = trie->insert(key, key_size, &key_pos);
511 if (added) {
512 *added = res ? 1 : 0;
513 }
514 return trie->get_key(key_pos).id();
515 } catch (const grn::dat::SizeError &) {
516 if (!grn_dat_rebuild_trie(ctx, dat)) {
517 return GRN_ID_NIL;
518 }
519 grn::dat::Trie * const new_trie = static_cast<grn::dat::Trie *>(dat->trie);
520 grn::dat::UInt32 key_pos;
521 const bool res = new_trie->insert(key, key_size, &key_pos);
522 if (added) {
523 *added = res ? 1 : 0;
524 }
525 return new_trie->get_key(key_pos).id();
526 } catch (const grn::dat::Exception &ex) {
527 ERR(grn_dat_translate_error_code(ex.code()),
528 "grn::dat::Trie::insert failed: %s",
529 ex.what());
530 return GRN_ID_NIL;
531 }
532}
533
534int
535grn_dat_get_key(grn_ctx *ctx, grn_dat *dat, grn_id id, void *keybuf, int bufsize)
536{
537 if (!grn_dat_open_trie_if_needed(ctx, dat)) {
538 return 0;
539 }
540 const grn::dat::Trie * const trie = static_cast<const grn::dat::Trie *>(dat->trie);
541 if (!trie) {
542 return 0;
543 }
544 const grn::dat::Key &key = trie->ith_key(id);
545 if (!key.is_valid()) {
546 return 0;
547 }
548 if (keybuf && (bufsize >= (int)key.length())) {
549 grn_memcpy(keybuf, key.ptr(), key.length());
550 }
551 return (int)key.length();
552}
553
554int
555grn_dat_get_key2(grn_ctx *ctx, grn_dat *dat, grn_id id, grn_obj *bulk)
556{
557 if (!grn_dat_open_trie_if_needed(ctx, dat)) {
558 return 0;
559 }
560 const grn::dat::Trie * const trie = static_cast<const grn::dat::Trie *>(dat->trie);
561 if (!trie) {
562 return 0;
563 }
564 const grn::dat::Key &key = trie->ith_key(id);
565 if (!key.is_valid()) {
566 return 0;
567 }
568 if (bulk->header.impl_flags & GRN_OBJ_REFER) {
569 bulk->u.b.head = static_cast<char *>(const_cast<void *>(key.ptr()));
570 bulk->u.b.curr = bulk->u.b.head + key.length();
571 } else {
572 grn_bulk_write(ctx, bulk, static_cast<const char *>(key.ptr()), key.length());
573 }
574 return (int)key.length();
575}
576
577grn_rc
578grn_dat_delete_by_id(grn_ctx *ctx, grn_dat *dat, grn_id id,
579 grn_table_delete_optarg *optarg)
580{
581 if (!grn_dat_open_trie_if_needed(ctx, dat)) {
582 return ctx->rc;
583 } else if (!dat->trie || (id == GRN_ID_NIL)) {
584 return GRN_INVALID_ARGUMENT;
585 }
586
587 if (optarg && optarg->func) {
588 const grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
589 if (!trie->ith_entry(id).is_valid()) {
590 return GRN_INVALID_ARGUMENT;
591 } else if (!optarg->func(ctx, reinterpret_cast<grn_obj *>(dat), id, optarg->func_arg)) {
592 return GRN_SUCCESS;
593 }
594 }
595
596 try {
597 grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
598 if (!trie->remove(id)) {
599 return GRN_INVALID_ARGUMENT;
600 }
601 } catch (const grn::dat::Exception &ex) {
602 ERR(grn_dat_translate_error_code(ex.code()),
603 "grn::dat::Trie::remove failed: %s",
604 ex.what());
605 return ctx->rc;
606 }
607 return GRN_SUCCESS;
608}
609
610grn_rc
611grn_dat_delete(grn_ctx *ctx, grn_dat *dat, const void *key, unsigned int key_size,
612 grn_table_delete_optarg *optarg)
613{
614 if (!grn_dat_open_trie_if_needed(ctx, dat)) {
615 return ctx->rc;
616 } else if (!dat->trie || !key || !key_size) {
617 return GRN_INVALID_ARGUMENT;
618 }
619
620 if (optarg && optarg->func) {
621 try {
622 const grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
623 grn::dat::UInt32 key_pos;
624 if (!trie->search(key, key_size, &key_pos)) {
625 return GRN_INVALID_ARGUMENT;
626 } else if (!optarg->func(ctx, reinterpret_cast<grn_obj *>(dat),
627 trie->get_key(key_pos).id(), optarg->func_arg)) {
628 return GRN_SUCCESS;
629 }
630 } catch (const grn::dat::Exception &ex) {
631 ERR(grn_dat_translate_error_code(ex.code()),
632 "grn::dat::Trie::search failed: %s",
633 ex.what());
634 return ctx->rc;
635 }
636 }
637
638 try {
639 grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
640 if (!trie->remove(key, key_size)) {
641 return GRN_INVALID_ARGUMENT;
642 }
643 } catch (const grn::dat::Exception &ex) {
644 ERR(grn_dat_translate_error_code(ex.code()),
645 "grn::dat::Trie::remove failed: %s",
646 ex.what());
647 return ctx->rc;
648 }
649 return GRN_SUCCESS;
650}
651
652grn_rc
653grn_dat_update_by_id(grn_ctx *ctx, grn_dat *dat, grn_id src_key_id,
654 const void *dest_key, unsigned int dest_key_size)
655{
656 if (!dest_key_size) {
657 return GRN_INVALID_ARGUMENT;
658 } else if (!grn_dat_open_trie_if_needed(ctx, dat)) {
659 return ctx->rc;
660 } else if (!dat->trie) {
661 return GRN_INVALID_ARGUMENT;
662 }
663 try {
664 try {
665 grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
666 if (!trie->update(src_key_id, dest_key, dest_key_size)) {
667 return GRN_INVALID_ARGUMENT;
668 }
669 } catch (const grn::dat::SizeError &) {
670 if (!grn_dat_rebuild_trie(ctx, dat)) {
671 return ctx->rc;
672 }
673 grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
674 if (!trie->update(src_key_id, dest_key, dest_key_size)) {
675 return GRN_INVALID_ARGUMENT;
676 }
677 }
678 } catch (const grn::dat::Exception &ex) {
679 ERR(grn_dat_translate_error_code(ex.code()),
680 "grn::dat::Trie::update failed: %s",
681 ex.what());
682 return ctx->rc;
683 }
684 return GRN_SUCCESS;
685}
686
687grn_rc
688grn_dat_update(grn_ctx *ctx, grn_dat *dat,
689 const void *src_key, unsigned int src_key_size,
690 const void *dest_key, unsigned int dest_key_size)
691{
692 if (!dest_key_size) {
693 return GRN_INVALID_ARGUMENT;
694 } else if (!grn_dat_open_trie_if_needed(ctx, dat)) {
695 return ctx->rc;
696 } else if (!dat->trie) {
697 return GRN_INVALID_ARGUMENT;
698 }
699 try {
700 try {
701 grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
702 if (!trie->update(src_key, src_key_size, dest_key, dest_key_size)) {
703 return GRN_INVALID_ARGUMENT;
704 }
705 } catch (const grn::dat::SizeError &) {
706 if (!grn_dat_rebuild_trie(ctx, dat)) {
707 return ctx->rc;
708 }
709 grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
710 if (!trie->update(src_key, src_key_size, dest_key, dest_key_size)) {
711 return GRN_INVALID_ARGUMENT;
712 }
713 }
714 } catch (const grn::dat::Exception &ex) {
715 ERR(grn_dat_translate_error_code(ex.code()),
716 "grn::dat::Trie::update failed: %s",
717 ex.what());
718 return ctx->rc;
719 }
720 return GRN_SUCCESS;
721}
722
723int
724grn_dat_scan(grn_ctx *ctx, grn_dat *dat, const char *str,
725 unsigned int str_size, grn_dat_scan_hit *scan_hits,
726 unsigned int max_num_scan_hits, const char **str_rest)
727{
728 if (!grn_dat_open_trie_if_needed(ctx, dat) || !str ||
729 !(dat->obj.header.flags & GRN_OBJ_KEY_VAR_SIZE) || !scan_hits) {
730 if (str_rest) {
731 *str_rest = str;
732 }
733 return -1;
734 }
735
736 grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
737 if (!trie) {
738 if (str_rest) {
739 *str_rest = str + str_size;
740 }
741 return 0;
742 }
743
744 if (!max_num_scan_hits || !str_size) {
745 if (str_rest) {
746 *str_rest = str;
747 }
748 return 0;
749 }
750
751 unsigned int num_scan_hits = 0;
752 try {
753 if (dat->normalizer) {
754 int flags = GRN_STRING_WITH_CHECKS;
755 grn_obj * const normalized_string = grn_string_open(ctx, str, str_size,
756 dat->normalizer,
757 flags);
758 if (!normalized_string) {
759 if (str_rest) {
760 *str_rest = str;
761 }
762 return -1;
763 }
764 grn_string_get_normalized(ctx, normalized_string, &str, &str_size, NULL);
765 const short *checks = grn_string_get_checks(ctx, normalized_string);
766 unsigned int offset = 0;
767 while (str_size) {
768 if (*checks) {
769 grn::dat::UInt32 key_pos;
770 if (trie->lcp_search(str, str_size, &key_pos)) {
771 const grn::dat::Key &key = trie->get_key(key_pos);
772 const grn::dat::UInt32 key_length = key.length();
773 if ((key_length == str_size) || (checks[key_length])) {
774 unsigned int length = 0;
775 for (grn::dat::UInt32 i = 0; i < key_length; ++i) {
776 if (checks[i] > 0) {
777 length += checks[i];
778 }
779 }
780 scan_hits[num_scan_hits].id = key.id();
781 scan_hits[num_scan_hits].offset = offset;
782 scan_hits[num_scan_hits].length = length;
783 offset += length;
784 str += key_length;
785 str_size -= key_length;
786 checks += key_length;
787 if (++num_scan_hits >= max_num_scan_hits) {
788 break;
789 }
790 continue;
791 }
792 }
793 if (*checks > 0) {
794 offset += *checks;
795 }
796 }
797 ++str;
798 --str_size;
799 ++checks;
800 }
801 if (str_rest) {
802 grn_string_get_original(ctx, normalized_string, str_rest, NULL);
803 *str_rest += offset;
804 }
805 grn_obj_close(ctx, normalized_string);
806 } else {
807 const char * const begin = str;
808 while (str_size) {
809 grn::dat::UInt32 key_pos;
810 if (trie->lcp_search(str, str_size, &key_pos)) {
811 const grn::dat::Key &key = trie->get_key(key_pos);
812 scan_hits[num_scan_hits].id = key.id();
813 scan_hits[num_scan_hits].offset = str - begin;
814 scan_hits[num_scan_hits].length = key.length();
815 str += key.length();
816 str_size -= key.length();
817 if (++num_scan_hits >= max_num_scan_hits) {
818 break;
819 }
820 } else {
821 const int char_length = grn_charlen(ctx, str, str + str_size);
822 if (char_length) {
823 str += char_length;
824 str_size -= char_length;
825 } else {
826 ++str;
827 --str_size;
828 }
829 }
830 }
831 if (str_rest) {
832 *str_rest = str;
833 }
834 }
835 } catch (const grn::dat::Exception &ex) {
836 ERR(grn_dat_translate_error_code(ex.code()),
837 "grn::dat::lcp_search failed: %s",
838 ex.what());
839 if (str_rest) {
840 *str_rest = str;
841 }
842 return -1;
843 }
844 return static_cast<int>(num_scan_hits);
845}
846
847grn_id
848grn_dat_lcp_search(grn_ctx *ctx, grn_dat *dat,
849 const void *key, unsigned int key_size)
850{
851 if (!grn_dat_open_trie_if_needed(ctx, dat) || !key ||
852 !(dat->obj.header.flags & GRN_OBJ_KEY_VAR_SIZE)) {
853 return GRN_ID_NIL;
854 }
855
856 grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
857 if (!trie) {
858 return GRN_ID_NIL;
859 }
860
861 try {
862 grn::dat::UInt32 key_pos;
863 if (!trie->lcp_search(key, key_size, &key_pos)) {
864 return GRN_ID_NIL;
865 }
866 return trie->get_key(key_pos).id();
867 } catch (const grn::dat::Exception &ex) {
868 ERR(grn_dat_translate_error_code(ex.code()),
869 "grn::dat::PrefixCursor::open failed: %s",
870 ex.what());
871 return GRN_ID_NIL;
872 }
873}
874
875unsigned int
876grn_dat_size(grn_ctx *ctx, grn_dat *dat)
877{
878 if (!grn_dat_open_trie_if_needed(ctx, dat)) {
879 return 0;
880 }
881 const grn::dat::Trie * const trie = static_cast<const grn::dat::Trie *>(dat->trie);
882 if (trie) {
883 return trie->num_keys();
884 }
885 return 0;
886}
887
888grn_dat_cursor *
889grn_dat_cursor_open(grn_ctx *ctx, grn_dat *dat,
890 const void *min, unsigned int min_size,
891 const void *max, unsigned int max_size,
892 int offset, int limit, int flags)
893{
894 if (!grn_dat_open_trie_if_needed(ctx, dat)) {
895 return NULL;
896 }
897
898 grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
899 if (!trie) {
900 grn_dat_cursor * const dc =
901 static_cast<grn_dat_cursor *>(GRN_MALLOC(sizeof(grn_dat_cursor)));
902 if (dc) {
903 grn_dat_cursor_init(ctx, dc);
904 }
905 return dc;
906 }
907
908 grn_dat_cursor * const dc =
909 static_cast<grn_dat_cursor *>(GRN_MALLOC(sizeof(grn_dat_cursor)));
910 if (!dc) {
911 return NULL;
912 }
913 grn_dat_cursor_init(ctx, dc);
914
915 try {
916 if ((flags & GRN_CURSOR_BY_ID) != 0) {
917 dc->cursor = grn::dat::CursorFactory::open(*trie,
918 min, min_size, max, max_size, offset, limit,
919 grn::dat::ID_RANGE_CURSOR |
920 ((flags & GRN_CURSOR_DESCENDING) ? grn::dat::DESCENDING_CURSOR : 0) |
921 ((flags & GRN_CURSOR_GT) ? grn::dat::EXCEPT_LOWER_BOUND : 0) |
922 ((flags & GRN_CURSOR_LT) ? grn::dat::EXCEPT_UPPER_BOUND : 0));
923 } else if ((flags & GRN_CURSOR_PREFIX) != 0) {
924 if (max && max_size) {
925 if ((dat->obj.header.flags & GRN_OBJ_KEY_VAR_SIZE) != 0) {
926 dc->cursor = grn::dat::CursorFactory::open(*trie,
927 NULL, min_size, max, max_size, offset, limit,
928 grn::dat::PREFIX_CURSOR | grn::dat::DESCENDING_CURSOR);
929 } else {
930 // TODO: near
931 }
932 } else if (min && min_size) {
933 if ((flags & GRN_CURSOR_RK) != 0) {
934 // TODO: rk search
935 } else {
936 dc->cursor = grn::dat::CursorFactory::open(*trie,
937 min, min_size, NULL, 0, offset, limit,
938 grn::dat::PREDICTIVE_CURSOR |
939 ((flags & GRN_CURSOR_DESCENDING) ? grn::dat::DESCENDING_CURSOR : 0) |
940 ((flags & GRN_CURSOR_GT) ? grn::dat::EXCEPT_EXACT_MATCH : 0));
941 }
942 }
943 } else {
944 dc->cursor = grn::dat::CursorFactory::open(*trie,
945 min, min_size, max, max_size, offset, limit,
946 grn::dat::KEY_RANGE_CURSOR |
947 ((flags & GRN_CURSOR_DESCENDING) ? grn::dat::DESCENDING_CURSOR : 0) |
948 ((flags & GRN_CURSOR_GT) ? grn::dat::EXCEPT_LOWER_BOUND : 0) |
949 ((flags & GRN_CURSOR_LT) ? grn::dat::EXCEPT_UPPER_BOUND : 0));
950 }
951 } catch (const grn::dat::Exception &ex) {
952 ERR(grn_dat_translate_error_code(ex.code()),
953 "grn::dat::CursorFactory::open failed: %s",
954 ex.what());
955 GRN_FREE(dc);
956 return NULL;
957 }
958 if (!dc->cursor) {
959 ERR(GRN_INVALID_ARGUMENT, "unsupported query");
960 GRN_FREE(dc);
961 return NULL;
962 }
963 dc->dat = dat;
964 return dc;
965}
966
967grn_id
968grn_dat_cursor_next(grn_ctx *ctx, grn_dat_cursor *c)
969{
970 if (!c || !c->cursor) {
971 return GRN_ID_NIL;
972 }
973 try {
974 grn::dat::Cursor * const cursor = static_cast<grn::dat::Cursor *>(c->cursor);
975 const grn::dat::Key &key = cursor->next();
976 c->key = &key;
977 c->curr_rec = key.is_valid() ? key.id() : GRN_ID_NIL;
978 } catch (const grn::dat::Exception &ex) {
979 ERR(grn_dat_translate_error_code(ex.code()),
980 "grn::dat::Cursor::next failed: %s",
981 ex.what());
982 return GRN_ID_NIL;
983 }
984 return c->curr_rec;
985}
986
987void
988grn_dat_cursor_close(grn_ctx *ctx, grn_dat_cursor *c)
989{
990 if (c) {
991 grn_dat_cursor_fin(ctx, c);
992 GRN_FREE(c);
993 }
994}
995
996int
997grn_dat_cursor_get_key(grn_ctx *ctx, grn_dat_cursor *c, const void **key)
998{
999 if (c) {
1000 const grn::dat::Key &key_ref = *static_cast<const grn::dat::Key *>(c->key);
1001 if (key_ref.is_valid()) {
1002 *key = key_ref.ptr();
1003 return (int)key_ref.length();
1004 }
1005 }
1006 return 0;
1007}
1008
1009grn_rc
1010grn_dat_cursor_delete(grn_ctx *ctx, grn_dat_cursor *c,
1011 grn_table_delete_optarg *optarg)
1012{
1013 if (!c || !c->cursor) {
1014 return GRN_INVALID_ARGUMENT;
1015 } else if (!grn_dat_open_trie_if_needed(ctx, c->dat)) {
1016 return ctx->rc;
1017 }
1018 grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(c->dat->trie);
1019 if (!trie) {
1020 return GRN_INVALID_ARGUMENT;
1021 }
1022 try {
1023 if (trie->remove(c->curr_rec)) {
1024 return GRN_SUCCESS;
1025 }
1026 } catch (const grn::dat::Exception &ex) {
1027 ERR(grn_dat_translate_error_code(ex.code()),
1028 "grn::dat::Trie::remove failed: %s",
1029 ex.what());
1030 return GRN_INVALID_ARGUMENT;
1031 }
1032 return GRN_INVALID_ARGUMENT;
1033}
1034
1035grn_id
1036grn_dat_curr_id(grn_ctx *ctx, grn_dat *dat)
1037{
1038 if (!grn_dat_open_trie_if_needed(ctx, dat)) {
1039 return GRN_ID_NIL;
1040 }
1041 const grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
1042 if (trie) {
1043 return trie->max_key_id();
1044 }
1045 return GRN_ID_NIL;
1046}
1047
1048grn_rc
1049grn_dat_truncate(grn_ctx *ctx, grn_dat *dat)
1050{
1051 if (!grn_dat_open_trie_if_needed(ctx, dat)) {
1052 return ctx->rc;
1053 }
1054 const grn::dat::Trie * const trie = static_cast<const grn::dat::Trie *>(dat->trie);
1055 if (!trie || !trie->max_key_id()) {
1056 return GRN_SUCCESS;
1057 }
1058
1059 char trie_path[PATH_MAX];
1060 grn_dat_generate_trie_path(grn_io_path(dat->io), trie_path, dat->header->file_id + 1);
1061 try {
1062 grn::dat::Trie().create(trie_path);
1063 } catch (const grn::dat::Exception &ex) {
1064 const grn_rc error_code = grn_dat_translate_error_code(ex.code());
1065 ERR(error_code, "grn::dat::Trie::create failed: %s", ex.what());
1066 return error_code;
1067 }
1068 ++dat->header->file_id;
1069 if (!grn_dat_open_trie_if_needed(ctx, dat)) {
1070 return ctx->rc;
1071 }
1072 return GRN_SUCCESS;
1073}
1074
1075const char *
1076_grn_dat_key(grn_ctx *ctx, grn_dat *dat, grn_id id, uint32_t *key_size)
1077{
1078 if (!grn_dat_open_trie_if_needed(ctx, dat)) {
1079 *key_size = 0;
1080 return NULL;
1081 }
1082 const grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
1083 if (!trie) {
1084 *key_size = 0;
1085 return NULL;
1086 }
1087 const grn::dat::Key &key = trie->ith_key(id);
1088 if (!key.is_valid()) {
1089 *key_size = 0;
1090 return NULL;
1091 }
1092 *key_size = key.length();
1093 return static_cast<const char *>(key.ptr());
1094}
1095
1096grn_id
1097grn_dat_next(grn_ctx *ctx, grn_dat *dat, grn_id id)
1098{
1099 if (!grn_dat_open_trie_if_needed(ctx, dat)) {
1100 return GRN_ID_NIL;
1101 }
1102 const grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
1103 if (!trie) {
1104 return GRN_ID_NIL;
1105 }
1106 while (id < trie->max_key_id()) {
1107 if (trie->ith_key(++id).is_valid()) {
1108 return id;
1109 }
1110 }
1111 return GRN_ID_NIL;
1112}
1113
1114grn_id
1115grn_dat_at(grn_ctx *ctx, grn_dat *dat, grn_id id)
1116{
1117 if (!grn_dat_open_trie_if_needed(ctx, dat)) {
1118 return GRN_ID_NIL;
1119 }
1120 const grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
1121 if (!trie) {
1122 return GRN_ID_NIL;
1123 }
1124 const grn::dat::Key &key = trie->ith_key(id);
1125 if (!key.is_valid()) {
1126 return GRN_ID_NIL;
1127 }
1128 return id;
1129}
1130
1131grn_rc
1132grn_dat_clear_status_flags(grn_ctx *ctx, grn_dat *dat)
1133{
1134 if (!grn_dat_open_trie_if_needed(ctx, dat)) {
1135 return ctx->rc;
1136 }
1137 grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
1138 if (!trie) {
1139 return GRN_INVALID_ARGUMENT;
1140 }
1141 trie->clear_status_flags();
1142 return GRN_SUCCESS;
1143}
1144
1145grn_rc
1146grn_dat_repair(grn_ctx *ctx, grn_dat *dat)
1147{
1148 if (!grn_dat_open_trie_if_needed(ctx, dat)) {
1149 return ctx->rc;
1150 }
1151 const grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
1152 if (!trie) {
1153 return GRN_INVALID_ARGUMENT;
1154 }
1155
1156 char trie_path[PATH_MAX];
1157 grn_dat_generate_trie_path(grn_io_path(dat->io), trie_path, dat->header->file_id + 1);
1158 try {
1159 grn::dat::Trie().repair(*trie, trie_path);
1160 } catch (const grn::dat::Exception &ex) {
1161 const grn_rc error_code = grn_dat_translate_error_code(ex.code());
1162 ERR(error_code, "grn::dat::Trie::create failed: %s", ex.what());
1163 return error_code;
1164 }
1165 ++dat->header->file_id;
1166 if (!grn_dat_open_trie_if_needed(ctx, dat)) {
1167 return ctx->rc;
1168 }
1169 return GRN_SUCCESS;
1170}
1171
1172grn_rc
1173grn_dat_flush(grn_ctx *ctx, grn_dat *dat)
1174{
1175 if (!dat->io) {
1176 return GRN_SUCCESS;
1177 }
1178
1179 grn_rc rc = grn_io_flush(ctx, dat->io);
1180 if (rc != GRN_SUCCESS) {
1181 return rc;
1182 }
1183
1184 if (dat->trie) {
1185 grn::dat::Trie * const trie = static_cast<grn::dat::Trie *>(dat->trie);
1186 try {
1187 trie->flush();
1188 } catch (const grn::dat::Exception &ex) {
1189 const grn_rc error_code = grn_dat_translate_error_code(ex.code());
1190 if (error_code == GRN_INPUT_OUTPUT_ERROR) {
1191 SERR("grn::dat::Trie::flush failed: %s", ex.what());
1192 } else {
1193 ERR(error_code, "grn::dat::Trie::flush failed: %s", ex.what());
1194 }
1195 return error_code;
1196 }
1197 }
1198
1199 return GRN_SUCCESS;
1200}
1201
1202grn_rc
1203grn_dat_dirty(grn_ctx *ctx, grn_dat *dat)
1204{
1205 if (!dat->io) {
1206 return GRN_SUCCESS;
1207 }
1208
1209 grn_rc rc = GRN_SUCCESS;
1210
1211 {
1212 CriticalSection critical_section(&dat->lock);
1213 if (!dat->is_dirty) {
1214 uint32_t n_dirty_opens;
1215 dat->is_dirty = GRN_TRUE;
1216 GRN_ATOMIC_ADD_EX(&(dat->header->n_dirty_opens), 1, n_dirty_opens);
1217 rc = grn_io_flush(ctx, dat->io);
1218 }
1219 }
1220
1221 return rc;
1222}
1223
1224grn_bool
1225grn_dat_is_dirty(grn_ctx *ctx, grn_dat *dat)
1226{
1227 if (!dat->header) {
1228 return GRN_FALSE;
1229 }
1230
1231 return dat->header->n_dirty_opens > 0;
1232}
1233
1234grn_rc
1235grn_dat_clean(grn_ctx *ctx, grn_dat *dat)
1236{
1237 grn_rc rc = GRN_SUCCESS;
1238
1239 if (!dat->io) {
1240 return rc;
1241 }
1242
1243 {
1244 CriticalSection critical_section(&dat->lock);
1245 if (dat->is_dirty) {
1246 uint32_t n_dirty_opens;
1247 dat->is_dirty = GRN_FALSE;
1248 GRN_ATOMIC_ADD_EX(&(dat->header->n_dirty_opens), -1, n_dirty_opens);
1249 rc = grn_io_flush(ctx, dat->io);
1250 }
1251 }
1252
1253 return rc;
1254}
1255
1256grn_rc
1257grn_dat_clear_dirty(grn_ctx *ctx, grn_dat *dat)
1258{
1259 grn_rc rc = GRN_SUCCESS;
1260
1261 if (!dat->io) {
1262 return rc;
1263 }
1264
1265 {
1266 CriticalSection critical_section(&dat->lock);
1267 dat->is_dirty = GRN_FALSE;
1268 dat->header->n_dirty_opens = 0;
1269 rc = grn_io_flush(ctx, dat->io);
1270 }
1271
1272 return rc;
1273}
1274
1275grn_bool
1276grn_dat_is_corrupt(grn_ctx *ctx, grn_dat *dat)
1277{
1278 if (!dat->io) {
1279 return GRN_FALSE;
1280 }
1281
1282 {
1283 CriticalSection critical_section(&dat->lock);
1284
1285 if (grn_io_is_corrupt(ctx, dat->io)) {
1286 return GRN_TRUE;
1287 }
1288
1289 if (dat->header->file_id == 0) {
1290 return GRN_FALSE;
1291 }
1292
1293 char trie_path[PATH_MAX];
1294 grn_dat_generate_trie_path(grn_io_path(dat->io),
1295 trie_path,
1296 dat->header->file_id);
1297 struct stat stat;
1298 if (::stat(trie_path, &stat) != 0) {
1299 SERR("[dat][corrupt] used path doesn't exist: <%s>",
1300 trie_path);
1301 return GRN_TRUE;
1302 }
1303 }
1304
1305 return GRN_FALSE;
1306}
1307
1308size_t
1309grn_dat_get_disk_usage(grn_ctx *ctx, grn_dat *dat)
1310{
1311 if (!dat->io) {
1312 return 0;
1313 }
1314
1315 {
1316 CriticalSection critical_section(&dat->lock);
1317 size_t usage;
1318
1319 usage = grn_io_get_disk_usage(ctx, dat->io);
1320
1321 if (dat->header->file_id == 0) {
1322 return usage;
1323 }
1324
1325 char trie_path[PATH_MAX];
1326 grn_dat_generate_trie_path(grn_io_path(dat->io),
1327 trie_path,
1328 dat->header->file_id);
1329 struct stat stat;
1330 if (::stat(trie_path, &stat) == 0) {
1331 usage += stat.st_size;
1332 }
1333
1334 return usage;
1335 }
1336}
1337
1338} // extern "C"
1339