1#include "ggml.h"
2#include "ggml-backend.h"
3#include "ggml-impl.h"
4#include "gguf.h"
5
6#include <cinttypes>
7#include <cstddef>
8#include <cstdint>
9#include <cstdio>
10#include <cstdlib>
11#include <cstring>
12#include <map>
13#include <new>
14#include <stdexcept>
15#include <string>
16#include <vector>
17
18template <typename T>
19struct type_to_gguf_type;
20
21template <>
22struct type_to_gguf_type<uint8_t> {
23 static constexpr enum gguf_type value = GGUF_TYPE_UINT8;
24};
25
26template <>
27struct type_to_gguf_type<int8_t> {
28 static constexpr enum gguf_type value = GGUF_TYPE_INT8;
29};
30
31template <>
32struct type_to_gguf_type<uint16_t> {
33 static constexpr enum gguf_type value = GGUF_TYPE_UINT16;
34};
35
36template <>
37struct type_to_gguf_type<int16_t> {
38 static constexpr enum gguf_type value = GGUF_TYPE_INT16;
39};
40
41template <>
42struct type_to_gguf_type<uint32_t> {
43 static constexpr enum gguf_type value = GGUF_TYPE_UINT32;
44};
45
46template <>
47struct type_to_gguf_type<int32_t> {
48 static constexpr enum gguf_type value = GGUF_TYPE_INT32;
49};
50
51template <>
52struct type_to_gguf_type<float> {
53 static constexpr enum gguf_type value = GGUF_TYPE_FLOAT32;
54};
55
56template <>
57struct type_to_gguf_type<bool> {
58 static constexpr enum gguf_type value = GGUF_TYPE_BOOL;
59};
60
61template <>
62struct type_to_gguf_type<std::string> {
63 static constexpr enum gguf_type value = GGUF_TYPE_STRING;
64};
65
66template <>
67struct type_to_gguf_type<uint64_t> {
68 static constexpr enum gguf_type value = GGUF_TYPE_UINT64;
69};
70
71template <>
72struct type_to_gguf_type<int64_t> {
73 static constexpr enum gguf_type value = GGUF_TYPE_INT64;
74};
75
76template <>
77struct type_to_gguf_type<double> {
78 static constexpr enum gguf_type value = GGUF_TYPE_FLOAT64;
79};
80
81static const std::map<gguf_type, size_t> GGUF_TYPE_SIZE = {
82 {GGUF_TYPE_UINT8, sizeof(uint8_t)},
83 {GGUF_TYPE_INT8, sizeof(int8_t)},
84 {GGUF_TYPE_UINT16, sizeof(uint16_t)},
85 {GGUF_TYPE_INT16, sizeof(int16_t)},
86 {GGUF_TYPE_UINT32, sizeof(uint32_t)},
87 {GGUF_TYPE_INT32, sizeof(int32_t)},
88 {GGUF_TYPE_FLOAT32, sizeof(float)},
89 {GGUF_TYPE_BOOL, sizeof(int8_t)},
90 {GGUF_TYPE_STRING, 0}, // undefined
91 {GGUF_TYPE_ARRAY, 0}, // undefined
92 {GGUF_TYPE_UINT64, sizeof(uint64_t)},
93 {GGUF_TYPE_INT64, sizeof(int64_t)},
94 {GGUF_TYPE_FLOAT64, sizeof(double)},
95};
96static_assert(GGUF_TYPE_COUNT == 13, "GGUF_TYPE_COUNT != 13");
97
98static const std::map<gguf_type, const char *> GGUF_TYPE_NAME = {
99 {GGUF_TYPE_UINT8, "u8"},
100 {GGUF_TYPE_INT8, "i8"},
101 {GGUF_TYPE_UINT16, "u16"},
102 {GGUF_TYPE_INT16, "i16"},
103 {GGUF_TYPE_UINT32, "u32"},
104 {GGUF_TYPE_INT32, "i32"},
105 {GGUF_TYPE_FLOAT32, "f32"},
106 {GGUF_TYPE_BOOL, "bool"},
107 {GGUF_TYPE_STRING, "str"},
108 {GGUF_TYPE_ARRAY, "arr"},
109 {GGUF_TYPE_UINT64, "u64"},
110 {GGUF_TYPE_INT64, "i64"},
111 {GGUF_TYPE_FLOAT64, "f64"},
112};
113static_assert(GGUF_TYPE_COUNT == 13, "GGUF_TYPE_COUNT != 13");
114
115size_t gguf_type_size(enum gguf_type type) {
116 auto it = GGUF_TYPE_SIZE.find(x: type);
117 return it == GGUF_TYPE_SIZE.end() ? 0 : it->second;
118}
119
120struct gguf_kv {
121 std::string key;
122
123 bool is_array;
124 enum gguf_type type;
125
126 std::vector<int8_t> data;
127 std::vector<std::string> data_string;
128
129 template <typename T>
130 gguf_kv(const std::string & key, const T value)
131 : key(key), is_array(false), type(type_to_gguf_type<T>::value) {
132 GGML_ASSERT(!key.empty());
133 data.resize(new_size: sizeof(T));
134 memcpy(data.data(), &value, sizeof(T));
135 }
136
137 template <typename T>
138 gguf_kv(const std::string & key, const std::vector<T> & value)
139 : key(key), is_array(true), type(type_to_gguf_type<T>::value) {
140 GGML_ASSERT(!key.empty());
141 data.resize(value.size()*sizeof(T));
142 for (size_t i = 0; i < value.size(); ++i) {
143 const T tmp = value[i];
144 memcpy(data.data() + i*sizeof(T), &tmp, sizeof(T));
145 }
146 }
147
148 gguf_kv(const std::string & key, const std::string & value)
149 : key(key), is_array(false), type(GGUF_TYPE_STRING) {
150 GGML_ASSERT(!key.empty());
151 data_string.push_back(x: value);
152 }
153
154 gguf_kv(const std::string & key, const std::vector<std::string> & value)
155 : key(key), is_array(true), type(GGUF_TYPE_STRING) {
156 GGML_ASSERT(!key.empty());
157 data_string = value;
158 }
159
160 const std::string & get_key() const {
161 return key;
162 }
163
164 const enum gguf_type & get_type() const {
165 return type;
166 }
167
168 size_t get_ne() const {
169 if (type == GGUF_TYPE_STRING) {
170 const size_t ne = data_string.size();
171 GGML_ASSERT(is_array || ne == 1);
172 return ne;
173 }
174 const size_t type_size = gguf_type_size(type);
175 GGML_ASSERT(data.size() % type_size == 0);
176 const size_t ne = data.size() / type_size;
177 GGML_ASSERT(is_array || ne == 1);
178 return ne;
179 }
180
181 template <typename T>
182 const T & get_val(const size_t i = 0) const {
183 GGML_ASSERT(type_to_gguf_type<T>::value == type);
184 if constexpr (std::is_same<T, std::string>::value) {
185 GGML_ASSERT(data_string.size() >= i+1);
186 return data_string[i];
187 }
188 const size_t type_size = gguf_type_size(type);
189 GGML_ASSERT(data.size() % type_size == 0);
190 GGML_ASSERT(data.size() >= (i+1)*type_size);
191 return reinterpret_cast<const T *>(data.data())[i];
192 }
193
194 void cast(const enum gguf_type new_type) {
195 const size_t new_type_size = gguf_type_size(type: new_type);
196 GGML_ASSERT(data.size() % new_type_size == 0);
197 type = new_type;
198 }
199};
200
201struct gguf_tensor_info {
202 struct ggml_tensor t; // for holding the equivalent info
203 uint64_t offset; // offset from start of `data`, must be a multiple of `ALIGNMENT`
204};
205
206struct gguf_context {
207 uint32_t version = GGUF_VERSION;
208
209 std::vector<struct gguf_kv> kv;
210 std::vector<struct gguf_tensor_info> info;
211
212 size_t alignment = GGUF_DEFAULT_ALIGNMENT;
213 size_t offset = 0; // offset of `data` from beginning of file
214 size_t size = 0; // size of `data` in bytes
215
216 void * data = nullptr;
217};
218
219struct gguf_reader {
220 FILE * file;
221
222 gguf_reader(FILE * file) : file(file) {}
223
224 template <typename T>
225 bool read(T & dst) const {
226 return fread(&dst, 1, sizeof(dst), file) == sizeof(dst);
227 }
228
229 template <typename T>
230 bool read(std::vector<T> & dst, const size_t n) const {
231 dst.resize(n);
232 for (size_t i = 0; i < dst.size(); ++i) {
233 if constexpr (std::is_same<T, bool>::value) {
234 bool tmp;
235 if (!read(dst&: tmp)) {
236 return false;
237 }
238 dst[i] = tmp;
239 } else {
240 if (!read(dst[i])) {
241 return false;
242 }
243 }
244 }
245 return true;
246 }
247
248 bool read(bool & dst) const {
249 int8_t tmp = -1;
250 if (!read(dst&: tmp)) {
251 return false;
252 }
253 dst = tmp != 0;
254 return true;
255 }
256
257 bool read(enum ggml_type & dst) const {
258 int32_t tmp = -1;
259 if (!read(dst&: tmp)) {
260 return false;
261 }
262 dst = ggml_type(tmp);
263 return true;
264 }
265
266 bool read(enum gguf_type & dst) const {
267 int32_t tmp = -1;
268 if (!read(dst&: tmp)) {
269 return false;
270 }
271 dst = gguf_type(tmp);
272 return true;
273 }
274
275 bool read(std::string & dst) const {
276 uint64_t size = 0;
277 if (!read(dst&: size)) {
278 return false;
279 }
280 dst.resize(n: size);
281 return fread(ptr: dst.data(), size: 1, n: dst.length(), stream: file) == dst.length();
282 }
283
284 bool read(void * dst, const size_t size) const {
285 return fread(ptr: dst, size: 1, n: size, stream: file) == size;
286 }
287};
288
289struct gguf_context * gguf_init_empty(void) {
290 return new gguf_context;
291}
292
293template<typename T>
294bool gguf_read_emplace_helper(const struct gguf_reader & gr, std::vector<struct gguf_kv> & kv, const std::string & key, const bool is_array, const size_t n) {
295 if (is_array) {
296 std::vector<T> value;
297 try {
298 if (!gr.read(value, n)) {
299 return false;
300 }
301 } catch (std::length_error &) {
302 GGML_LOG_ERROR("%s: encountered length_error while reading value for key '%s'\n", __func__, key.c_str());
303 return false;
304 } catch (std::bad_alloc &) {
305 GGML_LOG_ERROR("%s: encountered bad_alloc error while reading value for key '%s'\n", __func__, key.c_str());
306 return false;
307 }
308 kv.emplace_back(key, value);
309 } else {
310 T value;
311 if (!gr.read(value)) {
312 return false;
313 }
314 kv.emplace_back(key, value);
315 }
316 return true;
317}
318
319struct gguf_context * gguf_init_from_file_impl(FILE * file, struct gguf_init_params params) {
320 const struct gguf_reader gr(file);
321 struct gguf_context * ctx = new gguf_context;
322
323 bool ok = true;
324
325 // file magic
326 {
327 std::vector<char> magic;
328 ok = ok && gr.read(dst&: magic, n: 4);
329
330 if (!ok) {
331 GGML_LOG_ERROR("%s: failed to read magic\n", __func__);
332 gguf_free(ctx);
333 return nullptr;
334 }
335
336 for (uint32_t i = 0; i < magic.size(); i++) {
337 if (magic[i] != GGUF_MAGIC[i]) {
338 char c0 = isprint(magic[0]) ? magic[0] : '?';
339 char c1 = isprint(magic[1]) ? magic[1] : '?';
340 char c2 = isprint(magic[2]) ? magic[2] : '?';
341 char c3 = isprint(magic[3]) ? magic[3] : '?';
342 GGML_LOG_ERROR("%s: invalid magic characters: '%c%c%c%c', expected 'GGUF'\n", __func__, c0, c1, c2, c3);
343 gguf_free(ctx);
344 return nullptr;
345 }
346 }
347 }
348
349 // header
350 int64_t n_kv = 0;
351 int64_t n_tensors = 0;
352
353 if (ok && gr.read(dst&: ctx->version)) {
354 if (ok && ctx->version == 0) {
355 GGML_LOG_ERROR("%s: bad GGUF version: %" PRIu32 "\n", __func__, ctx->version);
356 ok = false;
357 }
358
359 /*
360 * bit layout is different when reading non-native endian models.
361 * assuming that the GGUF version is 3, the non-native endian model
362 * would read it as 0x30000000. we can use the AND operation against
363 * the last 4 hexadecimal digits to check if the model is the same
364 * endianness as the host system.
365 */
366 if (ok && (ctx->version & 0x0000FFFF) == 0x00000000) {
367 GGML_LOG_ERROR("%s: failed to load model: this GGUF file version %" PRIu32 " is extremely large, is there a mismatch between the host and model endianness?\n", __func__, ctx->version);
368 ok = false;
369 }
370
371 if (ok && ctx->version == 1) {
372 GGML_LOG_ERROR("%s: GGUFv1 is no longer supported, please use a more up-to-date version\n", __func__);
373 ok = false;
374 }
375 if (ok && ctx->version > GGUF_VERSION) {
376 GGML_LOG_ERROR("%s: this GGUF file is version %" PRIu32 " but this software only supports up to version %d\n",
377 __func__, ctx->version, GGUF_VERSION);
378 ok = false;
379 }
380 } else {
381 ok = false;
382 }
383
384 if (ok && gr.read(dst&: n_tensors)) {
385 static_assert(sizeof(size_t) <= 8 && sizeof(gguf_tensor_info) >= 2, "int64_t insufficient for indexing");
386 if (n_tensors < 0 || n_tensors > int64_t(SIZE_MAX/sizeof(gguf_tensor_info))) {
387 GGML_LOG_ERROR("%s: number of tensors is %" PRIi64 " but must be in [0, %zu]\n",
388 __func__, n_tensors, SIZE_MAX/sizeof(gguf_tensor_info));
389 ok = false;
390 }
391 } else {
392 ok = false;
393 }
394
395 if (ok && gr.read(dst&: n_kv)) {
396 static_assert(sizeof(size_t) <= 8 && sizeof(gguf_tensor_info) >= 2, "int64_t insufficient for indexing");
397 if (n_kv < 0 || n_kv > int64_t(SIZE_MAX/sizeof(gguf_kv))) {
398 GGML_LOG_ERROR("%s: number of key value pairs is %" PRIi64 " but must be in [0, %zu]\n",
399 __func__, n_kv, SIZE_MAX/sizeof(gguf_kv));
400 ok = false;
401 }
402 } else {
403 ok = false;
404 }
405
406 if (!ok) {
407 GGML_LOG_ERROR("%s: failed to read header\n", __func__);
408 gguf_free(ctx);
409 return nullptr;
410 }
411
412 // KV pairs
413 {
414 for (int64_t i = 0; ok && i < n_kv; ++i) {
415 std::string key;
416 gguf_type type = gguf_type(-1);
417 bool is_array = false;
418 uint64_t n = 1;
419
420 try {
421 ok = ok && gr.read(dst&: key);
422 } catch (std::length_error &) {
423 GGML_LOG_ERROR("%s: encountered length_error while reading key %" PRIi64 "\n", __func__, i);
424 ok = false;
425 } catch (std::bad_alloc &) {
426 GGML_LOG_ERROR("%s: encountered bad_alloc error while reading key %" PRIi64 "\n", __func__, i);
427 ok = false;
428 }
429 for (size_t j = 0; ok && j < ctx->kv.size(); ++j) {
430 if (key == ctx->kv[j].key) {
431 GGML_LOG_ERROR("%s: duplicate key '%s' for tensors %zu and %" PRIi64 " \n", __func__, key.c_str(), j, i);
432 ok = false;
433 }
434 }
435 if (!ok) {
436 break;
437 }
438
439 ok = ok && gr.read(dst&: type);
440 if (type == GGUF_TYPE_ARRAY) {
441 is_array = true;
442 ok = ok && gr.read(dst&: type);
443 ok = ok && gr.read(dst&: n);
444 }
445 if (!ok) {
446 break;
447 }
448
449 switch (type) {
450 case GGUF_TYPE_UINT8: ok = ok && gguf_read_emplace_helper<uint8_t> (gr, kv&: ctx->kv, key, is_array, n); break;
451 case GGUF_TYPE_INT8: ok = ok && gguf_read_emplace_helper<int8_t> (gr, kv&: ctx->kv, key, is_array, n); break;
452 case GGUF_TYPE_UINT16: ok = ok && gguf_read_emplace_helper<uint16_t> (gr, kv&: ctx->kv, key, is_array, n); break;
453 case GGUF_TYPE_INT16: ok = ok && gguf_read_emplace_helper<int16_t> (gr, kv&: ctx->kv, key, is_array, n); break;
454 case GGUF_TYPE_UINT32: ok = ok && gguf_read_emplace_helper<uint32_t> (gr, kv&: ctx->kv, key, is_array, n); break;
455 case GGUF_TYPE_INT32: ok = ok && gguf_read_emplace_helper<int32_t> (gr, kv&: ctx->kv, key, is_array, n); break;
456 case GGUF_TYPE_FLOAT32: ok = ok && gguf_read_emplace_helper<float> (gr, kv&: ctx->kv, key, is_array, n); break;
457 case GGUF_TYPE_BOOL: ok = ok && gguf_read_emplace_helper<bool> (gr, kv&: ctx->kv, key, is_array, n); break;
458 case GGUF_TYPE_STRING: ok = ok && gguf_read_emplace_helper<std::string>(gr, kv&: ctx->kv, key, is_array, n); break;
459 case GGUF_TYPE_UINT64: ok = ok && gguf_read_emplace_helper<uint64_t> (gr, kv&: ctx->kv, key, is_array, n); break;
460 case GGUF_TYPE_INT64: ok = ok && gguf_read_emplace_helper<int64_t> (gr, kv&: ctx->kv, key, is_array, n); break;
461 case GGUF_TYPE_FLOAT64: ok = ok && gguf_read_emplace_helper<double> (gr, kv&: ctx->kv, key, is_array, n); break;
462 case GGUF_TYPE_ARRAY:
463 default:
464 {
465 GGML_LOG_ERROR("%s: key '%s' has invalid GGUF type %d\n", __func__, key.c_str(), type);
466 ok = false;
467 } break;
468 }
469 }
470
471 if (!ok) {
472 GGML_LOG_ERROR("%s: failed to read key-value pairs\n", __func__);
473 gguf_free(ctx);
474 return nullptr;
475 }
476 GGML_ASSERT(int64_t(ctx->kv.size()) == n_kv);
477
478 const int alignment_idx = gguf_find_key(ctx, GGUF_KEY_GENERAL_ALIGNMENT);
479 ctx->alignment = alignment_idx == -1 ? GGUF_DEFAULT_ALIGNMENT : gguf_get_val_u32(ctx, key_id: alignment_idx);
480
481 if (ctx->alignment == 0 || (ctx->alignment & (ctx->alignment - 1)) != 0) {
482 GGML_LOG_ERROR("%s: alignment %zu is not a power of 2\n", __func__, ctx->alignment);
483 gguf_free(ctx);
484 return nullptr;
485 }
486 }
487
488 // read the tensor info
489 for (int64_t i = 0; ok && i < n_tensors; ++i) {
490 struct gguf_tensor_info info;
491
492 // tensor name
493 {
494 std::string name;
495 try {
496 ok = ok && gr.read(dst&: name);
497 } catch (std::length_error &) {
498 GGML_LOG_ERROR("%s: encountered length_error while reading tensor name %" PRIi64 "\n", __func__, i);
499 ok = false;
500 } catch (std::bad_alloc &) {
501 GGML_LOG_ERROR("%s: encountered bad_alloc error while reading tensor name %" PRIi64 "\n", __func__, i);
502 ok = false;
503 }
504 if (name.length() >= GGML_MAX_NAME) {
505 GGML_LOG_ERROR("%s: tensor name %" PRIi64 " is too long: %zu >= %d\n", __func__, i, name.length(), GGML_MAX_NAME);
506 ok = false;
507 break;
508 }
509 ggml_set_name(tensor: &info.t, name: name.c_str());
510
511 // make sure there are no duplicate tensor names
512 for (int64_t j = 0; ok && j < i; ++j) {
513 if (strcmp(s1: info.t.name, s2: ctx->info[j].t.name) == 0) {
514 GGML_LOG_ERROR("%s: duplicate tensor name '%s' for tensors %" PRIi64 " and %" PRIi64 "\n", __func__, info.t.name, j, i);
515 ok = false;
516 break;
517 }
518 }
519 }
520 if (!ok) {
521 break;
522 }
523
524 // tensor shape
525 {
526 uint32_t n_dims = 0;
527 ok = ok && gr.read(dst&: n_dims);
528 if (n_dims > GGML_MAX_DIMS) {
529 GGML_LOG_ERROR("%s: tensor '%s' has invalid number of dimensions: %" PRIu32 " > %" PRIu32 "\n",
530 __func__, info.t.name, n_dims, GGML_MAX_DIMS);
531 ok = false;
532 break;
533 }
534 for (uint32_t j = 0; ok && j < GGML_MAX_DIMS; ++j) {
535 info.t.ne[j] = 1;
536 if (j < n_dims) {
537 ok = ok && gr.read(dst&: info.t.ne[j]);
538 }
539
540 // check that all ne are non-negative
541 if (info.t.ne[j] < 0) {
542 GGML_LOG_ERROR("%s: tensor '%s' dimension %" PRIu32 " has invalid number of elements: %" PRIi64 " < 0\n",
543 __func__, info.t.name, j, info.t.ne[j]);
544 ok = false;
545 break;
546 }
547 }
548
549 // check that the total number of elements is representable
550 if (ok && ((INT64_MAX/info.t.ne[1] <= info.t.ne[0]) ||
551 (INT64_MAX/info.t.ne[2] <= info.t.ne[0]*info.t.ne[1]) ||
552 (INT64_MAX/info.t.ne[3] <= info.t.ne[0]*info.t.ne[1]*info.t.ne[2]))) {
553
554 GGML_LOG_ERROR("%s: total number of elements in tensor '%s' with shape "
555 "(%" PRIi64 ", %" PRIi64 ", %" PRIi64 ", %" PRIi64 ") is >= %" PRIi64 "\n",
556 __func__, info.t.name, info.t.ne[0], info.t.ne[1], info.t.ne[2], info.t.ne[3], INT64_MAX);
557 ok = false;
558 break;
559 }
560 }
561 if (!ok) {
562 break;
563 }
564
565 // tensor type
566 {
567 ok = ok && gr.read(dst&: info.t.type);
568
569 // check that tensor type is within defined range
570 if (info.t.type < 0 || info.t.type >= GGML_TYPE_COUNT) {
571 GGML_LOG_ERROR("%s: tensor '%s' has invalid ggml type %d (%s)\n",
572 __func__, info.t.name, info.t.type, ggml_type_name(info.t.type));
573 ok = false;
574 break;
575 }
576 const size_t type_size = ggml_type_size(type: info.t.type);
577 const int64_t blck_size = ggml_blck_size(type: info.t.type);
578
579 // check that row size is divisible by block size
580 if (blck_size == 0 || info.t.ne[0] % blck_size != 0) {
581 GGML_LOG_ERROR("%s: tensor '%s' of type %d (%s) has %" PRId64 " elements per row, "
582 "not a multiple of block size (%" PRId64 ")\n",
583 __func__, info.t.name, (int) info.t.type, ggml_type_name(info.t.type), info.t.ne[0], blck_size);
584 ok = false;
585 break;
586 }
587
588 // calculate byte offsets given the tensor shape and type
589 info.t.nb[0] = type_size;
590 info.t.nb[1] = info.t.nb[0]*(info.t.ne[0]/blck_size);
591 for (int j = 2; j < GGML_MAX_DIMS; ++j) {
592 info.t.nb[j] = info.t.nb[j - 1]*info.t.ne[j - 1];
593 }
594 }
595 if (!ok) {
596 break;
597 }
598
599 // tensor data offset within buffer
600 ok = ok && gr.read(dst&: info.offset);
601
602 ctx->info.push_back(x: info);
603 }
604
605 if (!ok) {
606 GGML_LOG_ERROR("%s: failed to read tensor info\n", __func__);
607 gguf_free(ctx);
608 return nullptr;
609 }
610 GGML_ASSERT(int64_t(ctx->info.size()) == n_tensors);
611
612 // we require the data section to be aligned, so take into account any padding
613 if (fseek(stream: file, GGML_PAD(ftell(file), ctx->alignment), SEEK_SET) != 0) {
614 GGML_LOG_ERROR("%s: failed to seek to beginning of data section\n", __func__);
615 gguf_free(ctx);
616 return nullptr;
617 }
618
619 // store the current file offset - this is where the data section starts
620 ctx->offset = ftell(stream: file);
621
622 // compute the total size of the data section, taking into account the alignment
623 {
624 ctx->size = 0;
625 for (size_t i = 0; i < ctx->info.size(); ++i) {
626 const gguf_tensor_info & ti = ctx->info[i];
627 if (ti.offset != ctx->size) {
628 GGML_LOG_ERROR("%s: tensor '%s' has offset %" PRIu64 ", expected %zu\n",
629 __func__, ti.t.name, ti.offset, ctx->size);
630 GGML_LOG_ERROR("%s: failed to read tensor data\n", __func__);
631 gguf_free(ctx);
632 return nullptr;
633 }
634 size_t padded_size = GGML_PAD(ggml_nbytes(&ti.t), ctx->alignment);
635 if (SIZE_MAX - ctx->size < padded_size) {
636 GGML_LOG_ERROR("%s: tensor '%s' size overflow, cannot accumulate size %zu + %zu\n",
637 __func__, ti.t.name, ctx->size, padded_size);
638 gguf_free(ctx);
639 return nullptr;
640 }
641 ctx->size += padded_size;
642 }
643 }
644
645 // load the tensor data only if requested
646 if (params.ctx != nullptr) {
647 // if the provided gguf_context is no_alloc, then we create "empty" tensors and do not read the binary blob
648 // otherwise, we load the binary blob into the created ggml_context as well, and point the "data" members of
649 // the ggml_tensor structs to the appropriate locations in the binary blob
650
651 // compute the exact size needed for the new ggml_context
652 const size_t mem_size =
653 params.no_alloc ?
654 (n_tensors )*ggml_tensor_overhead() :
655 (n_tensors + 1)*ggml_tensor_overhead() + ctx->size;
656
657 struct ggml_init_params pdata = {
658 /*mem_size =*/ mem_size,
659 /*mem_buffer =*/ nullptr,
660 /*no_alloc =*/ params.no_alloc,
661 };
662
663 *params.ctx = ggml_init(params: pdata);
664 if (*params.ctx == nullptr) {
665 GGML_LOG_ERROR("%s: failed to initialize ggml context for storing tensors\n", __func__);
666 gguf_free(ctx);
667 return nullptr;
668 }
669
670 struct ggml_context * ctx_data = *params.ctx;
671
672 struct ggml_tensor * data = nullptr;
673
674 if (!params.no_alloc) {
675 data = ggml_new_tensor_1d(ctx: ctx_data, type: GGML_TYPE_I8, ne0: ctx->size);
676
677 ok = ok && data != nullptr;
678
679 if (ok) {
680 ggml_set_name(tensor: data, name: "GGUF tensor data binary blob");
681 }
682
683 // read the binary blob with the tensor data
684 ok = ok && gr.read(dst: data->data, size: ctx->size);
685
686 if (!ok) {
687 GGML_LOG_ERROR("%s: failed to read tensor data binary blob\n", __func__);
688 ggml_free(ctx: ctx_data);
689 *params.ctx = nullptr;
690 gguf_free(ctx);
691 return nullptr;
692 }
693
694 ctx->data = data->data;
695 }
696
697 ggml_set_no_alloc(ctx: ctx_data, no_alloc: true);
698
699 // create the tensors
700 for (size_t i = 0; i < ctx->info.size(); ++i) {
701 const struct gguf_tensor_info & info = ctx->info[i];
702
703 struct ggml_tensor * cur = ggml_new_tensor(ctx: ctx_data, type: info.t.type, GGML_MAX_DIMS, ne: info.t.ne);
704
705 ok = ok && cur != nullptr;
706
707 if (!ok) {
708 break;
709 }
710
711 ggml_set_name(tensor: cur, name: info.t.name);
712
713 // point the data member to the appropriate location in the binary blob using the tensor info
714 if (!params.no_alloc) {
715 cur->data = (char *) data->data + info.offset;
716 }
717 }
718
719 if (!ok) {
720 GGML_LOG_ERROR("%s: failed to create tensors\n", __func__);
721 ggml_free(ctx: ctx_data);
722 *params.ctx = nullptr;
723 gguf_free(ctx);
724 return nullptr;
725 }
726
727 ggml_set_no_alloc(ctx: ctx_data, no_alloc: params.no_alloc);
728 }
729
730 return ctx;
731}
732
733struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params) {
734 FILE * file = ggml_fopen(fname, mode: "rb");
735
736 if (!file) {
737 GGML_LOG_ERROR("%s: failed to open GGUF file '%s'\n", __func__, fname);
738 return nullptr;
739 }
740
741 struct gguf_context * result = gguf_init_from_file_impl(file, params);
742 fclose(stream: file);
743 return result;
744}
745
746void gguf_free(struct gguf_context * ctx) {
747 if (ctx == nullptr) {
748 return;
749 }
750 delete ctx;
751}
752
753const char * gguf_type_name(enum gguf_type type) {
754 auto it = GGUF_TYPE_NAME.find(x: type);
755 return it == GGUF_TYPE_NAME.end() ? nullptr : it->second;
756}
757
758uint32_t gguf_get_version(const struct gguf_context * ctx) {
759 return ctx->version;
760}
761
762size_t gguf_get_alignment(const struct gguf_context * ctx) {
763 return ctx->alignment;
764}
765
766size_t gguf_get_data_offset(const struct gguf_context * ctx) {
767 return ctx->offset;
768}
769
770int64_t gguf_get_n_kv(const struct gguf_context * ctx) {
771 return ctx->kv.size();
772}
773
774int64_t gguf_find_key(const struct gguf_context * ctx, const char * key) {
775 // return -1 if key not found
776 int64_t keyfound = -1;
777
778 const int64_t n_kv = gguf_get_n_kv(ctx);
779
780 for (int64_t i = 0; i < n_kv; ++i) {
781 if (strcmp(s1: key, s2: gguf_get_key(ctx, key_id: i)) == 0) {
782 keyfound = i;
783 break;
784 }
785 }
786
787 return keyfound;
788}
789
790const char * gguf_get_key(const struct gguf_context * ctx, int64_t key_id) {
791 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
792 return ctx->kv[key_id].get_key().c_str();
793}
794
795enum gguf_type gguf_get_kv_type(const struct gguf_context * ctx, int64_t key_id) {
796 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
797 return ctx->kv[key_id].is_array ? GGUF_TYPE_ARRAY : ctx->kv[key_id].get_type();
798}
799
800enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int64_t key_id) {
801 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
802 GGML_ASSERT(ctx->kv[key_id].is_array);
803 return ctx->kv[key_id].get_type();
804}
805
806const void * gguf_get_arr_data(const struct gguf_context * ctx, int64_t key_id) {
807 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
808 GGML_ASSERT(ctx->kv[key_id].get_type() != GGUF_TYPE_STRING);
809 return ctx->kv[key_id].data.data();
810}
811
812const char * gguf_get_arr_str(const struct gguf_context * ctx, int64_t key_id, size_t i) {
813 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
814 GGML_ASSERT(ctx->kv[key_id].get_type() == GGUF_TYPE_STRING);
815 return ctx->kv[key_id].data_string[i].c_str();
816}
817
818size_t gguf_get_arr_n(const struct gguf_context * ctx, int64_t key_id) {
819 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
820
821 if (ctx->kv[key_id].type == GGUF_TYPE_STRING) {
822 return ctx->kv[key_id].data_string.size();
823 }
824
825 const size_t type_size = gguf_type_size(type: ctx->kv[key_id].type);
826 GGML_ASSERT(ctx->kv[key_id].data.size() % type_size == 0);
827 return ctx->kv[key_id].data.size() / type_size;
828}
829
830uint8_t gguf_get_val_u8(const struct gguf_context * ctx, int64_t key_id) {
831 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
832 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
833 return ctx->kv[key_id].get_val<uint8_t>();
834}
835
836int8_t gguf_get_val_i8(const struct gguf_context * ctx, int64_t key_id) {
837 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
838 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
839 return ctx->kv[key_id].get_val<int8_t>();
840}
841
842uint16_t gguf_get_val_u16(const struct gguf_context * ctx, int64_t key_id) {
843 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
844 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
845 return ctx->kv[key_id].get_val<uint16_t>();
846}
847
848int16_t gguf_get_val_i16(const struct gguf_context * ctx, int64_t key_id) {
849 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
850 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
851 return ctx->kv[key_id].get_val<int16_t>();
852}
853
854uint32_t gguf_get_val_u32(const struct gguf_context * ctx, int64_t key_id) {
855 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
856 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
857 return ctx->kv[key_id].get_val<uint32_t>();
858}
859
860int32_t gguf_get_val_i32(const struct gguf_context * ctx, int64_t key_id) {
861 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
862 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
863 return ctx->kv[key_id].get_val<int32_t>();
864}
865
866float gguf_get_val_f32(const struct gguf_context * ctx, int64_t key_id) {
867 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
868 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
869 return ctx->kv[key_id].get_val<float>();
870}
871
872uint64_t gguf_get_val_u64(const struct gguf_context * ctx, int64_t key_id) {
873 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
874 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
875 return ctx->kv[key_id].get_val<uint64_t>();
876}
877
878int64_t gguf_get_val_i64(const struct gguf_context * ctx, int64_t key_id) {
879 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
880 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
881 return ctx->kv[key_id].get_val<int64_t>();
882}
883
884double gguf_get_val_f64(const struct gguf_context * ctx, int64_t key_id) {
885 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
886 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
887 return ctx->kv[key_id].get_val<double>();
888}
889
890bool gguf_get_val_bool(const struct gguf_context * ctx, int64_t key_id) {
891 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
892 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
893 return ctx->kv[key_id].get_val<bool>();
894}
895
896const char * gguf_get_val_str(const struct gguf_context * ctx, int64_t key_id) {
897 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
898 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
899 return ctx->kv[key_id].get_val<std::string>().c_str();
900}
901
902const void * gguf_get_val_data(const struct gguf_context * ctx, int64_t key_id) {
903 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
904 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
905 GGML_ASSERT(ctx->kv[key_id].get_type() != GGUF_TYPE_STRING);
906 return ctx->kv[key_id].data.data();
907}
908
909int64_t gguf_get_n_tensors(const struct gguf_context * ctx) {
910 return ctx->info.size();
911}
912
913int64_t gguf_find_tensor(const struct gguf_context * ctx, const char * name) {
914 // return -1 if tensor not found
915 int64_t tensor_id = -1;
916
917 const int64_t n_tensors = gguf_get_n_tensors(ctx);
918
919 for (int64_t i = 0; i < n_tensors; ++i) {
920 if (strcmp(s1: name, s2: gguf_get_tensor_name(ctx, tensor_id: i)) == 0) {
921 tensor_id = i;
922 break;
923 }
924 }
925
926 return tensor_id;
927}
928
929size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id) {
930 GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));
931 return ctx->info[tensor_id].offset;
932}
933
934const char * gguf_get_tensor_name(const struct gguf_context * ctx, int64_t tensor_id) {
935 GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));
936 return ctx->info[tensor_id].t.name;
937}
938
939enum ggml_type gguf_get_tensor_type(const struct gguf_context * ctx, int64_t tensor_id) {
940 GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));
941 return ctx->info[tensor_id].t.type;
942}
943
944size_t gguf_get_tensor_size(const struct gguf_context * ctx, int64_t tensor_id) {
945 GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));
946 return ggml_nbytes(tensor: &ctx->info[tensor_id].t);
947}
948
949int64_t gguf_remove_key(struct gguf_context * ctx, const char * key) {
950 const int64_t key_id = gguf_find_key(ctx, key);
951 if (key_id >= 0) {
952 ctx->kv.erase(position: ctx->kv.begin() + key_id);
953 }
954 return key_id;
955}
956
957template<typename T>
958static void gguf_check_reserved_keys(const std::string & key, const T val) {
959 if (key == GGUF_KEY_GENERAL_ALIGNMENT) {
960 if constexpr (std::is_same<T, uint32_t>::value) {
961 GGML_ASSERT(val > 0 && (val & (val - 1)) == 0 && GGUF_KEY_GENERAL_ALIGNMENT " must be power of 2");
962 } else {
963 GGML_UNUSED(val);
964 GGML_ABORT(GGUF_KEY_GENERAL_ALIGNMENT " must be type u32");
965 }
966 }
967}
968
969void gguf_set_val_u8(struct gguf_context * ctx, const char * key, uint8_t val) {
970 gguf_check_reserved_keys(key, val);
971 gguf_remove_key(ctx, key);
972 ctx->kv.emplace_back(args&: key, args&: val);
973}
974
975void gguf_set_val_i8(struct gguf_context * ctx, const char * key, int8_t val) {
976 gguf_check_reserved_keys(key, val);
977 gguf_remove_key(ctx, key);
978 ctx->kv.emplace_back(args&: key, args&: val);
979}
980
981void gguf_set_val_u16(struct gguf_context * ctx, const char * key, uint16_t val) {
982 gguf_check_reserved_keys(key, val);
983 gguf_remove_key(ctx, key);
984 ctx->kv.emplace_back(args&: key, args&: val);
985}
986
987void gguf_set_val_i16(struct gguf_context * ctx, const char * key, int16_t val) {
988 gguf_check_reserved_keys(key, val);
989 gguf_remove_key(ctx, key);
990 ctx->kv.emplace_back(args&: key, args&: val);
991}
992
993void gguf_set_val_u32(struct gguf_context * ctx, const char * key, uint32_t val) {
994 gguf_check_reserved_keys(key, val);
995 gguf_remove_key(ctx, key);
996 ctx->kv.emplace_back(args&: key, args&: val);
997}
998
999void gguf_set_val_i32(struct gguf_context * ctx, const char * key, int32_t val) {
1000 gguf_check_reserved_keys(key, val);
1001 gguf_remove_key(ctx, key);
1002 ctx->kv.emplace_back(args&: key, args&: val);
1003}
1004
1005void gguf_set_val_f32(struct gguf_context * ctx, const char * key, float val) {
1006 gguf_check_reserved_keys(key, val);
1007 gguf_remove_key(ctx, key);
1008 ctx->kv.emplace_back(args&: key, args&: val);
1009}
1010
1011void gguf_set_val_u64(struct gguf_context * ctx, const char * key, uint64_t val) {
1012 gguf_check_reserved_keys(key, val);
1013 gguf_remove_key(ctx, key);
1014 ctx->kv.emplace_back(args&: key, args&: val);
1015}
1016
1017void gguf_set_val_i64(struct gguf_context * ctx, const char * key, int64_t val) {
1018 gguf_check_reserved_keys(key, val);
1019 gguf_remove_key(ctx, key);
1020 ctx->kv.emplace_back(args&: key, args&: val);
1021}
1022
1023void gguf_set_val_f64(struct gguf_context * ctx, const char * key, double val) {
1024 gguf_check_reserved_keys(key, val);
1025 gguf_remove_key(ctx, key);
1026 ctx->kv.emplace_back(args&: key, args&: val);
1027}
1028
1029void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val) {
1030 gguf_check_reserved_keys(key, val);
1031 gguf_remove_key(ctx, key);
1032 ctx->kv.emplace_back(args&: key, args&: val);
1033}
1034
1035void gguf_set_val_str(struct gguf_context * ctx, const char * key, const char * val) {
1036 gguf_check_reserved_keys(key, val);
1037 gguf_remove_key(ctx, key);
1038 ctx->kv.emplace_back(args&: key, args: std::string(val));
1039}
1040
1041void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, size_t n) {
1042 gguf_check_reserved_keys(key, val: data);
1043 gguf_remove_key(ctx, key);
1044
1045 const size_t nbytes = n*gguf_type_size(type);
1046 std::vector<int8_t> tmp(nbytes);
1047 if (!tmp.empty()) {
1048 memcpy(dest: tmp.data(), src: data, n: nbytes);
1049 }
1050 ctx->kv.emplace_back(args&: key, args&: tmp);
1051 ctx->kv.back().cast(new_type: type);
1052}
1053
1054void gguf_set_arr_str(struct gguf_context * ctx, const char * key, const char ** data, size_t n) {
1055 gguf_check_reserved_keys(key, val: data);
1056 gguf_remove_key(ctx, key);
1057
1058 std::vector<std::string> tmp(n);
1059 for (size_t i = 0; i < n; ++i) {
1060 tmp[i] = data[i];
1061 }
1062 ctx->kv.emplace_back(args&: key, args&: tmp);
1063}
1064
1065// set or add KV pairs from another context
1066void gguf_set_kv(struct gguf_context * ctx, const struct gguf_context * src) {
1067 const int64_t n_kv = gguf_get_n_kv(ctx: src);
1068 for (int64_t i = 0; i < n_kv; ++i) {
1069 const struct gguf_kv & kv = src->kv[i];
1070
1071 if (!kv.is_array) {
1072 switch (kv.get_type()) {
1073 case GGUF_TYPE_UINT8: gguf_set_val_u8 (ctx, key: kv.get_key().c_str(), val: kv.get_val<uint8_t>()); break;
1074 case GGUF_TYPE_INT8: gguf_set_val_i8 (ctx, key: kv.get_key().c_str(), val: kv.get_val<int8_t>()); break;
1075 case GGUF_TYPE_UINT16: gguf_set_val_u16 (ctx, key: kv.get_key().c_str(), val: kv.get_val<uint16_t>()); break;
1076 case GGUF_TYPE_INT16: gguf_set_val_i16 (ctx, key: kv.get_key().c_str(), val: kv.get_val<int16_t>()); break;
1077 case GGUF_TYPE_UINT32: gguf_set_val_u32 (ctx, key: kv.get_key().c_str(), val: kv.get_val<uint32_t>()); break;
1078 case GGUF_TYPE_INT32: gguf_set_val_i32 (ctx, key: kv.get_key().c_str(), val: kv.get_val<int32_t>()); break;
1079 case GGUF_TYPE_FLOAT32: gguf_set_val_f32 (ctx, key: kv.get_key().c_str(), val: kv.get_val<float>()); break;
1080 case GGUF_TYPE_UINT64: gguf_set_val_u64 (ctx, key: kv.get_key().c_str(), val: kv.get_val<uint64_t>()); break;
1081 case GGUF_TYPE_INT64: gguf_set_val_i64 (ctx, key: kv.get_key().c_str(), val: kv.get_val<int64_t>()); break;
1082 case GGUF_TYPE_FLOAT64: gguf_set_val_f64 (ctx, key: kv.get_key().c_str(), val: kv.get_val<double>()); break;
1083 case GGUF_TYPE_BOOL: gguf_set_val_bool(ctx, key: kv.get_key().c_str(), val: kv.get_val<bool>()); break;
1084 case GGUF_TYPE_STRING: gguf_set_val_str (ctx, key: kv.get_key().c_str(), val: kv.get_val<std::string>().c_str()); break;
1085 case GGUF_TYPE_ARRAY:
1086 default: GGML_ABORT("invalid type");
1087 }
1088 continue;
1089 }
1090
1091 const size_t ne = kv.get_ne();
1092
1093 switch (kv.get_type()) {
1094 case GGUF_TYPE_UINT8:
1095 case GGUF_TYPE_INT8:
1096 case GGUF_TYPE_UINT16:
1097 case GGUF_TYPE_INT16:
1098 case GGUF_TYPE_UINT32:
1099 case GGUF_TYPE_INT32:
1100 case GGUF_TYPE_FLOAT32:
1101 case GGUF_TYPE_UINT64:
1102 case GGUF_TYPE_INT64:
1103 case GGUF_TYPE_FLOAT64:
1104 case GGUF_TYPE_BOOL: {
1105 gguf_set_arr_data(ctx, key: kv.get_key().c_str(), type: kv.get_type(), data: kv.data.data(), n: ne);
1106 } break;
1107 case GGUF_TYPE_STRING: {
1108 std::vector<const char *> tmp(ne);
1109 for (size_t j = 0; j < ne; ++j) {
1110 tmp[j] = kv.data_string[j].c_str();
1111 }
1112 gguf_set_arr_str(ctx, key: kv.get_key().c_str(), data: tmp.data(), n: ne);
1113 } break;
1114 case GGUF_TYPE_ARRAY:
1115 default: GGML_ABORT("invalid type");
1116 }
1117 }
1118}
1119
1120void gguf_add_tensor(
1121 struct gguf_context * ctx,
1122 const struct ggml_tensor * tensor) {
1123 GGML_ASSERT(tensor);
1124 if (gguf_find_tensor(ctx, name: tensor->name) != -1) {
1125 GGML_ABORT("duplicate tensor name: %s", tensor->name);
1126 }
1127
1128 struct gguf_tensor_info ti;
1129 ti.t = *tensor;
1130 ti.offset = ctx->info.empty() ? 0 :
1131 ctx->info.back().offset + GGML_PAD(ggml_nbytes(&ctx->info.back().t), ctx->alignment);
1132 ctx->info.push_back(x: ti);
1133}
1134
1135void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type) {
1136 const int64_t tensor_id = gguf_find_tensor(ctx, name);
1137 if (tensor_id < 0) {
1138 GGML_ABORT("tensor not found: %s", name);
1139 }
1140 struct ggml_tensor * tensor = &ctx->info[tensor_id].t;
1141 const size_t type_size = ggml_type_size(type);
1142 const int64_t blck_size = ggml_blck_size(type);
1143
1144 tensor->type = type;
1145 GGML_ASSERT(tensor->ne[0] % blck_size == 0 && "tensor row size not divisible by block size of new type");
1146
1147 tensor->nb[0] = type_size;
1148 tensor->nb[1] = tensor->nb[0]*(tensor->ne[0]/blck_size);
1149 for (int i = 2; i < GGML_MAX_DIMS; i++) {
1150 tensor->nb[i] = tensor->nb[i - 1]*tensor->ne[i - 1];
1151 }
1152
1153 // update offsets
1154 const int64_t n_tensors = gguf_get_n_tensors(ctx);
1155 for (int64_t i = tensor_id + 1; i < n_tensors; ++i) {
1156 ctx->info[i].offset = ctx->info[i - 1].offset + GGML_PAD(ggml_nbytes(&ctx->info[i - 1].t), ctx->alignment);
1157 }
1158}
1159
1160void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data) {
1161 const int64_t tensor_id = gguf_find_tensor(ctx, name);
1162 if (tensor_id < 0) {
1163 GGML_ABORT("tensor not found: %s", name);
1164 }
1165
1166 ctx->info[tensor_id].t.data = (void *)(uintptr_t)data; // double cast suppresses warning about casting away const
1167}
1168
1169struct gguf_writer_base {
1170 size_t written_bytes {0u};
1171
1172 ~gguf_writer_base(void) {}
1173
1174 // we bet on devirtualization
1175 virtual void write(int8_t val) = 0;
1176 virtual void write(const std::vector<int8_t> & val) = 0;
1177 virtual void write_tensor_data(const struct gguf_tensor_info & info, size_t offset_data, size_t alignment) = 0;
1178
1179 template <typename T>
1180 void write(const T & val) {
1181 for (size_t i = 0; i < sizeof(val); ++i) {
1182 write(val: reinterpret_cast<const int8_t *>(&val)[i]);
1183 }
1184 }
1185
1186 void write(const bool & val) {
1187 const int8_t val8 = val ? 1 : 0;
1188 write(val: val8);
1189 }
1190
1191 void write(const std::string & val) {
1192 {
1193 const uint64_t n = val.length();
1194 write(val: n);
1195 }
1196 for (size_t i = 0; i < val.length(); ++i) {
1197 write(val: (val.data())[i]);
1198 }
1199 }
1200
1201 void write(const char * val) {
1202 write(val: std::string(val));
1203 }
1204
1205 void write(const enum ggml_type & val) {
1206 write(val: int32_t(val));
1207 }
1208
1209 void write(const enum gguf_type & val) {
1210 write(val: int32_t(val));
1211 }
1212
1213 void write(const struct gguf_kv & kv) {
1214 const uint64_t ne = kv.get_ne();
1215
1216 write(val: kv.get_key());
1217
1218 if (kv.is_array) {
1219 write(val: GGUF_TYPE_ARRAY);
1220 write(val: kv.get_type());
1221 write(val: ne);
1222 } else {
1223 write(val: kv.get_type());
1224 }
1225
1226 switch (kv.get_type()) {
1227 case GGUF_TYPE_UINT8:
1228 case GGUF_TYPE_INT8:
1229 case GGUF_TYPE_UINT16:
1230 case GGUF_TYPE_INT16:
1231 case GGUF_TYPE_UINT32:
1232 case GGUF_TYPE_INT32:
1233 case GGUF_TYPE_FLOAT32:
1234 case GGUF_TYPE_UINT64:
1235 case GGUF_TYPE_INT64:
1236 case GGUF_TYPE_FLOAT64: {
1237 write(val: kv.data);
1238 } break;
1239 case GGUF_TYPE_BOOL: {
1240 for (size_t i = 0; i < ne; ++i) {
1241 write(val: kv.get_val<bool>(i));
1242 }
1243 } break;
1244 case GGUF_TYPE_STRING: {
1245 for (size_t i = 0; i < ne; ++i) {
1246 write(val: kv.get_val<std::string>(i));
1247 }
1248 } break;
1249 case GGUF_TYPE_ARRAY:
1250 default: GGML_ABORT("invalid type");
1251 }
1252 }
1253
1254 void write_tensor_meta(const struct gguf_tensor_info & info) {
1255 write(val: info.t.name);
1256
1257 const uint32_t n_dims = ggml_n_dims(tensor: &info.t);
1258 write(val: n_dims);
1259
1260 for (uint32_t j = 0; j < n_dims; ++j) {
1261 write(val: info.t.ne[j]);
1262 }
1263 write(val: info.t.type);
1264 write(val: info.offset);
1265 }
1266
1267 void pad(const size_t alignment) {
1268 while (written_bytes % alignment != 0) {
1269 const int8_t zero = 0;
1270 write(val: zero);
1271 }
1272 }
1273};
1274
1275// vector buffer based writer
1276struct gguf_writer_buf final : public gguf_writer_base {
1277 std::vector<int8_t> & buf;
1278
1279 gguf_writer_buf(std::vector<int8_t> & buf) : buf(buf) {}
1280
1281 using gguf_writer_base::write;
1282
1283 void write(const int8_t val) override {
1284 buf.push_back(x: val);
1285 written_bytes++;
1286 }
1287
1288 void write(const std::vector<int8_t> & val) override {
1289 buf.insert(position: buf.end(), first: val.begin(), last: val.end());
1290 written_bytes += val.size();
1291 }
1292
1293 void write_tensor_data(const struct gguf_tensor_info & info, const size_t offset_data, const size_t alignment) override {
1294 GGML_ASSERT(buf.size() - offset_data == info.offset);
1295
1296 GGML_ASSERT(ggml_is_contiguous(&info.t));
1297 const size_t offset = buf.size();
1298 const size_t nbytes = ggml_nbytes(tensor: &info.t);
1299
1300 buf.resize(new_size: offset + nbytes);
1301 if (info.t.buffer) {
1302 ggml_backend_tensor_get(tensor: &info.t, data: buf.data() + offset, offset: 0, size: nbytes);
1303 } else {
1304 GGML_ASSERT(info.t.data);
1305 memcpy(dest: buf.data() + offset, src: info.t.data, n: nbytes);
1306 }
1307 written_bytes += nbytes;
1308
1309 pad(alignment);
1310 }
1311};
1312
1313// file based writer
1314struct gguf_writer_file final : public gguf_writer_base {
1315 FILE * file;
1316
1317 gguf_writer_file(FILE* file) : file(file) {}
1318
1319 using gguf_writer_base::write;
1320
1321 void write(const int8_t val) override {
1322 const auto real_val = static_cast<uint8_t>(val);
1323 const auto ret = fputc(c: real_val, stream: file);
1324 written_bytes++;
1325 if (ret != real_val) {
1326 throw std::runtime_error("unexpected fputc result '" + std::to_string(val: ret) + "' instead of '" + std::to_string(val: (int)real_val) + "'");
1327 }
1328 }
1329
1330 void write(const std::vector<int8_t> & val) override {
1331 const auto ret = fwrite(ptr: val.data(), size: 1, n: val.size(), s: file);
1332 written_bytes += val.size();
1333 if (ret != val.size()) {
1334 throw std::runtime_error("unexpected fwrite number of bytes written, '" + std::to_string(val: ret) + "' instead of '" + std::to_string(val: val.size()) + "'");
1335 }
1336 }
1337
1338 void write_tensor_data(const struct gguf_tensor_info & info, const size_t offset_data, const size_t alignment) override {
1339 GGML_ASSERT(written_bytes - offset_data == info.offset);
1340
1341 GGML_ASSERT(ggml_is_contiguous(&info.t));
1342 const size_t nbytes = ggml_nbytes(tensor: &info.t);
1343
1344 std::vector<int8_t> buf(nbytes);
1345 if (info.t.buffer) {
1346 ggml_backend_tensor_get(tensor: &info.t, data: buf.data(), offset: 0, size: nbytes);
1347 } else {
1348 GGML_ASSERT(info.t.data);
1349 memcpy(dest: buf.data(), src: info.t.data, n: nbytes);
1350 }
1351 write(val: buf);
1352
1353 pad(alignment);
1354 }
1355};
1356
1357template <typename writer_t>
1358static void gguf_write_out(const struct gguf_context * ctx, writer_t & gw, bool only_meta) {
1359 const int64_t n_kv = gguf_get_n_kv(ctx);
1360 const int64_t n_tensors = gguf_get_n_tensors(ctx);
1361
1362 // write header
1363 gw.write(GGUF_MAGIC[0]);
1364 gw.write(GGUF_MAGIC[1]);
1365 gw.write(GGUF_MAGIC[2]);
1366 gw.write(GGUF_MAGIC[3]);
1367 gw.write(ctx->version);
1368 gw.write(n_tensors);
1369 gw.write(n_kv);
1370
1371 // write key-value pairs
1372 for (int64_t i = 0; i < n_kv; ++i) {
1373 gw.write(ctx->kv[i]);
1374 }
1375
1376 // write tensor info
1377 for (int64_t i = 0; i < n_tensors; ++i) {
1378 gw.write_tensor_meta(ctx->info[i]);
1379 }
1380
1381 // we require the data section to be aligned
1382 gw.pad(ctx->alignment);
1383
1384 if (only_meta) {
1385 return;
1386 }
1387
1388 const size_t offset_data = gw.written_bytes;
1389
1390 // write tensor data
1391 for (int64_t i = 0; i < n_tensors; ++i) {
1392 gw.write_tensor_data(ctx->info[i], offset_data, ctx->alignment);
1393 }
1394}
1395
1396void gguf_write_to_buf(const struct gguf_context * ctx, std::vector<int8_t> & buf, bool only_meta) {
1397 gguf_writer_buf gw(buf);
1398 gguf_write_out(ctx, gw, only_meta);
1399}
1400
1401bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta) {
1402 FILE * file = ggml_fopen(fname, mode: "wb");
1403
1404 if (!file) {
1405 GGML_LOG_ERROR("%s: failed to open file '%s' for writing GGUF data\n", __func__, fname);
1406 return false;
1407 }
1408
1409 try {
1410 gguf_writer_file gw(file);
1411 gguf_write_out(ctx, gw, only_meta);
1412 } catch (const std::runtime_error& ex) {
1413 GGML_LOG_ERROR("%s: failed to write GGUF data into '%s': %s\n", __func__, fname, ex.what());
1414 fclose(stream: file);
1415 return false;
1416 }
1417
1418 fclose(stream: file);
1419 return true;
1420}
1421
1422size_t gguf_get_meta_size(const struct gguf_context * ctx) {
1423 // only return size
1424 std::vector<int8_t> buf;
1425 gguf_write_to_buf(ctx, buf, /*only_meta =*/ true);
1426 return buf.size();
1427}
1428
1429void gguf_get_meta_data(const struct gguf_context * ctx, void * data) {
1430 std::vector<int8_t> buf;
1431 gguf_write_to_buf(ctx, buf, /*only_meta =*/ true);
1432 memcpy(dest: data, src: buf.data(), n: buf.size());
1433}
1434