1// This file defines tests for various GGML ops and backends.
2// For the forward pass it asserts that the results of multiple backends computing the same GGML ops are consistent.
3// For the backward pass it asserts that the gradients from backpropagation are consistent
4// with the gradients obtained via the method of finite differences ("grad" mode, this is optional).
5// It is also possible to check the performance ("perf" mode).
6//
7// this file has three sections: Section 1 does general setup, section 2 defines the GGML ops to be tested,
8// and section 3 defines which tests to run.
9// Quick start for adding a new GGML op: Go to section 2 and create a struct that inherits from test_case,
10// then go to section 3 and add an instantiation of your struct.
11
12
13// ##############################
14// ## Section 1: General Setup ##
15// ##############################
16
17
18#include <ggml.h>
19#include <ggml-alloc.h>
20#include <ggml-backend.h>
21#include <ggml-cpp.h>
22
23#include <algorithm>
24#include <array>
25#include <cfloat>
26#include <cinttypes>
27#include <cstdarg>
28#include <cstdint>
29#include <cstdio>
30#include <cstdlib>
31#include <cstring>
32#include <ctime>
33#include <future>
34#include <memory>
35#include <random>
36#include <regex>
37#include <set>
38#include <string>
39#include <string_view>
40#include <thread>
41#include <vector>
42
43static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) {
44 size_t nels = ggml_nelements(tensor);
45 std::vector<float> data(nels);
46 {
47 // parallel initialization
48 static const size_t n_threads = std::thread::hardware_concurrency();
49 // static RNG initialization (revisit if n_threads stops being constant)
50 static std::vector<std::default_random_engine> generators = []() {
51 std::random_device rd;
52 std::vector<std::default_random_engine> vec;
53 vec.reserve(n: n_threads);
54 //for (size_t i = 0; i < n_threads; i++) { vec.emplace_back(1234 + i); } // fixed seed
55 for (size_t i = 0; i < n_threads; i++) { vec.emplace_back(args: rd()); }
56 return vec;
57 }();
58
59 auto init_thread = [&](size_t ith, size_t start, size_t end) {
60 std::uniform_real_distribution<float> distribution(min, max);
61 auto & gen = generators[ith];
62 for (size_t i = start; i < end; i++) {
63 data[i] = distribution(gen);
64 }
65 };
66
67 std::vector<std::future<void>> tasks;
68 tasks.reserve(n: n_threads);
69 for (size_t i = 0; i < n_threads; i++) {
70 size_t start = i*nels/n_threads;
71 size_t end = (i+1)*nels/n_threads;
72 tasks.push_back(x: std::async(policy: std::launch::async, fn&: init_thread, args&: i, args&: start, args&: end));
73 }
74 for (auto & t : tasks) {
75 t.get();
76 }
77 }
78
79 if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_I32) {
80 ggml_backend_tensor_set(tensor, data: data.data(), offset: 0, size: nels * sizeof(float));
81 } else if (ggml_is_quantized(type: tensor->type) || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) {
82 GGML_ASSERT(nels % ggml_blck_size(tensor->type) == 0);
83
84 // dummy importance matrix
85 std::vector<float> imatrix(tensor->ne[0], 1.0f);
86 const float * im = imatrix.data();
87 if (!ggml_quantize_requires_imatrix(type: tensor->type)) {
88 // when the imatrix is optional, we want to test both quantization with and without imatrix
89 // use one of the random numbers to decide
90 if (data[0] > 0.5f*(min + max)) {
91 im = nullptr;
92 }
93 }
94
95 std::vector<uint8_t> dataq(ggml_row_size(type: tensor->type, ne: nels));
96 {
97 // parallel quantization by block
98 size_t blck_size = ggml_blck_size(type: tensor->type);
99 size_t n_blocks = nels / blck_size;
100
101 auto quantize_thread = [&](size_t start, size_t end) {
102 ggml_quantize_chunk(type: tensor->type, src: data.data(), dst: dataq.data(),
103 start: start * blck_size, nrows: end - start, n_per_row: blck_size, imatrix: im);
104 };
105
106 const size_t min_blocks_per_thread = 1;
107 const size_t n_threads = std::min<size_t>(a: std::thread::hardware_concurrency()/2,
108 b: std::max<size_t>(a: 1, b: n_blocks / min_blocks_per_thread));
109 std::vector<std::future<void>> tasks;
110 tasks.reserve(n: n_threads);
111 for (size_t i = 0; i < n_threads; i++) {
112 size_t start = i*n_blocks/n_threads;
113 size_t end = (i+1)*n_blocks/n_threads;
114 tasks.push_back(x: std::async(policy: std::launch::async, fn&: quantize_thread, args&: start, args&: end));
115 }
116 for (auto & t : tasks) {
117 t.get();
118 }
119 }
120 ggml_backend_tensor_set(tensor, data: dataq.data(), offset: 0, size: dataq.size());
121 } else if (tensor->type == GGML_TYPE_I8 || tensor->type == GGML_TYPE_I16 || tensor->type == GGML_TYPE_I32) {
122 // This is going to create some weird integers though.
123 ggml_backend_tensor_set(tensor, data: data.data(), offset: 0, size: ggml_nbytes(tensor));
124 } else if (tensor->type == GGML_TYPE_I64) {
125 // Integers with a size of 8 bytes can be set by mirroring the float data, the specific values are again not really meaningful.
126 const size_t nbytes_half = ggml_nbytes(tensor)/2;
127 ggml_backend_tensor_set(tensor, data: data.data(), offset: 0*nbytes_half, size: nbytes_half);
128 ggml_backend_tensor_set(tensor, data: data.data(), offset: 1*nbytes_half, size: nbytes_half);
129 } else {
130 GGML_ABORT("fatal error");
131 }
132}
133
134// generate an F16 mask where certain blocks are randomly masked with -INF value
135static void init_tensor_kq_mask(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) {
136 GGML_ASSERT(tensor->type == GGML_TYPE_F16);
137
138 GGML_TENSOR_LOCALS( int32_t, ne, tensor, ne);
139
140 std::vector<float> data_f32(ne0*ne1*ne2*ne3);
141 std::vector<ggml_fp16_t> data_f16(ne0*ne1*ne2*ne3);
142
143 std::random_device rd;
144 std::mt19937 gen(rd());
145 std::uniform_real_distribution<float> dis(min, max);
146
147 for (size_t i = 0; i < data_f32.size(); i++) {
148 data_f32[i] = dis(gen);
149 }
150
151 // block size
152 const int blck0 = 128;
153 const int blck1 = 64;
154
155 // number of INF blocks
156 const int n_inf_blocks = 0.1*(ne0*ne1*ne2*ne3)/(blck0*blck1);
157
158 for (int b = 0; b < n_inf_blocks; b++) {
159 const int p3 = (rd() % ne3);
160 const int p2 = (rd() % ne2);
161 const int p1 = (rd() % ne1);
162 const int p0 = (rd() % ne0);
163
164 for (int i1 = 0; i1 < blck1 && p1 + i1 < ne1; i1++) {
165 const int idx = p3*ne2*ne1*ne0 + p2*ne1*ne0 + (p1 + i1)*ne0 + p0;
166
167 for (int i0 = 0; i0 < blck0 && p0 + i0 < ne0; i0++) {
168 data_f32[idx + i0] = -INFINITY;
169 }
170 }
171 }
172
173 ggml_fp32_to_fp16_row(data_f32.data(), data_f16.data(), ne0*ne1*ne2*ne3);
174
175 ggml_backend_tensor_set(tensor, data: data_f16.data(), offset: 0, size: data_f16.size()*sizeof(ggml_fp16_t));
176}
177
178static std::vector<float> tensor_to_float(const ggml_tensor * t) {
179 std::vector<float> tv;
180 tv.reserve(n: ggml_nelements(tensor: t));
181
182 std::vector<uint8_t> buf(ggml_nbytes(tensor: t));
183 ggml_backend_tensor_get(tensor: t, data: buf.data(), offset: 0, size: ggml_nbytes(tensor: t));
184
185 const auto * tt = ggml_get_type_traits(type: t->type);
186 size_t bs = ggml_blck_size(type: t->type);
187 std::vector<float> vq(ggml_blck_size(type: t->type));
188 bool quantized = ggml_is_quantized(type: t->type);
189
190 // access elements by index to avoid gaps in views
191 for (int64_t i3 = 0; i3 < t->ne[3]; i3++) {
192 for (int64_t i2 = 0; i2 < t->ne[2]; i2++) {
193 for (int64_t i1 = 0; i1 < t->ne[1]; i1++) {
194 for (int64_t i0 = 0; i0 < t->ne[0]; i0 += bs) {
195 size_t i = i3*t->nb[3] + i2*t->nb[2] + i1*t->nb[1] + i0/bs*t->nb[0];
196 if (t->type == GGML_TYPE_F16) {
197 tv.push_back(x: ggml_fp16_to_fp32(*(ggml_fp16_t*)&buf[i]));
198 } else if (t->type == GGML_TYPE_BF16) {
199 tv.push_back(x: ggml_bf16_to_fp32(*(ggml_bf16_t*)&buf[i]));
200 } else if (t->type == GGML_TYPE_F32) {
201 tv.push_back(x: *(float *) &buf[i]);
202 } else if (t->type == GGML_TYPE_I64) {
203 tv.push_back(x: (float)*(int64_t *) &buf[i]);
204 } else if (t->type == GGML_TYPE_I32) {
205 tv.push_back(x: (float)*(int32_t *) &buf[i]);
206 } else if (t->type == GGML_TYPE_I16) {
207 tv.push_back(x: (float)*(int16_t *) &buf[i]);
208 } else if (t->type == GGML_TYPE_I8) {
209 tv.push_back(x: (float)*(int8_t *) &buf[i]);
210 } else if (quantized) {
211 tt->to_float(&buf[i], vq.data(), bs);
212 tv.insert(position: tv.end(), first: vq.begin(), last: vq.end());
213 } else {
214 GGML_ABORT("fatal error");
215 }
216 }
217 }
218 }
219 }
220
221 return tv;
222}
223
224// normalized mean squared error = mse(a, b) / mse(a, 0)
225static double nmse(const float * a, const float * b, size_t n) {
226 double mse_a_b = 0.0;
227 double mse_a_0 = 0.0;
228
229 for (size_t i = 0; i < n; i++) {
230 float a_i = a[i];
231 float b_i = b[i];
232
233 mse_a_b += (a_i - b_i) * (a_i - b_i);
234 mse_a_0 += a_i * a_i;
235 }
236
237 return mse_a_b / mse_a_0;
238}
239
240// maximum absolute asymmetry between a and b
241// asymmetry: (a - b) / (a + b)
242// This is more stable than relative error if one of the values fluctuates towards zero.
243// n: number of values to compare.
244// expected_vals: optional vector of expected values for a. If expected_vals is not empty, filter out all comparisons where
245// a does not match any of the expected values. Needed for noncontinuous gradients where the numerical calculation can fail.
246static double mean_abs_asymm(const float * a, const float * b, const size_t n, const std::vector<float> & expected_vals) {
247 double sum = 0.0f;
248
249 size_t nvalid = 0;
250 for (size_t i = 0; i < n; i++) {
251 if (!expected_vals.empty()) {
252 bool matches_any = false;
253 for (const float & ev : expected_vals) {
254 if (fabsf(x: a[i] - ev) < 1e-3f) {
255 matches_any = true;
256 break;
257 }
258 }
259 if (!matches_any) {
260 continue;
261 }
262 }
263
264 const float asymm = (a[i] - b[i]) / (a[i] + b[i]);
265
266 sum += fabsf(x: asymm);
267 nvalid++;
268 }
269
270 return sum/nvalid;
271}
272
273// utils for printing the variables of the test cases
274
275template<typename T>
276static std::string var_to_str(const T & x) {
277 return std::to_string(x);
278}
279
280template<typename T, size_t N>
281static std::string var_to_str(const T (&x)[N]) {
282 std::string s = "[";
283 for (size_t i = 0; i < N; i++) {
284 if (i > 0) {
285 s += ",";
286 }
287 s += var_to_str(x[i]);
288 }
289 s += "]";
290 return s;
291}
292
293template<typename T, size_t N>
294static std::string var_to_str(const std::array<T, N> & x) {
295 std::string s = "[";
296 for (size_t i = 0; i < N; i++) {
297 if (i > 0) {
298 s += ",";
299 }
300 s += var_to_str(x[i]);
301 }
302 s += "]";
303 return s;
304}
305
306static std::string var_to_str(ggml_type type) {
307 return ggml_type_name(type);
308}
309
310static std::string var_to_str(ggml_prec prec) {
311 return prec == GGML_PREC_F32 ? "f32" : "def";
312}
313
314static std::string var_to_str(ggml_op_pool pool) {
315 switch (pool) {
316 case GGML_OP_POOL_AVG: return "avg";
317 case GGML_OP_POOL_MAX: return "max";
318 default: return std::to_string(val: pool);
319 }
320}
321
322static std::string var_to_str(ggml_scale_mode mode) {
323 switch (mode) {
324 case GGML_SCALE_MODE_NEAREST: return "nearest";
325 case GGML_SCALE_MODE_BILINEAR: return "bilinear";
326 default: return std::to_string(val: mode);
327 }
328}
329
330#define VAR_TO_STR(x) (#x "=" + var_to_str(x))
331
332#define VARS_TO_STR1(a) VAR_TO_STR(a)
333#define VARS_TO_STR2(a, b) VAR_TO_STR(a) + "," + VAR_TO_STR(b)
334#define VARS_TO_STR3(a, b, c) VAR_TO_STR(a) + "," + VARS_TO_STR2(b, c)
335#define VARS_TO_STR4(a, b, c, d) VAR_TO_STR(a) + "," + VARS_TO_STR3(b, c, d)
336#define VARS_TO_STR5(a, b, c, d, e) VAR_TO_STR(a) + "," + VARS_TO_STR4(b, c, d, e)
337#define VARS_TO_STR6(a, b, c, d, e, f) VAR_TO_STR(a) + "," + VARS_TO_STR5(b, c, d, e, f)
338#define VARS_TO_STR7(a, b, c, d, e, f, g) VAR_TO_STR(a) + "," + VARS_TO_STR6(b, c, d, e, f, g)
339#define VARS_TO_STR8(a, b, c, d, e, f, g, h) VAR_TO_STR(a) + "," + VARS_TO_STR7(b, c, d, e, f, g, h)
340#define VARS_TO_STR9(a, b, c, d, e, f, g, h, i) VAR_TO_STR(a) + "," + VARS_TO_STR8(b, c, d, e, f, g, h, i)
341#define VARS_TO_STR10(a, b, c, d, e, f, g, h, i, j) VAR_TO_STR(a) + "," + VARS_TO_STR9(b, c, d, e, f, g, h, i, j)
342#define VARS_TO_STR11(a, b, c, d, e, f, g, h, i, j, k) VAR_TO_STR(a) + "," + VARS_TO_STR10(b, c, d, e, f, g, h, i, j, k)
343#define VARS_TO_STR12(a, b, c, d, e, f, g, h, i, j, k, l) VAR_TO_STR(a) + "," + VARS_TO_STR11(b, c, d, e, f, g, h, i, j, k, l)
344#define VARS_TO_STR13(a, b, c, d, e, f, g, h, i, j, k, l, m) VAR_TO_STR(a) + "," + VARS_TO_STR12(b, c, d, e, f, g, h, i, j, k, l, m)
345#define VARS_TO_STR14(a, b, c, d, e, f, g, h, i, j, k, l, m, n) VAR_TO_STR(a) + "," + VARS_TO_STR13(b, c, d, e, f, g, h, i, j, k, l, m, n)
346#define VARS_TO_STR15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) VAR_TO_STR(a) + "," + VARS_TO_STR14(b, c, d, e, f, g, h, i, j, k, l, m, n, o)
347#define VARS_TO_STR16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) VAR_TO_STR(a) + "," + VARS_TO_STR15(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p)
348
349#ifdef GGML_USE_SYCL
350static bool inline _isinf(float f) {
351 return (*(uint32_t *)&f & 0x7fffffff) == 0x7f800000;
352}
353#else
354static bool inline _isinf(float f) { return std::isinf(x: f); }
355#endif
356
357// accept FLT_MAX as infinity
358static bool isinf_or_max(float f) {
359 return _isinf(f) || f == FLT_MAX || f == -FLT_MAX;
360}
361
362static bool ggml_is_view_op(enum ggml_op op) {
363 return op == GGML_OP_VIEW || op == GGML_OP_RESHAPE || op == GGML_OP_PERMUTE || op == GGML_OP_TRANSPOSE;
364}
365
366enum test_mode {
367 MODE_TEST,
368 MODE_PERF,
369 MODE_GRAD,
370 MODE_SUPPORT,
371};
372
373// Output format support similar to llama-bench
374enum output_formats { CONSOLE, SQL, CSV };
375
376static const char * output_format_str(output_formats format) {
377 switch (format) {
378 case CONSOLE:
379 return "console";
380 case SQL:
381 return "sql";
382 case CSV:
383 return "csv";
384 default:
385 GGML_ABORT("invalid output format");
386 }
387}
388
389static bool output_format_from_str(const std::string & s, output_formats & format) {
390 if (s == "console") {
391 format = CONSOLE;
392 } else if (s == "sql") {
393 format = SQL;
394 } else if (s == "csv") {
395 format = CSV;
396 } else {
397 return false;
398 }
399 return true;
400}
401
402// Test result structure for SQL output
403struct test_result {
404 std::string test_time;
405 std::string build_commit;
406 std::string backend_name;
407 std::string op_name;
408 std::string op_params;
409 std::string test_mode;
410 bool supported;
411 bool passed;
412 std::string error_message;
413 double time_us;
414 double flops;
415 double bandwidth_gb_s;
416 size_t memory_kb;
417 int n_runs;
418 std::string device_description;
419 std::string backend_reg_name;
420
421 test_result() {
422 // Initialize with default values
423 time_us = 0.0;
424 flops = 0.0;
425 bandwidth_gb_s = 0.0;
426 memory_kb = 0;
427 n_runs = 0;
428 supported = false;
429 passed = false;
430
431 // Set test time
432 time_t t = time(NULL);
433 char buf[32];
434 std::strftime(s: buf, maxsize: sizeof(buf), format: "%FT%TZ", tp: gmtime(timer: &t));
435 test_time = buf;
436
437 // Set build info
438 build_commit = ggml_commit();
439 }
440
441 test_result(const std::string & backend_name, const std::string & op_name, const std::string & op_params,
442 const std::string & test_mode, bool supported, bool passed, const std::string & error_message = "",
443 double time_us = 0.0, double flops = 0.0, double bandwidth_gb_s = 0.0, size_t memory_kb = 0,
444 int n_runs = 0, const std::string & device_description = "", const std::string & backend_reg_name = "") :
445 backend_name(backend_name),
446 op_name(op_name),
447 op_params(op_params),
448 test_mode(test_mode),
449 supported(supported),
450 passed(passed),
451 error_message(error_message),
452 time_us(time_us),
453 flops(flops),
454 bandwidth_gb_s(bandwidth_gb_s),
455 memory_kb(memory_kb),
456 n_runs(n_runs),
457 device_description(device_description),
458 backend_reg_name(backend_reg_name) {
459 // Set test time
460 time_t t = time(NULL);
461 char buf[32];
462 std::strftime(s: buf, maxsize: sizeof(buf), format: "%FT%TZ", tp: gmtime(timer: &t));
463 test_time = buf;
464
465 // Set build info
466 build_commit = ggml_commit();
467 }
468
469 static const std::vector<std::string> & get_fields() {
470 static const std::vector<std::string> fields = {
471 "test_time", "build_commit", "backend_name", "op_name", "op_params", "test_mode", "supported",
472 "passed", "error_message", "time_us", "flops", "bandwidth_gb_s", "memory_kb", "n_runs",
473 "device_description", "backend_reg_name"
474 };
475 return fields;
476 }
477
478 enum field_type { STRING, BOOL, INT, FLOAT };
479
480 static field_type get_field_type(const std::string & field) {
481 if (field == "supported" || field == "passed") {
482 return BOOL;
483 }
484 if (field == "memory_kb" || field == "n_runs") {
485 return INT;
486 }
487 if (field == "time_us" || field == "flops" || field == "bandwidth_gb_s") {
488 return FLOAT;
489 }
490 return STRING;
491 }
492
493 std::vector<std::string> get_values() const {
494 return { test_time,
495 build_commit,
496 backend_name,
497 op_name,
498 op_params,
499 test_mode,
500 std::to_string(val: supported),
501 std::to_string(val: passed),
502 error_message,
503 std::to_string(val: time_us),
504 std::to_string(val: flops),
505 std::to_string(val: bandwidth_gb_s),
506 std::to_string(val: memory_kb),
507 std::to_string(val: n_runs),
508 device_description,
509 backend_reg_name };
510 }
511};
512
513// Printer classes for different output formats
514enum class test_status_t { NOT_SUPPORTED, OK, FAIL, SKIPPED };
515
516struct test_operation_info {
517 std::string op_name;
518 std::string op_params;
519 std::string backend_name;
520 test_status_t status = test_status_t::OK;
521 std::string failure_reason;
522
523 // Additional information fields that were previously in separate structs
524 std::string error_component;
525 std::string error_details;
526
527 // Gradient info
528 int64_t gradient_index = -1;
529 std::string gradient_param_name;
530 float gradient_value = 0.0f;
531
532 // MAA error info
533 double maa_error = 0.0;
534 double maa_threshold = 0.0;
535
536 // Flags for different types of information
537 bool has_error = false;
538 bool has_gradient_info = false;
539 bool has_maa_error = false;
540 bool is_compare_failure = false;
541 bool is_large_tensor_skip = false;
542
543 test_operation_info() = default;
544
545 test_operation_info(const std::string & op_name, const std::string & op_params, const std::string & backend_name,
546 test_status_t status = test_status_t::OK, const std::string & failure_reason = "") :
547 op_name(op_name),
548 op_params(op_params),
549 backend_name(backend_name),
550 status(status),
551 failure_reason(failure_reason) {}
552
553 // Set error information
554 void set_error(const std::string & component, const std::string & details) {
555 has_error = true;
556 error_component = component;
557 error_details = details;
558 if (status == test_status_t::OK) {
559 status = test_status_t::FAIL;
560 }
561 }
562
563 // Set gradient information
564 void set_gradient_info(int64_t index, const std::string & param_name, float value) {
565 has_gradient_info = true;
566 gradient_index = index;
567 gradient_param_name = param_name;
568 gradient_value = value;
569 if (status == test_status_t::OK) {
570 status = test_status_t::FAIL;
571 }
572 }
573
574 // Set MAA error information
575 void set_maa_error(double error, double threshold) {
576 has_maa_error = true;
577 maa_error = error;
578 maa_threshold = threshold;
579 if (status == test_status_t::OK) {
580 status = test_status_t::FAIL;
581 }
582 }
583
584 // Set compare failure
585 void set_compare_failure() {
586 is_compare_failure = true;
587 if (status == test_status_t::OK) {
588 status = test_status_t::FAIL;
589 }
590 }
591
592 // Set large tensor skip
593 void set_large_tensor_skip() { is_large_tensor_skip = true; }
594};
595
596struct test_summary_info {
597 size_t tests_passed;
598 size_t tests_total;
599 bool is_backend_summary = false; // true for backend summary, false for test summary
600
601 test_summary_info() = default;
602
603 test_summary_info(size_t tests_passed, size_t tests_total, bool is_backend_summary = false) :
604 tests_passed(tests_passed),
605 tests_total(tests_total),
606 is_backend_summary(is_backend_summary) {}
607};
608
609struct testing_start_info {
610 size_t device_count;
611
612 testing_start_info() = default;
613
614 testing_start_info(size_t device_count) : device_count(device_count) {}
615};
616
617struct backend_init_info {
618 size_t device_index;
619 size_t total_devices;
620 std::string device_name;
621 bool skipped = false;
622 std::string skip_reason;
623 std::string description;
624 size_t memory_total_mb = 0;
625 size_t memory_free_mb = 0;
626 bool has_memory_info = false;
627
628 backend_init_info() = default;
629
630 backend_init_info(size_t device_index, size_t total_devices, const std::string & device_name, bool skipped = false,
631 const std::string & skip_reason = "", const std::string & description = "",
632 size_t memory_total_mb = 0, size_t memory_free_mb = 0, bool has_memory_info = false) :
633 device_index(device_index),
634 total_devices(total_devices),
635 device_name(device_name),
636 skipped(skipped),
637 skip_reason(skip_reason),
638 description(description),
639 memory_total_mb(memory_total_mb),
640 memory_free_mb(memory_free_mb),
641 has_memory_info(has_memory_info) {}
642};
643
644struct backend_status_info {
645 std::string backend_name;
646 test_status_t status;
647
648 backend_status_info() = default;
649
650 backend_status_info(const std::string & backend_name, test_status_t status) :
651 backend_name(backend_name),
652 status(status) {}
653};
654
655struct overall_summary_info {
656 size_t backends_passed;
657 size_t backends_total;
658 bool all_passed;
659
660 overall_summary_info() = default;
661
662 overall_summary_info(size_t backends_passed, size_t backends_total, bool all_passed) :
663 backends_passed(backends_passed),
664 backends_total(backends_total),
665 all_passed(all_passed) {}
666};
667
668struct printer {
669 virtual ~printer() {}
670
671 FILE * fout = stdout;
672
673 virtual void print_header() {}
674
675 virtual void print_test_result(const test_result & result) = 0;
676
677 virtual void print_footer() {}
678
679 virtual void print_operation(const test_operation_info & info) { (void) info; }
680
681 virtual void print_summary(const test_summary_info & info) { (void) info; }
682
683 virtual void print_testing_start(const testing_start_info & info) { (void) info; }
684
685 virtual void print_backend_init(const backend_init_info & info) { (void) info; }
686
687 virtual void print_backend_status(const backend_status_info & info) { (void) info; }
688
689 virtual void print_overall_summary(const overall_summary_info & info) { (void) info; }
690
691 virtual void print_failed_tests(const std::vector<std::string> & failed_tests) { (void) failed_tests; }
692};
693
694struct console_printer : public printer {
695 void print_test_result(const test_result & result) override {
696 if (result.test_mode == "test") {
697 print_test_console(result);
698 } else if (result.test_mode == "perf") {
699 print_perf_console(result);
700 } else if (result.test_mode == "support") {
701 print_support_console(result);
702 }
703 }
704
705 void print_operation(const test_operation_info & info) override {
706 printf(format: " %s(%s): ", info.op_name.c_str(), info.op_params.c_str());
707 fflush(stdout);
708
709 // Handle large tensor skip first
710 if (info.is_large_tensor_skip) {
711 printf(format: "skipping large tensors for speed \n");
712 return;
713 }
714
715 // Handle not supported status
716 if (info.status == test_status_t::NOT_SUPPORTED) {
717 if (!info.failure_reason.empty()) {
718 printf(format: "not supported [%s]\n", info.failure_reason.c_str());
719 } else {
720 printf(format: "not supported [%s]\n", info.backend_name.c_str());
721 }
722 return;
723 }
724
725 // Handle errors and additional information
726 if (info.has_error) {
727 if (info.error_component == "allocation") {
728 fprintf(stderr, format: "failed to allocate tensors [%s] ", info.backend_name.c_str());
729 } else if (info.error_component == "backend") {
730 fprintf(stderr, format: " Failed to initialize %s backend\n", info.backend_name.c_str());
731 } else {
732 fprintf(stderr, format: "Error in %s: %s\n", info.error_component.c_str(), info.error_details.c_str());
733 }
734 }
735
736 // Handle gradient info
737 if (info.has_gradient_info) {
738 printf(format: "[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", info.op_name.c_str(), info.gradient_index,
739 info.gradient_param_name.c_str(), info.gradient_value);
740 }
741
742 // Handle MAA error
743 if (info.has_maa_error) {
744 printf(format: "[%s] MAA = %.9f > %.9f ", info.op_name.c_str(), info.maa_error, info.maa_threshold);
745 }
746
747 // Handle compare failure
748 if (info.is_compare_failure) {
749 printf(format: "compare failed ");
750 }
751
752 // Print final status
753 if (info.status == test_status_t::OK) {
754 printf(format: "\033[1;32mOK\033[0m\n");
755 } else {
756 printf(format: "\033[1;31mFAIL\033[0m\n");
757 }
758 }
759
760 void print_summary(const test_summary_info & info) override {
761 if (info.is_backend_summary) {
762 printf(format: "%zu/%zu backends passed\n", info.tests_passed, info.tests_total);
763 } else {
764 printf(format: " %zu/%zu tests passed\n", info.tests_passed, info.tests_total);
765 }
766 }
767
768 void print_backend_status(const backend_status_info & info) override {
769 printf(format: " Backend %s: ", info.backend_name.c_str());
770 if (info.status == test_status_t::OK) {
771 printf(format: "\033[1;32mOK\033[0m\n");
772 } else {
773 printf(format: "\033[1;31mFAIL\033[0m\n");
774 }
775 }
776
777 void print_testing_start(const testing_start_info & info) override {
778 printf(format: "Testing %zu devices\n\n", info.device_count);
779 }
780
781 void print_backend_init(const backend_init_info & info) override {
782 printf(format: "Backend %zu/%zu: %s\n", info.device_index + 1, info.total_devices, info.device_name.c_str());
783
784 if (info.skipped) {
785 printf(format: " %s\n", info.skip_reason.c_str());
786 return;
787 }
788
789 if (!info.description.empty()) {
790 printf(format: " Device description: %s\n", info.description.c_str());
791 }
792
793 if (info.has_memory_info) {
794 printf(format: " Device memory: %zu MB (%zu MB free)\n", info.memory_total_mb, info.memory_free_mb);
795 }
796
797 printf(format: "\n");
798 }
799
800 void print_overall_summary(const overall_summary_info & info) override {
801 printf(format: "%zu/%zu backends passed\n", info.backends_passed, info.backends_total);
802 if (info.all_passed) {
803 printf(format: "\033[1;32mOK\033[0m\n");
804 } else {
805 printf(format: "\033[1;31mFAIL\033[0m\n");
806 }
807 }
808
809 void print_failed_tests(const std::vector<std::string> & failed_tests) override {
810 if (failed_tests.empty()) {
811 return;
812 }
813
814 printf(format: "\nFailing tests:\n");
815 for (const auto & test_name : failed_tests) {
816 printf(format: " %s\n", test_name.c_str());
817 }
818 }
819
820 private:
821 void print_test_console(const test_result & result) {
822 printf(format: " %s(%s): ", result.op_name.c_str(), result.op_params.c_str());
823 fflush(stdout);
824
825 if (!result.supported) {
826 printf(format: "not supported [%s] ", result.backend_name.c_str());
827 printf(format: "\n");
828 return;
829 }
830
831 if (result.passed) {
832 printf(format: "\033[1;32mOK\033[0m\n");
833 } else {
834 printf(format: "\033[1;31mFAIL\033[0m\n");
835 }
836 }
837
838 void print_perf_console(const test_result & result) {
839 int len = printf(format: " %s(%s): ", result.op_name.c_str(), result.op_params.c_str());
840 fflush(stdout);
841
842 if (!result.supported) {
843 printf(format: "not supported\n");
844 return;
845 }
846
847 // align while also leaving some margin for variations in parameters
848 int align = 8;
849 int last = (len + align - 1) / align * align;
850 if (last - len < 5) {
851 last += align;
852 }
853 printf(format: "%*s", last - len, "");
854
855 printf(format: " %8d runs - %8.2f us/run - ", result.n_runs, result.time_us);
856
857 if (result.flops > 0) {
858 auto format_flops = [](double flops) -> std::string {
859 char buf[256];
860 if (flops >= 1e12) {
861 snprintf(s: buf, maxlen: sizeof(buf), format: "%6.2f TFLOP", flops / 1e12);
862 } else if (flops >= 1e9) {
863 snprintf(s: buf, maxlen: sizeof(buf), format: "%6.2f GFLOP", flops / 1e9);
864 } else if (flops >= 1e6) {
865 snprintf(s: buf, maxlen: sizeof(buf), format: "%6.2f MFLOP", flops / 1e6);
866 } else {
867 snprintf(s: buf, maxlen: sizeof(buf), format: "%6.2f kFLOP", flops / 1e3);
868 }
869 return buf;
870 };
871 uint64_t op_flops_per_run = result.flops * result.time_us / 1e6;
872 printf(format: "%s/run - \033[1;34m%sS\033[0m", format_flops(op_flops_per_run).c_str(),
873 format_flops(result.flops).c_str());
874 } else {
875 printf(format: "%8zu kB/run - \033[1;34m%7.2f GB/s\033[0m", result.memory_kb, result.bandwidth_gb_s);
876 }
877 printf(format: "\n");
878 }
879
880 void print_support_console(const test_result & result) {
881 printf(format: " %s(%s): ", result.op_name.c_str(), result.op_params.c_str());
882 fflush(stdout);
883
884 if (result.supported) {
885 printf(format: "\033[1;32mSUPPORTED\033[0m\n");
886 } else {
887 printf(format: "\033[1;31mNOT SUPPORTED\033[0m\n");
888 }
889 }
890};
891
892struct sql_printer : public printer {
893 static std::string get_sql_field_type(const std::string & field) {
894 switch (test_result::get_field_type(field)) {
895 case test_result::STRING:
896 return "TEXT";
897 case test_result::BOOL:
898 case test_result::INT:
899 return "INTEGER";
900 case test_result::FLOAT:
901 return "REAL";
902 default:
903 GGML_ABORT("invalid field type");
904 }
905 }
906
907 void print_header() override {
908 std::vector<std::string> fields = test_result::get_fields();
909 fprintf(stream: fout, format: "CREATE TABLE IF NOT EXISTS test_backend_ops (\n");
910 for (size_t i = 0; i < fields.size(); i++) {
911 fprintf(stream: fout, format: " %s %s%s\n", fields[i].c_str(), get_sql_field_type(field: fields[i]).c_str(),
912 i < fields.size() - 1 ? "," : "");
913 }
914 fprintf(stream: fout, format: ");\n\n");
915 }
916
917 void print_test_result(const test_result & result) override {
918 fprintf(stream: fout, format: "INSERT INTO test_backend_ops (");
919 std::vector<std::string> fields = test_result::get_fields();
920 for (size_t i = 0; i < fields.size(); i++) {
921 fprintf(stream: fout, format: "%s%s", fields[i].c_str(), i < fields.size() - 1 ? ", " : "");
922 }
923 fprintf(stream: fout, format: ") VALUES (");
924 std::vector<std::string> values = result.get_values();
925 for (size_t i = 0; i < values.size(); i++) {
926 fprintf(stream: fout, format: "'%s'%s", values[i].c_str(), i < values.size() - 1 ? ", " : "");
927 }
928 fprintf(stream: fout, format: ");\n");
929 }
930};
931
932struct csv_printer : public printer {
933 void print_header() override {
934
935 std::vector<std::string> fields = test_result::get_fields();
936 std::vector<std::string> fields_csv = get_fields_csv();
937 for (size_t i = 0; i < fields.size(); i++) {
938 if (std::find(first: std::begin(cont&: fields_csv), last: std::end(cont&: fields_csv), val: fields[i]) == std::end(cont&: fields_csv)) {
939 continue;
940 }
941 printf(format: "\"%s\"%s", fields[i].c_str(), i < fields.size() - 1 ? "," : "");
942 }
943 printf(format: "\n");
944 }
945
946 void print_test_result(const test_result & result) override {
947
948 std::vector<std::string> values = result.get_values();
949 std::vector<std::string> fields = test_result::get_fields();
950 std::vector<std::string> fields_csv = get_fields_csv();
951
952 for (size_t i = 0; i < values.size(); i++) {
953
954 if (std::find(first: std::begin(cont&: fields_csv), last: std::end(cont&: fields_csv), val: fields[i]) == std::end(cont&: fields_csv)) {
955 continue;
956 }
957
958 // Escape quotes and wrap in quotes for CSV
959 std::string escaped_value = values[i];
960 size_t pos = 0;
961 while ((pos = escaped_value.find(s: "\"", pos: pos)) != std::string::npos) {
962 escaped_value.replace(pos: pos, n1: 1, s: "\"\"");
963 pos += 2;
964 }
965 printf(format: "\"%s\"%s", escaped_value.c_str(), i < values.size() - 1 ? "," : "");
966 }
967 printf(format: "\n");
968 }
969
970 static std::vector<std::string> get_fields_csv() {
971 return {
972 "op_name",
973 "op_params",
974 "supported",
975 "error_message",
976 "test_mode",
977 "backend_reg_name",
978 "backend_name",
979 };
980 }
981
982};
983
984static std::unique_ptr<printer> create_printer(output_formats format) {
985 switch (format) {
986 case CONSOLE:
987 return std::make_unique<console_printer>();
988 case SQL:
989 return std::make_unique<sql_printer>();
990 case CSV:
991 return std::make_unique<csv_printer>();
992 }
993 GGML_ABORT("invalid output format");
994}
995
996struct test_case {
997 virtual ~test_case() {}
998
999 virtual std::string op_desc(ggml_tensor * t) {
1000 return ggml_op_desc(t);
1001 }
1002
1003 virtual std::string vars() {
1004 return "";
1005 }
1006
1007 virtual ggml_tensor * build_graph(ggml_context * ctx) = 0;
1008
1009 virtual double max_nmse_err() {
1010 return 1e-7;
1011 }
1012
1013 virtual double max_maa_err() {
1014 return 1e-4;
1015 }
1016
1017 virtual float grad_eps() {
1018 return 1e-1f;
1019 }
1020
1021 // If false, estimate gradient with 2 points, neglects 3rd order derivative and higher.
1022 // If true, estimate gradient with 4 points, neglects 5th order derivative and higher.
1023 virtual bool grad_precise() {
1024 return false;
1025 }
1026
1027 // Skip gradient checks if total number of gradients to be checked is larger than this (to speed up the tests).
1028 virtual int64_t grad_nmax() {
1029 return 10000;
1030 }
1031
1032 // No effect if empty.
1033 // If not empty, skip all gradient checks where the numerical result does not match any of the values.
1034 // Needed for dealing with noncontinuous gradients (e.g. ReLU) where estimation using finite differences is unreliable.
1035 virtual std::vector<float> grad_expect() {
1036 return {};
1037 }
1038
1039 virtual void initialize_tensors(ggml_context * ctx) {
1040 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, tensor: t)) {
1041 init_tensor_uniform(tensor: t);
1042 }
1043 }
1044
1045 virtual size_t op_size(ggml_tensor * t) {
1046 size_t size = ggml_nbytes(tensor: t);
1047 // add source tensors
1048 for (int i = 0; i < GGML_MAX_SRC; i++) {
1049 if (t->src[i] != NULL) {
1050 size += ggml_nbytes(tensor: t->src[i]);
1051 }
1052 }
1053 return size;
1054 }
1055
1056 virtual uint64_t op_flops(ggml_tensor * t) {
1057 GGML_UNUSED(t);
1058 return 0;
1059 }
1060
1061 virtual bool run_whole_graph() { return false; }
1062
1063 ggml_cgraph * gf = nullptr;
1064 ggml_cgraph * gb = nullptr;
1065
1066 static const int sentinel_size = 1024;
1067
1068 test_mode mode;
1069
1070 std::vector<ggml_tensor *> sentinels;
1071
1072 std::string current_op_name;
1073
1074 void add_sentinel(ggml_context * ctx) {
1075 if (mode == MODE_PERF || mode == MODE_GRAD || mode == MODE_SUPPORT) {
1076 return;
1077 }
1078 ggml_tensor * sentinel = ::ggml_new_tensor_1d(ctx, type: GGML_TYPE_F32, ne0: sentinel_size);
1079 ggml_format_name(tensor: sentinel, fmt: "sent_%zu", sentinels.size());
1080 sentinels.push_back(x: sentinel);
1081 }
1082
1083 // hijack ggml_new_tensor to add sentinels after each tensor to check for overflows in the backend
1084
1085 ggml_tensor * ggml_new_tensor(ggml_context * ctx, ggml_type type, int n_dims, const int64_t * ne) {
1086 ggml_tensor * t = ::ggml_new_tensor(ctx, type, n_dims, ne);
1087 add_sentinel(ctx);
1088 return t;
1089 }
1090
1091 ggml_tensor * ggml_new_tensor_1d(ggml_context * ctx, ggml_type type, int64_t ne0) {
1092 ggml_tensor * t = ::ggml_new_tensor_1d(ctx, type, ne0);
1093 add_sentinel(ctx);
1094 return t;
1095 }
1096
1097 ggml_tensor * ggml_new_tensor_2d(ggml_context * ctx, ggml_type type, int64_t ne0, int64_t ne1) {
1098 ggml_tensor * t = ::ggml_new_tensor_2d(ctx, type, ne0, ne1);
1099 add_sentinel(ctx);
1100 return t;
1101 }
1102
1103 ggml_tensor * ggml_new_tensor_3d(ggml_context * ctx, ggml_type type, int64_t ne0, int64_t ne1, int64_t ne2) {
1104 ggml_tensor * t = ::ggml_new_tensor_3d(ctx, type, ne0, ne1, ne2);
1105 add_sentinel(ctx);
1106 return t;
1107 }
1108
1109 ggml_tensor * ggml_new_tensor_4d(ggml_context * ctx, ggml_type type, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3) {
1110 ggml_tensor * t = ::ggml_new_tensor_4d(ctx, type, ne0, ne1, ne2, ne3);
1111 add_sentinel(ctx);
1112 return t;
1113 }
1114
1115 // Checks an op against the test filter, which is a comma separated list of OP names or specific variations
1116 bool matches_filter(ggml_tensor * op, const char * op_names_filter) {
1117 if (op_names_filter) {
1118 const auto op_name = op_desc(t: op);
1119 const auto op_full_name = op_name + "(" + vars() + ")";
1120 std::string_view filter(op_names_filter);
1121 while (!filter.empty()) {
1122 auto comma_pos = filter.find_first_of(c: ',');
1123 const auto lparen_pos = filter.find_first_of(c: '(');
1124 if (lparen_pos < comma_pos) {
1125 auto rparen_pos = filter.find_first_of(c: ')');
1126 comma_pos = filter.find_first_of(c: ',', pos: rparen_pos);
1127 const auto op_filter = filter.substr(pos: 0, n: comma_pos);
1128 if (op_filter == op_full_name) {
1129 return true;
1130 }
1131 } else {
1132 const auto op_filter = filter.substr(pos: 0, n: comma_pos);
1133 if (op_filter == op_name) {
1134 return true;
1135 }
1136 }
1137 filter = comma_pos != std::string_view::npos ? filter.substr(pos: comma_pos + 1) : "";
1138 }
1139 return false;
1140 } else {
1141 return true;
1142 }
1143 }
1144
1145 test_status_t eval(ggml_backend_t backend1,
1146 ggml_backend_t backend2,
1147 const char * op_names_filter,
1148 printer * output_printer) {
1149 mode = MODE_TEST;
1150
1151 ggml_init_params params = {
1152 /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead(),
1153 /* .mem_base = */ NULL,
1154 /* .no_alloc = */ true,
1155 };
1156 ggml_context * ctx = ggml_init(params);
1157 GGML_ASSERT(ctx);
1158
1159 gf = ggml_new_graph(ctx);
1160
1161 // pre-graph sentinel
1162 add_sentinel(ctx);
1163
1164 ggml_tensor * out = build_graph(ctx);
1165 current_op_name = op_desc(t: out);
1166
1167 if (!matches_filter(op: out, op_names_filter)) {
1168 //printf(" %s: skipping\n", op_desc(out).c_str());
1169 ggml_free(ctx);
1170 return test_status_t::SKIPPED;
1171 }
1172
1173 // check if the backends support the ops
1174 bool supported = true;
1175 for (ggml_backend_t backend : {backend1, backend2}) {
1176 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
1177 if (!ggml_backend_supports_op(backend, op: t)) {
1178 supported = false;
1179 break;
1180 }
1181 }
1182 }
1183
1184 if (!supported) {
1185 // Create test result for unsupported operation
1186 test_result result(ggml_backend_name(backend: backend1), current_op_name, vars(), "test",
1187 false, false, "not supported");
1188
1189 if (output_printer) {
1190 output_printer->print_test_result(result);
1191 }
1192
1193 ggml_free(ctx);
1194 return test_status_t::NOT_SUPPORTED;
1195 }
1196
1197 // post-graph sentinel
1198 add_sentinel(ctx);
1199
1200 // allocate
1201 ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend: backend1);
1202
1203 if (buf == NULL) {
1204 printf(format: "failed to allocate tensors [%s] ", ggml_backend_name(backend: backend1));
1205 ggml_free(ctx);
1206 return test_status_t::FAIL;
1207 }
1208
1209 // build graph
1210 ggml_build_forward_expand(cgraph: gf, tensor: out);
1211
1212 // add sentinels as graph nodes so that they are checked in the callback
1213 for (ggml_tensor * sentinel : sentinels) {
1214 ggml_graph_add_node(cgraph: gf, tensor: sentinel);
1215 }
1216
1217 // randomize tensors
1218 initialize_tensors(ctx);
1219
1220 // compare
1221 struct callback_userdata {
1222 bool ok;
1223 double max_err;
1224 ggml_backend_t backend1;
1225 ggml_backend_t backend2;
1226 };
1227
1228 callback_userdata ud {
1229 .ok: true,
1230 .max_err: max_nmse_err(),
1231 .backend1: backend1,
1232 .backend2: backend2
1233 };
1234
1235 auto callback = [](int index, ggml_tensor * t1, ggml_tensor * t2, void * user_data) -> bool {
1236 callback_userdata * ud = (callback_userdata *) user_data;
1237 const char * bn1 = ggml_backend_name(backend: ud->backend1);
1238 const char * bn2 = ggml_backend_name(backend: ud->backend2);
1239
1240 if (t1->op == GGML_OP_NONE) {
1241 // sentinels must be unchanged
1242 std::vector<uint8_t> t1_data(ggml_nbytes(tensor: t1));
1243 std::vector<uint8_t> t2_data(ggml_nbytes(tensor: t2));
1244 ggml_backend_tensor_get(tensor: t1, data: t1_data.data(), offset: 0, size: ggml_nbytes(tensor: t1));
1245 ggml_backend_tensor_get(tensor: t2, data: t2_data.data(), offset: 0, size: ggml_nbytes(tensor: t2));
1246
1247 if (memcmp(s1: t1_data.data(), s2: t2_data.data(), n: ggml_nbytes(tensor: t1)) != 0) {
1248 printf(format: "sentinel mismatch: %s ", t1->name);
1249 ud->ok = false;
1250 return true;
1251 }
1252 }
1253
1254 std::vector<float> f1 = tensor_to_float(t: t1);
1255 std::vector<float> f2 = tensor_to_float(t: t2);
1256
1257 for (size_t i = 0; i < f1.size(); i++) {
1258 // check for nans
1259 if (std::isnan(x: f1[i]) || std::isnan(x: f2[i])) {
1260 printf(format: "[%s] NaN at index %zu (%s=%f %s=%f) ", ggml_op_desc(t: t1), i, bn1, f1[i], bn2, f2[i]);
1261 ud->ok = false;
1262 return true;
1263 }
1264 // check for infs: both must be inf of the same sign, or both must be finite
1265 if (isinf_or_max(f: f1[i]) || isinf_or_max(f: f2[i])) {
1266 if (isinf_or_max(f: f1[i]) && isinf_or_max(f: f2[i])) {
1267 if (std::signbit(x: f1[i]) != std::signbit(x: f2[i])) {
1268 printf(format: "[%s] inf sign mismatch: %s=%f %s=%f ", ggml_op_desc(t: t1), bn1, f1[i], bn2, f2[i]);
1269 ud->ok = false;
1270 return true;
1271 }
1272 } else {
1273 printf(format: "[%s] inf mismatch: %s=%f %s=%f ", ggml_op_desc(t: t1), bn1, f1[i], bn2, f2[i]);
1274 ud->ok = false;
1275 return true;
1276 }
1277 }
1278 }
1279
1280 double err = nmse(a: f1.data(), b: f2.data(), n: f1.size());
1281 if (err > ud->max_err) {
1282 printf(format: "[%s] NMSE = %.9f > %.9f ", ggml_op_desc(t: t1), err, ud->max_err);
1283 //for (int i = 0; i < (int) f1.size(); i++) {
1284 // printf("%5d %9.6f %9.6f, diff = %9.6f\n", i, f1[i], f2[i], f1[i] - f2[i]);
1285 //}
1286 //printf("\n");
1287 //exit(1);
1288 ud->ok = false;
1289 }
1290 return true;
1291
1292 GGML_UNUSED(index);
1293 };
1294
1295 const bool cmp_ok = ggml_backend_compare_graph_backend(backend1, backend2, graph: gf, callback, user_data: &ud, test_node: run_whole_graph() ? out : nullptr);
1296
1297 ggml_backend_buffer_free(buffer: buf);
1298
1299 ggml_free(ctx);
1300
1301 // Create test result
1302 bool test_passed = ud.ok && cmp_ok;
1303 std::string error_msg = test_passed ? "" : (!cmp_ok ? "compare failed" : "test failed");
1304 test_result result(ggml_backend_name(backend: backend1), current_op_name, vars(), "test", supported, test_passed,
1305 error_msg);
1306
1307 if (output_printer) {
1308 output_printer->print_test_result(result);
1309 }
1310
1311 return test_passed ? test_status_t::OK : test_status_t::FAIL;
1312 }
1313
1314 bool eval_perf(ggml_backend_t backend, const char * op_names_filter, printer * output_printer) {
1315 mode = MODE_PERF;
1316
1317 static const size_t graph_nodes = 8192;
1318
1319 ggml_init_params params = {
1320 /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead_custom(size: graph_nodes, grads: false),
1321 /* .mem_base = */ NULL,
1322 /* .no_alloc = */ true,
1323 };
1324 ggml_context_ptr ctx(ggml_init(params)); // smart ptr
1325 GGML_ASSERT(ctx);
1326
1327 ggml_tensor * out = build_graph(ctx: ctx.get());
1328 current_op_name = op_desc(t: out);
1329 if (!matches_filter(op: out, op_names_filter)) {
1330 //printf(" %s: skipping\n", op_desc(out).c_str());
1331 return true;
1332 }
1333
1334 if (!ggml_backend_supports_op(backend, op: out)) {
1335 // Create test result for unsupported performance test
1336 test_result result(ggml_backend_name(backend), current_op_name, vars(), "perf", false, false,
1337 "not supported");
1338
1339 output_printer->print_test_result(result);
1340
1341 return true;
1342 }
1343
1344 // allocate
1345 ggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx: ctx.get(), backend)); // smart ptr
1346
1347 if (buf == NULL) {
1348 printf(format: "failed to allocate tensors\n");
1349 return false;
1350 }
1351
1352 // randomize tensors
1353 initialize_tensors(ctx: ctx.get());
1354
1355 // build graph
1356 ggml_cgraph * gf = ggml_new_graph_custom(ctx: ctx.get(), size: graph_nodes, grads: false);
1357 ggml_build_forward_expand(cgraph: gf, tensor: out);
1358
1359 // warmup run
1360 ggml_status status = ggml_backend_graph_compute(backend, cgraph: gf);
1361 if (status != GGML_STATUS_SUCCESS) {
1362 fprintf(stderr, format: "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
1363 return false;
1364 }
1365
1366 // determine number of runs
1367 int n_runs;
1368 bool is_cpu = ggml_backend_dev_type(device: ggml_backend_get_device(backend)) == GGML_BACKEND_DEVICE_TYPE_CPU;
1369 if (op_flops(t: out) > 0) {
1370 // based on flops
1371 const uint64_t GFLOP = 1000 * 1000 * 1000;
1372 const uint64_t target_flops_cpu = 8ULL * GFLOP;
1373 const uint64_t target_flops_gpu = 100ULL * GFLOP;
1374 uint64_t target_flops = is_cpu ? target_flops_cpu : target_flops_gpu;
1375 n_runs = std::min<int>(a: ggml_graph_size(cgraph: gf) - ggml_graph_n_nodes(cgraph: gf), b: target_flops / op_flops(t: out)) + 1;
1376 } else {
1377 // based on memory size
1378 const size_t GB = 1ULL << 30;
1379 const size_t target_size_cpu = 8 * GB;
1380 const size_t target_size_gpu = 32 * GB;
1381 size_t target_size = is_cpu ? target_size_cpu : target_size_gpu;
1382 n_runs = std::min<int>(a: ggml_graph_size(cgraph: gf) - ggml_graph_n_nodes(cgraph: gf), b: target_size / op_size(t: out)) + 1;
1383 }
1384
1385 // duplicate the op
1386 for (int i = 1; i < n_runs; i++) {
1387 ggml_graph_add_node(cgraph: gf, tensor: out);
1388 }
1389
1390 // calculate memory
1391 size_t mem = n_runs * op_size(t: out);
1392 auto tensor_op_size = [](ggml_tensor * t) {
1393 size_t size = ggml_nbytes(tensor: t);
1394 // add source tensors
1395 for (int i = 0; i < GGML_MAX_SRC; i++) {
1396 if (t->src[i] != NULL) {
1397 size += ggml_nbytes(tensor: t->src[i]);
1398 }
1399 }
1400 return size;
1401 };
1402 for (int i = 0; i < ggml_graph_n_nodes(cgraph: gf); ++i) {
1403 if (ggml_is_view_op(op: ggml_graph_node(cgraph: gf, i)->op) || ggml_graph_node(cgraph: gf, i) == out) {
1404 continue;
1405 }
1406 mem += tensor_op_size(ggml_graph_node(cgraph: gf, i));
1407 }
1408
1409 // run
1410 int64_t total_time_us = 0;
1411 int64_t total_mem = 0;
1412 int total_runs = 0;
1413 do {
1414 int64_t start_time = ggml_time_us();
1415 ggml_status status = ggml_backend_graph_compute(backend, cgraph: gf);
1416 if (status != GGML_STATUS_SUCCESS) {
1417 fprintf(stderr, format: "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
1418 return false;
1419 }
1420 int64_t end_time = ggml_time_us();
1421
1422 total_time_us += end_time - start_time;
1423 total_mem += mem;
1424 total_runs += n_runs;
1425 } while (total_time_us < 1000*1000); // run for at least 1 second
1426
1427 // Create test result
1428 double avg_time_us = (double) total_time_us / total_runs;
1429 double calculated_flops = (op_flops(t: out) > 0) ? (op_flops(t: out) * total_runs) / (total_time_us / 1e6) : 0.0;
1430 double calculated_bandwidth =
1431 (op_flops(t: out) == 0) ? total_mem / (total_time_us / 1e6) / 1024.0 / 1024.0 / 1024.0 : 0.0;
1432 size_t calculated_memory_kb = op_size(t: out) / 1024;
1433
1434 test_result result(ggml_backend_name(backend), current_op_name, vars(), "perf", true, true, "", avg_time_us,
1435 calculated_flops, calculated_bandwidth, calculated_memory_kb, total_runs);
1436
1437 if (output_printer) {
1438 output_printer->print_test_result(result);
1439 }
1440
1441 return true;
1442 }
1443
1444 bool eval_support(ggml_backend_t backend, const char * op_names_filter, printer * output_printer) {
1445 mode = MODE_SUPPORT;
1446
1447 static const size_t graph_nodes = 8192;
1448
1449 ggml_init_params params = {
1450 /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead_custom(size: graph_nodes, grads: false),
1451 /* .mem_base = */ NULL,
1452 /* .no_alloc = */ true,
1453 };
1454 ggml_context_ptr ctx(ggml_init(params)); // smart ptr
1455 GGML_ASSERT(ctx);
1456
1457 gf = ggml_new_graph_custom(ctx: ctx.get(), size: graph_nodes, grads: false);
1458
1459 ggml_tensor * out = build_graph(ctx: ctx.get());
1460 current_op_name = op_desc(t: out);
1461
1462 if (!matches_filter(op: out, op_names_filter)) {
1463 return true;
1464 }
1465
1466 bool supported = ggml_backend_supports_op(backend, op: out);
1467
1468 std::string device_desc = ggml_backend_dev_description(device: ggml_backend_get_device(backend));
1469 std::string backend_reg_name = ggml_backend_reg_name(reg: ggml_backend_dev_backend_reg(device: ggml_backend_get_device(backend)));
1470
1471 test_result result(ggml_backend_name(backend), current_op_name, vars(), "support", supported, supported,
1472 supported ? "yes" : "no", 0.0, 0.0, 0.0, 0, 0, device_desc, backend_reg_name);
1473
1474 output_printer->print_test_result(result);
1475
1476 return true;
1477 }
1478
1479 bool eval_grad(ggml_backend_t backend, const char * op_names_filter, printer * output_printer) {
1480 mode = MODE_GRAD;
1481 const std::vector<float> expect = grad_expect();
1482
1483 ggml_init_params params = {
1484 /* .mem_size = */ ggml_tensor_overhead()*128 + 2*ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE, grads: true),
1485 /* .mem_base = */ NULL,
1486 /* .no_alloc = */ true,
1487 };
1488 ggml_context_ptr ctx(ggml_init(params)); // smart ptr
1489 GGML_ASSERT(ctx);
1490
1491 gf = ggml_new_graph_custom(ctx: ctx.get(), GGML_DEFAULT_GRAPH_SIZE, grads: true);
1492 gb = ggml_new_graph_custom(ctx: ctx.get(), GGML_DEFAULT_GRAPH_SIZE, grads: true);
1493
1494 ggml_tensor * out = build_graph(ctx: ctx.get());
1495
1496 if (!matches_filter(op: out, op_names_filter) || out->op == GGML_OP_OPT_STEP_ADAMW) {
1497 return true;
1498 }
1499
1500 if (out->type != GGML_TYPE_F32) {
1501 output_printer->print_operation(info: test_operation_info(op_desc(t: out), vars(), ggml_backend_name(backend),
1502 test_status_t::NOT_SUPPORTED,
1503 out->name + std::string("->type != FP32")));
1504 return true;
1505 }
1506
1507 // Print operation info first
1508 output_printer->print_operation(info: test_operation_info(op_desc(t: out), vars(), ggml_backend_name(backend)));
1509
1510 // check if the backend supports the ops
1511 bool supported = true;
1512 bool any_params = false;
1513 std::string failure_reason;
1514
1515 for (ggml_tensor * t = ggml_get_first_tensor(ctx: ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx: ctx.get(), tensor: t)) {
1516 if (!ggml_backend_supports_op(backend, op: t)) {
1517 supported = false;
1518 failure_reason = ggml_backend_name(backend);
1519 break;
1520 }
1521 if ((t->flags & GGML_TENSOR_FLAG_PARAM)) {
1522 any_params = true;
1523 if (t->type != GGML_TYPE_F32) {
1524 supported = false;
1525 failure_reason = std::string(t->name) + "->type != FP32";
1526 break;
1527 }
1528 }
1529 }
1530 if (!any_params) {
1531 supported = false;
1532 failure_reason = op_desc(t: out);
1533 }
1534
1535 if (!supported) {
1536 output_printer->print_operation(info: test_operation_info(op_desc(t: out), vars(), ggml_backend_name(backend),
1537 test_status_t::NOT_SUPPORTED, failure_reason));
1538 return true;
1539 }
1540
1541 int64_t ngrads = 0;
1542 for (ggml_tensor * t = ggml_get_first_tensor(ctx: ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx: ctx.get(), tensor: t)) {
1543 if (t->flags & GGML_TENSOR_FLAG_PARAM) {
1544 ngrads += ggml_nelements(tensor: t);
1545 }
1546 }
1547 if (ngrads > grad_nmax()) {
1548 test_operation_info info(op_desc(t: out), vars(), ggml_backend_name(backend));
1549 info.set_large_tensor_skip();
1550 output_printer->print_operation(info);
1551 return true;
1552 }
1553
1554
1555 if (!ggml_is_scalar(tensor: out)) {
1556 out = ggml_sum(ctx: ctx.get(), a: out);
1557 ggml_set_name(tensor: out, name: "sum_of_out");
1558 }
1559 ggml_set_loss(tensor: out);
1560
1561 ggml_build_forward_expand(cgraph: gf, tensor: out);
1562 ggml_graph_cpy(src: gf, dst: gb);
1563 ggml_build_backward_expand(ctx: ctx.get(), cgraph: gb, grad_accs: nullptr);
1564 if (expect.size() != 1 || expect[0] != 0.0f) {
1565 GGML_ASSERT(ggml_graph_n_nodes(gb) > ggml_graph_n_nodes(gf));
1566 for (ggml_tensor * t = ggml_get_first_tensor(ctx: ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx: ctx.get(), tensor: t)) {
1567 GGML_ASSERT(!(t->flags & GGML_TENSOR_FLAG_PARAM) || ggml_graph_get_grad(gb, t)->op != GGML_OP_NONE);
1568 }
1569 }
1570
1571 for (ggml_tensor * t = ggml_get_first_tensor(ctx: ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx: ctx.get(), tensor: t)) {
1572 if (!ggml_backend_supports_op(backend, op: t)) {
1573 output_printer->print_operation(info: test_operation_info(op_desc(t: out), vars(), ggml_backend_name(backend),
1574 test_status_t::NOT_SUPPORTED,
1575 ggml_backend_name(backend)));
1576 supported = false;
1577 break;
1578 }
1579 if ((t->flags & GGML_TENSOR_FLAG_PARAM) && t->type != GGML_TYPE_F32) {
1580 output_printer->print_operation(info: test_operation_info(op_desc(t: out), vars(), ggml_backend_name(backend),
1581 test_status_t::NOT_SUPPORTED,
1582 std::string(t->name) + "->type != FP32"));
1583 supported = false;
1584 break;
1585 }
1586 }
1587 if (!supported) {
1588 return true;
1589 }
1590
1591 // allocate
1592 ggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx: ctx.get(), backend)); // smart ptr
1593 if (buf == NULL) {
1594 test_operation_info info(op_desc(t: out), vars(), ggml_backend_name(backend));
1595 info.set_error(component: "allocation", details: "");
1596 output_printer->print_operation(info);
1597 return false;
1598 }
1599
1600 initialize_tensors(ctx: ctx.get()); // Randomizes all tensors (including gradients).
1601 ggml_graph_reset(cgraph: gb); // Sets gradients to 1 if loss, 0 otherwise.
1602
1603 ggml_status status = ggml_backend_graph_compute(backend, cgraph: gf);
1604 if (status != GGML_STATUS_SUCCESS) {
1605 fprintf(stderr, format: "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
1606 return false;
1607 }
1608 status = ggml_backend_graph_compute(backend, cgraph: gb);
1609 if (status != GGML_STATUS_SUCCESS) {
1610 fprintf(stderr, format: "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
1611 return false;
1612 }
1613
1614 bool ok = true;
1615 for (struct ggml_tensor * t = ggml_get_first_tensor(ctx: ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx: ctx.get(), tensor: t)) {
1616 if (!(t->flags & GGML_TENSOR_FLAG_PARAM)) {
1617 continue;
1618 }
1619
1620 const char * bn = ggml_backend_name(backend);
1621 const int64_t ne = ggml_nelements(tensor: t);
1622
1623 std::vector<float> ga;
1624 struct ggml_tensor * grad = ggml_graph_get_grad(cgraph: gb, node: t);
1625 if (grad) {
1626 ga = tensor_to_float(t: grad);
1627 } else {
1628 ga.resize(new_size: ne); // default value is 0.0f
1629 }
1630
1631 for (int64_t i = 0; i < ne; ++i) { // gradient algebraic
1632 // check for nans
1633 if (!std::isfinite(x: ga[i])) {
1634 test_operation_info info(op_desc(t: out), vars(), ggml_backend_name(backend));
1635 info.set_gradient_info(index: i, param_name: bn, value: ga[i]);
1636 output_printer->print_operation(info);
1637 ok = false;
1638 break;
1639 }
1640 }
1641 if (!ok) {
1642 break;
1643 }
1644
1645 std::vector<float> gn(ne); // gradient numeric
1646 GGML_ASSERT(ga.size() == gn.size());
1647
1648 std::vector<float> x0 = tensor_to_float(t); // original t data
1649 GGML_ASSERT(ggml_is_scalar(out));
1650 GGML_ASSERT(out->type == GGML_TYPE_F32);
1651
1652 const float eps = grad_eps();
1653 for (int64_t i = 0; i < ne; ++i) {
1654 const float xiu = x0[i] + 1.0f*eps; // x, index i, up
1655 const float xiuh = x0[i] + 0.5f*eps; // x, index i, up half
1656 const float xidh = x0[i] - 0.5f*eps; // x, index i, down half
1657 const float xid = x0[i] - 1.0f*eps; // x, index i, down
1658
1659 float fu, fuh, fdh, fd; // output values for xiu, xiuh, xid, xidh
1660
1661 ggml_backend_tensor_set(tensor: t, data: &xiu, offset: i*sizeof(float), size: sizeof(float));
1662 status = ggml_backend_graph_compute(backend, cgraph: gf);
1663 if (status != GGML_STATUS_SUCCESS) {
1664 fprintf(stderr, format: "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
1665 return false;
1666 }
1667 ggml_backend_tensor_get(tensor: out, data: &fu, offset: 0, size: ggml_nbytes(tensor: out));
1668
1669 ggml_backend_tensor_set(tensor: t, data: &xid, offset: i*sizeof(float), size: sizeof(float));
1670 status = ggml_backend_graph_compute(backend, cgraph: gf);
1671 if (status != GGML_STATUS_SUCCESS) {
1672 fprintf(stderr, format: "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
1673 return false;
1674 }
1675 ggml_backend_tensor_get(tensor: out, data: &fd, offset: 0, size: ggml_nbytes(tensor: out));
1676
1677 if (grad_precise()) {
1678 ggml_backend_tensor_set(tensor: t, data: &xiuh, offset: i*sizeof(float), size: sizeof(float));
1679 status = ggml_backend_graph_compute(backend, cgraph: gf);
1680 if (status != GGML_STATUS_SUCCESS) {
1681 fprintf(stderr, format: "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
1682 return false;
1683 }
1684 ggml_backend_tensor_get(tensor: out, data: &fuh, offset: 0, size: ggml_nbytes(tensor: out));
1685
1686 ggml_backend_tensor_set(tensor: t, data: &xidh, offset: i*sizeof(float), size: sizeof(float));
1687 status = ggml_backend_graph_compute(backend, cgraph: gf);
1688 if (status != GGML_STATUS_SUCCESS) {
1689 fprintf(stderr, format: "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
1690 return false;
1691 }
1692 ggml_backend_tensor_get(tensor: out, data: &fdh, offset: 0, size: ggml_nbytes(tensor: out));
1693
1694 gn[i] = (8.0*(double)fuh + (double)fd - (8.0*(double)fdh + (double)fu)) / (6.0*(double)eps);
1695 } else {
1696 gn[i] = (fu - fd) / (2.0f*eps);
1697 }
1698
1699 ggml_backend_tensor_set(tensor: t, data: x0.data(), offset: 0, size: ggml_nbytes(tensor: t));
1700 }
1701
1702 const double err = mean_abs_asymm(a: gn.data(), b: ga.data(), n: gn.size(), expected_vals: expect);
1703 if (err > max_maa_err()) {
1704 test_operation_info info(op_desc(t: out), vars(), ggml_backend_name(backend));
1705 info.set_maa_error(error: err, threshold: max_maa_err());
1706 output_printer->print_operation(info);
1707 ok = false;
1708 break;
1709 }
1710 if (!ok) {
1711 break;
1712 }
1713 }
1714
1715 // Create final test result
1716 test_operation_info final_info(op_desc(t: out), vars(), ggml_backend_name(backend));
1717 if (!ok) {
1718 final_info.set_compare_failure();
1719 }
1720 final_info.status = ok ? test_status_t::OK : test_status_t::FAIL;
1721 output_printer->print_operation(info: final_info);
1722
1723 if (ok) {
1724 return true;
1725 }
1726
1727 return false;
1728 }
1729};
1730
1731
1732// ###################################
1733// ## Section 2: GGML Op Defintions ##
1734// ###################################
1735
1736
1737// The following is an example showing the bare minimum for creating a test for a GGML op.
1738
1739// GGML_OP_EXAMPLE
1740struct test_example : public test_case {
1741 // Always define these 2 or variants thereof:
1742 const ggml_type type; // The type of the input tensors.
1743 const std::array<int64_t, 4> ne; // The shape of the input tensors.
1744 // For some ops it's necessary to define multiple types or shapes for the inputs.
1745 // Or they may need additional parameters.
1746
1747 // Put all parameters needed to fully define the test into one of the VARS_TO_STR macros.
1748 // In most cases these are just the properties of the struct that you defined above.
1749 // This is needed for info prints.
1750 std::string vars() override {
1751 return VARS_TO_STR2(type, ne);
1752 }
1753
1754 // Define a constructor for the struct.
1755 // In most cases it will be sufficient to have the same arguments as the struct has properties
1756 // and just use initializer lists.
1757 test_example(ggml_type type = GGML_TYPE_F32,
1758 std::array<int64_t, 4> ne = {10, 5, 4, 3})
1759 : type(type), ne(ne) {}
1760
1761 // Define how a simple GGML compute graph can be constructed for the new GGML op.
1762 ggml_tensor * build_graph(ggml_context * ctx) override {
1763 // Step 1: create input tensors that don't depend on any other tensors:
1764 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
1765 ggml_set_name(tensor: a, name: "a"); // Setting names is optional but it's useful for debugging.
1766
1767 ggml_tensor * b = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
1768 ggml_set_name(tensor: b, name: "b");
1769
1770 // Step 2: use the op that you want to test in the GGML compute graph.
1771 ggml_tensor * out = ggml_add(ctx, a, b); // For this example we're just doing a simple addition.
1772 ggml_set_name(tensor: out, name: "out");
1773
1774 // Step 3: return the output tensor.
1775 return out;
1776 }
1777 // In order to also check the gradients for your op, add calls like ggml_set_param(a)
1778 // immediately after you create the tensors.
1779 // This is optional and only makes sense if a backward pass has actually been implemented for the new op.
1780};
1781
1782
1783// GGML_OP_UNARY
1784struct test_unary : public test_case {
1785 const ggml_unary_op op;
1786 const ggml_type type;
1787 const std::array<int64_t, 4> ne_a;
1788 int v; // view (1 : non-contiguous a)
1789
1790 std::string vars() override {
1791 return VARS_TO_STR3(type, ne_a, v);
1792 }
1793
1794 test_unary(ggml_unary_op op,
1795 ggml_type type = GGML_TYPE_F32,
1796 std::array<int64_t, 4> ne_a = {128, 2, 2, 2},
1797 int v = 0)
1798 : op(op), type(type), ne_a(ne_a), v(v) {}
1799
1800 ggml_tensor * build_graph(ggml_context * ctx) override {
1801 const bool grad_supported = op == GGML_UNARY_OP_ABS || op == GGML_UNARY_OP_SGN || op == GGML_UNARY_OP_NEG ||
1802 op == GGML_UNARY_OP_STEP || op == GGML_UNARY_OP_RELU || op == GGML_UNARY_OP_SILU;
1803
1804 ggml_tensor * a;
1805 if (v & 1) {
1806 auto ne = ne_a; ne[0] *= 3;
1807 a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
1808 if (grad_supported) {
1809 ggml_set_param(tensor: a);
1810 }
1811 ggml_set_name(tensor: a, name: "a");
1812
1813 a = ggml_view_4d(ctx, a, ne0: ne_a[0], ne1: ne_a[1], ne2: ne_a[2], ne3: ne_a[3], nb1: a->nb[1], nb2: a->nb[2], nb3: a->nb[3], offset: 0);
1814 ggml_set_name(tensor: a, name: "view_of_a");
1815 } else {
1816 a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_a.data());
1817 if (grad_supported) {
1818 ggml_set_param(tensor: a);
1819 }
1820 ggml_set_name(tensor: a, name: "a");
1821 }
1822
1823 ggml_tensor * out = ggml_unary(ctx, a, op);
1824 ggml_set_name(tensor: out, name: "out");
1825
1826 return out;
1827 }
1828
1829 void initialize_tensors(ggml_context * ctx) override {
1830 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
1831 // test extended range of values to check for NaNs in GELU
1832 init_tensor_uniform(tensor: t, min: -150.f, max: 150.f);
1833 }
1834 }
1835
1836 float grad_eps() override {
1837 return 15.0f;
1838 }
1839
1840 std::vector<float> grad_expect() override {
1841 if (op == GGML_UNARY_OP_ABS) {
1842 return {-1.0f, 1.0f};
1843 }
1844 if (op == GGML_UNARY_OP_SGN || op == GGML_UNARY_OP_STEP) {
1845 return {0.0f};
1846 }
1847 if (op == GGML_UNARY_OP_RELU) {
1848 return {0.0f, 1.0f};
1849 }
1850 return {};
1851 }
1852
1853};
1854
1855// GGML_OP_GLU
1856struct test_glu : public test_case {
1857 const ggml_glu_op op;
1858 const ggml_type type;
1859 const std::array<int64_t, 4> ne_a;
1860 int v; // view (1 : non-contiguous a)
1861 bool swapped;
1862
1863 std::string vars() override {
1864 return VARS_TO_STR4(type, ne_a, v, swapped);
1865 }
1866
1867 test_glu(ggml_glu_op op,
1868 ggml_type type = GGML_TYPE_F32,
1869 std::array<int64_t, 4> ne_a = {128, 2, 2, 2},
1870 int v = 0,
1871 bool swapped = false)
1872 : op(op), type(type), ne_a(ne_a), v(v), swapped(swapped) {}
1873
1874 ggml_tensor * build_graph(ggml_context * ctx) override {
1875 ggml_tensor * a;
1876 if (v & 1) {
1877 auto ne = ne_a; ne[0] *= 3;
1878 a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
1879 ggml_set_name(tensor: a, name: "a");
1880
1881 a = ggml_view_4d(ctx, a, ne0: ne_a[0], ne1: ne_a[1], ne2: ne_a[2], ne3: ne_a[3], nb1: a->nb[1], nb2: a->nb[2], nb3: a->nb[3], offset: 0);
1882 ggml_set_name(tensor: a, name: "view_of_a");
1883 } else {
1884 a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_a.data());
1885 ggml_set_name(tensor: a, name: "a");
1886 }
1887
1888 ggml_tensor * out = ggml_glu(ctx, a, op, swapped);
1889 ggml_set_name(tensor: out, name: "out");
1890
1891 return out;
1892 }
1893
1894 void initialize_tensors(ggml_context * ctx) override {
1895 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
1896 // test extended range of values to check for NaNs in GELU
1897 init_tensor_uniform(tensor: t, min: -150.f, max: 150.f);
1898 }
1899 }
1900};
1901
1902struct test_glu_split : public test_case {
1903 const ggml_glu_op op;
1904 const ggml_type type;
1905 const std::array<int64_t, 4> ne_a;
1906 int v; // view (1 : non-contiguous a)
1907
1908 std::string vars() override {
1909 return VARS_TO_STR3(type, ne_a, v) + ",split";
1910 }
1911
1912 test_glu_split(ggml_glu_op op,
1913 ggml_type type = GGML_TYPE_F32,
1914 std::array<int64_t, 4> ne_a = {128, 2, 2, 2},
1915 int v = 0)
1916 : op(op), type(type), ne_a(ne_a), v(v) {}
1917
1918 ggml_tensor * build_graph(ggml_context * ctx) override {
1919 ggml_tensor * a;
1920 ggml_tensor * b;
1921 if (v & 1) {
1922 auto ne = ne_a; ne[0] *= 3;
1923 a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
1924 ggml_set_param(tensor: a);
1925 ggml_set_name(tensor: a, name: "a");
1926
1927 a = ggml_view_4d(ctx, a, ne0: ne_a[0], ne1: ne_a[1], ne2: ne_a[2], ne3: ne_a[3], nb1: a->nb[1], nb2: a->nb[2], nb3: a->nb[3], offset: 0);
1928 ggml_set_name(tensor: a, name: "view_of_a");
1929
1930 b = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
1931 ggml_set_param(tensor: b);
1932 ggml_set_name(tensor: b, name: "b");
1933
1934 b = ggml_view_4d(ctx, a: b, ne0: ne_a[0], ne1: ne_a[1], ne2: ne_a[2], ne3: ne_a[3], nb1: b->nb[1], nb2: b->nb[2], nb3: b->nb[3], offset: 0);
1935 ggml_set_name(tensor: a, name: "view_of_b");
1936 } else {
1937 a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_a.data());
1938 ggml_set_param(tensor: a);
1939 ggml_set_name(tensor: a, name: "a");
1940
1941 b = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_a.data());
1942 ggml_set_param(tensor: b);
1943 ggml_set_name(tensor: b, name: "b");
1944 }
1945
1946 ggml_tensor * out = ggml_glu_split(ctx, a, b, op);
1947 ggml_set_name(tensor: out, name: "out");
1948
1949 return out;
1950 }
1951
1952 void initialize_tensors(ggml_context * ctx) override {
1953 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
1954 // test extended range of values to check for NaNs in GELU
1955 init_tensor_uniform(tensor: t, min: -150.f, max: 150.f);
1956 }
1957 }
1958};
1959
1960struct test_swiglu_oai : public test_case {
1961 const ggml_type type;
1962 const std::array<int64_t, 4> ne_a;
1963 int v; // view (1 : non-contiguous a)
1964 float alpha;
1965 float limit;
1966
1967 std::string vars() override {
1968 return VARS_TO_STR5(type, ne_a, v, alpha, limit);
1969 }
1970
1971 test_swiglu_oai(ggml_type type = GGML_TYPE_F32,
1972 std::array<int64_t, 4> ne_a = {128, 2, 2, 2},
1973 int v = 0,
1974 float alpha = 1.702f,
1975 float limit = 7.0f)
1976 : type(type), ne_a(ne_a), v(v), alpha(alpha), limit(limit) {}
1977
1978 ggml_tensor * build_graph(ggml_context * ctx) override {
1979 ggml_tensor * a;
1980 ggml_tensor * b;
1981 if (v & 1) {
1982 auto ne = ne_a; ne[0] *= 3;
1983 a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
1984 ggml_set_param(tensor: a);
1985 ggml_set_name(tensor: a, name: "a");
1986
1987 a = ggml_view_4d(ctx, a, ne0: ne_a[0], ne1: ne_a[1], ne2: ne_a[2], ne3: ne_a[3], nb1: a->nb[1], nb2: a->nb[2], nb3: a->nb[3], offset: 0);
1988 ggml_set_name(tensor: a, name: "view_of_a");
1989
1990 b = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
1991 ggml_set_param(tensor: b);
1992 ggml_set_name(tensor: b, name: "b");
1993
1994 b = ggml_view_4d(ctx, a: b, ne0: ne_a[0], ne1: ne_a[1], ne2: ne_a[2], ne3: ne_a[3], nb1: b->nb[1], nb2: b->nb[2], nb3: b->nb[3], offset: 0);
1995 ggml_set_name(tensor: a, name: "view_of_b");
1996 } else {
1997 a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_a.data());
1998 ggml_set_param(tensor: a);
1999 ggml_set_name(tensor: a, name: "a");
2000
2001 b = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_a.data());
2002 ggml_set_param(tensor: b);
2003 ggml_set_name(tensor: b, name: "b");
2004 }
2005
2006 ggml_tensor * out = ggml_swiglu_oai(ctx, a, b, alpha, limit);
2007 ggml_set_name(tensor: out, name: "out");
2008
2009 return out;
2010 }
2011
2012 void initialize_tensors(ggml_context * ctx) override {
2013 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
2014 // test extended range of values to check for NaNs in GELU
2015 init_tensor_uniform(tensor: t, min: -150.f, max: 150.f);
2016 }
2017 }
2018};
2019
2020// GGML_OP_GET_ROWS
2021struct test_get_rows : public test_case {
2022 const ggml_type type;
2023 const int n; // cols
2024 const int m; // rows
2025 const int r; // rows to get
2026 const int be1; // batch size
2027 const int be2; // batch size
2028 const bool v; // view (non-contiguous src1)
2029
2030 std::string vars() override {
2031 return VARS_TO_STR7(type, n, m, r, be1, be2, v);
2032 }
2033
2034 test_get_rows(ggml_type type = GGML_TYPE_F32, int n = 10, int m = 5, int r = 3, int be1 = 1, int be2 = 1, bool v = false)
2035 : type(type), n(n), m(m), r(r), be1(be1), be2(be2), v(v) {}
2036
2037 ggml_tensor * build_graph(ggml_context * ctx) override {
2038 ggml_tensor * in = ggml_new_tensor_4d(ctx, type, ne0: n, ne1: m, ne2: be1, ne3: be2);
2039 ggml_set_name(tensor: in, name: "in");
2040
2041 ggml_tensor * rows = ggml_new_tensor_3d(ctx, type: GGML_TYPE_I32, ne0: r, ne1: be1, ne2: be2);
2042 ggml_set_name(tensor: rows, name: "rows");
2043 if (v) {
2044 rows = ggml_view_3d(ctx, a: rows, ne0: r/2, ne1: be1, ne2: be2, nb1: rows->nb[1], nb2: rows->nb[2], offset: 0);
2045 ggml_set_name(tensor: rows, name: "view_of_rows");
2046 }
2047
2048 const bool grad_supported = ggml_is_matrix(tensor: in) && ggml_is_vector(tensor: rows);
2049 if (grad_supported) {
2050 ggml_set_param(tensor: in);
2051 // rows is a constant input -> no gradients
2052 }
2053
2054 ggml_tensor * out = ggml_get_rows(ctx, a: in, b: rows);
2055 ggml_set_name(tensor: out, name: "out");
2056
2057 return out;
2058 }
2059
2060 void initialize_tensors(ggml_context * ctx) override {
2061 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
2062 if (t->type == GGML_TYPE_I32) {
2063 if (ggml_is_view_op(op: t->op)) { continue; }
2064 // rows
2065 std::vector<int> data(r*be1*be2);
2066 for (int i = 0; i < r*be1*be2; i++) {
2067 data[i] = rand() % m;
2068 }
2069 ggml_backend_tensor_set(tensor: t, data: data.data(), offset: 0, size: r * be1 * be2 * sizeof(int));
2070 } else {
2071 init_tensor_uniform(tensor: t);
2072 }
2073 }
2074 }
2075};
2076
2077// GGML_OP_GET_ROWS_BACK
2078struct test_get_rows_back : public test_case {
2079 const ggml_type type;
2080 const int n; // cols
2081 const int m; // rows
2082 const int r; // rows to get
2083 const int b; // batch size
2084 const bool v; // view (non-contiguous src1)
2085
2086 std::string vars() override {
2087 return VARS_TO_STR6(type, n, m, r, b, v);
2088 }
2089
2090 test_get_rows_back(ggml_type type = GGML_TYPE_F32, int n = 10, int m = 5, int r = 3, int b = 1, bool v = false)
2091 : type(type), n(n), m(m), r(r), b(b), v(v) {}
2092
2093 ggml_tensor * build_graph(ggml_context * ctx) override {
2094 ggml_tensor * in_forward = ggml_new_tensor_3d(ctx, type, ne0: n, ne1: m, ne2: b);
2095 ggml_set_name(tensor: in_forward, name: "in_forward");
2096
2097 ggml_tensor * rows = ggml_new_tensor_2d(ctx, type: GGML_TYPE_I32, ne0: r, ne1: b);
2098 ggml_set_name(tensor: rows, name: "rows");
2099 if (v) {
2100 rows = ggml_view_2d(ctx, a: rows, ne0: r/2, ne1: b, nb1: rows->nb[1], offset: 0);
2101 ggml_set_name(tensor: rows, name: "view_of_rows");
2102 }
2103
2104 ggml_tensor * grad = ggml_new_tensor_3d(ctx, type, ne0: n, ne1: r, ne2: b);
2105 ggml_set_name(tensor: grad, name: "grad");
2106
2107 ggml_tensor * out = ggml_get_rows_back(ctx, a: grad, b: rows, c: in_forward);
2108 ggml_set_name(tensor: out, name: "out");
2109
2110 return out;
2111 }
2112
2113 void initialize_tensors(ggml_context * ctx) override {
2114 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
2115 if (t->type == GGML_TYPE_I32) {
2116 if (ggml_is_view_op(op: t->op)) { continue; }
2117 // rows
2118 std::vector<int> data(r*b);
2119 for (int i = 0; i < r*b; i++) {
2120 data[i] = rand() % m;
2121 }
2122 ggml_backend_tensor_set(tensor: t, data: data.data(), offset: 0, size: r * b * sizeof(int));
2123 } else {
2124 init_tensor_uniform(tensor: t);
2125 }
2126 }
2127 }
2128};
2129
2130static void init_set_rows_row_ids(ggml_tensor * t, int num_rows) {
2131 std::random_device rd;
2132 std::default_random_engine rng(rd());
2133 for (int i2 = 0; i2 < t->ne[2]; i2++) {
2134 for (int i1 = 0; i1 < t->ne[1]; i1++) {
2135 // generate a shuffled subset of row indices
2136 std::vector<int64_t> data(num_rows);
2137 for (int i = 0; i < num_rows; i++) {
2138 data[i] = i;
2139 }
2140 std::shuffle(first: data.begin(), last: data.end(), g&: rng);
2141 data.resize(new_size: t->ne[0]);
2142
2143 const size_t offs = i1*t->nb[1] + i2*t->nb[2];
2144 if (t->type == GGML_TYPE_I32) {
2145 // TODO: Make a template or something
2146 std::vector<int32_t> data_i32(t->ne[0]);
2147 for (int i = 0; i < t->ne[0]; i++) {
2148 data_i32[i] = static_cast<int32_t>(data[i]);
2149 }
2150 ggml_backend_tensor_set(tensor: t, data: data_i32.data(), offset: offs, size: t->ne[0]*sizeof(int32_t));
2151 } else {
2152 ggml_backend_tensor_set(tensor: t, data: data.data(), offset: offs, size: t->ne[0]*sizeof(int64_t));
2153 }
2154 }
2155 }
2156}
2157
2158// GGML_OP_SET_ROWS
2159struct test_set_rows : public test_case {
2160 const ggml_type type;
2161 const ggml_type type_idx;
2162 const std::array<int64_t, 4> ne;
2163 const std::array<int, 2> nr23; // broadcast only dims 2 and 3
2164 const int r; // rows to set
2165 const bool v; // view (non-contiguous src1)
2166
2167 std::string vars() override {
2168 return VARS_TO_STR6(type, type_idx, ne, nr23, r, v);
2169 }
2170
2171 test_set_rows(ggml_type type,
2172 ggml_type type_idx,
2173 std::array<int64_t, 4> ne,
2174 std::array<int, 2> nr23,
2175 int r, bool v = false)
2176 : type(type), type_idx(type_idx), ne(ne), nr23(nr23), r(r), v(v) {}
2177
2178 ggml_tensor * build_graph(ggml_context * ctx) override {
2179 ggml_tensor * dst = ggml_new_tensor_4d(ctx, type, ne0: ne[0], ne1: ne[1], ne2: ne[2]*nr23[0], ne3: ne[3]*nr23[1]);
2180 ggml_set_name(tensor: dst, name: "dst");
2181
2182 ggml_tensor * src = ggml_new_tensor_4d(ctx, type: GGML_TYPE_F32, ne0: ne[0], ne1: r, ne2: ne[2]*nr23[0], ne3: ne[3]*nr23[1]);
2183 ggml_set_name(tensor: src, name: "src");
2184
2185 ggml_tensor * row_idxs = ggml_new_tensor_3d(ctx, type: type_idx, ne0: r, ne1: ne[2], ne2: ne[3]);
2186 ggml_set_name(tensor: row_idxs, name: "row_idxs");
2187
2188 if (v) {
2189 src = ggml_view_4d(ctx, a: src, ne0: ne[0], ne1: r/2, ne2: ne[2]*nr23[0], ne3: ne[3]*nr23[1], nb1: src->nb[1], nb2: src->nb[2], nb3: src->nb[3], offset: 0);
2190 row_idxs = ggml_view_3d(ctx, a: row_idxs, ne0: r/2, ne1: ne[2], ne2: ne[3], nb1: row_idxs->nb[1], nb2: row_idxs->nb[2], offset: 0);
2191 ggml_set_name(tensor: row_idxs, name: "view_of_rows");
2192 }
2193
2194 ggml_tensor * out = ggml_set_rows(ctx, a: dst, b: src, c: row_idxs);
2195 ggml_set_name(tensor: out, name: "out");
2196
2197 return out;
2198 }
2199
2200 void initialize_tensors(ggml_context * ctx) override {
2201 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
2202 if (t->type == GGML_TYPE_I64 || t->type == GGML_TYPE_I32) {
2203 if (ggml_is_view_op(op: t->op)) {
2204 continue;
2205 }
2206
2207 init_set_rows_row_ids(t, num_rows: ne[1]);
2208 } else {
2209 init_tensor_uniform(tensor: t);
2210 }
2211 }
2212 }
2213
2214 double max_nmse_err() override {
2215 if (type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || type == GGML_TYPE_IQ4_NL ||
2216 type == GGML_TYPE_Q5_0 || type == GGML_TYPE_Q5_1 || type == GGML_TYPE_Q8_0) {
2217 // estimate what the max nmse error would be if one quantized value is
2218 // off by one. The test values are distributed in [-1,1], so it'll be
2219 // roughly (2.0 / 2^bits)^2, divided by the mean square value of the reference,
2220 // which is roughly 0.25 times the number of elements.
2221 double err_estimate = 1.0f/8.0f;
2222 if (type == GGML_TYPE_Q5_0 || type == GGML_TYPE_Q5_1) {
2223 err_estimate /= 2.0f;
2224 }
2225 if (type == GGML_TYPE_Q8_0) {
2226 err_estimate /= 8.0f;
2227 }
2228 err_estimate *= err_estimate;
2229 err_estimate /= 0.25f*float(ne[0] * r * ne[2]*nr23[0] * ne[3]*nr23[1]);
2230 return err_estimate;
2231 }
2232 return 1e-7;
2233 }
2234};
2235
2236// GGML_OP_ROPE + GGML_OP_VIEW + GGML_OP_SET_ROWS
2237struct test_rope_set_rows : public test_case {
2238 const ggml_type type;
2239 const ggml_type type_idx;
2240 const std::array<int64_t, 4> ne;
2241 int mode;
2242
2243 std::string vars() override {
2244 return VARS_TO_STR4(type, type_idx, ne, mode);
2245 }
2246
2247 std::string op_desc(ggml_tensor * t) override {
2248 GGML_UNUSED(t);
2249 return "ROPE_SET_ROWS";
2250 }
2251
2252 bool run_whole_graph() override { return true; }
2253
2254 test_rope_set_rows(ggml_type type,
2255 ggml_type type_idx,
2256 std::array<int64_t, 4> ne,
2257 int mode)
2258 : type(type), type_idx(type_idx), ne(ne), mode(mode) {}
2259
2260 ggml_tensor * build_graph(ggml_context * ctx) override {
2261 ggml_tensor * src = ggml_new_tensor_4d(ctx, type: GGML_TYPE_F32, ne0: ne[0], ne1: ne[1], ne2: ne[2], ne3: 1);
2262 ggml_set_name(tensor: src, name: "src");
2263
2264 ggml_tensor * pos = ggml_new_tensor_1d(ctx, type: GGML_TYPE_I32, ne0: ne[2]);
2265
2266 ggml_tensor * rope = ggml_rope(ctx, a: src, b: pos, n_dims: ne[0], mode);
2267
2268 ggml_tensor * view = ggml_view_2d(ctx, a: rope, ne0: ne[0] * ne[1], ne1: ne[2], nb1: rope->nb[2], offset: 0);
2269
2270 ggml_tensor * dst = ggml_new_tensor_4d(ctx, type, ne0: ne[0] * ne[1], ne1: ne[2] * ne[3], ne2: 1, ne3: 1);
2271 ggml_set_name(tensor: dst, name: "dst");
2272
2273 ggml_tensor * row_idxs = ggml_new_tensor_3d(ctx, type: type_idx, ne0: ne[2], ne1: 1, ne2: 1);
2274 ggml_set_name(tensor: row_idxs, name: "row_idxs");
2275
2276 ggml_tensor * out = ggml_set_rows(ctx, a: dst, b: view, c: row_idxs);
2277 ggml_set_name(tensor: out, name: "out");
2278
2279 return out;
2280 }
2281
2282 void initialize_tensors(ggml_context * ctx) override {
2283 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
2284 if (t->type == GGML_TYPE_I64 || t->type == GGML_TYPE_I32) {
2285 if (ggml_is_view_op(op: t->op)) {
2286 continue;
2287 }
2288
2289 init_set_rows_row_ids(t, num_rows: ne[2]);
2290 } else {
2291 init_tensor_uniform(tensor: t);
2292 }
2293 }
2294 }
2295};
2296
2297// GGML_OP_RMS_NORM + GGML_OP_MUL + GGML_OP_ROPE (+ GGML_OP_VIEW + GGML_OP_SET_ROWS)
2298struct test_rms_norm_mul_rope : public test_case {
2299 const std::array<int64_t, 4> ne;
2300 const float eps;
2301 const bool multi_add; // test a sequence of adds feeding into rms_norm
2302 const bool set_rows;
2303 int mode;
2304
2305 std::string op_desc(ggml_tensor * t) override {
2306 GGML_UNUSED(t);
2307 return "RMS_NORM_MUL_ROPE";
2308 }
2309
2310 bool run_whole_graph() override { return true; }
2311
2312 std::string vars() override {
2313 return VARS_TO_STR5(ne, eps, multi_add, set_rows, mode);
2314 }
2315
2316 test_rms_norm_mul_rope(std::array<int64_t, 4> ne, float eps = 1e-6f, bool multi_add = false,
2317 bool set_rows = false, int mode = GGML_ROPE_TYPE_NORMAL)
2318 : ne(ne), eps(eps), multi_add(multi_add), set_rows(set_rows), mode(mode) {}
2319
2320 ggml_tensor * build_graph(ggml_context * ctx) override {
2321 ggml_tensor * a = ggml_new_tensor_4d(ctx, type: GGML_TYPE_F32, ne0: ne[0], ne1: ne[1], ne2: ne[2], ne3: 1);
2322 ggml_tensor * b = ggml_new_tensor_4d(ctx, type: GGML_TYPE_F32, ne0: ne[0], ne1: ne[1], ne2: ne[2], ne3: 1);
2323 ggml_tensor * c = ggml_new_tensor_4d(ctx, type: GGML_TYPE_F32, ne0: ne[0], ne1: ne[1], ne2: ne[2], ne3: 1);
2324
2325 if (multi_add) {
2326 a = ggml_add(ctx, a: ggml_add(ctx, a, b), b: c);
2327 }
2328
2329 a = ggml_mul(ctx, a: ggml_rms_norm(ctx, a, eps), b);
2330
2331 ggml_tensor * pos = ggml_new_tensor_1d(ctx, type: GGML_TYPE_I32, ne0: ne[2]);
2332
2333 ggml_tensor * rope = ggml_rope(ctx, a, b: pos, n_dims: ne[0], mode);
2334
2335 ggml_tensor * out;
2336
2337 if (set_rows) {
2338 ggml_tensor * view = ggml_view_2d(ctx, a: rope, ne0: ne[0] * ne[1], ne1: ne[2], nb1: rope->nb[2], offset: 0);
2339
2340 ggml_tensor * dst = ggml_new_tensor_4d(ctx, type: GGML_TYPE_F16, ne0: ne[0] * ne[1], ne1: ne[2] * ne[3], ne2: 1, ne3: 1);
2341 ggml_set_name(tensor: dst, name: "dst");
2342
2343 ggml_tensor * row_idxs = ggml_new_tensor_3d(ctx, type: GGML_TYPE_I64, ne0: ne[2], ne1: 1, ne2: 1);
2344 ggml_set_name(tensor: row_idxs, name: "row_idxs");
2345
2346 out = ggml_set_rows(ctx, a: dst, b: view, c: row_idxs);
2347 ggml_set_name(tensor: out, name: "out");
2348 } else {
2349 out = rope;
2350 }
2351
2352 return out;
2353 }
2354
2355 void initialize_tensors(ggml_context * ctx) override {
2356 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
2357 if (t->type == GGML_TYPE_I64 || t->type == GGML_TYPE_I32) {
2358 if (ggml_is_view_op(op: t->op)) {
2359 continue;
2360 }
2361
2362 init_set_rows_row_ids(t, num_rows: ne[2]);
2363 } else {
2364 init_tensor_uniform(tensor: t);
2365 }
2366 }
2367 }
2368};
2369
2370// GGML_OP_ARGMAX
2371struct test_argmax : public test_case {
2372 const ggml_type type;
2373 const std::array<int64_t, 4> ne;
2374
2375 std::string vars() override {
2376 return VARS_TO_STR2(type, ne);
2377 }
2378
2379 test_argmax(ggml_type type = GGML_TYPE_F32,
2380 std::array<int64_t, 4> ne = {10, 100, 1, 1})
2381 : type(type), ne(ne) {}
2382
2383 ggml_tensor * build_graph(ggml_context * ctx) override {
2384 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
2385 ggml_set_name(tensor: a, name: "a");
2386
2387 ggml_tensor * out = ggml_argmax(ctx, a);
2388 ggml_set_name(tensor: out, name: "out");
2389
2390 return out;
2391 }
2392
2393 void initialize_tensors(ggml_context * ctx) override {
2394 std::random_device rd;
2395 std::default_random_engine rng(rd());
2396 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
2397 if (t->type == GGML_TYPE_F32) {
2398 // initialize with unique values to avoid ties
2399 for (int64_t r = 0; r < ggml_nrows(tensor: t); r++) {
2400 std::vector<float> data(t->ne[0]);
2401 for (int i = 0; i < t->ne[0]; i++) {
2402 data[i] = i;
2403 }
2404 std::shuffle(first: data.begin(), last: data.end(), g&: rng);
2405 ggml_backend_tensor_set(tensor: t, data: data.data(), offset: r * t->nb[1], size: t->ne[0] * sizeof(float));
2406 }
2407 } else {
2408 init_tensor_uniform(tensor: t);
2409 }
2410 }
2411 }
2412
2413 double max_nmse_err() override {
2414 return 0.0;
2415 }
2416};
2417
2418// GGML_OP_COUNT_EQUAL
2419struct test_count_equal : public test_case {
2420 const ggml_type type;
2421 const std::array<int64_t, 4> ne;
2422
2423 std::string vars() override {
2424 return VARS_TO_STR2(type, ne);
2425 }
2426
2427 test_count_equal(ggml_type type = GGML_TYPE_F32,
2428 std::array<int64_t, 4> ne = {4, 500, 1, 1})
2429 : type(type), ne(ne) {}
2430
2431 ggml_tensor * build_graph(ggml_context * ctx) override {
2432 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
2433 ggml_set_name(tensor: a, name: "a");
2434
2435 ggml_tensor * a_argmax = ggml_argmax(ctx, a);
2436 ggml_set_name(tensor: a_argmax, name: "a_argmax");
2437
2438 ggml_tensor * b = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
2439 ggml_set_name(tensor: b, name: "b");
2440
2441 ggml_tensor * b_argmax = ggml_argmax(ctx, a: b);
2442 ggml_set_name(tensor: b_argmax, name: "b_argmax");
2443
2444 ggml_tensor * out = ggml_count_equal(ctx, a: a_argmax, b: b_argmax);
2445 ggml_set_name(tensor: out, name: "out");
2446
2447 return out;
2448 }
2449
2450 double max_nmse_err() override {
2451 return 0.0;
2452 }
2453
2454 void initialize_tensors(ggml_context * ctx) override {
2455 std::random_device rd;
2456 std::default_random_engine rng(rd());
2457 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
2458 if (t->type == GGML_TYPE_F32) {
2459 // initialize with unique values to avoid ties
2460 for (int64_t r = 0; r < ggml_nrows(tensor: t); r++) {
2461 std::vector<float> data(t->ne[0]);
2462 for (int i = 0; i < t->ne[0]; i++) {
2463 data[i] = i;
2464 }
2465 std::shuffle(first: data.begin(), last: data.end(), g&: rng);
2466 ggml_backend_tensor_set(tensor: t, data: data.data(), offset: r * t->nb[1], size: t->ne[0] * sizeof(float));
2467 }
2468 } else {
2469 init_tensor_uniform(tensor: t);
2470 }
2471 }
2472 }
2473};
2474
2475// GGML_OP_REPEAT
2476struct test_repeat : public test_case {
2477 const ggml_type type;
2478 const std::array<int64_t, 4> ne;
2479 const std::array<int, 4> nr;
2480
2481 std::string vars() override {
2482 return VARS_TO_STR3(type, ne, nr);
2483 }
2484
2485 size_t op_size(ggml_tensor * t) override {
2486 return ggml_nbytes(tensor: t) * 2;
2487 }
2488
2489 test_repeat(ggml_type type = GGML_TYPE_F32,
2490 std::array<int64_t, 4> ne = {10, 5, 4, 3},
2491 std::array<int, 4> nr = {2, 2, 2, 2})
2492 : type(type), ne(ne), nr(nr) {}
2493
2494 ggml_tensor * build_graph(ggml_context * ctx) override {
2495 ggml_tensor * target = ggml_new_tensor_4d(ctx, type, ne0: ne[0]*nr[0], ne1: ne[1]*nr[1], ne2: ne[2]*nr[2], ne3: ne[3]*nr[3]);
2496 ggml_set_name(tensor: target, name: "target");
2497
2498 ggml_tensor * src = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
2499 ggml_set_param(tensor: src);
2500 ggml_set_name(tensor: src, name: "src");
2501
2502 ggml_tensor * out = ggml_repeat(ctx, a: src, b: target);
2503 ggml_set_name(tensor: out, name: "out");
2504
2505 return out;
2506 }
2507};
2508
2509// GGML_OP_REPEAT_BACK
2510struct test_repeat_back : public test_case {
2511 const ggml_type type;
2512 const std::array<int64_t, 4> ne;
2513 const std::array<int, 4> nr;
2514 const bool v; // whether src is a noncontiguous view
2515
2516 std::string vars() override {
2517 return VARS_TO_STR4(type, ne, nr, v);
2518 }
2519
2520 size_t op_size(ggml_tensor * t) override {
2521 return ggml_nbytes(tensor: t) * 2;
2522 }
2523
2524 test_repeat_back(ggml_type type = GGML_TYPE_F32,
2525 std::array<int64_t, 4> ne = {8, 6, 4, 2},
2526 std::array<int, 4> nr = {2, 2, 2, 2},
2527 bool v = false)
2528 : type(type), ne(ne), nr(nr), v(v) {}
2529
2530 ggml_tensor * build_graph(ggml_context * ctx) override {
2531 ggml_tensor * src = ggml_new_tensor_4d(ctx, type, ne0: ne[0]*nr[0], ne1: ne[1]*nr[1], ne2: ne[2]*nr[2], ne3: ne[3]*nr[3]);
2532 ggml_set_name(tensor: src, name: "src");
2533
2534 if (v) {
2535 GGML_ASSERT(ne[0] % 2 == 0);
2536 GGML_ASSERT(ne[1] % 2 == 0);
2537 GGML_ASSERT(ne[2] % 2 == 0);
2538 GGML_ASSERT(ne[3] % 2 == 0);
2539 GGML_ASSERT(nr[0] % 2 == 0 || nr[0] == 1);
2540 GGML_ASSERT(nr[1] % 2 == 0 || nr[1] == 1);
2541 GGML_ASSERT(nr[2] % 2 == 0 || nr[2] == 1);
2542 GGML_ASSERT(nr[3] % 2 == 0 || nr[3] == 1);
2543
2544 const int64_t ne00 = nr[0] == 1 ? src->ne[0] : src->ne[0] / 2;
2545 const int64_t ne01 = nr[1] == 1 ? src->ne[1] : src->ne[1] / 2;
2546 const int64_t ne02 = nr[2] == 1 ? src->ne[2] : src->ne[2] / 2;
2547 const int64_t ne03 = nr[3] == 1 ? src->ne[3] : src->ne[3] / 2;
2548
2549 src = ggml_view_4d(ctx, a: src, ne0: ne00, ne1: ne01, ne2: ne02, ne3: ne03, nb1: src->nb[1], nb2: src->nb[2], nb3: src->nb[3], offset: 0);
2550 }
2551
2552 ggml_tensor * target = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
2553 ggml_set_name(tensor: target, name: "target");
2554
2555 ggml_tensor * out = ggml_repeat_back(ctx, a: src, b: target);
2556 ggml_set_name(tensor: out, name: "out");
2557
2558 return out;
2559 }
2560};
2561
2562// GGML_OP_DUP
2563struct test_dup : public test_case {
2564 const ggml_type type;
2565 const std::array<int64_t, 4> ne;
2566 const std::array<int64_t, 4> permute;
2567 bool _use_permute;
2568
2569 std::string vars() override {
2570 std::string v = VARS_TO_STR2(type, ne);
2571 if (_use_permute) v += "," + VAR_TO_STR(permute);
2572 return v;
2573 }
2574
2575 test_dup(ggml_type type = GGML_TYPE_F32,
2576 std::array<int64_t, 4> ne = {10, 10, 20, 1},
2577 std::array<int64_t, 4> permute = {0, 0, 0, 0})
2578 : type(type), ne(ne), permute(permute),
2579 _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0) {}
2580
2581 ggml_tensor * build_graph(ggml_context * ctx) override {
2582 ggml_tensor * src = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
2583 ggml_set_param(tensor: src);
2584 ggml_set_name(tensor: src, name: "src");
2585
2586 if (_use_permute) {
2587 src = ggml_permute(ctx, a: src, axis0: permute[0], axis1: permute[1], axis2: permute[2], axis3: permute[3]);
2588 ggml_set_name(tensor: src, name: "src_permuted");
2589 }
2590
2591 ggml_tensor * out = ggml_dup(ctx, a: src);
2592 ggml_set_name(tensor: out, name: "out");
2593
2594 return out;
2595 }
2596};
2597
2598// GGML_OP_SET
2599struct test_set : public test_case {
2600 const ggml_type type_src;
2601 const ggml_type type_dst;
2602 const std::array<int64_t, 4> ne;
2603 const int dim;
2604
2605 std::string vars() override {
2606 return VARS_TO_STR4(type_src, type_dst, ne, dim);
2607 }
2608
2609 size_t op_size(ggml_tensor * t) override {
2610 return ggml_nbytes(tensor: t) + ggml_nbytes(tensor: t->src[0]);
2611 }
2612
2613 test_set(ggml_type type_src = GGML_TYPE_F32, ggml_type type_dst = GGML_TYPE_F32,
2614 std::array<int64_t, 4> ne = {6, 5, 4, 3}, int dim = 1)
2615 : type_src(type_src), type_dst(type_dst), ne(ne), dim(dim) {}
2616
2617 ggml_tensor * build_graph(ggml_context * ctx) override {
2618 ggml_tensor * src = ggml_new_tensor(ctx, type: type_src, n_dims: 4, ne: ne.data());
2619 ggml_set_param(tensor: src);
2620 ggml_set_name(tensor: src, name: "src");
2621
2622 auto ne_dst = ne;
2623 for (int i = 0; i < dim; ++i) {
2624 ne_dst[i] *= 2;
2625 }
2626 ggml_tensor* dst = ggml_new_tensor(ctx, type: type_dst, n_dims: 4, ne: ne_dst.data());
2627 ggml_set_param(tensor: dst);
2628 ggml_set_name(tensor: dst, name: "dst");
2629
2630 size_t offset = 0;
2631 for (int i = 0; i < dim; ++i) {
2632 offset += ((ne_dst[i] - ne[i])/2)*dst->nb[i];
2633 }
2634 ggml_tensor * out = ggml_set(ctx, a: dst, b: src,
2635 // The backward pass requires setting a contiguous region:
2636 nb1: src->nb[1], nb2: src->nb[2], nb3: src->nb[3], offset);
2637 ggml_set_name(tensor: out, name: "out");
2638
2639 return out;
2640 }
2641};
2642
2643// GGML_OP_CPY
2644struct test_cpy : public test_case {
2645 const ggml_type type_src;
2646 const ggml_type type_dst;
2647 const std::array<int64_t, 4> ne;
2648 const std::array<int64_t, 4> permute_src;
2649 const std::array<int64_t, 4> permute_dst;
2650 bool _src_use_permute;
2651 bool _dst_use_permute;
2652 bool _src_transpose;
2653
2654 std::string vars() override {
2655 return VARS_TO_STR6(type_src, type_dst, ne, permute_src, permute_dst, _src_transpose);
2656 }
2657
2658 double max_nmse_err() override {
2659 if (type_src == type_dst) {
2660 return 0.0;
2661 }
2662 if (type_dst == GGML_TYPE_Q4_0 || type_dst == GGML_TYPE_Q4_1 || type_dst == GGML_TYPE_IQ4_NL ||
2663 type_dst == GGML_TYPE_Q5_0 || type_dst == GGML_TYPE_Q5_1 || type_dst == GGML_TYPE_Q8_0) {
2664 // estimate what the max nmse error would be if one quantized value is
2665 // off by one. The test values are distributed in [-150,150], so it'll be
2666 // roughly (150*2.0 / 2^bits)^2, divided by the mean square value of the reference,
2667 // which is roughly 0.25*150^2 times the number of elements.
2668 double err_estimate = 1.0f/8.0f * 150.0f;
2669 if (type_dst == GGML_TYPE_IQ4_NL) {
2670 // iq4_nl values are a bit more spread out
2671 err_estimate *= 2.0f;
2672 }
2673 if (type_dst == GGML_TYPE_Q5_0 || type_dst == GGML_TYPE_Q5_1) {
2674 err_estimate /= 2.0f;
2675 }
2676 if (type_dst == GGML_TYPE_Q8_0) {
2677 err_estimate /= 8.0f;
2678 }
2679 err_estimate *= err_estimate;
2680 err_estimate /= (150.0f*150.0f*0.25f)*float(ne[0] * ne[1] * ne[2] * ne[3]);
2681 return err_estimate;
2682 }
2683 return 1e-6;
2684 }
2685
2686 size_t op_size(ggml_tensor * t) override {
2687 return ggml_nbytes(tensor: t) + ggml_nbytes(tensor: t->src[0]);
2688 }
2689
2690 test_cpy(ggml_type type_src = GGML_TYPE_F32, ggml_type type_dst = GGML_TYPE_F32,
2691 std::array<int64_t, 4> ne = {10, 10, 10, 1},
2692 std::array<int64_t, 4> permute_src = {0, 0, 0, 0},
2693 std::array<int64_t, 4> permute_dst = {0, 0, 0, 0},
2694 bool transpose_src = false)
2695 : type_src(type_src), type_dst(type_dst), ne(ne), permute_src(permute_src), permute_dst(permute_dst),
2696 _src_use_permute(permute_src[0] + permute_src[1] + permute_src[2] + permute_src[3] > 0),
2697 _dst_use_permute(permute_dst[0] + permute_dst[1] + permute_dst[2] + permute_dst[3] > 0),
2698 _src_transpose(transpose_src){}
2699
2700 ggml_tensor * build_graph(ggml_context * ctx) override {
2701 ggml_tensor * src = ggml_new_tensor(ctx, type: type_src, n_dims: 4, ne: ne.data());
2702 ggml_set_param(tensor: src);
2703 ggml_set_name(tensor: src, name: "src");
2704
2705 if (_src_use_permute) {
2706 src = ggml_permute(ctx, a: src, axis0: permute_src[0], axis1: permute_src[1], axis2: permute_src[2], axis3: permute_src[3]);
2707 ggml_set_name(tensor: src, name: "src_permuted");
2708 }
2709
2710 if (_src_transpose) {
2711 src = ggml_transpose(ctx, a: src);
2712 ggml_set_name(tensor: src, name: "src_transposed");
2713 }
2714
2715 ggml_tensor * dst = ggml_new_tensor(ctx, type: type_dst, n_dims: 4, ne: src->ne);
2716 ggml_set_name(tensor: dst, name: "dst");
2717
2718 if (_dst_use_permute) {
2719 dst = ggml_permute(ctx, a: dst, axis0: permute_dst[0], axis1: permute_dst[1], axis2: permute_dst[2], axis3: permute_dst[3]);
2720 ggml_set_name(tensor: dst, name: "dst_permuted");
2721 }
2722
2723 ggml_tensor * out = ggml_cpy(ctx, a: src, b: dst);
2724 ggml_set_name(tensor: out, name: "out");
2725
2726 return out;
2727 }
2728
2729 void initialize_tensors(ggml_context * ctx) override {
2730 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
2731 // test extended range of values to check if casting between f32 and i32 is consistent
2732 init_tensor_uniform(tensor: t, min: -150.f, max: 150.f);
2733 }
2734 }
2735};
2736
2737// GGML_OP_CONT
2738struct test_cont : public test_case {
2739 const ggml_type type;
2740 const std::array<int64_t, 4> ne;
2741
2742 std::string vars() override {
2743 return VARS_TO_STR2(type, ne);
2744 }
2745
2746 test_cont(ggml_type type = GGML_TYPE_F32,
2747 std::array<int64_t, 4> ne = {10, 10, 10, 1})
2748 : type(type), ne(ne) {}
2749
2750 ggml_tensor * build_graph(ggml_context * ctx) override {
2751 ggml_tensor * src = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
2752 ggml_set_param(tensor: src);
2753 ggml_set_name(tensor: src, name: "src");
2754
2755 src = ggml_transpose(ctx, a: src);
2756 ggml_set_name(tensor: src, name: "src_transposed");
2757
2758 ggml_tensor * out = ggml_cont(ctx, a: src);
2759 ggml_set_name(tensor: out, name: "out");
2760
2761 return out;
2762 }
2763};
2764
2765// GGML_OP_ADD
2766// GGML_OP_SUB
2767// GGML_OP_MUL
2768// GGML_OP_DIV
2769struct test_bin_bcast : public test_case {
2770 using op_t = ggml_tensor * (*) (ggml_context *, ggml_tensor *, ggml_tensor *);
2771 op_t op;
2772 const ggml_type type;
2773 const std::array<int64_t, 4> ne;
2774 const std::array<int, 4> nr;
2775 int nf; // number of fused ops, nf == 1 -> single op (no fusion)
2776
2777 bool run_whole_graph() override { return true; }
2778
2779 std::string vars() override {
2780 return VARS_TO_STR4(type, ne, nr, nf);
2781 }
2782
2783 size_t op_size(ggml_tensor * t) override {
2784 return ggml_nbytes(tensor: t) * 3;
2785 }
2786
2787 test_bin_bcast(op_t op, ggml_type type = GGML_TYPE_F32,
2788 std::array<int64_t, 4> ne = {10, 10, 1, 1},
2789 std::array<int, 4> nr = {1, 2, 1, 1},
2790 int nf = 1)
2791 : op(op), type(type), ne(ne), nr(nr), nf(nf) {}
2792
2793 ggml_tensor * build_graph(ggml_context * ctx) override {
2794 GGML_ASSERT(nf <= 16);
2795
2796 ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne0: ne[0]*nr[0], ne1: ne[1]*nr[1], ne2: ne[2]*nr[2], ne3: ne[3]*nr[3]);
2797 ggml_set_name(tensor: a, name: "a");
2798
2799 ggml_tensor * b[16];
2800 for (int i = 0; i < nf; ++i) {
2801 b[i] = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
2802 ggml_set_name(tensor: b[i], name: (std::string("b") + std::to_string(val: i)).c_str());
2803 }
2804
2805 // The backward pass supports broadcasting only for GGML_ADD:
2806 const bool grad_supported = op == ggml_add && ggml_are_same_shape(t0: a, t1: b[0]) && nf == 1;
2807 if (grad_supported) {
2808 ggml_set_param(tensor: a);
2809 ggml_set_param(tensor: b[0]);
2810 }
2811
2812 ggml_tensor * out = a;
2813
2814 for (int i = 0; i < nf; ++i) {
2815 out = op(ctx, out, b[i]);
2816 }
2817
2818 ggml_set_name(tensor: out, name: "out");
2819
2820 return out;
2821 }
2822
2823 void initialize_tensors(ggml_context * ctx) override {
2824 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
2825 if (op == ggml_mul || op == ggml_div) {
2826 // MUL and DIV have numerical issues around zero:
2827 init_tensor_uniform(tensor: t, min: 0.9f, max: 1.1f);
2828 } else {
2829 init_tensor_uniform(tensor: t);
2830 }
2831 }
2832 }
2833
2834 float grad_eps() override {
2835 return 0.1f * (op == ggml_mul ? ne[0]*ne[1]*ne[2]*ne[3] : 1);
2836 }
2837
2838 bool grad_precise() override {
2839 return op == ggml_div;
2840 }
2841
2842 double max_maa_err() override {
2843 return op == ggml_add ? 1e-4 : 1e-3;
2844 }
2845};
2846
2847// GGML_OP_ADD_ID
2848struct test_add_id : public test_case {
2849 const ggml_type type_a;
2850 const ggml_type type_b;
2851 const int64_t n_embd;
2852 const int64_t n_experts;
2853 const int64_t n_experts_used;
2854 const int64_t n_token;
2855
2856 std::string vars() override {
2857 return VARS_TO_STR6(type_a, type_b, n_embd, n_experts, n_experts_used, n_token);
2858 }
2859
2860 size_t op_size(ggml_tensor * t) override {
2861 return ggml_nbytes(tensor: t) + ggml_nbytes(tensor: t->src[0]) + ggml_nbytes(tensor: t->src[2]);
2862 }
2863
2864 test_add_id(ggml_type type_a = GGML_TYPE_F32,
2865 ggml_type type_b = GGML_TYPE_F32,
2866 int64_t n_embd = 128,
2867 int64_t n_experts = 16,
2868 int64_t n_experts_used = 8,
2869 int64_t n_token = 10)
2870 : type_a(type_a), type_b(type_b), n_embd(n_embd),
2871 n_experts(n_experts), n_experts_used(n_experts_used), n_token(n_token) {}
2872
2873 ggml_tensor * build_graph(ggml_context * ctx) override {
2874 ggml_tensor * a = ggml_new_tensor_3d(ctx, type: type_a, ne0: n_embd, ne1: n_experts_used, ne2: n_token);
2875 ggml_tensor * b = ggml_new_tensor_2d(ctx, type: type_b, ne0: n_embd, ne1: n_experts);
2876 ggml_tensor * ids = ggml_new_tensor_2d(ctx, type: GGML_TYPE_I32, ne0: n_experts, ne1: n_token);
2877 if (n_experts_used != n_experts) {
2878 ids = ggml_view_2d(ctx, a: ids, ne0: n_experts_used, ne1: n_token, nb1: ids->nb[1], offset: 0);
2879 ggml_set_name(tensor: ids, name: "view_of_ids");
2880 }
2881
2882 ggml_tensor * out = ggml_add_id(ctx, a, b, ids);
2883 ggml_set_name(tensor: out, name: "out");
2884 return out;
2885 }
2886
2887 void initialize_tensors(ggml_context * ctx) override {
2888 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
2889 if (t->type == GGML_TYPE_I32) {
2890 if (ggml_is_view_op(op: t->op)) { continue; }
2891 std::random_device rd;
2892 std::default_random_engine rng(rd());
2893 // ids
2894 for (int64_t r = 0; r < ggml_nrows(tensor: t); r++) {
2895 std::vector<int32_t> data(t->ne[0]);
2896 for (int i = 0; i < t->ne[0]; i++) {
2897 data[i] = i % n_experts;
2898 }
2899 std::shuffle(first: data.begin(), last: data.end(), g&: rng);
2900 ggml_backend_tensor_set(tensor: t, data: data.data(), offset: r * t->nb[1], size: t->ne[0] * sizeof(int32_t));
2901 }
2902 } else {
2903 init_tensor_uniform(tensor: t);
2904 }
2905 }
2906 }
2907};
2908
2909// GGML_OP_ADD1
2910struct test_add1 : public test_case {
2911 const ggml_type type;
2912 const std::array<int64_t, 4> ne;
2913
2914 std::string vars() override {
2915 return VARS_TO_STR2(type, ne);
2916 }
2917
2918 test_add1(ggml_type type = GGML_TYPE_F32,
2919 std::array<int64_t, 4> ne = {10, 5, 4, 3})
2920 : type(type), ne(ne) {}
2921
2922 ggml_tensor * build_graph(ggml_context * ctx) override {
2923 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
2924 ggml_set_param(tensor: a);
2925 ggml_set_name(tensor: a, name: "a");
2926
2927 ggml_tensor * b = ggml_new_tensor_1d(ctx, type, ne0: 1);
2928 // ggml_set_param(b); // TODO: implement
2929 ggml_set_name(tensor: b, name: "b");
2930
2931 ggml_tensor * out = ggml_add1(ctx, a, b);
2932 ggml_set_name(tensor: out, name: "out");
2933
2934 return out;
2935 }
2936
2937 float grad_eps() override {
2938 return 0.1f * ne[0]*ne[1]*ne[2]*ne[3];
2939 }
2940};
2941
2942// GGML_OP_SCALE
2943struct test_scale : public test_case {
2944 const ggml_type type;
2945 const std::array<int64_t, 4> ne;
2946 float scale;
2947 float bias;
2948 bool inplace;
2949
2950 std::string vars() override {
2951 return VARS_TO_STR5(type, ne, scale, bias, inplace);
2952 }
2953
2954 test_scale(ggml_type type = GGML_TYPE_F32,
2955 std::array<int64_t, 4> ne = {10, 10, 10, 10},
2956 float scale = 2.0f,
2957 float bias = 0.0f,
2958 bool inplace = false)
2959 : type(type), ne(ne), scale(scale), bias(bias), inplace(inplace) {}
2960
2961 ggml_tensor * build_graph(ggml_context * ctx) override {
2962 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
2963 ggml_set_param(tensor: a);
2964 ggml_set_name(tensor: a, name: "a");
2965
2966 ggml_tensor * out;
2967 if (inplace) {
2968 out = ggml_scale_bias_inplace(ctx, a, s: scale, b: bias);
2969 } else {
2970 out = ggml_scale_bias(ctx, a, s: scale, b: bias);
2971 }
2972 ggml_set_name(tensor: out, name: "out");
2973
2974 return out;
2975 }
2976};
2977
2978// GGML_OP_SCALE + GGML_UNARY_OP_TANH + GGML_OP_SCALE
2979struct test_softcap : public test_case {
2980 const ggml_type type;
2981 const std::array<int64_t, 4> ne;
2982 float softcap;
2983
2984 std::string op_desc(ggml_tensor * t) override {
2985 GGML_UNUSED(t);
2986 return "SOFTCAP";
2987 }
2988
2989 bool run_whole_graph() override { return true; }
2990
2991 std::string vars() override {
2992 return VARS_TO_STR3(type, ne, softcap);
2993 }
2994
2995 test_softcap(ggml_type type = GGML_TYPE_F32,
2996 std::array<int64_t, 4> ne = {10, 10, 10, 10},
2997 float softcap = 30.0f)
2998 : type(type), ne(ne), softcap(softcap) {}
2999
3000 ggml_tensor * build_graph(ggml_context * ctx) override {
3001 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3002
3003 ggml_set_param(tensor: a);
3004 ggml_set_name(tensor: a, name: "a");
3005
3006 ggml_tensor * out = ggml_scale(ctx, a: ggml_tanh(ctx, a: ggml_scale(ctx, a, s: 1.0f / softcap)), s: softcap);
3007 ggml_set_name(tensor: out, name: "out");
3008
3009 return out;
3010 }
3011};
3012
3013// GGML_OP_SILU_BACK
3014struct test_silu_back : public test_case {
3015 const ggml_type type;
3016 const std::array<int64_t, 4> ne;
3017 float eps;
3018
3019 std::string vars() override {
3020 return VARS_TO_STR3(type, ne, eps);
3021 }
3022
3023 test_silu_back(ggml_type type = GGML_TYPE_F32,
3024 std::array<int64_t, 4> ne = {64, 5, 4, 3},
3025 float eps = 1e-6f)
3026 : type(type), ne(ne), eps(eps) {}
3027
3028 ggml_tensor * build_graph(ggml_context * ctx) override {
3029 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3030 ggml_set_name(tensor: a, name: "a");
3031
3032 ggml_tensor * grad = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3033 ggml_set_name(tensor: grad, name: "grad");
3034
3035 ggml_tensor * out = ggml_silu_back(ctx, a, b: grad);
3036 ggml_set_name(tensor: out, name: "out");
3037
3038 return out;
3039 }
3040
3041 bool grad_precise() override {
3042 return true;
3043 }
3044};
3045
3046// GGML_OP_NORM
3047struct test_norm : public test_case {
3048 const ggml_type type;
3049 const std::array<int64_t, 4> ne;
3050 const bool v; // whether a is a non-contiguous view
3051 const float eps;
3052
3053 std::string vars() override {
3054 return VARS_TO_STR4(type, ne, v, eps);
3055 }
3056
3057 test_norm(ggml_type type = GGML_TYPE_F32,
3058 std::array<int64_t, 4> ne = {64, 5, 4, 3},
3059 bool v = false,
3060 float eps = 1e-6f)
3061 : type(type), ne(ne), v(v), eps(eps) {}
3062
3063 ggml_tensor * build_graph(ggml_context * ctx) override {
3064 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3065 ggml_set_name(tensor: a, name: "a");
3066
3067 if (v) {
3068 a = ggml_view_4d(ctx, a, ne0: a->ne[0]/2, ne1: a->ne[1]/2, ne2: a->ne[2]/2, ne3: a->ne[3]/2, nb1: a->nb[1], nb2: a->nb[2], nb3: a->nb[3], offset: 0);
3069 ggml_set_name(tensor: a, name: "view of a");
3070 }
3071
3072 ggml_tensor * out = ggml_norm(ctx, a, eps);
3073 ggml_set_name(tensor: out, name: "out");
3074
3075 return out;
3076 }
3077};
3078
3079// GGML_OP_NORM + GGML_OP_MUL + GGML_OP_ADD
3080struct test_norm_mul_add : public test_case {
3081 const ggml_type type;
3082 const std::array<int64_t, 4> ne;
3083 float eps;
3084 const bool broadcast;
3085
3086 std::string op_desc(ggml_tensor * t) override {
3087 GGML_UNUSED(t);
3088 return "NORM_MUL_ADD";
3089 }
3090
3091 bool run_whole_graph() override { return true; }
3092
3093 std::string vars() override {
3094 return VARS_TO_STR4(type, ne, eps, broadcast);
3095 }
3096
3097 test_norm_mul_add(ggml_type type = GGML_TYPE_F32,
3098 std::array<int64_t, 4> ne = {128, 2, 1, 1},
3099 float eps = 1e-5f,
3100 bool broadcast = false)
3101 : type(type), ne(ne), eps(eps), broadcast(broadcast) {}
3102
3103 ggml_tensor * build_graph(ggml_context * ctx) override {
3104 std::array<int64_t, 4> broadcast_dims = {ne[0], ne[1] * 2, ne[2] * 2, ne[3] * 2};
3105
3106 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: broadcast ? broadcast_dims.data() : ne.data());
3107 ggml_tensor * w = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3108 ggml_tensor * b = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3109 ggml_set_param(tensor: a); ggml_set_param(tensor: w); ggml_set_param(tensor: b);
3110 ggml_set_name(tensor: a, name: "a"); ggml_set_name(tensor: w, name: "w"); ggml_set_name(tensor: b, name: "b");
3111
3112 // Use a, w and b early to avoid OP_NONE in graph
3113 a = ggml_add(ctx, a: ggml_add(ctx, a, b: w), b);
3114
3115 ggml_tensor * n = ggml_norm(ctx, a, eps);
3116 ggml_tensor * m = ggml_mul(ctx, a: n, b: w);
3117 ggml_tensor * out = ggml_add(ctx, a: m, b);
3118 ggml_set_name(tensor: out, name: "out");
3119 return out;
3120 }
3121};
3122// GGML_OP_RMS_NORM
3123struct test_rms_norm : public test_case {
3124 const ggml_type type;
3125 const std::array<int64_t, 4> ne;
3126 const bool v; // whether a is a non-contiguous view
3127 const float eps;
3128 const bool inplace; // whether to do the operation inplace
3129
3130 std::string vars() override {
3131 return VARS_TO_STR5(type, ne, v, eps, inplace);
3132 }
3133
3134 test_rms_norm(ggml_type type = GGML_TYPE_F32,
3135 std::array<int64_t, 4> ne = {64, 5, 4, 3},
3136 bool v = false,
3137 float eps = 1e-6f,
3138 bool inplace = false)
3139 : type(type), ne(ne), v(v), eps(eps), inplace(inplace) {}
3140
3141 ggml_tensor * build_graph(ggml_context * ctx) override {
3142 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3143 ggml_set_param(tensor: a);
3144 ggml_set_name(tensor: a, name: "a");
3145
3146 if (v) {
3147 a = ggml_view_4d(ctx, a, ne0: a->ne[0]/2, ne1: a->ne[1]/2, ne2: a->ne[2]/2, ne3: a->ne[3]/2, nb1: a->nb[1], nb2: a->nb[2], nb3: a->nb[3], offset: 0);
3148 ggml_set_name(tensor: a, name: "view of a");
3149 }
3150
3151 ggml_tensor * out;
3152 if (inplace) {
3153 out = ggml_rms_norm_inplace(ctx, a, eps);
3154 } else {
3155 out = ggml_rms_norm(ctx, a, eps);
3156 }
3157 ggml_set_name(tensor: out, name: "out");
3158
3159 return out;
3160 }
3161
3162 void initialize_tensors(ggml_context * ctx) override {
3163 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
3164 init_tensor_uniform(tensor: t, min: -10.f, max: 10.f);
3165 }
3166 }
3167
3168 float grad_eps() override {
3169 return 1.0f;
3170 }
3171
3172 bool grad_precise() override {
3173 return true;
3174 }
3175};
3176
3177// GGML_OP_RMS_NORM_BACK
3178struct test_rms_norm_back : public test_case {
3179 const ggml_type type;
3180 const std::array<int64_t, 4> ne;
3181 const float eps;
3182
3183 std::string vars() override {
3184 return VARS_TO_STR3(type, ne, eps);
3185 }
3186
3187 test_rms_norm_back(ggml_type type = GGML_TYPE_F32,
3188 std::array<int64_t, 4> ne = {64, 5, 4, 3},
3189 float eps = 1e-6f)
3190 : type(type), ne(ne), eps(eps) {}
3191
3192 ggml_tensor * build_graph(ggml_context * ctx) override {
3193 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3194 ggml_set_name(tensor: a, name: "a");
3195
3196 ggml_tensor * b = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3197 ggml_set_name(tensor: b, name: "b");
3198
3199 ggml_tensor * out = ggml_rms_norm_back(ctx, a, b, eps);
3200 ggml_set_name(tensor: out, name: "out");
3201
3202 return out;
3203 }
3204
3205 void initialize_tensors(ggml_context * ctx) override {
3206 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
3207 init_tensor_uniform(tensor: t, min: -10.f, max: 10.f);
3208 }
3209 }
3210};
3211
3212// GGML_OP_RMS_NORM + GGML_OP_MUL + GGML_OP_ADD
3213struct test_rms_norm_mul_add : public test_case {
3214 const ggml_type type;
3215 const std::array<int64_t, 4> ne;
3216 const float eps;
3217 const bool broadcast;
3218 const bool multi_add; // test a sequence of adds feeding into rms_norm
3219
3220 std::string op_desc(ggml_tensor * t) override {
3221 GGML_UNUSED(t);
3222 return "RMS_NORM_MUL_ADD";
3223 }
3224
3225 bool run_whole_graph() override { return true; }
3226
3227 std::string vars() override {
3228 return VARS_TO_STR5(type, ne, eps, broadcast, multi_add);
3229 }
3230
3231 test_rms_norm_mul_add(ggml_type type = GGML_TYPE_F32,
3232 std::array<int64_t, 4> ne = {64, 5, 4, 3},
3233 float eps = 1e-6f, bool broadcast = false, bool multi_add = false)
3234 : type(type), ne(ne), eps(eps), broadcast(broadcast), multi_add(multi_add) {}
3235
3236 ggml_tensor * build_graph(ggml_context * ctx) override {
3237 std::array<int64_t, 4> broadcast_dims = {ne[0]*2, ne[1]*3, ne[2]*3, ne[3]*4};
3238
3239 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: broadcast ? broadcast_dims.data() : ne.data());
3240 ggml_tensor * b = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3241 ggml_tensor * c = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3242
3243 ggml_set_param(tensor: a);
3244 ggml_set_name(tensor: a, name: "a");
3245 ggml_set_param(tensor: b);
3246 ggml_set_name(tensor: b, name: "b");
3247 ggml_set_param(tensor: c);
3248 ggml_set_name(tensor: c, name: "c");
3249
3250 // Use a, b and c early, so we don't end up with an OP_NONE between rms_norm and mul
3251 a = ggml_add(ctx, a: ggml_add(ctx, a, b), b: c);
3252 if (multi_add) {
3253 a = ggml_add(ctx, a: ggml_add(ctx, a, b), b: c);
3254 }
3255 ggml_tensor * out = ggml_add(ctx, a: ggml_mul(ctx, a: ggml_rms_norm(ctx, a, eps), b), b: c);
3256 ggml_set_name(tensor: out, name: "out");
3257
3258 return out;
3259 }
3260
3261 void initialize_tensors(ggml_context * ctx) override {
3262 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
3263 init_tensor_uniform(tensor: t, min: -10.f, max: 10.f);
3264 }
3265 }
3266
3267 float grad_eps() override {
3268 return 1.0f;
3269 }
3270
3271 bool grad_precise() override {
3272 return true;
3273 }
3274};
3275
3276// GGML_OP_SSM_CONV
3277struct test_ssm_conv : public test_case {
3278 const ggml_type type;
3279 const std::array<int64_t, 4> ne_a;
3280 const std::array<int64_t, 4> ne_b;
3281
3282 std::string vars() override {
3283 return VARS_TO_STR3(type, ne_a, ne_b);
3284 }
3285
3286 test_ssm_conv(ggml_type type = GGML_TYPE_F32,
3287 std::array<int64_t, 4> ne_a = {10, 10, 10, 1},
3288 std::array<int64_t, 4> ne_b = {3, 3, 1, 1})
3289 : type(type), ne_a(ne_a), ne_b(ne_b) {}
3290
3291 ggml_tensor * build_graph(ggml_context * ctx) override {
3292 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_a.data());
3293 ggml_tensor * b = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_b.data());
3294 ggml_tensor * out = ggml_ssm_conv(ctx, sx: a, c: b);
3295 return out;
3296 }
3297};
3298
3299// GGML_OP_SSM_SCAN
3300struct test_ssm_scan : public test_case {
3301 const ggml_type type;
3302
3303 const int64_t d_state;
3304 const int64_t head_dim;
3305 const int64_t n_head;
3306 const int64_t n_group;
3307 const int64_t n_seq_tokens;
3308 const int64_t n_seqs;
3309
3310 std::string vars() override {
3311 return VARS_TO_STR7(type, d_state, head_dim, n_head, n_group, n_seq_tokens, n_seqs);
3312 }
3313
3314 test_ssm_scan(ggml_type type = GGML_TYPE_F32,
3315 int64_t d_state = 32,
3316 int64_t head_dim = 1, // non-zero for Mamba-2
3317 int64_t n_head = 32,
3318 int64_t n_group = 1,
3319 int64_t n_seq_tokens = 32,
3320 int64_t n_seqs = 32)
3321 : type(type), d_state(d_state), head_dim(head_dim), n_head(n_head), n_group(n_group), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs) {}
3322
3323 ggml_tensor * build_graph(ggml_context * ctx) override {
3324 ggml_tensor * s = ggml_new_tensor_4d(ctx, type, ne0: d_state, ne1: head_dim, ne2: n_head, ne3: n_seqs);
3325 ggml_tensor * x = ggml_new_tensor_4d(ctx, type, ne0: head_dim, ne1: n_head, ne2: n_seq_tokens, ne3: n_seqs);
3326 ggml_tensor * dt = ggml_new_tensor_3d(ctx, type, ne0: n_head, ne1: n_seq_tokens, ne2: n_seqs);
3327 ggml_tensor * A = ggml_new_tensor_2d(ctx, type, ne0: (head_dim > 1) ? 1 : d_state, ne1: n_head);
3328 ggml_tensor * B = ggml_new_tensor_4d(ctx, type, ne0: d_state, ne1: n_group, ne2: n_seq_tokens, ne3: n_seqs);
3329 ggml_tensor * C = ggml_new_tensor_4d(ctx, type, ne0: d_state, ne1: n_group, ne2: n_seq_tokens, ne3: n_seqs);
3330 ggml_tensor * ids = ggml_new_tensor_1d(ctx, type: GGML_TYPE_I32, ne0: n_seqs);
3331 ggml_tensor * out = ggml_ssm_scan(ctx, s, x, dt, A, B, C, ids);
3332 return out;
3333 }
3334
3335 // similar to test_mul_mat_id
3336 void initialize_tensors(ggml_context * ctx) override {
3337 std::random_device rd;
3338 std::default_random_engine rng(rd());
3339 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
3340 if (t->type == GGML_TYPE_I32) {
3341 if (ggml_is_view_op(op: t->op)) { continue; }
3342 // ids
3343 for (int64_t r = 0; r < ggml_nrows(tensor: t); r++) {
3344 std::vector<int32_t> data(t->ne[0]);
3345 for (int i = 0; i < t->ne[0]; i++) {
3346 data[i] = i;
3347 }
3348 std::shuffle(first: data.begin(), last: data.end(), g&: rng);
3349 ggml_backend_tensor_set(tensor: t, data: data.data(), offset: r * t->nb[1], size: t->ne[0] * sizeof(int32_t));
3350 }
3351 } else {
3352 init_tensor_uniform(tensor: t);
3353 }
3354 }
3355 }
3356};
3357
3358// GGML_OP_RWKV_WKV6
3359struct test_rwkv_wkv6 : public test_case {
3360 const ggml_type type;
3361
3362 const int64_t head_count;
3363 const int64_t head_size;
3364 const int64_t n_seq_tokens;
3365 const int64_t n_seqs;
3366
3367 std::string vars() override {
3368 return VARS_TO_STR5(type, head_count, head_size, n_seq_tokens, n_seqs);
3369 }
3370
3371 test_rwkv_wkv6(ggml_type type = GGML_TYPE_F32,
3372 int64_t head_count = 32, int64_t head_size = 64, int64_t n_seq_tokens = 32, int64_t n_seqs = 32)
3373 : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs) {}
3374
3375 ggml_tensor * build_graph(ggml_context * ctx) override {
3376 const int64_t n_tokens = n_seq_tokens * n_seqs;
3377 ggml_tensor * r = ggml_new_tensor(ctx, type, n_dims: 3, ne: std::vector<int64_t>{ head_size, head_count, n_tokens }.data());
3378 ggml_tensor * k = ggml_new_tensor(ctx, type, n_dims: 3, ne: std::vector<int64_t>{ head_size, head_count, n_tokens }.data());
3379 ggml_tensor * v = ggml_new_tensor(ctx, type, n_dims: 3, ne: std::vector<int64_t>{ head_size, head_count, n_tokens }.data());
3380 ggml_tensor * tf = ggml_new_tensor(ctx, type, n_dims: 2, ne: std::vector<int64_t>{ head_size, head_count }.data());
3381 ggml_tensor * td = ggml_new_tensor(ctx, type, n_dims: 3, ne: std::vector<int64_t>{ head_size, head_count, n_tokens }.data());
3382 ggml_tensor * s = ggml_new_tensor(ctx, type, n_dims: 2, ne: std::vector<int64_t>{ head_size * head_size * head_count, n_seqs }.data());
3383 ggml_tensor * out = ggml_rwkv_wkv6(ctx, k, v, r, tf, td, state: s);
3384 return out;
3385 }
3386};
3387
3388// GGML_OP_GATED_LINEAR_ATTN
3389struct test_gla : public test_case {
3390 const ggml_type type;
3391
3392 const int64_t head_count;
3393 const int64_t head_size;
3394 const int64_t n_seq_tokens;
3395 const int64_t n_seqs;
3396
3397 std::string vars() override {
3398 return VARS_TO_STR5(type, head_count, head_size, n_seq_tokens, n_seqs);
3399 }
3400
3401 test_gla(ggml_type type = GGML_TYPE_F32,
3402 int64_t head_count = 32, int64_t head_size = 64, int64_t n_seq_tokens = 32, int64_t n_seqs = 32)
3403 : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs) {}
3404
3405 ggml_tensor * build_graph(ggml_context * ctx) override {
3406 const int64_t n_tokens = n_seq_tokens * n_seqs;
3407 ggml_tensor * q = ggml_new_tensor(ctx, type, n_dims: 3, ne: std::vector<int64_t>{ head_size, head_count, n_tokens }.data());
3408 ggml_tensor * k = ggml_new_tensor(ctx, type, n_dims: 3, ne: std::vector<int64_t>{ head_size, head_count, n_tokens }.data());
3409 ggml_tensor * v = ggml_new_tensor(ctx, type, n_dims: 3, ne: std::vector<int64_t>{ head_size, head_count, n_tokens }.data());
3410 ggml_tensor * g = ggml_new_tensor(ctx, type, n_dims: 3, ne: std::vector<int64_t>{ head_size, head_count, n_tokens }.data());
3411 ggml_tensor * s = ggml_new_tensor(ctx, type, n_dims: 2, ne: std::vector<int64_t>{ head_size * head_size * head_count, n_seqs }.data());
3412 ggml_tensor * out = ggml_gated_linear_attn(ctx, k, v, q, g, state: s, scale: pow(x: head_size, y: -0.5));
3413 return out;
3414 }
3415};
3416
3417// GGML_OP_RWKV_WKV7
3418struct test_rwkv_wkv7 : public test_case {
3419 const ggml_type type;
3420
3421 const int64_t head_count;
3422 const int64_t head_size;
3423 const int64_t n_seq_tokens;
3424 const int64_t n_seqs;
3425
3426 std::string vars() override {
3427 return VARS_TO_STR5(type, head_count, head_size, n_seq_tokens, n_seqs);
3428 }
3429
3430 test_rwkv_wkv7(ggml_type type = GGML_TYPE_F32,
3431 int64_t head_count = 32, int64_t head_size = 64, int64_t n_seq_tokens = 32, int64_t n_seqs = 32)
3432 : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs) {}
3433
3434 ggml_tensor * build_graph(ggml_context * ctx) override {
3435 const int64_t n_tokens = n_seq_tokens * n_seqs;
3436 ggml_tensor * r = ggml_new_tensor(ctx, type, n_dims: 3, ne: std::vector<int64_t>{ head_size, head_count, n_tokens }.data());
3437 ggml_tensor * w = ggml_new_tensor(ctx, type, n_dims: 3, ne: std::vector<int64_t>{ head_size, head_count, n_tokens }.data());
3438 ggml_tensor * k = ggml_new_tensor(ctx, type, n_dims: 3, ne: std::vector<int64_t>{ head_size, head_count, n_tokens }.data());
3439 ggml_tensor * v = ggml_new_tensor(ctx, type, n_dims: 3, ne: std::vector<int64_t>{ head_size, head_count, n_tokens }.data());
3440 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 3, ne: std::vector<int64_t>{ head_size, head_count, n_tokens }.data());
3441 ggml_tensor * b = ggml_new_tensor(ctx, type, n_dims: 3, ne: std::vector<int64_t>{ head_size, head_count, n_tokens }.data());
3442 // Outputs may become NaN with long seqlen without these normalization
3443 a = ggml_l2_norm(ctx, a, eps: 1e-7F);
3444 b = ggml_l2_norm(ctx, a: b, eps: 1e-7F);
3445 ggml_tensor * s = ggml_new_tensor(ctx, type, n_dims: 2, ne: std::vector<int64_t>{ head_size * head_size * head_count, n_seqs }.data());
3446 ggml_tensor * out = ggml_rwkv_wkv7(ctx, r, w, k, v, a, b, state: s);
3447 return out;
3448 }
3449};
3450
3451// GGML_OP_MUL_MAT
3452struct test_mul_mat : public test_case {
3453 const ggml_type type_a;
3454 const ggml_type type_b;
3455 const int64_t m;
3456 const int64_t n;
3457 const int64_t k;
3458 const std::array<int64_t, 2> bs; // dims 3 and 4
3459 const std::array<int64_t, 2> nr; // repeat in dims 3 and 4
3460 const std::array<int64_t, 4> per; // permutation of dimensions
3461 const int64_t k_v; // size of k in memory, resulting in a non-contiguous view for k_v > k, no view for k_v == 0
3462 const uint32_t o; // number of outputs
3463
3464 std::string vars() override {
3465 return VARS_TO_STR10(type_a, type_b, m, n, k, bs, nr, per, k_v, o);
3466 }
3467
3468 double max_nmse_err() override {
3469 return 5e-4;
3470 }
3471
3472 int64_t grad_nmax() override {
3473 return 20000;
3474 }
3475
3476 uint64_t op_flops(ggml_tensor * t) override {
3477 GGML_UNUSED(t);
3478 return 2 * m * n * k * bs[0] * nr[0] * bs[1] * nr[1];
3479 }
3480
3481 test_mul_mat(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32,
3482 int64_t m = 32, int64_t n = 32, int64_t k = 32,
3483 std::array<int64_t, 2> bs = {10, 10},
3484 std::array<int64_t, 2> nr = {2, 2},
3485 std::array<int64_t, 4> per = {0, 1, 2, 3},
3486 int64_t k_v = 0, uint32_t o = 1)
3487 : type_a(type_a), type_b(type_b), m(m), n(n), k(k), bs(bs), nr(nr), per(per), k_v(k_v), o(o) {}
3488
3489 ggml_tensor * build_graph(ggml_context * ctx) override {
3490 // C^T = A * B^T: (k, m) * (k, n) => (m, n)
3491 ggml_tensor * a;
3492 ggml_tensor * b;
3493
3494 const int npermuted = (per[0] != 0) + (per[1] != 1) + (per[2] != 2) + (per[3] != 3);
3495 if (npermuted > 0) {
3496 GGML_ASSERT(npermuted == 2);
3497 GGML_ASSERT(k_v == 0); // not handled
3498 GGML_ASSERT(!ggml_is_quantized(type_a) || per[0] == 0);
3499 GGML_ASSERT(!ggml_is_quantized(type_b) || per[0] == 0);
3500
3501 // Create tensors with the permuted dimensions, then permute them back to the dimensions given by m,n,k.
3502 const int64_t ne_a[4] = {k, m, bs[0], bs[1]};
3503 const int64_t ne_b[4] = {k, n, bs[0]*nr[0], bs[1]*nr[1]};
3504
3505 a = ggml_new_tensor_4d(ctx, type: type_a, ne0: ne_a[per[0]], ne1: ne_a[per[1]], ne2: ne_a[per[2]], ne3: ne_a[per[3]]);
3506 b = ggml_new_tensor_4d(ctx, type: type_b, ne0: ne_b[per[0]], ne1: ne_b[per[1]], ne2: ne_b[per[2]], ne3: ne_b[per[3]]);
3507 if (!ggml_is_quantized(type: type_a)) {
3508 if (bs[1] == 1 && nr[1] == 1) {
3509 ggml_set_param(tensor: a);
3510 }
3511 ggml_set_param(tensor: b);
3512 }
3513 ggml_set_name(tensor: a, name: "a");
3514 ggml_set_name(tensor: b, name: "b");
3515
3516 a = ggml_permute(ctx, a, axis0: per[0], axis1: per[1], axis2: per[2], axis3: per[3]);
3517 b = ggml_permute(ctx, a: b, axis0: per[0], axis1: per[1], axis2: per[2], axis3: per[3]);
3518 ggml_set_name(tensor: a, name: "a_permuted");
3519 ggml_set_name(tensor: b, name: "b_permuted");
3520 } else {
3521 const int64_t k_physical = k_v == 0 ? k : k_v;
3522 a = ggml_new_tensor_4d(ctx, type: type_a, ne0: k_physical, ne1: m, ne2: bs[0], ne3: bs[1]);
3523 b = ggml_new_tensor_4d(ctx, type: type_b, ne0: k_physical, ne1: n, ne2: bs[0]*nr[0], ne3: bs[1]*nr[1]);
3524
3525 if (!ggml_is_quantized(type: type_a)) {
3526 if (bs[1] == 1 && nr[1] == 1) {
3527 ggml_set_param(tensor: a);
3528 }
3529 ggml_set_param(tensor: b);
3530 }
3531
3532 if (k_v != 0) {
3533 GGML_ASSERT(k_v > k);
3534 a = ggml_view_4d(ctx, a, ne0: k, ne1: m, ne2: bs[0], ne3: bs[1], nb1: a->nb[1], nb2: a->nb[2], nb3: a->nb[3], offset: 0);
3535 b = ggml_view_4d(ctx, a: b, ne0: k, ne1: n, ne2: bs[0]*nr[0], ne3: bs[1]*nr[1], nb1: b->nb[1], nb2: b->nb[2], nb3: b->nb[3], offset: 0);
3536 }
3537 ggml_set_name(tensor: a, name: "a");
3538 ggml_set_name(tensor: b, name: "b");
3539 }
3540
3541 ggml_tensor * out = ggml_mul_mat(ctx, a, b);
3542 ggml_set_name(tensor: out, name: "out");
3543 for (uint32_t i = 1; i < o; ++i) {
3544 ggml_tensor * out2 = ggml_mul_mat(ctx, a, b);
3545 ggml_set_name(tensor: out2, name: "out2");
3546 out = ggml_add(ctx, a: out, b: out2);
3547 }
3548
3549 return out;
3550 }
3551
3552 bool run_whole_graph() override { return o > 1; }
3553
3554 std::string op_desc(ggml_tensor * t) override {
3555 GGML_UNUSED(t);
3556 return ggml_op_name(op: GGML_OP_MUL_MAT);
3557 }
3558};
3559
3560// GGML_OP_MUL_MAT_ID
3561struct test_mul_mat_id : public test_case {
3562 const ggml_type type_a;
3563 const ggml_type type_b;
3564 const int n_mats;
3565 const int n_used;
3566 const bool b; // broadcast b matrix
3567 const int64_t m;
3568 const int64_t n;
3569 const int64_t k;
3570 const uint32_t o; // number of outputs
3571
3572 std::string vars() override {
3573 return VARS_TO_STR9(type_a, type_b, n_mats, n_used, b, m, n, k, o);
3574 }
3575
3576 double max_nmse_err() override {
3577 return 5e-4;
3578 }
3579
3580 uint64_t op_flops(ggml_tensor * t) override {
3581 GGML_UNUSED(t);
3582 return 2 * m * k * n * n_used;
3583 }
3584
3585 test_mul_mat_id(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32,
3586 int n_mats = 8, int n_used = 2, bool b = false,
3587 int64_t m = 32, int64_t n = 32, int64_t k = 32, uint32_t o = 1)
3588 : type_a(type_a), type_b(type_b), n_mats(n_mats), n_used(n_used), b(b),
3589 m(m), n(n), k(k), o(o) {
3590 GGML_ASSERT(n_used <= n_mats);
3591 }
3592
3593 ggml_tensor * build_graph(ggml_context * ctx) override {
3594 // C^T = A * B^T: (k, m) * (k, n) => (m, n)
3595 ggml_tensor * as = ggml_new_tensor_3d(ctx, type: type_a, ne0: k, ne1: m, ne2: n_mats);
3596 ggml_set_name(tensor: as, name: "as");
3597
3598 ggml_tensor * ids = ggml_new_tensor_2d(ctx, type: GGML_TYPE_I32, ne0: n_mats, ne1: n);
3599 ggml_set_name(tensor: ids, name: "ids");
3600 if (n_used != n_mats) {
3601 ids = ggml_view_2d(ctx, a: ids, ne0: n_used, ne1: n, nb1: ids->nb[1], offset: 0);
3602 ggml_set_name(tensor: ids, name: "view_of_ids");
3603 }
3604
3605 ggml_tensor * b = ggml_new_tensor_3d(ctx, type: type_b, ne0: k, ne1: this->b ? 1 : n_used, ne2: n);
3606 ggml_set_name(tensor: b, name: "b");
3607
3608 ggml_tensor * out = ggml_mul_mat_id(ctx, as, b, ids);
3609 ggml_set_name(tensor: out, name: "out");
3610
3611 for (uint32_t i = 1; i < o; ++i) {
3612 ggml_tensor * a2 = ggml_new_tensor_3d(ctx, type: type_a, ne0: k, ne1: m, ne2: n_mats);
3613 ggml_tensor * out2 = ggml_mul_mat_id(ctx, as: a2, b, ids);
3614 ggml_set_name(tensor: out2, name: "out2");
3615 out = ggml_add(ctx, a: out, b: out2);
3616 }
3617
3618 return out;
3619 }
3620
3621 void initialize_tensors(ggml_context * ctx) override {
3622 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
3623 if (t->type == GGML_TYPE_I32) {
3624 if (ggml_is_view_op(op: t->op)) { continue; }
3625 std::random_device rd;
3626 std::default_random_engine rng(rd());
3627 // ids
3628 for (int64_t r = 0; r < ggml_nrows(tensor: t); r++) {
3629 std::vector<int32_t> data(t->ne[0]);
3630 for (int i = 0; i < t->ne[0]; i++) {
3631 data[i] = i % n_mats;
3632 }
3633 std::shuffle(first: data.begin(), last: data.end(), g&: rng);
3634 ggml_backend_tensor_set(tensor: t, data: data.data(), offset: r * t->nb[1], size: t->ne[0] * sizeof(int32_t));
3635 }
3636 } else {
3637 init_tensor_uniform(tensor: t);
3638 }
3639 }
3640 }
3641
3642 bool run_whole_graph() override { return o > 1; }
3643
3644 std::string op_desc(ggml_tensor * t) override {
3645 GGML_UNUSED(t);
3646 return ggml_op_name(op: GGML_OP_MUL_MAT_ID);
3647 }
3648};
3649
3650// GGML_OP_OUT_PROD
3651struct test_out_prod : public test_case {
3652 const ggml_type type_a;
3653 const ggml_type type_b;
3654 const int64_t m;
3655 const int64_t n;
3656 const int64_t k;
3657 const std::array<int64_t, 2> bs; // dims 3 and 4
3658 const std::array<int64_t, 2> nr; // repeat in dims 3 and 4
3659 const bool trans_b;
3660
3661 std::string vars() override {
3662 return VARS_TO_STR8(type_a, type_b, m, n, k, bs, nr, trans_b);
3663 }
3664
3665 double max_nmse_err() override {
3666 return 5e-4;
3667 }
3668
3669 test_out_prod(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32,
3670 int64_t m = 32, int64_t n = 32, int64_t k = 32,
3671 std::array<int64_t, 2> bs = {10, 10},
3672 std::array<int64_t, 2> nr = {2, 2},
3673 bool trans_b = false)
3674 : type_a(type_a), type_b(type_b), m(m), n(n), k(k), bs(bs), nr(nr), trans_b(trans_b) {}
3675
3676 ggml_tensor * build_graph(ggml_context * ctx) override {
3677 ggml_tensor * a = ggml_new_tensor_4d(ctx, type: type_a, ne0: m, ne1: k, ne2: bs[0], ne3: bs[1]);
3678 ggml_set_name(tensor: a, name: "a");
3679
3680 ggml_tensor * b;
3681 if (trans_b) {
3682 b = ggml_new_tensor_4d(ctx, type: type_b, ne0: k, ne1: n, ne2: bs[0]*nr[0], ne3: bs[1]*nr[1]);
3683 b = ggml_transpose(ctx, a: b);
3684 } else {
3685 b = ggml_new_tensor_4d(ctx, type: type_b, ne0: n, ne1: k, ne2: bs[0]*nr[0], ne3: bs[1]*nr[1]);
3686 }
3687 ggml_set_name(tensor: b, name: "b");
3688
3689 ggml_tensor * out = ggml_out_prod(ctx, a, b);
3690 ggml_set_name(tensor: out, name: "out");
3691
3692 return out;
3693 }
3694};
3695
3696// GGML_OP_SQR
3697struct test_sqr : public test_case {
3698 const ggml_type type;
3699 const std::array<int64_t, 4> ne;
3700
3701 std::string vars() override {
3702 return VARS_TO_STR2(type, ne);
3703 }
3704
3705 test_sqr(ggml_type type = GGML_TYPE_F32,
3706 std::array<int64_t, 4> ne = {10, 5, 4, 3})
3707 : type(type), ne(ne) {}
3708
3709 ggml_tensor * build_graph(ggml_context * ctx) override {
3710 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3711 ggml_set_param(tensor: a);
3712 ggml_set_name(tensor: a, name: "a");
3713
3714 ggml_tensor * out = ggml_sqr(ctx, a);
3715 ggml_set_name(tensor: out, name: "out");
3716
3717 return out;
3718 }
3719
3720 float grad_eps() override {
3721 return 0.1f * 0.25f*ne[0]*ne[1]*ne[2]*ne[3]; // 10% of expected value of sum.
3722 }
3723};
3724
3725// GGML_OP_SQRT
3726struct test_sqrt : public test_case {
3727 const ggml_type type;
3728 const std::array<int64_t, 4> ne;
3729
3730 std::string vars() override {
3731 return VARS_TO_STR2(type, ne);
3732 }
3733
3734 test_sqrt(ggml_type type = GGML_TYPE_F32,
3735 std::array<int64_t, 4> ne = {10, 3, 3, 2})
3736 : type(type), ne(ne) {}
3737
3738 ggml_tensor * build_graph(ggml_context * ctx) override {
3739 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3740 ggml_set_param(tensor: a);
3741 ggml_set_name(tensor: a, name: "a");
3742
3743 ggml_tensor * out = ggml_sqrt(ctx, a);
3744 ggml_set_name(tensor: out, name: "out");
3745
3746 return out;
3747 }
3748
3749 void initialize_tensors(ggml_context * ctx) override {
3750 // fill with positive values
3751 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
3752 init_tensor_uniform(tensor: t, min: 50.0f, max: 100.0f);
3753 }
3754 }
3755
3756 float grad_eps() override {
3757 return 20.0f;
3758 }
3759
3760 bool grad_precise() override {
3761 return true;
3762 }
3763};
3764
3765// GGML_OP_LOG
3766struct test_log : public test_case {
3767 const ggml_type type;
3768 const std::array<int64_t, 4> ne;
3769
3770 std::string vars() override {
3771 return VARS_TO_STR2(type, ne);
3772 }
3773
3774 test_log(ggml_type type = GGML_TYPE_F32,
3775 std::array<int64_t, 4> ne = {10, 5, 4, 3})
3776 : type(type), ne(ne) {}
3777
3778 ggml_tensor * build_graph(ggml_context * ctx) override {
3779 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3780 ggml_set_param(tensor: a);
3781 ggml_set_name(tensor: a, name: "a");
3782
3783 ggml_tensor * out = ggml_log(ctx, a);
3784 ggml_set_name(tensor: out, name: "out");
3785
3786 return out;
3787 }
3788
3789 void initialize_tensors(ggml_context * ctx) override {
3790 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
3791 // log(1) == 0, cluster values there to keep the sum low for better precision in the backward pass:
3792 init_tensor_uniform(tensor: t, min: 0.9f, max: 1.1f);
3793 }
3794 }
3795
3796 bool grad_precise() override {
3797 return true;
3798 }
3799};
3800
3801// GGML_OP_SIN
3802struct test_sin : public test_case {
3803 const ggml_type type;
3804 const std::array<int64_t, 4> ne;
3805
3806 std::string vars() override {
3807 return VARS_TO_STR2(type, ne);
3808 }
3809
3810 test_sin(ggml_type type = GGML_TYPE_F32,
3811 std::array<int64_t, 4> ne = {10, 2, 2, 2})
3812 : type(type), ne(ne) {}
3813
3814 ggml_tensor * build_graph(ggml_context * ctx) override {
3815 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3816 ggml_set_param(tensor: a);
3817 ggml_set_name(tensor: a, name: "a");
3818
3819 ggml_tensor * out = ggml_sin(ctx, a);
3820 ggml_set_name(tensor: out, name: "out");
3821
3822 return out;
3823 }
3824
3825 void initialize_tensors(ggml_context * ctx) override {
3826 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
3827 init_tensor_uniform(tensor: t, min: -6.5f, max: 6.5f); // Covers interval [-2*pi, 2*pi].
3828 }
3829 }
3830
3831 double max_maa_err() override {
3832 return 1e-3;
3833 }
3834
3835 float grad_eps() override {
3836 return 0.2f;
3837 }
3838
3839 bool grad_precise() override {
3840 return true;
3841 }
3842};
3843
3844// GGML_OP_COS
3845struct test_cos : public test_case {
3846 const ggml_type type;
3847 const std::array<int64_t, 4> ne;
3848
3849 std::string vars() override {
3850 return VARS_TO_STR2(type, ne);
3851 }
3852
3853 test_cos(ggml_type type = GGML_TYPE_F32,
3854 std::array<int64_t, 4> ne = {10, 2, 2, 2})
3855 : type(type), ne(ne) {}
3856
3857 ggml_tensor * build_graph(ggml_context * ctx) override {
3858 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3859 ggml_set_param(tensor: a);
3860 ggml_set_name(tensor: a, name: "a");
3861
3862 ggml_tensor * out = ggml_cos(ctx, a);
3863 ggml_set_name(tensor: out, name: "out");
3864
3865 return out;
3866 }
3867
3868 void initialize_tensors(ggml_context * ctx) override {
3869 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
3870 init_tensor_uniform(tensor: t, min: -6.5f, max: 6.5f); // Covers interval [-2*pi, 2*pi].
3871 }
3872 }
3873
3874 double max_maa_err() override {
3875 return 1e-3;
3876 }
3877
3878 float grad_eps() override {
3879 return 0.2f;
3880 }
3881
3882 bool grad_precise() override {
3883 return true;
3884 }
3885};
3886
3887// GGML_OP_CLAMP
3888struct test_clamp : public test_case {
3889 const ggml_type type;
3890 const std::array<int64_t, 4> ne;
3891 float min;
3892 float max;
3893
3894 std::string vars() override {
3895 return VARS_TO_STR4(type, ne, min, max);
3896 }
3897
3898 test_clamp(ggml_type type = GGML_TYPE_F32,
3899 std::array<int64_t, 4> ne = {10, 5, 4, 3},
3900 float min = -0.5f, float max = 0.5f)
3901 : type(type), ne(ne), min(min), max(max) {}
3902
3903 ggml_tensor * build_graph(ggml_context * ctx) override {
3904 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3905 ggml_set_name(tensor: a, name: "a");
3906
3907 ggml_tensor * out = ggml_clamp(ctx, a, min, max);
3908 ggml_set_name(tensor: out, name: "out");
3909
3910 return out;
3911 }
3912
3913 float grad_eps() override {
3914 return 1e-2f;
3915 }
3916
3917 std::vector<float> grad_expect() override {
3918 return {0.0f, 1.0f};
3919 }
3920};
3921
3922// GGML_OP_FLOOR
3923struct test_floor : public test_case {
3924 const ggml_type type;
3925 const std::array<int64_t, 4> ne;
3926
3927 std::string vars() override {
3928 return VARS_TO_STR2(type, ne);
3929 }
3930
3931 test_floor(ggml_type type = GGML_TYPE_F32,
3932 std::array<int64_t, 4> ne = {10, 2, 2, 2})
3933 : type(type), ne(ne) {}
3934
3935 ggml_tensor * build_graph(ggml_context * ctx) override {
3936 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3937 ggml_set_param(tensor: a);
3938 ggml_set_name(tensor: a, name: "a");
3939
3940 ggml_tensor * out = ggml_floor(ctx, a);
3941 ggml_set_name(tensor: out, name: "out");
3942
3943 return out;
3944 }
3945
3946 void initialize_tensors(ggml_context * ctx) override {
3947 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
3948 init_tensor_uniform(tensor: t, min: -10.0f, max: 10.0f);
3949 }
3950 }
3951};
3952
3953// GGML_OP_CEIL
3954struct test_ceil : public test_case {
3955 const ggml_type type;
3956 const std::array<int64_t, 4> ne;
3957
3958 std::string vars() override {
3959 return VARS_TO_STR2(type, ne);
3960 }
3961
3962 test_ceil(ggml_type type = GGML_TYPE_F32,
3963 std::array<int64_t, 4> ne = {10, 2, 2, 2})
3964 : type(type), ne(ne) {}
3965
3966 ggml_tensor * build_graph(ggml_context * ctx) override {
3967 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3968 ggml_set_param(tensor: a);
3969 ggml_set_name(tensor: a, name: "a");
3970
3971 ggml_tensor * out = ggml_ceil(ctx, a);
3972 ggml_set_name(tensor: out, name: "out");
3973
3974 return out;
3975 }
3976
3977 void initialize_tensors(ggml_context * ctx) override {
3978 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
3979 init_tensor_uniform(tensor: t, min: -10.0f, max: 10.0f);
3980 }
3981 }
3982};
3983
3984// GGML_OP_ROUND
3985struct test_round : public test_case {
3986 const ggml_type type;
3987 const std::array<int64_t, 4> ne;
3988
3989 std::string vars() override {
3990 return VARS_TO_STR2(type, ne);
3991 }
3992
3993 test_round(ggml_type type = GGML_TYPE_F32,
3994 std::array<int64_t, 4> ne = {10, 2, 2, 2})
3995 : type(type), ne(ne) {}
3996
3997 ggml_tensor * build_graph(ggml_context * ctx) override {
3998 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
3999 ggml_set_param(tensor: a);
4000 ggml_set_name(tensor: a, name: "a");
4001
4002 ggml_tensor * out = ggml_round(ctx, a);
4003 ggml_set_name(tensor: out, name: "out");
4004
4005 return out;
4006 }
4007
4008 void initialize_tensors(ggml_context * ctx) override {
4009 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
4010 init_tensor_uniform(tensor: t, min: -10.0f, max: 10.0f);
4011 }
4012 }
4013};
4014
4015// GGML_OP_TRUNC
4016struct test_trunc : public test_case {
4017 const ggml_type type;
4018 const std::array<int64_t, 4> ne;
4019
4020 std::string vars() override {
4021 return VARS_TO_STR2(type, ne);
4022 }
4023
4024 test_trunc(ggml_type type = GGML_TYPE_F32,
4025 std::array<int64_t, 4> ne = {10, 2, 2, 2})
4026 : type(type), ne(ne) {}
4027
4028 ggml_tensor * build_graph(ggml_context * ctx) override {
4029 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
4030 ggml_set_param(tensor: a);
4031 ggml_set_name(tensor: a, name: "a");
4032
4033 ggml_tensor * out = ggml_trunc(ctx, a);
4034 ggml_set_name(tensor: out, name: "out");
4035
4036 return out;
4037 }
4038
4039 void initialize_tensors(ggml_context * ctx) override {
4040 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
4041 init_tensor_uniform(tensor: t, min: -10.0f, max: 10.0f);
4042 }
4043 }
4044};
4045
4046// GGML_OP_DIAG_MASK_INF
4047struct test_diag_mask_inf : public test_case {
4048 const ggml_type type;
4049 const std::array<int64_t, 4> ne;
4050 const int n_past;
4051
4052 std::string vars() override {
4053 return VARS_TO_STR3(type, ne, n_past);
4054 }
4055
4056 test_diag_mask_inf(ggml_type type = GGML_TYPE_F32,
4057 std::array<int64_t, 4> ne = {10, 10, 3, 2},
4058 int n_past = 5)
4059 : type(type), ne(ne), n_past(n_past) {}
4060
4061 ggml_tensor * build_graph(ggml_context * ctx) override {
4062 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
4063 ggml_set_param(tensor: a);
4064 ggml_set_name(tensor: a, name: "a");
4065
4066 ggml_tensor * out = ggml_diag_mask_inf(ctx, a, n_past);
4067 ggml_set_name(tensor: out, name: "out");
4068
4069 return out;
4070 }
4071};
4072
4073// GGML_OP_SOFT_MAX
4074struct test_soft_max : public test_case {
4075 const ggml_type type;
4076 const std::array<int64_t, 4> ne;
4077 const bool mask;
4078 const bool sinks;
4079 const ggml_type m_prec;
4080 const std::array<int64_t, 2> nr23; // broadcast only dims 2 and 3
4081 const float scale;
4082 const float max_bias;
4083 const bool inplace;
4084
4085 std::string vars() override {
4086 return VARS_TO_STR9(type, ne, mask, sinks, m_prec, nr23, scale, max_bias, inplace);
4087 }
4088
4089 // the 1024 test with bias occasionally fails:
4090 // SOFT_MAX(type=f32,ne=[1024,16,1,1],mask=1,scale=1.000000,max_bias=8.000000): [SOFT_MAX] NMSE = 0.000000103 > 0.000000100 FAIL
4091 virtual double max_nmse_err() override {
4092 return 1e-6;
4093 }
4094
4095 test_soft_max(ggml_type type = GGML_TYPE_F32,
4096 std::array<int64_t, 4> ne = {10, 5, 4, 3},
4097 bool mask = false,
4098 bool sinks = false,
4099 ggml_type m_prec = GGML_TYPE_F32,
4100 std::array<int64_t, 2> nr23 = {1, 1},
4101 float scale = 1.0f,
4102 float max_bias = 0.0f,
4103 bool inplace = false)
4104 : type(type), ne(ne), mask(mask), sinks(sinks), m_prec(m_prec), nr23(nr23), scale(scale), max_bias(max_bias), inplace(inplace) {}
4105
4106 ggml_tensor * build_graph(ggml_context * ctx) override {
4107 ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne0: ne[0], ne1: ne[1], ne2: ne[2]*nr23[0], ne3: ne[3]*nr23[1]);
4108 ggml_set_param(tensor: a);
4109 ggml_set_name(tensor: a, name: "a");
4110
4111 ggml_tensor * mask = nullptr;
4112 if (this->mask) {
4113 mask = ggml_new_tensor_4d(ctx, type: m_prec, ne0: ne[0], ne1: ne[1], ne2: ne[2], ne3: ne[3]);
4114 ggml_set_name(tensor: mask, name: "mask");
4115 }
4116
4117 ggml_tensor * sinks = nullptr;
4118 if (this->sinks) {
4119 sinks = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F32, ne0: ne[2]*nr23[0]);
4120 ggml_set_name(tensor: sinks, name: "sinks");
4121 }
4122
4123 ggml_tensor * out;
4124 if (inplace) {
4125 out = ggml_soft_max_ext_inplace(ctx, a, mask, scale, max_bias);
4126 } else {
4127 out = ggml_soft_max_ext(ctx, a, mask, scale, max_bias);
4128 }
4129 ggml_soft_max_add_sinks(a: out, sinks);
4130 ggml_set_name(tensor: out, name: "out");
4131
4132 return out;
4133 }
4134
4135 bool grad_precise() override {
4136 return true;
4137 }
4138};
4139
4140// GGML_OP_SOFT_MAX_BACK
4141struct test_soft_max_back : public test_case {
4142 const ggml_type type;
4143 const std::array<int64_t, 4> ne;
4144 const float scale;
4145 const float max_bias;
4146
4147 std::string vars() override {
4148 return VARS_TO_STR4(type, ne, scale, max_bias);
4149 }
4150
4151 test_soft_max_back(ggml_type type = GGML_TYPE_F32,
4152 std::array<int64_t, 4> ne = {10, 5, 4, 3},
4153 float scale = 1.0f,
4154 float max_bias = 0.0f)
4155 : type(type), ne(ne), scale(scale), max_bias(max_bias) {}
4156
4157 ggml_tensor * build_graph(ggml_context * ctx) override {
4158 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
4159 ggml_set_name(tensor: a, name: "a");
4160
4161 ggml_tensor * b = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
4162 ggml_set_name(tensor: a, name: "a");
4163
4164 ggml_tensor * out = ggml_soft_max_ext_back(ctx, a, b, scale, max_bias);
4165 ggml_set_name(tensor: out, name: "out");
4166
4167 return out;
4168 }
4169};
4170
4171// GGML_OP_ROPE + GGML_OP_ROPE_BACK
4172struct test_rope : public test_case {
4173 const ggml_type type;
4174 const std::array<int64_t, 4> ne_a;
4175 int n_dims;
4176 int mode;
4177 int n_ctx; // used to generate positions
4178 float fs; // freq_scale
4179 float ef; // ext_factor
4180 float af; // attn_factor
4181 bool ff;
4182 int v; // view (1 : non-contiguous a)
4183 bool forward;
4184 bool inplace;
4185
4186 std::string vars() override {
4187 // forward can be inferred from the op, does not need to be printed
4188 return VARS_TO_STR11(type, ne_a, n_dims, mode, n_ctx, fs, ef, af, ff, v, inplace);
4189 }
4190
4191 test_rope(ggml_type type = GGML_TYPE_F32,
4192 std::array<int64_t, 4> ne_a = {10, 5, 3, 1},
4193 int n_dims = 10, int mode = GGML_ROPE_TYPE_NORMAL, int n_ctx = 512, float fs = 1.0f,
4194 float ef = 0.0f, float af = 0.0f, bool ff = false, int v = 0, bool forward = true, bool inplace = false)
4195 : type(type), ne_a(ne_a), n_dims(n_dims), mode(mode), n_ctx(n_ctx), fs(fs), ef(ef), af(af), ff(ff), v(v), forward(forward), inplace(inplace) {}
4196
4197 ggml_tensor * build_graph(ggml_context * ctx) override {
4198 ggml_tensor * a;
4199 if (v & 1) {
4200 auto ne = ne_a; ne[0] *= 2; ne[1] *= 4; ne[2] *= 3;
4201 a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
4202 if (forward) {
4203 ggml_set_param(tensor: a);
4204 }
4205 ggml_set_name(tensor: a, name: "a");
4206
4207 a = ggml_view_4d(ctx, a, ne0: ne_a[0], ne1: ne_a[1], ne2: ne_a[2], ne3: ne_a[3], nb1: a->nb[1], nb2: a->nb[2], nb3: a->nb[3], offset: 0);
4208 ggml_set_name(tensor: a, name: "view_of_a");
4209 } else {
4210 a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_a.data());
4211 if (forward) {
4212 ggml_set_param(tensor: a);
4213 }
4214 ggml_set_name(tensor: a, name: "a");
4215 }
4216
4217 const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE;
4218 const bool is_vision = mode == GGML_ROPE_TYPE_VISION;
4219
4220 ggml_tensor * pos;
4221 if (is_mrope || is_vision) {
4222 pos = ggml_new_tensor_1d(ctx, type: GGML_TYPE_I32, ne0: ne_a[2] * 4);
4223 } else {
4224 pos = ggml_new_tensor_1d(ctx, type: GGML_TYPE_I32, ne0: ne_a[2]);
4225 }
4226 ggml_set_name(tensor: pos, name: "pos");
4227
4228 ggml_tensor * freq = nullptr;
4229 if (ff) {
4230 freq = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F32, ne0: n_dims/2);
4231 ggml_set_name(tensor: freq, name: "freq");
4232 }
4233
4234 ggml_tensor * out;
4235 if (is_mrope) {
4236 if (is_vision) {
4237 GGML_ASSERT(n_dims/4 > 0);
4238 int rope_sections[4] = {n_dims/4, n_dims/4, 0, 0}; // Vision-RoPE only use first two dimension for image (x, y) coordinate
4239 if (forward) {
4240 if (inplace) {
4241 out = ggml_rope_multi_inplace(ctx, a, b: pos, c: freq, n_dims: n_dims/2, sections: rope_sections, mode, n_ctx_orig: 0, freq_base: 10000.0f, freq_scale: fs, ext_factor: ef, attn_factor: af, beta_fast: 1.0f, beta_slow: 1.0f);
4242 } else {
4243 out = ggml_rope_multi(ctx, a, b: pos, c: freq, n_dims: n_dims/2, sections: rope_sections, mode, n_ctx_orig: 0, freq_base: 10000.0f, freq_scale: fs, ext_factor: ef, attn_factor: af, beta_fast: 1.0f, beta_slow: 1.0f);
4244 }
4245 } else {
4246 out = ggml_rope_multi_back(ctx, a, b: pos, c: freq, n_dims: n_dims/2, sections: rope_sections, mode, n_ctx_orig: 0, freq_base: 10000.0f, freq_scale: fs, ext_factor: ef, attn_factor: af, beta_fast: 1.0f, beta_slow: 1.0f);
4247 }
4248 } else {
4249 GGML_ASSERT(n_dims/3 > 0);
4250 int rope_sections[4] = {n_dims/3, n_dims/3, n_dims/3, 0};
4251 if (forward) {
4252 if (inplace) {
4253 out = ggml_rope_multi_inplace(ctx, a, b: pos, c: freq, n_dims, sections: rope_sections, mode, n_ctx_orig: 0, freq_base: 10000.0f, freq_scale: fs, ext_factor: ef, attn_factor: af, beta_fast: 1.0f, beta_slow: 1.0f);
4254 } else {
4255 out = ggml_rope_multi(ctx, a, b: pos, c: freq, n_dims, sections: rope_sections, mode, n_ctx_orig: 0, freq_base: 10000.0f, freq_scale: fs, ext_factor: ef, attn_factor: af, beta_fast: 1.0f, beta_slow: 1.0f);
4256 }
4257 } else {
4258 out = ggml_rope_multi_back(ctx, a, b: pos, c: freq, n_dims, sections: rope_sections, mode, n_ctx_orig: 0, freq_base: 10000.0f, freq_scale: fs, ext_factor: ef, attn_factor: af, beta_fast: 1.0f, beta_slow: 1.0f);
4259 }
4260 }
4261 } else {
4262 if (forward) {
4263 if (inplace) {
4264 out = ggml_rope_ext_inplace(ctx, a, b: pos, c: freq, n_dims, mode, n_ctx_orig: 0, freq_base: 10000.0f, freq_scale: fs, ext_factor: ef, attn_factor: af, beta_fast: 1.0f, beta_slow: 1.0f);
4265 } else {
4266 out = ggml_rope_ext(ctx, a, b: pos, c: freq, n_dims, mode, n_ctx_orig: 0, freq_base: 10000.0f, freq_scale: fs, ext_factor: ef, attn_factor: af, beta_fast: 1.0f, beta_slow: 1.0f);
4267 }
4268 } else {
4269 out = ggml_rope_ext_back(ctx, a, b: pos, c: freq, n_dims, mode, n_ctx_orig: 0, freq_base: 10000.0f, freq_scale: fs, ext_factor: ef, attn_factor: af, beta_fast: 1.0f, beta_slow: 1.0f);
4270 }
4271
4272 // TODO: add test with a non-contiguous view as input ; this case is needed for build_rope_2d in clip.cpp
4273 }
4274 ggml_set_name(tensor: out, name: "out");
4275
4276 return out;
4277 }
4278
4279 void initialize_tensors(ggml_context * ctx) override {
4280 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
4281 if (t->type == GGML_TYPE_I32) {
4282 // pos
4283 const int num_pos_ids = (mode & GGML_ROPE_TYPE_MROPE) ? ne_a[2] * 4 : ne_a[2];
4284 std::vector<int> data(num_pos_ids);
4285 for (int i = 0; i < num_pos_ids; i++) {
4286 data[i] = rand() % n_ctx;
4287 }
4288 ggml_backend_tensor_set(tensor: t, data: data.data(), offset: 0, size: num_pos_ids * sizeof(int));
4289 } else {
4290 if (t->ne[0] == n_dims/2) {
4291 // frequency factors in the range [0.9f, 1.1f]
4292 init_tensor_uniform(tensor: t, min: 0.9f, max: 1.1f);
4293 } else {
4294 init_tensor_uniform(tensor: t);
4295 }
4296 }
4297 }
4298 }
4299
4300 double max_maa_err() override {
4301 return 1e-3;
4302 }
4303
4304 bool grad_precise() override {
4305 return true;
4306 }
4307};
4308
4309// GGML_OP_POOL2D
4310struct test_pool2d : public test_case {
4311 enum ggml_op_pool pool_type;
4312 const ggml_type type_input;
4313 const std::array<int64_t, 4> ne_input;
4314 // kernel size
4315 const int k0;
4316 const int k1;
4317 // stride
4318 const int s0;
4319 const int s1;
4320 // padding
4321 const int p0;
4322 const int p1;
4323
4324 std::string vars() override {
4325 return VARS_TO_STR9(pool_type, type_input, ne_input, k0, k1, s0, s1, p0, p1);
4326 }
4327
4328 test_pool2d(ggml_op_pool pool_type = GGML_OP_POOL_AVG,
4329 ggml_type type_input = GGML_TYPE_F32,
4330 std::array<int64_t, 4> ne_input = {10, 10, 3, 1}, // [input_width, input_height, input_channels, 1]
4331 int k0 = 3, int k1 = 3,
4332 int s0 = 1, int s1 = 1,
4333 int p0 = 1, int p1 = 1)
4334 : pool_type(pool_type), type_input(type_input), ne_input(ne_input), k0(k0), k1(k1), s0(s0), s1(s1), p0(p0), p1(p1) {}
4335
4336 ggml_tensor * build_graph(ggml_context * ctx) override {
4337 ggml_tensor * input = ggml_new_tensor(ctx, type: type_input, n_dims: 4, ne: ne_input.data());
4338 ggml_set_param(tensor: input);
4339 ggml_set_name(tensor: input, name: "input");
4340
4341 ggml_tensor * out = ggml_pool_2d(ctx, a: input, op: pool_type, k0, k1, s0, s1, p0, p1);
4342 ggml_set_name(tensor: out, name: "out");
4343
4344 return out;
4345 }
4346};
4347
4348// GGML_OP_CONV_TRANSPOSE_1D
4349struct test_conv_transpose_1d : public test_case {
4350 const std::array<int64_t, 4> ne_input;
4351 const std::array<int64_t, 4> ne_kernel;
4352
4353 const int s0; // stride
4354 const int p0; // padding
4355 const int d0; // dilation
4356
4357 std::string vars() override {
4358 return VARS_TO_STR5(ne_input, ne_kernel, s0, p0, d0);
4359 }
4360
4361 test_conv_transpose_1d(std::array<int64_t, 4> ne_input = {197, 32, 1, 1}, // [input_width, input_channels, 1 /* assert in cpu kernel*/, 1 (should be batch)]
4362 std::array<int64_t, 4> ne_kernel = {16, 32, 32, 1}, // [kernel_width, output_channels, input_channels, 1 (should be batch)]
4363 int s0 = 1, int p0 = 0, int d0 = 1)
4364 : ne_input(ne_input), ne_kernel(ne_kernel), s0(s0), p0(p0), d0(d0) {}
4365
4366 ggml_tensor * build_graph(ggml_context * ctx) override {
4367 ggml_tensor * input = ggml_new_tensor(ctx, type: GGML_TYPE_F32, n_dims: 4, ne: ne_input.data());
4368 ggml_set_name(tensor: input, name: "input");
4369
4370 ggml_tensor * kernel = ggml_new_tensor(ctx, type: GGML_TYPE_F32, n_dims: 4, ne: ne_kernel.data());
4371 ggml_set_name(tensor: kernel, name: "kernel");
4372
4373 ggml_tensor * out = ggml_conv_transpose_1d(ctx, a: kernel, b: input, s0, p0, d0);
4374 ggml_set_name(tensor: out, name: "out");
4375
4376 return out;
4377 }
4378};
4379
4380// GGML_OP_CONV_TRANSPOSE_2D
4381struct test_conv_transpose_2d : public test_case {
4382 const std::array<int64_t, 4> ne_input;
4383 const std::array<int64_t, 4> ne_kernel;
4384 const int stride;
4385
4386 std::string vars() override {
4387 return VARS_TO_STR3(ne_input, ne_kernel, stride);
4388 }
4389
4390 double max_nmse_err() override {
4391 return 5e-4; // The default 1e-7 is too small for Vulkan.
4392 }
4393
4394 test_conv_transpose_2d(std::array<int64_t, 4> ne_input = {10, 10, 3, 1}, // [input_width, input_height, input_channels, 1]
4395 std::array<int64_t, 4> ne_kernel = {3, 3, 3, 1}, // [kernel_width, kernel_height, input_channels, 1]
4396 int stride = 1)
4397 : ne_input(ne_input), ne_kernel(ne_kernel), stride(stride){}
4398
4399 ggml_tensor * build_graph(ggml_context * ctx) override {
4400 ggml_tensor * input = ggml_new_tensor(ctx, type: GGML_TYPE_F32, n_dims: 4, ne: ne_input.data());
4401 ggml_set_name(tensor: input, name: "input");
4402
4403 ggml_tensor * kernel = ggml_new_tensor(ctx, type: GGML_TYPE_F16, n_dims: 4, ne: ne_kernel.data());
4404 ggml_set_name(tensor: kernel, name: "kernel");
4405
4406 ggml_tensor * out = ggml_conv_transpose_2d_p0(ctx, a: kernel, b: input, stride);
4407 ggml_set_name(tensor: out, name: "out");
4408
4409 return out;
4410 }
4411};
4412
4413// GGML_OP_IM2COL
4414struct test_im2col : public test_case {
4415 const ggml_type type_input;
4416 const ggml_type type_kernel;
4417 const ggml_type dst_type;
4418 const std::array<int64_t, 4> ne_input;
4419 const std::array<int64_t, 4> ne_kernel;
4420 // stride
4421 const int s0;
4422 const int s1;
4423 // padding
4424 const int p0;
4425 const int p1;
4426 // dilation
4427 const int d0;
4428 const int d1;
4429 // mode
4430 const bool is_2D;
4431
4432 std::string vars() override {
4433 return VARS_TO_STR12(type_input, type_kernel, dst_type, ne_input, ne_kernel, s0, s1, p0, p1, d0, d1, is_2D);
4434 }
4435
4436 test_im2col(ggml_type type_input = GGML_TYPE_F32, ggml_type type_kernel = GGML_TYPE_F16, ggml_type dst_type = GGML_TYPE_F32,
4437 std::array<int64_t, 4> ne_input = {10, 10, 3, 1}, // [input_width, input_height, input_channels, 1]
4438 std::array<int64_t, 4> ne_kernel = {3, 3, 3, 1}, // [kernel_width, kernel_height, input_channels, 1]
4439 int s0 = 1, int s1 = 1,
4440 int p0 = 1, int p1 = 1,
4441 int d0 = 1, int d1 = 1,
4442 bool is_2D = true)
4443 : type_input(type_input), type_kernel(type_kernel), dst_type(dst_type), ne_input(ne_input), ne_kernel(ne_kernel), s0(s0), s1(s1), p0(p0), p1(p1), d0(d0), d1(d1), is_2D(is_2D) {}
4444
4445 ggml_tensor * build_graph(ggml_context * ctx) override {
4446 ggml_tensor * input = ggml_new_tensor(ctx, type: type_input, n_dims: 4, ne: ne_input.data());
4447 ggml_set_param(tensor: input);
4448 ggml_set_name(tensor: input, name: "input");
4449
4450 ggml_tensor * kernel = ggml_new_tensor(ctx, type: type_kernel, n_dims: 4, ne: ne_kernel.data());
4451 ggml_set_name(tensor: kernel, name: "kernel");
4452
4453 ggml_tensor * out = ggml_im2col(ctx, a: kernel, b: input, s0, s1, p0, p1, d0, d1, is_2D, dst_type);
4454 ggml_set_name(tensor: out, name: "out");
4455
4456 return out;
4457 }
4458};
4459
4460// GGML_OP_IM2COL_3D
4461struct test_im2col_3d : public test_case {
4462 const ggml_type type_input;
4463 const ggml_type type_kernel;
4464 const ggml_type dst_type;
4465 const std::array<int64_t, 4> ne_input;
4466 const std::array<int64_t, 4> ne_kernel;
4467 // stride
4468 const int s0;
4469 const int s1;
4470 const int s2;
4471 // padding
4472 const int p0;
4473 const int p1;
4474 const int p2;
4475 // dilation
4476 const int d0;
4477 const int d1;
4478 const int d2;
4479
4480 const int64_t IC;
4481 const bool v;
4482
4483 std::string vars() override {
4484 return VARS_TO_STR16(type_input, type_kernel, dst_type, ne_input, ne_kernel, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, v);
4485 }
4486
4487 test_im2col_3d(ggml_type type_input = GGML_TYPE_F32, ggml_type type_kernel = GGML_TYPE_F16, ggml_type dst_type = GGML_TYPE_F32,
4488 std::array<int64_t, 4> ne_input = {10, 10, 10, 9}, // [OC*IC, KD, KH, KW]
4489 std::array<int64_t, 4> ne_kernel = {3, 3, 3, 1}, // [N*IC, ID, IH, IW]
4490 int64_t IC = 3,
4491 int s0 = 1, int s1 = 1, int s2 = 1,
4492 int p0 = 1, int p1 = 1, int p2 = 1,
4493 int d0 = 1, int d1 = 1, int d2 = 1,
4494 bool v = false)
4495 : type_input(type_input), type_kernel(type_kernel), dst_type(dst_type), ne_input(ne_input), ne_kernel(ne_kernel), s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), IC(IC), v(v) {}
4496
4497 ggml_tensor * build_graph(ggml_context * ctx) override {
4498 ggml_tensor * input = ggml_new_tensor(ctx, type: type_input, n_dims: 4, ne: ne_input.data());
4499 ggml_set_param(tensor: input);
4500 ggml_set_name(tensor: input, name: "input");
4501
4502 if (v) {
4503 input = ggml_view_4d(ctx, a: input, ne0: ne_input[0] - 2, ne1: ne_input[1] - 2, ne2: ne_input[2] - 2, ne3: ne_input[3] - 2, nb1: input->nb[1], nb2: input->nb[2], nb3: input->nb[3], offset: 0);
4504 ggml_set_name(tensor: input, name: "view_of_input");
4505 }
4506
4507 ggml_tensor * kernel = ggml_new_tensor(ctx, type: type_kernel, n_dims: 4, ne: ne_kernel.data());
4508 ggml_set_name(tensor: kernel, name: "kernel");
4509
4510 ggml_tensor * out = ggml_im2col_3d(ctx, a: kernel, b: input, IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, dst_type);
4511 ggml_set_name(tensor: out, name: "out");
4512
4513 return out;
4514 }
4515};
4516
4517// CONV_2D
4518struct test_conv_2d : public test_case {
4519 const std::array<int64_t, 4> ne_input;
4520 const std::array<int64_t, 4> ne_kernel;
4521 const ggml_type type_kernel;
4522 const int stride0;
4523 const int stride1;
4524 const int padding0;
4525 const int padding1;
4526 const int dilation0;
4527 const int dilation1;
4528 // Whether the inputs are contiguous in the channel dim or the width dim
4529 const bool cwhn;
4530
4531 // If true, the direct CONV_2D will be used in the graph, otherwise it
4532 // uses ggml_conv_2d:
4533 // * if the program is called with -o CONV_2D_DIRECT_IMPL, the
4534 // CONV_2D graph will be built, while
4535 // * if the program is called with -o CONV_2D_INDIRECT_IMPL, the
4536 // IM2COL -> MUL_MM graph will be built.
4537
4538 std::string vars() override {
4539 return VARS_TO_STR10(ne_input, ne_kernel, type_kernel, stride0, stride1, padding0, padding1, dilation0, dilation1, cwhn);
4540 }
4541
4542 double max_nmse_err() override {
4543 return 5e-4;
4544 }
4545
4546 uint64_t op_flops(ggml_tensor * t) override {
4547 GGML_UNUSED(t);
4548 // Just counting matmul costs:
4549 // KxCRS @ CRSxNPQ = KxNPQ --> KxNPQx(CRS+CRS-1) flops
4550
4551 // Copied from ggml.c: int64_t ggml_calc_conv_output_size(int64_t ins, int64_t ks, int s, int p, int d)
4552 auto calc_conv_output_size = [](int64_t ins, int64_t ks, int s, int p, int d) -> int64_t {
4553 return (ins + 2 * p - d * (ks - 1) - 1) / s + 1;
4554 };
4555
4556 int64_t W = ne_input[0];
4557 int64_t H = ne_input[1];
4558 int64_t KW = ne_kernel[0];
4559 int64_t KH = ne_kernel[1];
4560 int64_t Cin = ne_kernel[2];
4561 int64_t Cout = ne_kernel[3];
4562 int64_t N = ne_input[3];
4563 int64_t OH = calc_conv_output_size(H, KH, stride0, padding0, dilation0);
4564 int64_t OW = calc_conv_output_size(W, KW, stride0, padding0, dilation0);
4565
4566 int64_t K = Cout;
4567 int64_t CRS = Cin * KH * KW;
4568 int64_t NPQ = N * OH * OW;
4569
4570 return K * NPQ * (2 * CRS - 1);
4571 }
4572
4573 test_conv_2d(std::array<int64_t, 4> ne_input = { 64, 64, 16, 1 },
4574 std::array<int64_t, 4> ne_kernel = { 3, 3, 1, 16 }, ggml_type type_kernel = GGML_TYPE_F32, int stride0 = 1,
4575 int stride1 = 1, int padding0 = 0, int padding1 = 0, int dilation0 = 1, int dilation1 = 1, bool cwhn = false) :
4576 ne_input(ne_input),
4577 ne_kernel(ne_kernel),
4578 type_kernel(type_kernel),
4579 stride0(stride0),
4580 stride1(stride1),
4581 padding0(padding0),
4582 padding1(padding1),
4583 dilation0(dilation0),
4584 dilation1(dilation1),
4585 cwhn(cwhn) {}
4586
4587 ggml_tensor * build_graph(ggml_context * ctx) override {
4588 ggml_tensor * input = ggml_new_tensor(ctx, type: GGML_TYPE_F32, n_dims: 4, ne: ne_input.data());
4589 ggml_set_name(tensor: input, name: "input");
4590
4591 ggml_tensor * kernel = ggml_new_tensor(ctx, type: type_kernel, n_dims: 4, ne: ne_kernel.data());
4592 ggml_set_name(tensor: kernel, name: "kernel");
4593
4594 if (cwhn) {
4595 // change memory layout to channel-most-contiguous (CWHN),
4596 // then permute it back so NE matches the original input
4597 input = ggml_cont(ctx, a: ggml_permute(ctx, a: input, axis0: 1, axis1: 2, axis2: 0, axis3: 3));
4598 input = ggml_permute(ctx, a: input, axis0: 2, axis1: 0, axis2: 1, axis3: 3);
4599 kernel = ggml_cont(ctx, a: ggml_permute(ctx, a: kernel, axis0: 2, axis1: 3, axis2: 1, axis3: 0));
4600 kernel = ggml_permute(ctx, a: kernel, axis0: 3, axis1: 2, axis2: 0, axis3: 1);
4601 }
4602
4603 ggml_tensor * out =
4604 ggml_conv_2d_direct(ctx, a: kernel, b: input, s0: stride0, s1: stride1, p0: padding0, p1: padding1, d0: dilation0, d1: dilation1);
4605 ggml_set_name(tensor: out, name: "out");
4606 return out;
4607 }
4608};
4609
4610// GGML_OP_CONV_2D_DW
4611struct test_conv_2d_dw : public test_case {
4612 const std::array<int64_t, 4> ne_input;
4613 const std::array<int64_t, 4> ne_kernel;
4614 const int stride;
4615 const int padding;
4616 const int dilation;
4617 const bool cwhn;
4618
4619 std::string vars() override {
4620 return VARS_TO_STR6(ne_input, ne_kernel, stride, padding, dilation, cwhn);
4621 }
4622
4623 test_conv_2d_dw(std::array<int64_t, 4> ne_input = {64, 64, 16, 1},
4624 std::array<int64_t, 4> ne_kernel = {3, 3, 1, 16},
4625 int stride = 1, int padding = 0, int dilation = 1, bool cwhn = false)
4626 : ne_input(ne_input), ne_kernel(ne_kernel), stride(stride), padding(padding), dilation(dilation), cwhn(cwhn) {}
4627
4628 ggml_tensor * build_graph(ggml_context * ctx) override {
4629 ggml_tensor * input = ggml_new_tensor(ctx, type: GGML_TYPE_F32, n_dims: 4, ne: ne_input.data());
4630 ggml_set_name(tensor: input, name: "input");
4631
4632 ggml_tensor * kernel = ggml_new_tensor(ctx, type: GGML_TYPE_F32, n_dims: 4, ne: ne_kernel.data());
4633 ggml_set_name(tensor: kernel, name: "kernel");
4634
4635 if (cwhn) {
4636 // change memory layout to channel-most-contiguous (CWHN),
4637 // then permute it back so NE matches the original input
4638 input = ggml_cont(ctx, a: ggml_permute(ctx, a: input, axis0: 1, axis1: 2, axis2: 0, axis3: 3));
4639 input = ggml_permute(ctx, a: input, axis0: 2, axis1: 0, axis2: 1, axis3: 3);
4640 kernel = ggml_cont(ctx, a: ggml_permute(ctx, a: kernel, axis0: 2, axis1: 3, axis2: 1, axis3: 0));
4641 kernel = ggml_permute(ctx, a: kernel, axis0: 3, axis1: 2, axis2: 0, axis3: 1);
4642 }
4643
4644 ggml_tensor * out = ggml_conv_2d_dw_direct(
4645 ctx, a: kernel, b: input,
4646 stride0: stride, stride1: stride, pad0: padding, pad1: padding, dilation0: dilation, dilation1: dilation);
4647 ggml_set_name(tensor: out, name: "out");
4648 return out;
4649 }
4650};
4651
4652// GGML_OP_CONV_3D
4653struct test_conv_3d : public test_case {
4654 // Logical 5D dimensions
4655 const int64_t N, IC, ID, IH, IW;
4656 const int64_t OC, KD, KH, KW;
4657 // Conv params
4658 const int s0, s1, s2;
4659 const int p0, p1, p2;
4660 const int d0, d1, d2;
4661 // Types
4662 const ggml_type type_kernel;
4663
4664 std::string op_desc(ggml_tensor * t) override {
4665 GGML_UNUSED(t);
4666 return "CONV_3D";
4667 }
4668
4669 std::string vars() override {
4670 return VARS_TO_STR11(N, IC, ID, IH, IW, OC, KD, KH, KW, s0, s1) + "," +
4671 VARS_TO_STR8(s2, p0, p1, p2, d0, d1, d2, type_kernel);
4672 }
4673
4674 double max_nmse_err() override {
4675 return 5e-4;
4676 }
4677
4678 uint64_t op_flops(ggml_tensor * t) override {
4679 GGML_UNUSED(t);
4680 auto calc_conv_output_size = [](int64_t ins, int64_t ks, int s, int p, int d) -> int64_t {
4681 return (ins + 2 * p - d * (ks - 1) - 1) / s + 1;
4682 };
4683 const int64_t OD = calc_conv_output_size(ID, KD, s2, p2, d2);
4684 const int64_t OH = calc_conv_output_size(IH, KH, s1, p1, d1);
4685 const int64_t OW = calc_conv_output_size(IW, KW, s0, p0, d0);
4686
4687 return (uint64_t)N * OC * OD * OH * OW * (2 * IC * KD * KH * KW - 1);
4688 }
4689
4690 test_conv_3d(
4691 int64_t N, int64_t IC, int64_t ID, int64_t IH, int64_t IW,
4692 int64_t OC, int64_t KD, int64_t KH, int64_t KW,
4693 int s0, int s1, int s2,
4694 int p0, int p1, int p2,
4695 int d0, int d1, int d2,
4696 ggml_type type_kernel
4697 ) : N(N), IC(IC), ID(ID), IH(IH), IW(IW),
4698 OC(OC), KD(KD), KH(KH), KW(KW),
4699 s0(s0), s1(s1), s2(s2),
4700 p0(p0), p1(p1), p2(p2),
4701 d0(d0), d1(d1), d2(d2),
4702 type_kernel(type_kernel) {}
4703
4704 ggml_tensor * build_graph(ggml_context * ctx) override {
4705 // GGML input tensor is packed as [W, H, D, C*N]
4706 const int64_t ne_input[] = {IW, IH, ID, IC * N};
4707 ggml_tensor * input = ggml_new_tensor(ctx, type: GGML_TYPE_F32, n_dims: 4, ne: ne_input);
4708 ggml_set_name(tensor: input, name: "input");
4709
4710 // GGML kernel tensor is packed as [KW, KH, KD, IC*OC]
4711 const int64_t ne_kernel[] = {KW, KH, KD, IC * OC};
4712 ggml_tensor * kernel = ggml_new_tensor(ctx, type: type_kernel, n_dims: 4, ne: ne_kernel);
4713 ggml_set_name(tensor: kernel, name: "kernel");
4714
4715 ggml_tensor * out = ggml_conv_3d_direct(ctx, a: kernel, b: input, s0, s1, s2, p0, p1, p2, d0, d1, d2, n_channels: (int)IC, n_batch: (int)N, n_channels_out: (int)OC);
4716 ggml_set_name(tensor: out, name: "out");
4717 return out;
4718 }
4719};
4720
4721// GGML_OP_CONCAT
4722struct test_concat : public test_case {
4723 const ggml_type type;
4724 const std::array<int64_t, 4> ne_a;
4725 const int64_t ne_b_d;
4726 const int dim;
4727 const int v; // view (1 << 0: non-cont a, 1 << 1: non-cont b)
4728
4729 std::string vars() override {
4730 return VARS_TO_STR5(type, ne_a, ne_b_d, dim, v);
4731 }
4732
4733 test_concat(ggml_type type = GGML_TYPE_F32,
4734 std::array<int64_t, 4> ne_a = {10, 5, 5, 5},
4735 int64_t ne_b_d = 5,
4736 int dim = 2, int v = 0)
4737 : type(type), ne_a(ne_a), ne_b_d(ne_b_d), dim(dim), v(v) {}
4738
4739 ggml_tensor * build_graph(ggml_context * ctx) override {
4740 auto ne_b = ne_a;
4741 ne_b[dim] = ne_b_d;
4742 ggml_tensor * a;
4743 if (v & 1) {
4744 auto ne = ne_a; ne[0] *= 2; ne[1] *= 4; ne[2] *= 3;
4745 a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
4746 ggml_set_name(tensor: a, name: "a");
4747
4748 a = ggml_view_4d(ctx, a, ne0: ne_a[0], ne1: ne_a[1], ne2: ne_a[2], ne3: ne_a[3], nb1: a->nb[1], nb2: a->nb[2], nb3: a->nb[3], offset: 0);
4749 ggml_set_name(tensor: a, name: "view_of_a");
4750 } else {
4751 a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_a.data());
4752 ggml_set_name(tensor: a, name: "a");
4753 }
4754 ggml_tensor * b;
4755 if (v & 2) {
4756 auto ne = ne_b; ne[0] *= 3; ne[1] *= 2; ne[2] *= 4;
4757 b = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
4758 ggml_set_name(tensor: b, name: "b");
4759
4760 b = ggml_view_4d(ctx, a: b, ne0: ne_b[0], ne1: ne_b[1], ne2: ne_b[2], ne3: ne_b[3], nb1: b->nb[1], nb2: b->nb[2], nb3: b->nb[3], offset: 0);
4761 ggml_set_name(tensor: b, name: "view_of_b");
4762 } else {
4763 b = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_b.data());
4764 ggml_set_name(tensor: b, name: "b");
4765 }
4766
4767 ggml_tensor * out = ggml_concat(ctx, a, b, dim);
4768 ggml_set_name(tensor: out, name: "out");
4769
4770 return out;
4771 }
4772};
4773
4774// GGML_OP_ARGSORT
4775struct test_argsort : public test_case {
4776 const ggml_type type;
4777 const std::array<int64_t, 4> ne;
4778 ggml_sort_order order;
4779
4780 std::string vars() override {
4781 return VARS_TO_STR3(type, ne, order);
4782 }
4783
4784 test_argsort(ggml_type type = GGML_TYPE_F32,
4785 std::array<int64_t, 4> ne = {16, 10, 10, 10},
4786 ggml_sort_order order = GGML_SORT_ORDER_ASC)
4787 : type(type), ne(ne), order(order) {}
4788
4789 ggml_tensor * build_graph(ggml_context * ctx) override {
4790 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
4791 ggml_set_name(tensor: a, name: "a");
4792
4793 ggml_tensor * out = ggml_argsort(ctx, a, order);
4794 ggml_set_name(tensor: out, name: "out");
4795
4796 return out;
4797 }
4798
4799 void initialize_tensors(ggml_context * ctx) override {
4800 std::random_device rd;
4801 std::default_random_engine rng(rd());
4802 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
4803 if (t->type == GGML_TYPE_I32) {
4804 // indices
4805 std::vector<int> data(ggml_nelements(tensor: t));
4806 for (int i = 0; i < ggml_nelements(tensor: t); i++) {
4807 data[i] = rand();
4808 }
4809 std::shuffle(first: data.begin(), last: data.end(), g&: rng);
4810 ggml_backend_tensor_set(tensor: t, data: data.data(), offset: 0, size: ne[0]*ne[1]*ne[2]*ne[3] * sizeof(int));
4811 } else if (t->type == GGML_TYPE_F32) {
4812 // initialize with unique values to avoid ties
4813 for (int64_t r = 0; r < ggml_nrows(tensor: t); r++) {
4814 std::vector<float> data(t->ne[0]);
4815 for (int i = 0; i < t->ne[0]; i++) {
4816 data[i] = i;
4817 }
4818 std::shuffle(first: data.begin(), last: data.end(), g&: rng);
4819 ggml_backend_tensor_set(tensor: t, data: data.data(), offset: r * t->nb[1], size: t->ne[0] * sizeof(float));
4820 }
4821 } else {
4822 GGML_ABORT("fatal error");
4823 }
4824 }
4825 }
4826};
4827
4828struct test_topk_moe: public test_case {
4829 const std::array<int64_t, 4> ne;
4830 const int n_expert_used;
4831 const bool with_norm;
4832 const bool delayed_softmax;
4833
4834 test_topk_moe(std::array<int64_t, 4> ne = { 10, 5, 1, 1 },
4835 int n_expert_used = 1,
4836 bool with_norm = false,
4837 bool delayed_softmax = false) :
4838 ne(ne),
4839 n_expert_used(n_expert_used),
4840 with_norm(with_norm),
4841 delayed_softmax(delayed_softmax) {
4842 GGML_ASSERT(n_expert_used <= ne[0]);
4843 GGML_ASSERT(!(with_norm && delayed_softmax));
4844 }
4845
4846 std::string vars() override { return VARS_TO_STR4(ne, n_expert_used, with_norm, delayed_softmax); }
4847
4848 std::string op_desc(ggml_tensor * t) override {
4849 GGML_UNUSED(t);
4850 return "TOPK_MOE";
4851 }
4852
4853 bool run_whole_graph() override { return true; }
4854
4855 ggml_tensor * build_graph(ggml_context * ctx) override {
4856 const int n_expert = ne[0];
4857 const int n_tokens = ne[1];
4858
4859 ggml_tensor * logits = ggml_new_tensor(ctx, type: GGML_TYPE_F32, n_dims: 4, ne: ne.data());
4860 ggml_tensor * probs = delayed_softmax ? logits : ggml_soft_max(ctx, a: logits);
4861 ggml_tensor * selected_experts = ggml_top_k(ctx, a: probs, k: n_expert_used); // [n_expert_used, n_tokens]
4862
4863 ggml_tensor * out = ggml_get_rows(ctx, a: ggml_reshape_3d(ctx, a: probs, ne0: 1, ne1: n_expert, ne2: n_tokens), b: selected_experts); // [1, n_expert_used, n_tokens]
4864
4865 if (delayed_softmax) {
4866 out = ggml_reshape_2d(ctx, a: out, ne0: n_expert_used, ne1: n_tokens);
4867 out = ggml_soft_max(ctx, a: out); // [n_expert_used, n_tokens]
4868 out = ggml_reshape_3d(ctx, a: out, ne0: 1, ne1: n_expert_used, ne2: n_tokens);
4869 }
4870
4871 if (with_norm) {
4872 out = ggml_reshape_2d(ctx, a: out, ne0: n_expert_used, ne1: n_tokens);
4873 ggml_tensor * weights_sum = ggml_sum_rows(ctx, a: out); // [1, n_tokens]
4874
4875 weights_sum = ggml_clamp(ctx, a: weights_sum, min: 6.103515625e-5, INFINITY);
4876 out = ggml_div(ctx, a: out, b: weights_sum); // [n_expert_used, n_tokens]
4877 out = ggml_reshape_3d(ctx, a: out, ne0: 1, ne1: n_expert_used, ne2: n_tokens);
4878 }
4879
4880 ggml_set_name(tensor: out, name: "out");
4881 return out;
4882 }
4883};
4884
4885struct test_mul_mat_vec_fusion : public test_case {
4886 const ggml_type type;
4887 const ggml_glu_op glu_op;
4888 const int64_t m;
4889 const int64_t n;
4890 const int64_t k;
4891 const bool use_id;
4892 const int n_mats;
4893 const int n_used;
4894 const bool b; // broadcast b matrix (only for use_id)
4895 const bool with_bias;
4896 const bool with_gate;
4897
4898 test_mul_mat_vec_fusion(ggml_type type, ggml_glu_op op, int64_t m, int64_t n, int64_t k,
4899 bool use_id = false, int n_mats = 1, int n_used = 1, bool b = false, bool with_bias = false, bool with_gate = true)
4900 : type(type), glu_op(op), m(m), n(n), k(k), use_id(use_id), n_mats(n_mats), n_used(n_used), b(b), with_bias(with_bias), with_gate(with_gate) {
4901 if (use_id) {
4902 GGML_ASSERT(n_used <= n_mats);
4903 }
4904 }
4905
4906 std::string vars() override {
4907 return VARS_TO_STR11(type, glu_op, m, n, k, use_id, n_mats, n_used, b, with_bias, with_gate);
4908 }
4909
4910 std::string op_desc(ggml_tensor * t) override {
4911 GGML_UNUSED(t);
4912 return "MUL_MAT_VEC_FUSION";
4913 }
4914
4915 bool run_whole_graph() override { return true; }
4916
4917 ggml_tensor * build_gate(ggml_context * ctx, ggml_tensor * ffn_gate, ggml_tensor * ffn_up) {
4918 ggml_tensor * out = nullptr;
4919 if (with_gate) {
4920 if (glu_op == GGML_GLU_OP_SWIGLU_OAI) {
4921 constexpr float alpha = 1.702f;
4922 constexpr float limit = 7.0f;
4923 out = ggml_swiglu_oai(ctx, a: ffn_gate, b: ffn_up, alpha, limit);
4924 } else {
4925 out = ggml_glu_split(ctx, a: ffn_gate, b: ffn_up, op: glu_op);
4926 }
4927 }
4928 return out;
4929 }
4930
4931 ggml_tensor * build_graph(ggml_context * ctx) override {
4932 if (!use_id) {
4933 const int channels = 4;
4934 const int samples = 2;
4935 std::array<int64_t, 4> ne = { k, m, channels, samples };
4936 std::array<int64_t, 4> ne0 = { k, n, channels, samples };
4937
4938 ggml_tensor * cur = ggml_new_tensor(ctx, type: GGML_TYPE_F32, n_dims: 4, ne: ne.data());
4939 ggml_tensor * gate = with_gate ? ggml_new_tensor(ctx, type, n_dims: 4, ne: ne0.data()) : nullptr;
4940 ggml_tensor * up = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne0.data());
4941
4942 ggml_tensor * ffn_up = ggml_mul_mat(ctx, a: up, b: cur);
4943 if (with_bias) {
4944 std::array<int64_t, 4> bias_ne = { ffn_up->ne[0], 1, channels, samples };
4945 ggml_tensor * up_bias = ggml_new_tensor(ctx, type: GGML_TYPE_F32, n_dims: 4, ne: bias_ne.data());
4946 ffn_up = ggml_add(ctx, a: ffn_up, b: up_bias);
4947 }
4948
4949 ggml_tensor * ffn_gate = with_gate ? ggml_mul_mat(ctx, a: gate, b: cur) : nullptr;
4950 if (with_bias && with_gate) {
4951 std::array<int64_t, 4> bias_ne = { ffn_gate->ne[0], 1, channels, samples };
4952 ggml_tensor * gate_bias = ggml_new_tensor(ctx, type: GGML_TYPE_F32, n_dims: 4, ne: bias_ne.data());
4953 ffn_gate = ggml_add(ctx, a: ffn_gate, b: gate_bias);
4954 }
4955
4956 ggml_tensor * out = with_gate ? build_gate(ctx, ffn_gate, ffn_up) : ffn_up;
4957 ggml_set_name(tensor: out, name: "out");
4958 return out;
4959 } else {
4960 ggml_tensor * gates = ggml_new_tensor_3d(ctx, type, ne0: k, ne1: n, ne2: n_mats);
4961 ggml_tensor * ups = ggml_new_tensor_3d(ctx, type, ne0: k, ne1: n, ne2: n_mats);
4962 ggml_tensor * ids = ggml_new_tensor_2d(ctx, type: GGML_TYPE_I32, ne0: n_mats, ne1: m);
4963
4964 if (n_used != n_mats) {
4965 ids = ggml_view_2d(ctx, a: ids, ne0: n_used, ne1: m, nb1: ids->nb[1], offset: 0);
4966 }
4967
4968 ggml_tensor * cur = ggml_new_tensor_3d(ctx, type: GGML_TYPE_F32, ne0: k, ne1: this->b ? 1 : n_used, ne2: m);
4969 ggml_set_name(tensor: cur, name: "cur");
4970
4971 ggml_tensor * ffn_up = ggml_mul_mat_id(ctx, as: ups, b: cur, ids);
4972 if (with_bias) {
4973 ggml_tensor * up_bias_param = ggml_new_tensor_2d(ctx, type: GGML_TYPE_F32, ne0: ffn_up->ne[0], ne1: n_mats);
4974 ffn_up = ggml_add_id(ctx, a: ffn_up, b: up_bias_param, ids);
4975 }
4976
4977 ggml_tensor * ffn_gate = with_gate? ggml_mul_mat_id(ctx, as: gates, b: cur, ids) : nullptr;
4978 if (with_bias && with_gate) {
4979 ggml_tensor * gate_bias_param = ggml_new_tensor_2d(ctx, type: GGML_TYPE_F32, ne0: ffn_gate->ne[0], ne1: n_mats);
4980 ffn_gate = ggml_add_id(ctx, a: ffn_gate, b: gate_bias_param, ids);
4981 }
4982
4983 ggml_tensor * out = with_gate ? build_gate(ctx, ffn_gate, ffn_up) : ffn_up;
4984 ggml_set_name(tensor: out, name: "out");
4985 return out;
4986 }
4987 }
4988
4989 void initialize_tensors(ggml_context * ctx) override {
4990 if (!use_id) {
4991 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
4992 init_tensor_uniform(tensor: t);
4993 }
4994 } else {
4995 std::random_device rd;
4996 std::default_random_engine rng(rd());
4997 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
4998 if (t->type == GGML_TYPE_I32) {
4999 if (ggml_is_view_op(op: t->op)) { continue; }
5000 // ids
5001 for (int64_t r = 0; r < ggml_nrows(tensor: t); r++) {
5002 std::vector<int32_t> data(t->ne[0]);
5003 for (int i = 0; i < t->ne[0]; i++) {
5004 data[i] = i % n_mats;
5005 }
5006 std::shuffle(first: data.begin(), last: data.end(), g&: rng);
5007 ggml_backend_tensor_set(tensor: t, data: data.data(), offset: r * t->nb[1], size: t->ne[0] * sizeof(int32_t));
5008 }
5009 } else {
5010 init_tensor_uniform(tensor: t);
5011 }
5012 }
5013 }
5014 }
5015
5016 double max_nmse_err() override {
5017 return 5e-3;
5018 }
5019};
5020
5021// GGML_OP_SUM
5022struct test_sum : public test_case {
5023 const ggml_type type;
5024 const std::array<int64_t, 4> ne;
5025 const std::array<int64_t, 4> permute;
5026 bool _use_permute;
5027
5028 std::string vars() override {
5029 std::string v = VARS_TO_STR2(type, ne);
5030 if (_use_permute) v += "," + VAR_TO_STR(permute);
5031 return v;
5032 }
5033
5034 test_sum(ggml_type type = GGML_TYPE_F32,
5035 std::array<int64_t, 4> ne = {10, 5, 4, 3},
5036 std::array<int64_t, 4> permute = {0, 0, 0, 0})
5037 : type(type), ne(ne), permute(permute),
5038 _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0) {}
5039
5040 ggml_tensor * build_graph(ggml_context * ctx) override {
5041 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
5042 ggml_set_param(tensor: a);
5043 ggml_set_name(tensor: a, name: "a");
5044
5045 if (_use_permute) {
5046 a = ggml_permute(ctx, a, axis0: permute[0], axis1: permute[1], axis2: permute[2], axis3: permute[3]);
5047 ggml_set_name(tensor: a, name: "a_permuted");
5048 }
5049
5050 ggml_tensor * out = ggml_sum(ctx, a);
5051 ggml_set_name(tensor: out, name: "out");
5052
5053 return out;
5054 }
5055
5056 float grad_eps() override {
5057 return 0.1f * sqrtf(x: ne[0]*ne[1]*ne[2]*ne[3]);
5058 }
5059};
5060
5061// GGML_OP_SUM_ROWS
5062struct test_sum_rows : public test_case {
5063 const ggml_type type;
5064 const std::array<int64_t, 4> ne;
5065 const bool permute;
5066 const bool slice;
5067
5068 std::string vars() override {
5069 return VARS_TO_STR4(type, ne, permute, slice);
5070 }
5071
5072 test_sum_rows(ggml_type type = GGML_TYPE_F32,
5073 std::array<int64_t, 4> ne = {10, 5, 4, 3},
5074 bool permute = false, bool slice = false)
5075 : type(type), ne(ne), permute(permute), slice(slice) {}
5076
5077 ggml_tensor * build_graph(ggml_context * ctx) override {
5078 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
5079 ggml_set_param(tensor: a);
5080 ggml_set_name(tensor: a, name: "a");
5081
5082 if (slice) {
5083 a = ggml_view_4d(ctx, a,
5084 ne0: ne[0], ne1: ne[1], ne2: ne[2] / 2, ne3: ne[3] - 1,
5085 nb1: a->nb[1], nb2: a->nb[2] * 2, nb3: a->nb[3], /*offset=*/a->nb[3]);
5086 }
5087 if (permute) {
5088 a = ggml_permute(ctx, a, axis0: 0, axis1: 2, axis2: 3, axis3: 1);
5089 }
5090
5091 ggml_tensor * out = ggml_sum_rows(ctx, a);
5092 ggml_set_name(tensor: out, name: "out");
5093
5094 return out;
5095 }
5096};
5097
5098// GGML_OP_MEAN
5099struct test_mean : public test_case {
5100 const ggml_type type;
5101 const std::array<int64_t, 4> ne;
5102
5103 std::string vars() override {
5104 return VARS_TO_STR2(type, ne);
5105 }
5106
5107 test_mean(ggml_type type = GGML_TYPE_F32,
5108 std::array<int64_t, 4> ne = {10, 5, 4, 3})
5109 : type(type), ne(ne) {}
5110
5111 ggml_tensor * build_graph(ggml_context * ctx) override {
5112 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
5113 ggml_set_param(tensor: a);
5114 ggml_set_name(tensor: a, name: "a");
5115
5116 ggml_tensor * out = ggml_mean(ctx, a);
5117 ggml_set_name(tensor: out, name: "out");
5118
5119 return out;
5120 }
5121
5122 float grad_eps() override {
5123 return 0.1f * ne[0]*ne[1]*ne[2]*ne[3];
5124 }
5125};
5126
5127// GGML_OP_UPSCALE
5128struct test_upscale : public test_case {
5129 const ggml_type type;
5130 const std::array<int64_t, 4> ne;
5131 const int32_t scale_factor;
5132 const bool transpose;
5133 const ggml_scale_mode mode;
5134
5135 std::string vars() override {
5136 return VARS_TO_STR5(type, ne, scale_factor, mode, transpose);
5137 }
5138
5139 test_upscale(ggml_type type = GGML_TYPE_F32,
5140 std::array<int64_t, 4> ne = {512, 512, 3, 1},
5141 int32_t scale_factor = 2, ggml_scale_mode mode = GGML_SCALE_MODE_NEAREST, bool transpose = false)
5142 : type(type), ne(ne), scale_factor(scale_factor), transpose(transpose), mode(mode) {}
5143
5144 ggml_tensor * build_graph(ggml_context * ctx) override {
5145 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
5146 ggml_set_name(tensor: a, name: "a");
5147
5148 if (transpose) {
5149 a = ggml_transpose(ctx, a);
5150 ggml_set_name(tensor: a, name: "a_transposed");
5151 }
5152
5153 ggml_tensor * out = ggml_upscale(ctx, a, scale_factor, mode);
5154 ggml_set_name(tensor: out, name: "out");
5155
5156 return out;
5157 }
5158};
5159
5160// GGML_OP_UPSCALE (via ggml_interpolate)
5161struct test_interpolate : public test_case {
5162 const ggml_type type;
5163 const std::array<int64_t, 4> ne;
5164 const std::array<int64_t, 4> ne_tgt;
5165 const uint32_t mode = GGML_SCALE_MODE_NEAREST;
5166
5167 std::string vars() override {
5168 return VARS_TO_STR4(type, ne, ne_tgt, mode);
5169 }
5170
5171 test_interpolate(ggml_type type = GGML_TYPE_F32,
5172 std::array<int64_t, 4> ne = {2, 5, 7, 11},
5173 std::array<int64_t, 4> ne_tgt = {5, 7, 11, 13},
5174 uint32_t mode = GGML_SCALE_MODE_NEAREST)
5175 : type(type), ne(ne), ne_tgt(ne_tgt), mode(mode) {}
5176
5177 ggml_tensor * build_graph(ggml_context * ctx) override {
5178 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
5179 ggml_set_name(tensor: a, name: "a");
5180
5181 ggml_tensor * out = ggml_interpolate(ctx, a, ne0: ne_tgt[0], ne1: ne_tgt[1],ne2: ne_tgt[2], ne3: ne_tgt[3], mode);
5182 ggml_set_name(tensor: out, name: "out");
5183
5184 return out;
5185 }
5186};
5187
5188// GGML_OP_GROUP_NORM
5189struct test_group_norm : public test_case {
5190 const ggml_type type;
5191 const std::array<int64_t, 4> ne;
5192 const int32_t num_groups;
5193 const float eps;
5194
5195 std::string vars() override {
5196 return VARS_TO_STR4(type, ne, num_groups, eps);
5197 }
5198
5199 test_group_norm(ggml_type type = GGML_TYPE_F32,
5200 std::array<int64_t, 4> ne = {64, 64, 320, 1},
5201 int32_t num_groups = 32,
5202 float eps = 1e-6f)
5203 : type(type), ne(ne), num_groups(num_groups), eps(eps) {}
5204
5205 ggml_tensor * build_graph(ggml_context * ctx) override {
5206 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
5207 ggml_set_name(tensor: a, name: "a");
5208
5209 ggml_tensor * out = ggml_group_norm(ctx, a, n_groups: num_groups, eps);
5210 ggml_set_name(tensor: out, name: "out");
5211
5212 return out;
5213 }
5214};
5215
5216// GGML_OP_GROUP_NORM + GGML_OP_MUL + GGML_OP_ADD
5217struct test_group_norm_mul_add : public test_case {
5218 const ggml_type type;
5219 const std::array<int64_t, 4> ne;
5220 int num_groups;
5221 float eps;
5222
5223 std::string op_desc(ggml_tensor * t) override {
5224 GGML_UNUSED(t);
5225 return "GROUP_NORM_MUL_ADD";
5226 }
5227
5228 bool run_whole_graph() override { return true; }
5229
5230 std::string vars() override {
5231 return VARS_TO_STR4(type, ne, num_groups, eps);
5232 }
5233
5234 test_group_norm_mul_add(ggml_type type = GGML_TYPE_F32,
5235 std::array<int64_t, 4> ne = {128, 1, 1, 1},
5236 int num_groups = 4,
5237 float eps = 1e-5f)
5238 : type(type), ne(ne), num_groups(num_groups), eps(eps) {}
5239
5240 ggml_tensor * build_graph(ggml_context * ctx) override {
5241 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
5242 ggml_tensor * w = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
5243 ggml_tensor * b = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
5244 ggml_set_param(tensor: a); ggml_set_param(tensor: w); ggml_set_param(tensor: b);
5245 ggml_set_name(tensor: a, name: "a"); ggml_set_name(tensor: w, name: "w"); ggml_set_name(tensor: b, name: "b");
5246 ggml_tensor * n = ggml_group_norm(ctx, a, n_groups: num_groups, eps);
5247 ggml_tensor * m = ggml_mul(ctx, a: n, b: w);
5248 ggml_tensor * out = ggml_add(ctx, a: m, b);
5249 ggml_set_name(tensor: out, name: "out");
5250 return out;
5251 }
5252};
5253
5254// GGML_OP_L2_NORM
5255struct test_l2_norm : public test_case {
5256 const ggml_type type;
5257 const std::array<int64_t, 4> ne;
5258 const float eps;
5259
5260 std::string vars() override {
5261 return VARS_TO_STR2(type, ne);
5262 }
5263
5264 test_l2_norm(ggml_type type = GGML_TYPE_F32,
5265 std::array<int64_t, 4> ne = {64, 64, 320, 1},
5266 float eps = 1e-12f)
5267 : type(type), ne(ne), eps(eps) {}
5268
5269 ggml_tensor * build_graph(ggml_context * ctx) override {
5270 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
5271 ggml_set_name(tensor: a, name: "a");
5272
5273 ggml_tensor * out = ggml_l2_norm(ctx, a, eps);
5274 ggml_set_name(tensor: out, name: "out");
5275
5276 return out;
5277 }
5278};
5279
5280// GGML_OP_ACC
5281struct test_acc : public test_case {
5282 const ggml_type type;
5283 const std::array<int64_t, 4> ne_a;
5284 const std::array<int64_t, 4> ne_b;
5285
5286 std::string vars() override {
5287 return VARS_TO_STR3(type, ne_a, ne_b);
5288 }
5289
5290 test_acc(ggml_type type = GGML_TYPE_F32,
5291 std::array<int64_t, 4> ne_a = {256, 17, 1, 1},
5292 std::array<int64_t, 4> ne_b = {256, 16, 1, 1})
5293 : type(type), ne_a(ne_a), ne_b(ne_b) {}
5294
5295 ggml_tensor * build_graph(ggml_context * ctx) override {
5296 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_a.data());
5297 ggml_set_param(tensor: a);
5298 ggml_set_name(tensor: a, name: "a");
5299
5300 ggml_tensor * b = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_b.data());
5301 ggml_set_param(tensor: b);
5302 ggml_set_name(tensor: b, name: "b");
5303
5304 ggml_tensor * out = ggml_acc(ctx, a, b, nb1: a->nb[1], nb2: a->nb[2], nb3: a->nb[3], offset: b->nb[1]);
5305 ggml_set_name(tensor: out, name: "out");
5306
5307 return out;
5308 }
5309};
5310
5311// GGML_OP_PAD
5312struct test_pad : public test_case {
5313 const ggml_type type;
5314 const std::array<int64_t, 4> ne_a;
5315 const int pad_0;
5316 const int pad_1;
5317
5318 std::string vars() override {
5319 return VARS_TO_STR4(type, ne_a, pad_0, pad_1);
5320 }
5321
5322 test_pad(ggml_type type = GGML_TYPE_F32,
5323 std::array<int64_t, 4> ne_a = {512, 512, 1, 1},
5324 int pad_0 = 1, int pad_1 = 1)
5325 : type(type), ne_a(ne_a), pad_0(pad_0), pad_1(pad_1) {}
5326
5327 ggml_tensor * build_graph(ggml_context * ctx) override {
5328 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_a.data());
5329 ggml_set_name(tensor: a, name: "a");
5330
5331 ggml_tensor * out = ggml_pad(ctx, a, p0: pad_0, p1: pad_1, p2: 0, p3: 0);
5332 ggml_set_name(tensor: out, name: "out");
5333
5334 return out;
5335 }
5336};
5337
5338struct test_pad_ext : public test_case {
5339 const ggml_type type;
5340 const std::array<int64_t, 4> ne_a;
5341 const int lp0;
5342 const int rp0;
5343 const int lp1;
5344 const int rp1;
5345 const int lp2;
5346 const int rp2;
5347 const int lp3;
5348 const int rp3;
5349 const bool v;
5350
5351 std::string vars() override {
5352 return VARS_TO_STR11(type, ne_a, lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3, v);
5353 }
5354
5355 test_pad_ext(ggml_type type = GGML_TYPE_F32,
5356 std::array<int64_t, 4> ne_a = {512, 512, 3, 1},
5357 int lp0 = 1, int rp0 = 1, int lp1 = 1, int rp1 = 1,
5358 int lp2 = 1, int rp2 = 1, int lp3 = 1, int rp3 = 1,
5359 bool v = false)
5360 : type(type), ne_a(ne_a), lp0(lp0), rp0(rp0), lp1(lp1), rp1(rp1), lp2(lp2), rp2(rp2), lp3(lp3), rp3(rp3), v(v) {}
5361
5362 ggml_tensor * build_graph(ggml_context * ctx) override {
5363 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_a.data());
5364 ggml_set_name(tensor: a, name: "a");
5365
5366 if (v) {
5367 a = ggml_view_4d(ctx, a, ne0: (a->ne[0] + 1) / 2, ne1: (a->ne[1] + 1) / 2, ne2: (a->ne[2] + 1) / 2, ne3: (a->ne[3] + 1) / 2, nb1: a->nb[1], nb2: a->nb[2], nb3: a->nb[3], offset: 0);
5368 ggml_set_name(tensor: a, name: "view of a");
5369 }
5370
5371 ggml_tensor * out = ggml_pad_ext(ctx, a, lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3);
5372 ggml_set_name(tensor: out, name: "out");
5373
5374 return out;
5375 }
5376};
5377
5378// GGML_OP_PAD_REFLECT_1D
5379struct test_pad_reflect_1d : public test_case {
5380 const ggml_type type;
5381 const std::array<int64_t, 4> ne_a;
5382 const int pad_0;
5383 const int pad_1;
5384
5385 std::string vars() override {
5386 return VARS_TO_STR4(type, ne_a, pad_0, pad_1);
5387 }
5388
5389 test_pad_reflect_1d(ggml_type type = GGML_TYPE_F32,
5390 std::array<int64_t, 4> ne_a = {512, 34, 2, 1},
5391 int pad_0 = 10, int pad_1 = 9)
5392 : type(type), ne_a(ne_a), pad_0(pad_0), pad_1(pad_1) {}
5393
5394 ggml_tensor * build_graph(ggml_context * ctx) override {
5395 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 2, ne: ne_a.data());
5396 ggml_set_name(tensor: a, name: "a");
5397
5398 ggml_tensor * out = ggml_pad_reflect_1d(ctx, a, p0: pad_0, p1: pad_1);
5399 ggml_set_name(tensor: out, name: "out");
5400
5401 return out;
5402 }
5403};
5404
5405// GGML_OP_ROLL
5406struct test_roll : public test_case {
5407 const int shift0;
5408 const int shift1;
5409 const int shift3;
5410 const int shift4;
5411
5412 std::string vars() override {
5413 return VARS_TO_STR4(shift0, shift1, shift3, shift4);
5414 }
5415
5416 test_roll(int shift0 = 3, int shift1 = -2, int shift3 = 1, int shift4 = -1)
5417 : shift0(shift0), shift1(shift1), shift3(shift3), shift4(shift4) {}
5418
5419 ggml_tensor * build_graph(ggml_context * ctx) override {
5420 int64_t ne[4] = {10, 5, 4, 3};
5421 ggml_tensor * a = ggml_new_tensor(ctx, type: GGML_TYPE_F32, n_dims: 4, ne);
5422 ggml_set_name(tensor: a, name: "a");
5423
5424 ggml_tensor * out = ggml_roll(ctx, a, shift0, shift1, shift2: shift3, shift3: shift4);
5425 ggml_set_name(tensor: out, name: "out");
5426
5427 return out;
5428 }
5429};
5430
5431// GGML_OP_ARANGE
5432struct test_arange : public test_case {
5433 const ggml_type type;
5434 const float start;
5435 const float stop;
5436 const float step;
5437
5438 std::string vars() override {
5439 return VARS_TO_STR4(type, start, stop, step);
5440 }
5441
5442 test_arange(ggml_type type = GGML_TYPE_F32,
5443 float start = 0.f, float stop = 10.f, float step = 1.f)
5444 : type(type), start(start), stop(stop), step(step) {}
5445
5446 ggml_tensor * build_graph(ggml_context * ctx) override {
5447 ggml_tensor * out = ggml_arange(ctx, start, stop, step);
5448 ggml_set_name(tensor: out, name: "out");
5449
5450 return out;
5451 }
5452};
5453
5454// GGML_OP_TIMESTEP_EMBEDDING
5455struct test_timestep_embedding : public test_case {
5456 const ggml_type type;
5457 const std::array<int64_t, 4> ne_a;
5458 const int dim;
5459 const int max_period;
5460
5461 std::string vars() override {
5462 return VARS_TO_STR4(type, ne_a, dim, max_period);
5463 }
5464
5465 test_timestep_embedding(ggml_type type = GGML_TYPE_F32,
5466 std::array<int64_t, 4> ne_a = {2, 1, 1, 1},
5467 int dim = 320, int max_period=10000)
5468 : type(type), ne_a(ne_a), dim(dim), max_period(max_period) {}
5469
5470 ggml_tensor * build_graph(ggml_context * ctx) override {
5471 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_a.data());
5472 ggml_set_name(tensor: a, name: "a");
5473
5474 ggml_tensor * out = ggml_timestep_embedding(ctx, timesteps: a, dim, max_period);
5475 ggml_set_name(tensor: out, name: "out");
5476
5477 return out;
5478 }
5479};
5480
5481// GGML_OP_LEAKY_RELU
5482struct test_leaky_relu : public test_case {
5483 const ggml_type type;
5484 const std::array<int64_t, 4> ne_a;
5485 const float negative_slope;
5486
5487 std::string vars() override {
5488 return VARS_TO_STR3(type, ne_a, negative_slope);
5489 }
5490
5491 test_leaky_relu(ggml_type type = GGML_TYPE_F32,
5492 std::array<int64_t, 4> ne_a = {10, 5, 4, 3},
5493 float negative_slope = 0.1f)
5494 : type(type), ne_a(ne_a), negative_slope(negative_slope) {}
5495
5496 ggml_tensor * build_graph(ggml_context * ctx) override {
5497 ggml_tensor * a = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne_a.data());
5498 ggml_set_name(tensor: a, name: "a");
5499
5500 ggml_tensor * out = ggml_leaky_relu(ctx, a, negative_slope, inplace: true);
5501 ggml_set_name(tensor: out, name: "out");
5502
5503 return out;
5504 }
5505};
5506
5507// GGML_OP_FLASH_ATTN_EXT
5508struct test_flash_attn_ext : public test_case {
5509 const int64_t hsk; // K head size
5510 const int64_t hsv; // V head size
5511 const int64_t nh; // num heads
5512 const std::array<int64_t, 2> nr23; // repeat in dim 2 and 3, tests for grouped-query attention
5513 const int64_t kv; // kv size
5514 const int64_t nb; // batch size
5515
5516 const bool mask; // use mask
5517 const bool sinks; // use sinks
5518
5519 const float max_bias; // ALiBi
5520 const float logit_softcap; // Gemma 2
5521
5522 const ggml_prec prec;
5523 const ggml_type type_KV;
5524 std::array<int32_t, 4> permute;
5525
5526 std::string vars() override {
5527 return VARS_TO_STR13(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, permute);
5528 }
5529
5530 double max_nmse_err() override {
5531 return 5e-4;
5532 }
5533
5534 uint64_t op_flops(ggml_tensor * t) override {
5535 GGML_UNUSED(t);
5536 // Just counting matmul costs:
5537 // Q*K^T is nb x hsk x kv, P*V is nb x kv x hsv, per head
5538 return (2 * nh*nr23[0] * nb * (hsk + hsv) * kv)*nr23[1];
5539 }
5540
5541 test_flash_attn_ext(int64_t hsk = 128, int64_t hsv = 128, int64_t nh = 32, std::array<int64_t, 2> nr23 = {1, 1}, int64_t kv = 96, int64_t nb = 8,
5542 bool mask = true, bool sinks = false, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_prec prec = GGML_PREC_F32,
5543 ggml_type type_KV = GGML_TYPE_F16, std::array<int32_t, 4> permute = {0, 1, 2, 3})
5544 : hsk(hsk), hsv(hsv), nh(nh), nr23(nr23), kv(kv), nb(nb), mask(mask), sinks(sinks), max_bias(max_bias), logit_softcap(logit_softcap), prec(prec), type_KV(type_KV), permute(permute) {}
5545
5546 ggml_tensor * build_graph(ggml_context * ctx) override {
5547 const int64_t hsk_padded = GGML_PAD(hsk, ggml_blck_size(type_KV));
5548 const int64_t hsv_padded = GGML_PAD(hsv, ggml_blck_size(type_KV));
5549
5550 auto const &create_permuted = [&](ggml_type type, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, bool is_view) -> ggml_tensor * {
5551 int64_t ne[4] = {ne0, ne1, ne2, ne3};
5552 int64_t ne_perm[4];
5553 for (int i = 0; i < 4; ++i) {
5554 ne_perm[permute[i]] = ne[i];
5555 }
5556 ggml_tensor * t;
5557 if (is_view) {
5558 ggml_tensor * t0 = ggml_new_tensor_4d(ctx, type, ne0: ne_perm[0], ne1: 2*ne_perm[1], ne2: ne_perm[2], ne3: ne_perm[3]);
5559 t = ggml_view_4d(ctx, a: t0, ne0: ne_perm[0], ne1: ne_perm[1], ne2: ne_perm[2], ne3: ne_perm[3], nb1: t0->nb[1], nb2: t0->nb[2], nb3: t0->nb[3], offset: 0);
5560 } else {
5561 t = ggml_new_tensor_4d(ctx, type, ne0: ne_perm[0], ne1: ne_perm[1], ne2: ne_perm[2], ne3: ne_perm[3]);
5562 }
5563 if (permute != std::array<int32_t, 4>{0, 1, 2, 3}) {
5564 t = ggml_permute(ctx, a: t, axis0: permute[0], axis1: permute[1], axis2: permute[2], axis3: permute[3]);
5565 }
5566 return t;
5567 };
5568
5569 ggml_tensor * q = create_permuted(GGML_TYPE_F32, hsk_padded, nb, nh*nr23[0], nr23[1], false);
5570 ggml_set_name(tensor: q, name: "q");
5571
5572 ggml_tensor * k = create_permuted(type_KV, hsk_padded, kv, nh, nr23[1], true); // the K tensor is usually a view of the K cache
5573 ggml_set_name(tensor: k, name: "k");
5574
5575 ggml_tensor * v = create_permuted(type_KV, hsv_padded, kv, nh, nr23[1], true); // the V tensor is usually a view of the V cache
5576 ggml_set_name(tensor: v, name: "v");
5577
5578 ggml_tensor * m = nullptr;
5579 if (mask) {
5580 m = ggml_new_tensor_4d(ctx, type: GGML_TYPE_F16, ne0: kv, GGML_PAD(nb, GGML_KQ_MASK_PAD), ne2: 1, ne3: nr23[1]);
5581 ggml_set_name(tensor: m, name: "m");
5582 }
5583
5584 ggml_tensor * s = nullptr;
5585 if (sinks) {
5586 s = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F32, ne0: q->ne[2]);
5587 ggml_set_name(tensor: s, name: "s");
5588 }
5589
5590 ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, mask: m, scale: 1.0f/sqrtf(x: hsk), max_bias, logit_softcap);
5591 ggml_flash_attn_ext_add_sinks(a: out, sinks: s);
5592 ggml_flash_attn_ext_set_prec (a: out, prec);
5593 ggml_set_name(tensor: out, name: "out");
5594
5595 return out;
5596 }
5597
5598 void initialize_tensors(ggml_context * ctx) override {
5599 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
5600 if (strcmp(s1: t->name, s2: "s") == 0) {
5601 // make the sink values more noticable in order to trigger a test failure when the implementation is wrong
5602 init_tensor_uniform(tensor: t, min: -10.0f, max: 10.0f);
5603 } else if (strcmp(s1: t->name, s2: "m") == 0) {
5604 init_tensor_kq_mask(tensor: t);
5605 } else {
5606 init_tensor_uniform(tensor: t);
5607 }
5608 }
5609 }
5610
5611 bool grad_precise() override {
5612 return true;
5613 }
5614};
5615
5616// GGML_OP_CROSS_ENTROPY_LOSS
5617struct test_cross_entropy_loss : public test_case {
5618 const ggml_type type;
5619 const std::array<int64_t, 4> ne;
5620
5621 std::string vars() override {
5622 return VARS_TO_STR2(type, ne);
5623 }
5624
5625 test_cross_entropy_loss(ggml_type type = GGML_TYPE_F32,
5626 std::array<int64_t, 4> ne = {10, 5, 4, 3})
5627 : type(type), ne(ne) {}
5628
5629 ggml_tensor * build_graph(ggml_context * ctx) override {
5630 ggml_tensor * logits = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
5631 ggml_set_param(tensor: logits);
5632 ggml_set_name(tensor: logits, name: "logits");
5633
5634 ggml_tensor * labels = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
5635 // The labels are assumed to be constant -> no gradients.
5636 ggml_set_name(tensor: labels, name: "labels");
5637
5638 // Ensure labels add up to 1:
5639 labels = ggml_soft_max(ctx, a: labels);
5640 ggml_set_name(tensor: labels, name: "labels_normalized");
5641
5642 ggml_tensor * out = ggml_cross_entropy_loss(ctx, a: logits, b: labels);
5643 ggml_set_name(tensor: out, name: "out");
5644
5645 return out;
5646 }
5647
5648 void initialize_tensors(ggml_context * ctx) override {
5649 // For larger abs. diffs between logits softmax is more linear, therefore more precise num. gradients.
5650 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
5651 init_tensor_uniform(tensor: t, min: -100.0f, max: 100.0f);
5652 }
5653 }
5654
5655 float grad_eps() override {
5656 return 1.0f;
5657 }
5658
5659 bool grad_precise() override {
5660 return true;
5661 }
5662};
5663
5664// GGML_OP_CROSS_ENTROPY_LOSS_BACK
5665struct test_cross_entropy_loss_back : public test_case {
5666 const ggml_type type;
5667 const std::array<int64_t, 4> ne;
5668
5669 std::string vars() override {
5670 return VARS_TO_STR2(type, ne);
5671 }
5672
5673 test_cross_entropy_loss_back(ggml_type type = GGML_TYPE_F32,
5674 std::array<int64_t, 4> ne = {10, 5, 4, 3})
5675 : type(type), ne(ne) {}
5676
5677 ggml_tensor * build_graph(ggml_context * ctx) override {
5678 ggml_tensor * grad = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F32, ne0: 1);
5679 ggml_set_name(tensor: grad, name: "grad");
5680
5681 ggml_tensor * logits = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
5682 ggml_set_name(tensor: logits, name: "logits");
5683
5684 ggml_tensor * labels = ggml_new_tensor(ctx, type, n_dims: 4, ne: ne.data());
5685 ggml_set_name(tensor: labels, name: "labels");
5686
5687 // Ensure labels add up to 1:
5688 labels = ggml_soft_max(ctx, a: labels);
5689 ggml_set_name(tensor: labels, name: "labels_normalized");
5690
5691 ggml_tensor * out = ggml_cross_entropy_loss_back(ctx, a: grad, b: logits, c: labels);
5692 ggml_set_name(tensor: out, name: "out");
5693
5694 return out;
5695 }
5696};
5697
5698// GGML_OP_OPT_STEP_ADAMW
5699struct test_opt_step_adamw : public test_case {
5700 const ggml_type type;
5701 const std::array<int64_t, 4> ne;
5702
5703 std::string vars() override {
5704 return VARS_TO_STR2(type, ne);
5705 }
5706
5707 test_opt_step_adamw(ggml_type type = GGML_TYPE_F32,
5708 std::array<int64_t, 4> ne = {10, 5, 4, 3})
5709 : type(type), ne(ne) {}
5710
5711 ggml_tensor * build_graph(ggml_context * ctx) override {
5712 ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne0: ne[0], ne1: ne[1], ne2: ne[2], ne3: ne[3]);
5713 ggml_set_param(tensor: a); // Despite tensor a having gradients the output tensor will not.
5714 ggml_set_name(tensor: a, name: "a");
5715
5716 ggml_tensor * grad = ggml_new_tensor_4d(ctx, type, ne0: ne[0], ne1: ne[1], ne2: ne[2], ne3: ne[3]);
5717 ggml_set_name(tensor: grad, name: "grad");
5718
5719 ggml_tensor * grad_m = ggml_new_tensor_4d(ctx, type, ne0: ne[0], ne1: ne[1], ne2: ne[2], ne3: ne[3]);
5720 ggml_set_name(tensor: grad_m, name: "grad_m");
5721
5722 ggml_tensor * grad_v = ggml_new_tensor_4d(ctx, type, ne0: ne[0], ne1: ne[1], ne2: ne[2], ne3: ne[3]);
5723 ggml_set_name(tensor: grad_v, name: "grad_v");
5724
5725 ggml_tensor * adamw_params = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F32, ne0: 7);
5726 ggml_set_name(tensor: adamw_params, name: "adamw_params");
5727
5728 ggml_tensor * out = ggml_opt_step_adamw(ctx, a, grad, m: grad_m, v: grad_v, adamw_params);
5729 ggml_set_name(tensor: out, name: "out");
5730
5731 return out;
5732 }
5733
5734 void initialize_tensors(ggml_context * ctx) override {
5735 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
5736 init_tensor_uniform(tensor: t, min: 0.0f, max: 1.0f); // grad_v and adamw_params need non-negative values.
5737 }
5738 }
5739
5740 bool grad_precise() override {
5741 return true;
5742 }
5743};
5744
5745struct test_opt_step_sgd : public test_case {
5746 const ggml_type type;
5747 const std::array<int64_t, 4> ne;
5748
5749 std::string vars() override { return VARS_TO_STR2(type, ne); }
5750
5751 test_opt_step_sgd(ggml_type type = GGML_TYPE_F32,
5752 std::array<int64_t, 4> ne = { 10, 5, 4, 3 })
5753 : type(type), ne(ne) {}
5754
5755 ggml_tensor * build_graph(ggml_context * ctx) override {
5756 ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne0: ne[0], ne1: ne[1], ne2: ne[2], ne3: ne[3]);
5757 ggml_set_param(tensor: a); // Despite tensor a having gradients the output tensor will not.
5758 ggml_set_name(tensor: a, name: "a");
5759
5760 ggml_tensor * grad = ggml_new_tensor_4d(ctx, type, ne0: ne[0], ne1: ne[1], ne2: ne[2], ne3: ne[3]);
5761 ggml_set_name(tensor: grad, name: "grad");
5762
5763 ggml_tensor * sgd_params = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F32, ne0: 2);
5764 ggml_set_name(tensor: sgd_params, name: "sgd_params");
5765
5766 ggml_tensor * out = ggml_opt_step_sgd(ctx, a, grad, sgd_params);
5767
5768 ggml_set_name(tensor: out, name: "out");
5769
5770 return out;
5771 }
5772
5773 void initialize_tensors(ggml_context * ctx) override {
5774 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
5775 init_tensor_uniform(tensor: t, min: 0.0f, max: 1.0f); // sgd_params need non-negative values.
5776 }
5777 }
5778
5779 bool grad_precise() override {
5780 return true;
5781 }
5782};
5783
5784enum llm_norm_type {
5785 LLM_NORM,
5786 LLM_NORM_RMS,
5787};
5788
5789struct llama_hparams {
5790 uint32_t n_vocab;
5791 uint32_t n_embd;
5792 uint32_t n_head;
5793 uint32_t n_head_kv;
5794 static constexpr uint32_t n_layer = 1;
5795 uint32_t n_rot;
5796 uint32_t n_embd_head; // dimension of values (d_v)
5797 uint32_t n_ff;
5798
5799 float f_norm_eps;
5800 float f_norm_rms_eps;
5801
5802 // cparams
5803 static constexpr uint32_t n_ctx = 512; // user-specified context size
5804 static constexpr uint32_t n_ctx_orig = n_ctx;
5805
5806 // batch
5807 int32_t n_tokens;
5808
5809 // llm_build_context
5810 static constexpr int32_t n_kv = 32; // size of KV cache to consider (n_kv <= n_ctx
5811 static constexpr int32_t kv_head = 1; // index of where we store new KV data in the cache
5812
5813 uint32_t n_embd_gqa() const { // dimension of key embeddings across all k-v heads
5814 return n_embd_head * n_head_kv;
5815 }
5816};
5817
5818// LLM base class
5819struct test_llm : public test_case {
5820 llama_hparams hp;
5821
5822protected:
5823 test_llm(llama_hparams hp)
5824 : hp(std::move(hp)) {
5825 }
5826
5827public:
5828 struct ggml_tensor * llm_build_norm(
5829 struct ggml_context * ctx,
5830 struct ggml_tensor * cur,
5831 struct ggml_tensor * mw,
5832 struct ggml_tensor * mb,
5833 llm_norm_type type) {
5834 switch (type) {
5835 case LLM_NORM: cur = ggml_norm (ctx, a: cur, eps: hp.f_norm_eps); break;
5836 case LLM_NORM_RMS: cur = ggml_rms_norm(ctx, a: cur, eps: hp.f_norm_rms_eps); break;
5837 }
5838 cur = ggml_mul(ctx, a: cur, b: mw);
5839 if (mb) {
5840 cur = ggml_add(ctx, a: cur, b: mb);
5841 }
5842 return cur;
5843 }
5844
5845 void llm_build_kv_store(
5846 struct ggml_context * ctx,
5847 struct ggml_tensor * k_l,
5848 struct ggml_tensor * v_l,
5849 struct ggml_tensor * k_cur,
5850 struct ggml_tensor * v_cur) {
5851 // compute the transposed [n_tokens, n_embd] V matrix
5852 struct ggml_tensor * v_cur_t = ggml_transpose(ctx, a: ggml_reshape_2d(ctx, a: v_cur, ne0: hp.n_embd_gqa(), ne1: hp.n_tokens));
5853
5854 struct ggml_tensor * k_cache_view = ggml_view_1d(ctx, a: k_l, ne0: hp.n_tokens*hp.n_embd_gqa(),
5855 offset: (ggml_row_size(type: k_l->type, ne: hp.n_embd_gqa()))*hp.kv_head);
5856
5857 struct ggml_tensor * v_cache_view = ggml_view_2d(ctx, a: v_l, ne0: hp.n_tokens, ne1: hp.n_embd_gqa(),
5858 nb1: ( hp.n_ctx)*ggml_element_size(tensor: v_l),
5859 offset: (hp.kv_head)*ggml_element_size(tensor: v_l));
5860
5861 // important: storing RoPE-ed version of K in the KV cache!
5862 ggml_cpy(ctx, a: k_cur, b: k_cache_view);
5863 ggml_cpy(ctx, a: v_cur_t, b: v_cache_view);
5864 }
5865
5866 struct ggml_tensor * llm_build_kqv(
5867 struct ggml_context * ctx,
5868 struct ggml_tensor * k_l,
5869 struct ggml_tensor * v_l,
5870 struct ggml_tensor * q_cur,
5871 struct ggml_tensor * kq_mask,
5872 float kq_scale) {
5873 struct ggml_tensor * q = ggml_permute(ctx, a: q_cur, axis0: 0, axis1: 2, axis2: 1, axis3: 3);
5874
5875 struct ggml_tensor * k =
5876 ggml_view_3d(ctx, a: k_l,
5877 ne0: hp.n_embd_head, ne1: hp.n_kv, ne2: hp.n_head_kv,
5878 nb1: ggml_row_size(type: k_l->type, ne: hp.n_embd_gqa()),
5879 nb2: ggml_row_size(type: k_l->type, ne: hp.n_embd_head),
5880 offset: 0);
5881
5882 struct ggml_tensor * kq = ggml_mul_mat(ctx, a: k, b: q);
5883
5884 kq = ggml_soft_max_ext(ctx, a: kq, mask: kq_mask, scale: kq_scale, max_bias: 0.0f);
5885
5886 // split cached v into n_head heads
5887 struct ggml_tensor * v =
5888 ggml_view_3d(ctx, a: v_l,
5889 ne0: hp.n_kv, ne1: hp.n_embd_head, ne2: hp.n_head_kv,
5890 nb1: ggml_element_size(tensor: v_l)*hp.n_ctx,
5891 nb2: ggml_element_size(tensor: v_l)*hp.n_ctx*hp.n_embd_head,
5892 offset: 0);
5893
5894 struct ggml_tensor * kqv = ggml_mul_mat(ctx, a: v, b: kq);
5895
5896 struct ggml_tensor * kqv_merged = ggml_permute(ctx, a: kqv, axis0: 0, axis1: 2, axis2: 1, axis3: 3);
5897
5898 struct ggml_tensor * cur = ggml_cont_2d(ctx, a: kqv_merged, ne0: hp.n_embd_head*hp.n_head, ne1: hp.n_tokens);
5899
5900 struct ggml_tensor * wo = ggml_new_tensor_2d(ctx, type: GGML_TYPE_Q4_0, ne0: hp.n_embd, ne1: hp.n_embd);
5901 cur = ggml_mul_mat(ctx, a: wo, b: cur);
5902
5903 return cur;
5904 }
5905
5906 void initialize_tensors(ggml_context * ctx) override {
5907 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, tensor: t)) {
5908 if (t->type == GGML_TYPE_I32) {
5909 // pos
5910 std::vector<int> data(hp.n_tokens);
5911 for (int i = 0; i < hp.n_tokens; i++) {
5912 data[i] = rand() % hp.n_ctx;
5913 }
5914 ggml_backend_tensor_set(tensor: t, data: data.data(), offset: 0, size: hp.n_tokens * sizeof(int));
5915 } else {
5916 init_tensor_uniform(tensor: t);
5917 }
5918 }
5919 }
5920};
5921
5922// Llama
5923struct test_llama : public test_llm {
5924 static constexpr float freq_base = 10000.0f;
5925 static constexpr float freq_scale = 1.0f;
5926 static constexpr float ext_factor = 0.0f;
5927 static constexpr float attn_factor = 1.0f;
5928 static constexpr float beta_fast = 32.0f;
5929 static constexpr float beta_slow = 1.0f;
5930 bool fused;
5931
5932 std::string op_desc(ggml_tensor * t) override {
5933 GGML_UNUSED(t);
5934 return "LLAMA";
5935 }
5936
5937 std::string vars() override {
5938 auto n_tokens = hp.n_tokens;
5939 return VARS_TO_STR1(n_tokens);
5940 }
5941
5942 double max_nmse_err() override {
5943 return 2e-3;
5944 }
5945
5946 bool run_whole_graph() override { return fused; }
5947
5948 test_llama(int n_tokens = 1, bool fused = false)
5949 : test_llm({
5950 /*n_vocab =*/ 32000,
5951 /*n_embd =*/ 3200,
5952 /*n_head =*/ 32,
5953 /*n_head_kv =*/ 32,
5954 /*n_rot =*/ 100,
5955 /*n_embd_head =*/ 100,
5956 /*n_ff =*/ 8640,
5957 /*f_norm_eps =*/ 0.f,
5958 /*f_norm_rms_eps =*/ 1e-5f,
5959 /*n_tokens =*/ n_tokens,
5960 })
5961 , fused(fused)
5962 {
5963 }
5964
5965 ggml_tensor * build_graph(ggml_context * ctx) override {
5966 struct ggml_tensor * cur;
5967 struct ggml_tensor * inpL;
5968
5969 inpL = ggml_new_tensor_2d(ctx, type: GGML_TYPE_F32, ne0: hp.n_embd, ne1: hp.n_tokens);
5970
5971 // inp_pos - contains the positions
5972 struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx, type: GGML_TYPE_I32, ne0: hp.n_tokens);
5973
5974 // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
5975 struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx, type: GGML_TYPE_F16, ne0: hp.n_kv, ne1: hp.n_tokens, ne2: 1);
5976
5977 ggml_tensor * k_l = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F16, ne0: 1638400);
5978 ggml_tensor * v_l = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F16, ne0: 1638400);
5979
5980 for (uint32_t il = 0; il < hp.n_layer; ++il) {
5981 struct ggml_tensor * inpSA = inpL;
5982
5983 // norm
5984 ggml_tensor * attn_norm = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F32, ne0: hp.n_embd);
5985 cur = llm_build_norm(ctx, cur: inpL, mw: attn_norm, mb: nullptr, type: LLM_NORM_RMS);
5986
5987 // self-attention
5988 {
5989 ggml_tensor * wq = ggml_new_tensor_2d(ctx, type: GGML_TYPE_Q4_0, ne0: hp.n_embd, ne1: hp.n_embd);
5990 ggml_tensor * wk = ggml_new_tensor_2d(ctx, type: GGML_TYPE_Q4_0, ne0: hp.n_embd, ne1: hp.n_embd_gqa());
5991 ggml_tensor * wv = ggml_new_tensor_2d(ctx, type: GGML_TYPE_Q4_0, ne0: hp.n_embd, ne1: hp.n_embd_gqa());
5992
5993 // compute Q and K and RoPE them
5994 struct ggml_tensor * Qcur = ggml_mul_mat(ctx, a: wq, b: cur);
5995 struct ggml_tensor * Kcur = ggml_mul_mat(ctx, a: wk, b: cur);
5996 struct ggml_tensor * Vcur = ggml_mul_mat(ctx, a: wv, b: cur);
5997
5998 Qcur = ggml_rope_ext(
5999 ctx, a: ggml_reshape_3d(ctx, a: Qcur, ne0: hp.n_embd_head, ne1: hp.n_head, ne2: hp.n_tokens), b: inp_pos, c: nullptr,
6000 n_dims: hp.n_rot, mode: 0, n_ctx_orig: hp.n_ctx_orig, freq_base, freq_scale,
6001 ext_factor, attn_factor, beta_fast, beta_slow
6002 );
6003
6004 Kcur = ggml_rope_ext(
6005 ctx, a: ggml_reshape_3d(ctx, a: Kcur, ne0: hp.n_embd_head, ne1: hp.n_head_kv, ne2: hp.n_tokens), b: inp_pos, c: nullptr,
6006 n_dims: hp.n_rot, mode: 0, n_ctx_orig: hp.n_ctx_orig, freq_base, freq_scale,
6007 ext_factor, attn_factor, beta_fast, beta_slow
6008 );
6009
6010 llm_build_kv_store(ctx, k_l, v_l, k_cur: Kcur, v_cur: Vcur);
6011
6012 cur = llm_build_kqv(ctx, k_l, v_l, q_cur: Qcur, kq_mask: KQ_mask, kq_scale: 1.0f/sqrtf(x: float(hp.n_embd_head)));
6013 }
6014
6015 struct ggml_tensor * ffn_inp = ggml_add(ctx, a: cur, b: inpSA);
6016
6017 // feed-forward network
6018 ggml_tensor * ffn_norm = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F32, ne0: hp.n_embd);
6019 cur = llm_build_norm(ctx, cur: ffn_inp, mw: ffn_norm, mb: nullptr, type: LLM_NORM_RMS);
6020
6021 ggml_tensor * ffn_gate = ggml_new_tensor_2d(ctx, type: GGML_TYPE_Q4_0, ne0: hp.n_embd, ne1: hp.n_ff);
6022 ggml_tensor * ffn_down = ggml_new_tensor_2d(ctx, type: GGML_TYPE_Q4_0, ne0: hp.n_ff, ne1: hp.n_embd);
6023 ggml_tensor * ffn_up = ggml_new_tensor_2d(ctx, type: GGML_TYPE_Q4_0, ne0: hp.n_embd, ne1: hp.n_ff);
6024 struct ggml_tensor * tmp = ggml_mul_mat(ctx, a: ffn_up, b: cur);
6025 cur = ggml_mul_mat(ctx, a: ffn_gate, b: cur);
6026 cur = ggml_silu(ctx, a: cur);
6027 cur = ggml_mul(ctx, a: cur, b: tmp);
6028 cur = ggml_mul_mat(ctx, a: ffn_down, b: cur);
6029
6030 cur = ggml_add(ctx, a: cur, b: ffn_inp);
6031
6032 // input for next layer
6033 inpL = cur;
6034 }
6035
6036 cur = inpL;
6037
6038 ggml_tensor * output_norm = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F32, ne0: hp.n_embd);
6039 cur = llm_build_norm(ctx, cur, mw: output_norm, mb: nullptr, type: LLM_NORM_RMS);
6040
6041 // lm_head
6042 ggml_tensor * output = ggml_new_tensor_2d(ctx, type: GGML_TYPE_Q4_0, ne0: hp.n_embd, ne1: hp.n_vocab);
6043 cur = ggml_mul_mat(ctx, a: output, b: cur);
6044
6045 return cur;
6046 }
6047};
6048
6049// Falcon
6050struct test_falcon : public test_llm {
6051 static constexpr float freq_base = 10000.0f;
6052 static constexpr float freq_scale = 1.0f;
6053 static constexpr float ext_factor = 0.0f;
6054 static constexpr float attn_factor = 1.0f;
6055 static constexpr float beta_fast = 32.0f;
6056 static constexpr float beta_slow = 1.0f;
6057
6058 std::string op_desc(ggml_tensor * t) override {
6059 GGML_UNUSED(t);
6060 return "FALCON";
6061 }
6062
6063 std::string vars() override {
6064 auto n_tokens = hp.n_tokens;
6065 return VARS_TO_STR1(n_tokens);
6066 }
6067
6068 double max_nmse_err() override {
6069 return 2e-3;
6070 }
6071
6072 test_falcon(int n_tokens = 1)
6073 : test_llm({
6074 /*n_vocab =*/ 32000,
6075 /*n_embd =*/ 3200,
6076 /*n_head =*/ 50,
6077 /*n_head_kv =*/ 1,
6078 /*n_rot =*/ 64,
6079 /*n_embd_head =*/ 64,
6080 /*n_ff =*/ 8640,
6081 /*f_norm_eps =*/ 1e-5f,
6082 /*f_norm_rms_eps =*/ 0.f,
6083 /*n_tokens =*/ n_tokens,
6084 }) {
6085 }
6086
6087 ggml_tensor * build_graph(ggml_context * ctx) override {
6088 struct ggml_tensor * cur;
6089 struct ggml_tensor * inpL;
6090
6091 inpL = ggml_new_tensor_2d(ctx, type: GGML_TYPE_F32, ne0: hp.n_embd, ne1: hp.n_tokens);
6092
6093 // inp_pos - contains the positions
6094 struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx, type: GGML_TYPE_I32, ne0: hp.n_tokens);
6095
6096 // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
6097 struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx, type: GGML_TYPE_F16, ne0: hp.n_kv, ne1: hp.n_tokens, ne2: 1);
6098
6099 ggml_tensor * k_l = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F16, ne0: 1638400);
6100 ggml_tensor * v_l = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F16, ne0: 1638400);
6101
6102 for (uint32_t il = 0; il < hp.n_layer; ++il) {
6103 // norm
6104 ggml_tensor * attn_norm_w = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F32, ne0: hp.n_embd);
6105 ggml_tensor * attn_norm_b = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F32, ne0: hp.n_embd);
6106 ggml_tensor * attn_norm = llm_build_norm(ctx, cur: inpL, mw: attn_norm_w, mb: attn_norm_b, type: LLM_NORM);
6107
6108 // self-attention
6109 {
6110 cur = attn_norm;
6111
6112 ggml_tensor * wqkv = ggml_new_tensor_2d(ctx, type: GGML_TYPE_Q4_0, ne0: hp.n_embd, ne1: hp.n_embd + 2*hp.n_embd_gqa());
6113
6114 cur = ggml_mul_mat(ctx, a: wqkv, b: cur);
6115
6116 struct ggml_tensor * Qcur = ggml_cont(ctx, a: ggml_view_2d(ctx, a: cur, ne0: hp.n_embd, ne1: hp.n_tokens, nb1: cur->nb[1], offset: 0*sizeof(float)*(hp.n_embd)));
6117 struct ggml_tensor * Kcur = ggml_cont(ctx, a: ggml_view_2d(ctx, a: cur, ne0: hp.n_embd_gqa(), ne1: hp.n_tokens, nb1: cur->nb[1], offset: 1*sizeof(float)*(hp.n_embd)));
6118 struct ggml_tensor * Vcur = ggml_cont(ctx, a: ggml_view_2d(ctx, a: cur, ne0: hp.n_embd_gqa(), ne1: hp.n_tokens, nb1: cur->nb[1], offset: 1*sizeof(float)*(hp.n_embd + hp.n_embd_gqa())));
6119
6120 Qcur = ggml_reshape_3d(ctx, a: Qcur, ne0: hp.n_embd_head, ne1: hp.n_head, ne2: hp.n_tokens);
6121 Kcur = ggml_reshape_3d(ctx, a: Kcur, ne0: hp.n_embd_head, ne1: hp.n_head_kv, ne2: hp.n_tokens);
6122
6123 // using mode = 2 for neox mode
6124 Qcur = ggml_rope_ext(
6125 ctx, a: Qcur, b: inp_pos, c: nullptr, n_dims: hp.n_rot, mode: 2, n_ctx_orig: hp.n_ctx_orig,
6126 freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
6127 );
6128
6129 Kcur = ggml_rope_ext(
6130 ctx, a: Kcur, b: inp_pos, c: nullptr, n_dims: hp.n_rot, mode: 2, n_ctx_orig: hp.n_ctx_orig,
6131 freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
6132 );
6133
6134 llm_build_kv_store(ctx, k_l, v_l, k_cur: Kcur, v_cur: Vcur);
6135
6136 cur = llm_build_kqv(ctx, k_l, v_l, q_cur: Qcur, kq_mask: KQ_mask, kq_scale: 1.0f/sqrtf(x: float(hp.n_embd_head)));
6137 }
6138
6139 struct ggml_tensor * ffn_inp = cur;
6140
6141 // feed forward
6142 {
6143 ggml_tensor * ffn_up = ggml_new_tensor_2d(ctx, type: GGML_TYPE_Q4_0, ne0: hp.n_embd, ne1: hp.n_ff);
6144 ggml_tensor * ffn_down = ggml_new_tensor_2d(ctx, type: GGML_TYPE_Q4_0, ne0: hp.n_ff, ne1: hp.n_embd);
6145 cur = attn_norm;
6146 cur = ggml_mul_mat(ctx, a: ffn_up, b: cur);
6147 cur = ggml_gelu(ctx, a: cur);
6148 cur = ggml_mul_mat(ctx, a: ffn_down, b: cur);
6149 }
6150
6151 cur = ggml_add(ctx, a: cur, b: ffn_inp);
6152
6153 cur = ggml_add(ctx, a: cur, b: inpL);
6154
6155 // input for next layer
6156 inpL = cur;
6157 }
6158
6159 cur = inpL;
6160
6161 ggml_tensor * output_norm = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F32, ne0: hp.n_embd);
6162 ggml_tensor * output_norm_b = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F32, ne0: hp.n_embd);
6163 cur = llm_build_norm(ctx, cur, mw: output_norm, mb: output_norm_b, type: LLM_NORM);
6164
6165 // lm_head
6166 ggml_tensor * output = ggml_new_tensor_2d(ctx, type: GGML_TYPE_Q8_0, ne0: hp.n_embd, ne1: hp.n_vocab);
6167 cur = ggml_mul_mat(ctx, a: output, b: cur);
6168
6169 return cur;
6170 }
6171};
6172
6173
6174// ###########################################
6175// ## Section 3: GGML Op Test Instantiation ##
6176// ###########################################
6177static const ggml_type all_types[] = {
6178 GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16,
6179 GGML_TYPE_Q4_0, GGML_TYPE_Q4_1,
6180 GGML_TYPE_Q5_0, GGML_TYPE_Q5_1,
6181 GGML_TYPE_Q8_0,
6182 GGML_TYPE_MXFP4,
6183 GGML_TYPE_Q2_K, GGML_TYPE_Q3_K,
6184 GGML_TYPE_Q4_K, GGML_TYPE_Q5_K,
6185 GGML_TYPE_Q6_K,
6186 // GGML_TYPE_TQ1_0, GGML_TYPE_TQ2_0, // TODO: implement for all backends
6187 GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S,
6188 GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M,
6189 GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS,
6190};
6191
6192static const ggml_type base_types[] = {
6193 GGML_TYPE_F32, GGML_TYPE_F16,
6194 GGML_TYPE_Q8_0, // for I8MM tests
6195 GGML_TYPE_Q4_0,
6196 GGML_TYPE_Q4_1, // for I8MM tests
6197 GGML_TYPE_Q4_K,
6198 GGML_TYPE_MXFP4, // TODO: or "other"
6199 GGML_TYPE_IQ2_XXS
6200};
6201
6202static const ggml_type other_types[] = {
6203 GGML_TYPE_Q4_1,
6204 GGML_TYPE_Q5_0, GGML_TYPE_Q5_1,
6205 GGML_TYPE_Q8_0,
6206 GGML_TYPE_Q2_K, GGML_TYPE_Q3_K,
6207 GGML_TYPE_Q5_K,
6208 GGML_TYPE_Q6_K,
6209 // GGML_TYPE_TQ1_0, GGML_TYPE_TQ2_0, // TODO: implement for all backends
6210 GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S,
6211 GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M,
6212 GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS,
6213 GGML_TYPE_BF16,
6214};
6215
6216// Test cases for evaluation: should try to cover edge cases while using small input sizes to keep the runtime low
6217static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
6218 std::vector<std::unique_ptr<test_case>> test_cases;
6219 std::default_random_engine rng(0);
6220
6221 // unary ops
6222 for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
6223 for (int v : {0, 1}) {
6224 for (int op = 0; op < GGML_UNARY_OP_COUNT; op++) {
6225 test_cases.emplace_back(args: new test_unary((ggml_unary_op) op, type, { 128, 2, 2, 2 }, v));
6226 test_cases.emplace_back(args: new test_unary((ggml_unary_op) op, type, { 5, 7, 11, 13 }, v));
6227 }
6228 }
6229 }
6230
6231 // glu ops
6232 for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
6233 for (int v : {0, 1}) {
6234 for (int op = 0; op < GGML_GLU_OP_COUNT; op++) {
6235 if (op == GGML_GLU_OP_SWIGLU_OAI) {
6236 // SWIGLU_OAI is handled separately
6237 continue;
6238 }
6239
6240 for (bool swapped : {false, true}) {
6241 test_cases.emplace_back(args: new test_glu((ggml_glu_op) op, type, { 128, 2, 2, 2 }, v, swapped));
6242 test_cases.emplace_back(args: new test_glu((ggml_glu_op) op, type, { 5, 7, 11, 13 }, v, swapped));
6243 }
6244
6245 test_cases.emplace_back(args: new test_glu_split((ggml_glu_op) op, type, { 128, 2, 2, 2 }, v));
6246 test_cases.emplace_back(args: new test_glu_split((ggml_glu_op) op, type, { 5, 7, 11, 13 }, v));
6247 }
6248 }
6249 }
6250
6251 for (int v : {0, 1}) {
6252 for (float alpha : {.5f, 1.702f}) {
6253 for (float limit : {2.0f, 7.0f}) {
6254 test_cases.emplace_back(args: new test_swiglu_oai(GGML_TYPE_F32, { 128, 2, 2, 2 }, v, alpha, limit));
6255 }
6256 }
6257 }
6258
6259 for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_Q4_0}) {
6260 test_cases.emplace_back(args: new test_get_rows(type, 300*256, 5, 4, 1, 2, false));
6261 test_cases.emplace_back(args: new test_get_rows(type, 256, 80000, 70000, 2, 1, false));
6262 test_cases.emplace_back(args: new test_get_rows(type, 256, 5, 4, 700, 100, false));
6263 }
6264
6265 test_cases.emplace_back(args: new test_get_rows(GGML_TYPE_F32, 1, 8, 2, 1, 1, false));
6266 for (ggml_type type : all_types) {
6267 for (int b : {1, 7}) {
6268 for (bool v : {false, true}) {
6269 test_cases.emplace_back(args: new test_get_rows(type, 256, 5, 4, b, 1, v));
6270 }
6271 }
6272 }
6273 for (int b : {1, 7}) {
6274 for (bool v : {false, true}) {
6275 test_cases.emplace_back(args: new test_get_rows(GGML_TYPE_I32, 256, 5, 4, b, 1, v));
6276 }
6277 }
6278
6279 test_cases.emplace_back(args: new test_get_rows_back(GGML_TYPE_F32, 1, 8, 2, 1, false));
6280 for (ggml_type type : all_types) {
6281 for (bool v : {false, true}) {
6282 test_cases.emplace_back(args: new test_get_rows_back(type, 256, 5, 4, 1, v));
6283 }
6284 }
6285 for (bool v : {false, true}) {
6286 test_cases.emplace_back(args: new test_get_rows_back(GGML_TYPE_I32, 256, 5, 4, 1, v));
6287 }
6288
6289 test_cases.emplace_back(args: new test_set_rows(GGML_TYPE_F32, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, false));
6290 test_cases.emplace_back(args: new test_set_rows(GGML_TYPE_F32, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, false));
6291 test_cases.emplace_back(args: new test_set_rows(GGML_TYPE_Q8_0, GGML_TYPE_I32, { 256, 5, 1, 3 }, { 1, 1, }, 1, false));
6292 for (ggml_type type : all_types) {
6293 for (int b : {1, 7}) {
6294 for (bool v : {false, true}) {
6295 test_cases.emplace_back(args: new test_set_rows(type, GGML_TYPE_I64, { 256, 5, b, 3 }, { 1, 1, }, 1, v));
6296 test_cases.emplace_back(args: new test_set_rows(type, GGML_TYPE_I64, { 256, 11, 1, b }, { 2, 3, }, 7, v));
6297
6298 test_cases.emplace_back(args: new test_set_rows(type, GGML_TYPE_I64, { 3*ggml_blck_size(type), 3, b, 1 }, { 2, 3, }, 2, v));
6299
6300 if (ggml_blck_size(type) == 1) {
6301 test_cases.emplace_back(args: new test_set_rows(type, GGML_TYPE_I64, { 31, 3, b, 1 }, { 2, 3, }, 2, v));
6302 test_cases.emplace_back(args: new test_set_rows(type, GGML_TYPE_I64, { 33, 5, 1, b }, { 2, 3, }, 1, v));
6303 }
6304 }
6305 }
6306 }
6307
6308 for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX }) {
6309 for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
6310 test_cases.emplace_back(args: new test_rope_set_rows(type, GGML_TYPE_I64, { 128, 32, 1, 100 }, mode));
6311 test_cases.emplace_back(args: new test_rope_set_rows(type, GGML_TYPE_I64, { 128, 32, 512, 1 }, mode));
6312 }
6313 }
6314
6315 for (ggml_type type_input : {GGML_TYPE_F32}) {
6316 for (ggml_op_pool pool_type : {GGML_OP_POOL_AVG, GGML_OP_POOL_MAX}) {
6317 for (int k0 : {1, 3}) {
6318 for (int k1 : {1, 3}) {
6319 for (int s0 : {1, 2}) {
6320 for (int s1 : {1, 2}) {
6321 for (int p0 : {0, 1}) {
6322 for (int p1 : {0, 1}) {
6323 test_cases.emplace_back(args: new test_pool2d(pool_type, type_input, {10, 10, 3, 1}, k0, k1, s0, s1, p0, p1));
6324 }
6325 }
6326 }
6327 }
6328 }
6329 }
6330 }
6331 }
6332
6333#if 0
6334 // >4GB im2col destination. Too slow to run by default.
6335 // Test cases taken from Wan2.1 T2V 1.3B.
6336 test_cases.emplace_back(new test_im2col (GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {832, 480, 192, 4}, {3, 3, 192, 96}, 1, 1, 1, 1, 1, 1, true));
6337 test_cases.emplace_back(new test_im2col_3d(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {834, 482, 6, 96}, {3, 3,3, 9216}, 96, 1, 1, 1, 0, 0, 0, 1, 1, 1, false));
6338#endif
6339
6340 // im2col 1D
6341 test_cases.emplace_back(args: new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false));
6342 test_cases.emplace_back(args: new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false));
6343 test_cases.emplace_back(args: new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false));
6344 for (int s0 : {1, 3}) {
6345 for (int p0 : {0, 3}) {
6346 for (int d0 : {1, 3}) {
6347 test_cases.emplace_back(args: new test_im2col(
6348 GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {20, 2, 2, 1}, {3, 2, 2, 1},
6349 s0, 0, p0, 0, d0, 0, false));
6350 }
6351 }
6352 }
6353
6354 // im2col 2D
6355 test_cases.emplace_back(args: new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32));
6356 test_cases.emplace_back(args: new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32));
6357 test_cases.emplace_back(args: new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16));
6358 for (int s0 : {1, 3}) {
6359 for (int s1 : {1, 3}) {
6360 for (int p0 : {0, 3}) {
6361 for (int p1 : {0, 3}) {
6362 for (int d0 : {1, 3}) {
6363 for (int d1 : {1, 3}) {
6364 test_cases.emplace_back(args: new test_im2col(
6365 GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {20, 20, 2, 2}, {3, 3, 2, 2},
6366 s0, s1, p0, p1, d0, d1, true));
6367 }
6368 }
6369 }
6370 }
6371 }
6372 }
6373
6374 // extra tests for im2col 2D
6375 test_cases.emplace_back(args: new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 1, 32}, {3, 3, 1, 32}, 1, 1, 1, 1, 1, 1, true));
6376 test_cases.emplace_back(args: new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 2, 32}, {3, 3, 2, 32}, 1, 1, 1, 1, 1, 1, true));
6377 test_cases.emplace_back(args: new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 1, 1024}, {3, 3, 1, 1024}, 1, 1, 1, 1, 1, 1, true));
6378 test_cases.emplace_back(args: new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 2, 1024}, {3, 3, 2, 1024}, 1, 1, 1, 1, 1, 1, true));
6379 test_cases.emplace_back(args: new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 1, 2048}, {3, 3, 1, 2048}, 1, 1, 1, 1, 1, 1, true));
6380 test_cases.emplace_back(args: new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 2, 2048}, {3, 3, 2, 2048}, 1, 1, 1, 1, 1, 1, true));
6381 test_cases.emplace_back(args: new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 1, 2560}, {3, 3, 1, 2560}, 1, 1, 1, 1, 1, 1, true));
6382 test_cases.emplace_back(args: new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {12, 12, 2, 2560}, {3, 3, 2, 2560}, 1, 1, 1, 1, 1, 1, true));
6383 test_cases.emplace_back(args: new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {5, 5, 1, 32}, {3, 4, 1, 32}, 1, 1, 0, 0, 1, 1, true));
6384
6385 // im2col 3D
6386 test_cases.emplace_back(args: new test_im2col_3d(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32));
6387 test_cases.emplace_back(args: new test_im2col_3d(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32));
6388 test_cases.emplace_back(args: new test_im2col_3d(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16));
6389 for (int s0 : {1, 3}) {
6390 for (int s1 : {1, 3}) {
6391 for (int s2 : {1, 3}) {
6392 for (int p0 : {0, 3}) {
6393 for (int p1 : {0, 3}) {
6394 for (int p2 : {0, 3}) {
6395 for (int d0 : {1, 3}) {
6396 for (int d1 : {1, 3}) {
6397 for (int d2 : {1, 3}) {
6398 for (int IC : {1, 3}) {
6399 for (bool v : {false, true}) {
6400 test_cases.emplace_back(args: new test_im2col_3d(
6401 GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {20, 20, 10, 3}, {3, 3, 3, 3},
6402 IC, s0, s1, s2, p0, p1, p2, d0, d1, d2, v));
6403 }
6404 }
6405 }
6406 }
6407 }
6408 }
6409 }
6410 }
6411 }
6412 }
6413 }
6414
6415// Conv_2D test cases
6416#ifdef DETAILED_TESTS
6417 // Probably we do not have enough time to execute these in the pipeline.
6418 uint32_t iwh_idx = 0;
6419 uint32_t kwh_idx = 1;
6420 uint32_t Cout_idx = 2;
6421 uint32_t Cin_idx = 3;
6422 uint32_t B_idx = 4;
6423
6424 std::vector<std::array<int, 5>> cases = {
6425 //{IWH, KWH, Cout, Cin, B}
6426 // K=CRS=NPQ=4096 conv_2d matmul performance
6427 {19, 4, 4096, 256, 16},
6428 // K=128, CRS=128, NPQ=4096
6429 { 19, 4, 128, 8, 16},
6430 // K=130, CRS=128, NPQ=4096
6431 { 19, 4, 130, 8, 16},
6432 // Edge case: K x CRS is small
6433 { 19, 2, 4, 4, 16},
6434 // A ConvNet's first layer
6435 { 224, 3, 8, 3, 1 },
6436 // A ConvNet's first layer with 2x2 convolution, and 1 channel
6437 { 224, 2, 8, 1, 1 },
6438 // A ConvNet's first layer with 2x2 convolution, and 1 channel, several images in the batch
6439 { 224, 2, 8, 1, 8 },
6440 // A middle layer of a ConvNet
6441 { 58, 3, 64, 32, 1 },
6442 // A middle layer of a ConvNet, several images in the batch
6443 { 58, 3, 64, 32, 8 },
6444 // A deep layer of a ConvNet, several images in the batch
6445 { 16, 3, 256, 128, 8 }
6446 };
6447
6448 for (auto kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
6449 for (auto act_case : cases) {
6450 test_cases.emplace_back(new test_conv_2d(
6451 { act_case[iwh_idx], act_case[iwh_idx], act_case[Cin_idx], act_case[B_idx] },
6452 { act_case[kwh_idx], act_case[kwh_idx], act_case[Cin_idx], act_case[Cout_idx] },
6453 kernel_type, 1, 1, 0, 0, 1, 1, false));
6454 }
6455 }
6456#endif
6457
6458 // CONV_2D:
6459 auto calc_conv_output_size = [](int64_t ins, int64_t ks, int s, int p, int d) -> int64_t {
6460 return (ins + 2 * p - d * (ks - 1) - 1) / s + 1;
6461 };
6462
6463 //uint32_t s0 = 3;
6464 uint32_t s1 = 5;
6465 uint32_t p0 = 5;
6466 //uint32_t p1 = 2;
6467 uint32_t d0 = 2;
6468 uint32_t d1 = 4;
6469
6470 for (uint32_t s0 : { 1, 3 }) {
6471 for (uint32_t p1 : { 2, 5 }) {
6472 for (uint32_t Cin : { 1, 25 }) {
6473 for (uint32_t Cout : { 1, 12 }) {
6474 for (uint32_t KH : { 1, 2, 3, 11 }) {
6475 for (uint32_t KW : { 1, 2, 3, 11 }) {
6476 for (uint32_t H : { 1, 133 }) {
6477 for (uint32_t W : { 1, 141 }) {
6478 if (calc_conv_output_size(W, KW, s0, p0, d0) > 0 &&
6479 calc_conv_output_size(H, KH, s1, p1, d1) > 0) {
6480 for (auto kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
6481 test_cases.emplace_back(args: new test_conv_2d(
6482 { W, H, Cin, 2 }, { KW, KH, Cin, Cout }, kernel_type, s0, s1, p0, p1, d0, d1, false));
6483 }
6484 }
6485 }
6486 }
6487 }
6488 }
6489 }
6490 }
6491 }
6492 }
6493
6494 // sycl backend will limit task global_range < MAX_INT
6495 // test cases for 2D im2col with large input W and H (occurs in stable-diffusion)
6496 // however these cases need to alloc more memory which may fail in some devices (Intel Arc770, etc.)
6497 // these cases are verified (pass) in Intel(R) Data Center GPU Max 1100 (sycl backend) and NV A30 (cuda backend)
6498 // test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {1024, 1024, 256, 1}, {3, 3, 256, 1}, 1, 1, 1, 1, 1, 1, true));
6499 // test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32, {1024, 1024, 256, 1}, {3, 3, 256, 1}, 1, 1, 1, 1, 1, 1, true));
6500
6501 test_cases.emplace_back(args: new test_conv_2d_dw({17, 34, 9, 1}, {3, 3, 1, 9}, 1, 0, 1, false));
6502 test_cases.emplace_back(args: new test_conv_2d_dw({17, 34, 9, 1}, {3, 3, 1, 9}, 1, 0, 1, true));
6503 test_cases.emplace_back(args: new test_conv_2d_dw({32, 8, 64, 1}, {3, 3, 1, 64}, 2, 1, 1, false));
6504 test_cases.emplace_back(args: new test_conv_2d_dw({32, 8, 64, 1}, {3, 3, 1, 64}, 2, 1, 1, true));
6505
6506 // CONV_3D
6507 auto calc_conv_output_size_3d = [](int64_t ins, int64_t ks, int s, int p, int d) -> int64_t {
6508 return (ins + 2 * p - d * (ks - 1) - 1) / s + 1;
6509 };
6510
6511 for (ggml_type kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
6512 for (int N : {1, 2}) {
6513 for (int IC : {1, 3}) {
6514 for (int OC : {1, 4}) {
6515 for (int s0 : {1, 2}) {
6516 for (int p1 : {0, 1}) {
6517 for (int d2 : {1, 2}) {
6518 int64_t IW = 20, IH = 22, ID = 18;
6519 int64_t KW = 3, KH = 3, KD = 3;
6520 int s1 = s0, s2 = s0;
6521 int p0 = p1, p2 = p1;
6522 int d0 = d2, d1 = d2;
6523
6524 if (calc_conv_output_size_3d(IW, KW, s0, p0, d0) <= 0 ||
6525 calc_conv_output_size_3d(IH, KH, s1, p1, d1) <= 0 ||
6526 calc_conv_output_size_3d(ID, KD, s2, p2, d2) <= 0) {
6527 continue;
6528 }
6529 test_cases.emplace_back(args: new test_conv_3d(
6530 N, IC, ID, IH, IW,
6531 OC, KD, KH, KW,
6532 s0, s1, s2, p0, p1, p2, d0, d1, d2,
6533 kernel_type));
6534
6535 // Asymmetric kernel and params
6536 int64_t asym_KW = 5, asym_KH = 1, asym_KD = 3;
6537 int asym_s0 = 2, asym_s1 = 1, asym_s2 = 1;
6538 int asym_p0 = 2, asym_p1 = 0, asym_p2 = 1;
6539 int asym_d0 = 1, asym_d1 = 1, asym_d2 = 2;
6540
6541 if (calc_conv_output_size_3d(IW, asym_KW, asym_s0, asym_p0, asym_d0) <= 0 ||
6542 calc_conv_output_size_3d(IH, asym_KH, asym_s1, asym_p1, asym_d1) <= 0 ||
6543 calc_conv_output_size_3d(ID, asym_KD, asym_s2, asym_p2, asym_d2) <= 0) {
6544 continue;
6545 }
6546 test_cases.emplace_back(args: new test_conv_3d(
6547 N, IC, ID, IH, IW,
6548 OC, asym_KD, asym_KH, asym_KW,
6549 asym_s0, asym_s1, asym_s2, asym_p0, asym_p1, asym_p2, asym_d0, asym_d1, asym_d2,
6550 kernel_type));
6551 }
6552 }
6553 }
6554 }
6555 }
6556 }
6557 // Case with kernel size 1
6558 test_cases.emplace_back(args: new test_conv_3d(1, 4, 8, 8, 8, 8, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type));
6559 }
6560
6561 for(uint32_t Cout : {1, 9}){
6562 for(uint32_t Cin : {1, 7}){
6563 for(uint32_t K : {1, 3, 1337}){
6564 for(uint32_t L : {1, 2, 13}){
6565 for(uint32_t s0: {1, 2, 3}){
6566 test_cases.emplace_back(args: new test_conv_transpose_1d({L,Cin,1,1}, {K,Cout,Cin,1}, s0, 0, 1));
6567 }
6568 }
6569 }
6570 }
6571 }
6572
6573 test_cases.emplace_back(args: new test_conv_transpose_1d());
6574 test_cases.emplace_back(args: new test_conv_transpose_1d({3,2,1,1}, {2,3,2,1}, 3, 0, 1));
6575 test_cases.emplace_back(args: new test_conv_transpose_1d({3,2,1,1}, {2,3,2,1}, 2, 0, 1));
6576 test_cases.emplace_back(args: new test_conv_transpose_1d({3,2,1,1}, {2,3,2,1}, 1, 0, 1));
6577 test_cases.emplace_back(args: new test_conv_transpose_1d({3,2,1,1}, {3,2,2,1}, 2, 0, 1));
6578 test_cases.emplace_back(args: new test_conv_transpose_1d({3,2,1,1}, {3,2,2,1}, 1, 0, 1));
6579 test_cases.emplace_back(args: new test_conv_transpose_1d({3,2,1,1}, {3,1,2,1}, 1, 0, 1));
6580 test_cases.emplace_back(args: new test_conv_transpose_1d({2,1,1,1}, {3,1,1,1}, 1, 0, 1));
6581
6582 test_cases.emplace_back(args: new test_conv_transpose_2d({3, 2, 3, 1}, {2, 2, 1, 3}, 1));
6583 test_cases.emplace_back(args: new test_conv_transpose_2d({10, 10, 9, 1}, {3, 3, 1, 9}, 2));
6584
6585 test_cases.emplace_back(args: new test_count_equal(GGML_TYPE_F32, {4, 500, 1, 1}));
6586 test_cases.emplace_back(args: new test_count_equal(GGML_TYPE_F32, {4, 5000, 1, 1}));
6587
6588 test_cases.emplace_back(args: new test_argmax(GGML_TYPE_F32, {32, 1, 1, 1}));
6589 test_cases.emplace_back(args: new test_argmax(GGML_TYPE_F32, {32, 513, 1, 1}));
6590 test_cases.emplace_back(args: new test_argmax(GGML_TYPE_F32, {100, 10, 1, 1}));
6591 test_cases.emplace_back(args: new test_argmax(GGML_TYPE_F32, {1024, 10, 1, 1}));
6592 test_cases.emplace_back(args: new test_argmax(GGML_TYPE_F32, {1024, 12, 1, 1}));
6593 test_cases.emplace_back(args: new test_argmax(GGML_TYPE_F32, {2000, 10, 1, 1}));
6594 test_cases.emplace_back(args: new test_argmax(GGML_TYPE_F32, {5438, 3, 1, 1}));
6595
6596 for (int ne3 : {1, 3}) { // CUDA backward pass only supports ne3 == 1
6597 test_cases.emplace_back(args: new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 1, 1, 1}));
6598 test_cases.emplace_back(args: new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {2, 1, 1, 1}));
6599 test_cases.emplace_back(args: new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 2, 1, 1}));
6600 test_cases.emplace_back(args: new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 1, 2, 1}));
6601 test_cases.emplace_back(args: new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 1, 1, 2}));
6602 test_cases.emplace_back(args: new test_repeat(GGML_TYPE_I32, {10, 5, 4, ne3}, {2, 1, 1, 1}));
6603 test_cases.emplace_back(args: new test_repeat(GGML_TYPE_I16, {10, 5, 4, ne3}, {1, 1, 1, 2}));
6604 }
6605
6606 for (bool view : {false, true}) {
6607 test_cases.emplace_back(args: new test_repeat_back(GGML_TYPE_F32, {8, 6, 4, 2}, {1, 1, 1, 1}, view));
6608 test_cases.emplace_back(args: new test_repeat_back(GGML_TYPE_F32, {8, 6, 4, 2}, {2, 1, 1, 1}, view));
6609 test_cases.emplace_back(args: new test_repeat_back(GGML_TYPE_F32, {8, 6, 4, 2}, {1, 2, 1, 1}, view));
6610 test_cases.emplace_back(args: new test_repeat_back(GGML_TYPE_F32, {8, 6, 4, 2}, {1, 1, 2, 1}, view));
6611 test_cases.emplace_back(args: new test_repeat_back(GGML_TYPE_F32, {8, 6, 4, 2}, {1, 1, 1, 2}, view));
6612 }
6613
6614 test_cases.emplace_back(args: new test_dup(GGML_TYPE_F32));
6615 test_cases.emplace_back(args: new test_dup(GGML_TYPE_F16));
6616 test_cases.emplace_back(args: new test_dup(GGML_TYPE_I32));
6617 test_cases.emplace_back(args: new test_dup(GGML_TYPE_I16));
6618 test_cases.emplace_back(args: new test_dup(GGML_TYPE_F32, {10, 10, 5, 1}, {0, 2, 1, 3}));
6619 test_cases.emplace_back(args: new test_dup(GGML_TYPE_F16, {10, 10, 5, 1}, {0, 2, 1, 3})); // dup by rows
6620 test_cases.emplace_back(args: new test_dup(GGML_TYPE_F32, {10, 10, 5, 1}, {1, 0, 2, 3}));
6621 test_cases.emplace_back(args: new test_dup(GGML_TYPE_F16, {10, 10, 5, 1}, {1, 0, 2, 3})); // dup dst not-contiguous
6622 test_cases.emplace_back(args: new test_dup(GGML_TYPE_I16, {10, 8, 3, 1}, {0, 2, 1, 3}));
6623 test_cases.emplace_back(args: new test_dup(GGML_TYPE_I16, {10, 8, 3, 1}, {1, 2, 0, 3}));
6624
6625 for (int dim = 1; dim < GGML_MAX_DIMS; ++dim) {
6626 test_cases.emplace_back(args: new test_set(GGML_TYPE_F32, GGML_TYPE_F32, {6, 5, 4, 3}, dim));
6627 }
6628
6629 for (int dim = 1; dim < GGML_MAX_DIMS; ++dim) {
6630 test_cases.emplace_back(args: new test_set(GGML_TYPE_I32, GGML_TYPE_I32, {6, 5, 4, 3}, dim));
6631 }
6632
6633 // same-type copy
6634 for (ggml_type type : all_types) {
6635 const auto nk = ggml_blck_size(type);
6636
6637 for (int k = 1; k < 4; ++k) {
6638 test_cases.emplace_back(args: new test_cpy(type, type, {k*nk, 2, 3, 4}));
6639 test_cases.emplace_back(args: new test_cpy(type, type, {k*nk, 2, 3, 4}, {0, 2, 1, 3}));
6640 test_cases.emplace_back(args: new test_cpy(type, type, {k*nk, 2, 3, 4}, {0, 3, 1, 2}, {0, 2, 1, 3}));
6641 }
6642 }
6643
6644 for (ggml_type type_src : {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_F32}) {
6645 for (ggml_type type_dst : all_types) {
6646 test_cases.emplace_back(args: new test_cpy(type_src, type_dst, {256, 4, 4, 4}));
6647 test_cases.emplace_back(args: new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {0, 2, 1, 3})); // cpy by rows
6648 }
6649 }
6650 for (ggml_type type_src : all_types) {
6651 for (ggml_type type_dst : {GGML_TYPE_F32}) {
6652 test_cases.emplace_back(args: new test_cpy(type_src, type_dst, {256, 4, 4, 4}));
6653 test_cases.emplace_back(args: new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {0, 2, 1, 3})); // cpy by rows
6654 }
6655 }
6656 for (ggml_type type_src : {GGML_TYPE_F16, GGML_TYPE_F32}) {
6657 for (ggml_type type_dst : {GGML_TYPE_F16, GGML_TYPE_F32}) {
6658 test_cases.emplace_back(args: new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {1, 0, 2, 3})); // cpy not-contiguous
6659 }
6660 }
6661 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F32, GGML_TYPE_I32, {256, 2, 3, 4}));
6662 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F32, GGML_TYPE_I32, {256, 2, 3, 4}, {1, 0, 2, 3}));
6663 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_I32, GGML_TYPE_F32, {256, 2, 3, 4}));
6664 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_I32, GGML_TYPE_F32, {256, 2, 3, 4}, {1, 0, 2, 3}));
6665 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {256, 4, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
6666 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 4, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
6667 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 4, 3, 3}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
6668 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {256, 4, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
6669 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {256, 4, 1, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
6670 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 4, 1, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
6671 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {256, 4, 1, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
6672 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 1, 4, 1}, {1, 2, 0, 3}, {0, 0, 0, 0}));
6673
6674 test_cases.emplace_back(args: new test_cont());
6675 test_cases.emplace_back(args: new test_cont(GGML_TYPE_F32, {2, 1, 1 ,1}));
6676 test_cases.emplace_back(args: new test_cont(GGML_TYPE_F32, {2, 1, 3 ,5}));
6677 test_cases.emplace_back(args: new test_cont(GGML_TYPE_F32, {2, 3, 5 ,7}));
6678 test_cases.emplace_back(args: new test_cont(GGML_TYPE_F16, {2, 1, 1 ,1}));
6679 test_cases.emplace_back(args: new test_cont(GGML_TYPE_F16, {2, 1, 3 ,5}));
6680 test_cases.emplace_back(args: new test_cont(GGML_TYPE_F16, {2, 3, 5 ,7}));
6681 test_cases.emplace_back(args: new test_cont(GGML_TYPE_BF16, {2, 1, 1 ,1}));
6682 test_cases.emplace_back(args: new test_cont(GGML_TYPE_BF16, {2, 1, 3 ,5}));
6683 test_cases.emplace_back(args: new test_cont(GGML_TYPE_BF16, {2, 3, 5 ,7}));
6684
6685 auto add_test_bin_bcast = [&](ggml_type type, std::array<int64_t, 4> ne, std::array<int, 4> nr) {
6686 for (auto op : {ggml_add, ggml_sub, ggml_mul, ggml_div}) {
6687 test_cases.emplace_back(args: new test_bin_bcast(op, type, ne, nr));
6688 }
6689 };
6690 for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
6691 add_test_bin_bcast(type, {1, 1, 8, 1}, {1, 1, 1, 1});
6692 add_test_bin_bcast(type, {1, 1, 1, 1}, {32, 1, 1, 1});
6693 add_test_bin_bcast(type, {1, 1, 320, 320}, {1, 1, 1, 1});
6694 add_test_bin_bcast(type, {10, 5, 1, 1}, {1, 1, 1, 1});
6695 add_test_bin_bcast(type, {10, 5, 4, 1}, {1, 1, 1, 1});
6696 add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 1, 1, 1});
6697 add_test_bin_bcast(type, {10, 5, 4, 3}, {2, 1, 1, 1});
6698 add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 2, 1, 1});
6699 add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 1, 2, 1});
6700 add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 1, 1, 2});
6701 add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 1, 2, 2});
6702 add_test_bin_bcast(type, {10, 5, 4, 3}, {1, 2, 2, 2});
6703 add_test_bin_bcast(type, {10, 5, 4, 3}, {2, 2, 2, 2});
6704
6705 // test case for k_bin_bcast_unravel in CUDA backend
6706 add_test_bin_bcast(type, {1, 1, 65536, 1}, {256, 1, 1, 1});
6707
6708 // stable diffusion
6709 add_test_bin_bcast(type, {1280, 1, 1, 1}, {1, 1, 1, 1});
6710 add_test_bin_bcast(type, {1280, 1, 1, 1}, {1, 16, 16, 1});
6711 add_test_bin_bcast(type, {1280, 16, 16, 1}, {1, 1, 1, 1});
6712 add_test_bin_bcast(type, {1280, 1, 1, 1}, {1, 256, 1, 1});
6713 add_test_bin_bcast(type, {1, 1, 1280, 1}, {16, 16, 1, 1});
6714 add_test_bin_bcast(type, {16, 16, 1280, 1}, {1, 1, 1, 1});
6715 add_test_bin_bcast(type, {1, 1, 1920, 1}, {16, 16, 1, 1});
6716 add_test_bin_bcast(type, {1, 1, 2560, 1}, {16, 16, 1, 1});
6717 add_test_bin_bcast(type, {1, 1, 1280, 1}, {32, 32, 1, 1});
6718 add_test_bin_bcast(type, {1, 1, 1920, 1}, {32, 32, 1, 1});
6719 add_test_bin_bcast(type, {1, 1, 640, 1}, {32, 32, 1, 1});
6720 add_test_bin_bcast(type, {5120, 1, 1, 1}, {1, 256, 1, 1});
6721 add_test_bin_bcast(type, {640, 1, 1, 1}, {1, 1, 1, 1});
6722 add_test_bin_bcast(type, {64, 262144, 1, 1}, {1, 1, 1, 1});
6723 //add_test_bin_bcast(type, {3, 3, 2560, 1280}, {1, 1, 1, 1});
6724 //add_test_bin_bcast(type, {3, 3, 2560, 1280}, {2, 1, 1, 1});
6725 }
6726
6727 // single inplace tests, especially important for WebGPU backend since kernels for inplace vs. not are different
6728 test_cases.emplace_back(args: new test_bin_bcast(ggml_add_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16));
6729 test_cases.emplace_back(args: new test_bin_bcast(ggml_mul_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16));
6730 test_cases.emplace_back(args: new test_bin_bcast(ggml_sub_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16));
6731 test_cases.emplace_back(args: new test_bin_bcast(ggml_div_inplace, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16));
6732
6733 // fusion
6734 test_cases.emplace_back(args: new test_bin_bcast(ggml_add, GGML_TYPE_F32, {10, 5, 4, 3}, {2, 1, 1, 1}, 2));
6735 test_cases.emplace_back(args: new test_bin_bcast(ggml_add, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 2, 1, 1}, 3));
6736 test_cases.emplace_back(args: new test_bin_bcast(ggml_add, GGML_TYPE_F32, {10, 5, 4, 3}, {1, 1, 2, 1}, 4));
6737 test_cases.emplace_back(args: new test_bin_bcast(ggml_add, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 2}, 5));
6738 test_cases.emplace_back(args: new test_bin_bcast(ggml_add, GGML_TYPE_F32, {10, 5, 4, 3}, {1, 1, 2, 2}, 6));
6739 test_cases.emplace_back(args: new test_bin_bcast(ggml_add, GGML_TYPE_F32, {10, 5, 4, 3}, {1, 2, 2, 2}, 7));
6740 test_cases.emplace_back(args: new test_bin_bcast(ggml_add, GGML_TYPE_F32, {16, 5, 4, 3}, {2, 2, 2, 2}, 8));
6741 test_cases.emplace_back(args: new test_bin_bcast(ggml_add, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 16));
6742
6743 test_cases.emplace_back(args: new test_add1());
6744 test_cases.emplace_back(args: new test_scale());
6745 test_cases.emplace_back(args: new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f));
6746 test_cases.emplace_back(args: new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, true)); // inplace test
6747 test_cases.emplace_back(args: new test_scale(GGML_TYPE_F32, {100, 10, 10, 10}, 2.0f, 1.0f));
6748 test_cases.emplace_back(args: new test_softcap(GGML_TYPE_F32, {10, 10, 10, 10}, 50.0f));
6749 test_cases.emplace_back(args: new test_silu_back());
6750
6751 for (float eps : {0.0f, 1e-6f, 1e-4f, 1e-1f}) {
6752 for (bool v : {false, true}) {
6753 test_cases.emplace_back(args: new test_norm (GGML_TYPE_F32, {64, 5, 4, 3}, v, eps));
6754 test_cases.emplace_back(args: new test_rms_norm(GGML_TYPE_F32, {64, 5, 4, 3}, v, eps));
6755 }
6756 test_cases.emplace_back(args: new test_rms_norm_back(GGML_TYPE_F32, {64, 5, 4, 3}, eps));
6757 test_cases.emplace_back(args: new test_l2_norm (GGML_TYPE_F32, {64, 5, 4, 3}, eps));
6758 }
6759
6760 // in-place tests
6761 test_cases.emplace_back(args: new test_rms_norm(GGML_TYPE_F32, {64, 5, 4, 3}, false, 1e-6f, true));
6762
6763 for (float eps : {0.0f, 1e-6f, 1e-4f, 1e-1f, 1.0f}) {
6764 test_cases.emplace_back(args: new test_rms_norm_mul_add(GGML_TYPE_F32, {64, 5, 4, 3}, eps, false));
6765 test_cases.emplace_back(args: new test_rms_norm_mul_add(GGML_TYPE_F32, {64, 5, 4, 3}, eps, true));
6766 test_cases.emplace_back(args: new test_norm_mul_add(GGML_TYPE_F32, {64, 5, 4, 3}, eps, false));
6767 test_cases.emplace_back(args: new test_norm_mul_add(GGML_TYPE_F32, {64, 5, 4, 3}, eps, true));
6768 }
6769 for (uint32_t n : {1, 511, 1025, 8192, 33*512}) {
6770 for (bool multi_add : {false, true}) {
6771 test_cases.emplace_back(args: new test_rms_norm_mul_add(GGML_TYPE_F32, {n, 1, 1, 1}, 1e-6f, false, multi_add));
6772 }
6773 }
6774
6775 for (auto multi_add : {false, true}) {
6776 for (auto set_rows : {false, true}) {
6777 for (auto rope : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) {
6778 test_cases.emplace_back(args: new test_rms_norm_mul_rope({768, 1, 1, 1}, 1e-6f, multi_add, set_rows, rope));
6779 test_cases.emplace_back(args: new test_rms_norm_mul_rope({768, 3, 1, 1}, 1e-6f, multi_add, set_rows, rope));
6780 test_cases.emplace_back(args: new test_rms_norm_mul_rope({768, 3, 5, 1}, 1e-6f, multi_add, set_rows, rope));
6781 test_cases.emplace_back(args: new test_rms_norm_mul_rope({128, 32, 2, 1}, 1e-6f, multi_add, set_rows, rope));
6782 test_cases.emplace_back(args: new test_rms_norm_mul_rope({128, 4, 2, 1}, 1e-6f, multi_add, set_rows, rope));
6783 test_cases.emplace_back(args: new test_rms_norm_mul_rope({128, 32, 50, 1}, 1e-6f, multi_add, set_rows, rope));
6784 test_cases.emplace_back(args: new test_rms_norm_mul_rope({128, 4, 50, 1}, 1e-6f, multi_add, set_rows, rope));
6785 test_cases.emplace_back(args: new test_rms_norm_mul_rope({8192, 2, 2, 1}, 1e-6f, multi_add, set_rows, rope));
6786 test_cases.emplace_back(args: new test_rms_norm_mul_rope({8192, 2, 2, 1}, 1e-6f, multi_add, set_rows, rope));
6787 }
6788 }
6789 }
6790
6791 test_cases.emplace_back(args: new test_l2_norm(GGML_TYPE_F32, {64, 5, 4, 3}, 1e-12f));
6792
6793 for (int64_t d_conv : {3, 4}) {
6794 for (int64_t d_inner: {1024, 1536, 2048}) {
6795 test_cases.emplace_back(args: new test_ssm_conv(GGML_TYPE_F32, {4, d_inner, 1, 1}, {d_conv, d_inner, 1, 1}));
6796 test_cases.emplace_back(args: new test_ssm_conv(GGML_TYPE_F32, {8, d_inner, 1, 1}, {d_conv, d_inner, 1, 1}));
6797 test_cases.emplace_back(args: new test_ssm_conv(GGML_TYPE_F32, {4, d_inner, 4, 1}, {d_conv, d_inner, 1, 1}));
6798 }
6799 }
6800
6801 test_cases.emplace_back(args: new test_ssm_scan(GGML_TYPE_F32, 16, 1, 1024, 1, 32, 4)); // Mamba-1
6802 test_cases.emplace_back(args: new test_ssm_scan(GGML_TYPE_F32, 128, 64, 16, 2, 32, 4)); // Mamba-2
6803 test_cases.emplace_back(args: new test_ssm_scan(GGML_TYPE_F32, 256, 64, 8, 2, 32, 4)); // Falcon-H1
6804
6805 test_cases.emplace_back(args: new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 1, 1));
6806 test_cases.emplace_back(args: new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 32, 1));
6807 test_cases.emplace_back(args: new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 32, 4));
6808 test_cases.emplace_back(args: new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 128, 4));
6809
6810 test_cases.emplace_back(args: new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 1, 1));
6811 test_cases.emplace_back(args: new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 32, 1));
6812 test_cases.emplace_back(args: new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 32, 4));
6813 test_cases.emplace_back(args: new test_rwkv_wkv7(GGML_TYPE_F32, 32, 64, 128, 4));
6814
6815 test_cases.emplace_back(args: new test_gla(GGML_TYPE_F32, 32, 64, 1, 1));
6816 test_cases.emplace_back(args: new test_gla(GGML_TYPE_F32, 32, 64, 32, 1));
6817 test_cases.emplace_back(args: new test_gla(GGML_TYPE_F32, 32, 64, 32, 4));
6818 test_cases.emplace_back(args: new test_gla(GGML_TYPE_F32, 32, 64, 128, 4));
6819
6820#if 0
6821 // > 4GB A matrix. Too slow to be enabled by default.
6822 test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 900000, 3, 2592, {1, 1}, {1, 1}));
6823 test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1700000, 96, 2592, {1, 1}, {1, 1}));
6824 test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1700000, 3, 2592, {1, 1}, {1, 1}));
6825 test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1700000, 1, 2592, {1, 1}, {1, 1}));
6826#endif
6827
6828 for (ggml_type type_a : all_types) {
6829 for (int i = 1; i < 10; ++i) {
6830 test_cases.emplace_back(args: new test_mul_mat(type_a, GGML_TYPE_F32, 16, i, 256, { 1, 1}, {1, 1}));
6831 }
6832 }
6833
6834#if 0
6835 {
6836 // Test paths in OpenCL
6837 std::vector<int> ns = {32, 64, 128, 256, 512, 1024, 4096};
6838 std::vector<int> ks = {896, 1536, 4096};
6839 for (auto n : ns) {
6840 for (auto k : ks) {
6841 test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 1024, n, k, {1, 1}, {1, 1}));
6842 }
6843 }
6844 }
6845#endif
6846
6847#if 1
6848 for (ggml_type type_a : base_types) {
6849 for (ggml_type type_b : {GGML_TYPE_F32, GGML_TYPE_F16}) {
6850 std::vector<int> ks = { 256 };
6851 if (ggml_blck_size(type: type_a) == 1) {
6852 ks.push_back(x: 4);
6853 }
6854 for (auto k : ks) {
6855 // test cases without permutation
6856 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 1, k, {1, 1}, {1, 1}));
6857 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 1, k, {1, 1}, {2, 1}));
6858 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 1, k, {1, 1}, {1, 2}));
6859 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 1, k, {3, 1}, {1, 1}));
6860 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 1, k, {3, 1}, {2, 1}));
6861 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 1, k, {3, 2}, {1, 1}));
6862 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 1, k, {3, 2}, {2, 1}));
6863 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 1, k, {3, 2}, {1, 2}));
6864 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 1, k, {3, 2}, {2, 2}));
6865
6866 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 16, k, {1, 1}, {1, 1}));
6867 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 16, k, {1, 1}, {2, 1}));
6868 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 16, k, {1, 1}, {1, 2}));
6869 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 16, k, {3, 1}, {1, 1}));
6870 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 16, k, {3, 1}, {2, 1}));
6871 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 16, k, {3, 2}, {1, 1}));
6872 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 16, k, {3, 2}, {2, 1}));
6873 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 16, k, {3, 2}, {1, 2}));
6874 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 16, k, {3, 2}, {2, 2}));
6875
6876 // test cases with permutation
6877 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 1, k, {2, 3}, {1, 1}, {0, 2, 1, 3}));
6878 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 1, k, {2, 3}, {1, 1}, {0, 1, 3, 2}));
6879 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 1, k, {2, 3}, {1, 1}, {0, 3, 2, 1}));
6880
6881 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 8, k, {2, 3}, {1, 1}, {0, 2, 1, 3}));
6882 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 8, k, {2, 3}, {1, 1}, {0, 1, 3, 2}));
6883 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 8, k, {2, 3}, {1, 1}, {0, 3, 2, 1}));
6884
6885 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 16, k, {2, 3}, {1, 1}, {0, 2, 1, 3}));
6886 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 16, k, {2, 3}, {1, 1}, {0, 1, 3, 2}));
6887 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 16, k, {2, 3}, {1, 1}, {0, 3, 2, 1}));
6888 }
6889
6890 // test cases with large ne00/ne10 to cover stream-k fixup
6891 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 1, 1024, {3, 2}, {1, 1}));
6892 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 8, 1024, {3, 2}, {1, 1}));
6893 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 16, 1024, {3, 2}, {1, 1}));
6894
6895 // test cases with large batch size
6896 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 8, 256, {1536, 1}, {1, 1}));
6897 }
6898 }
6899 for (ggml_type type_a : other_types) {
6900 for (ggml_type type_b : {GGML_TYPE_F32}) {
6901 if (ggml_blck_size(type: type_a) != 256) {
6902 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 1, ggml_blck_size(type: type_a), {1, 1}, {1, 1}));
6903 }
6904 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 16, 1, 256, {1, 1}, {1, 1}));
6905 }
6906 }
6907#else
6908 // m = a rows
6909 // n = b rows
6910 // k = cols
6911 std::uniform_int_distribution<> dist_m(1, 128);
6912 std::uniform_int_distribution<> dist_n(16, 128);
6913 std::uniform_int_distribution<> dist_k(1, 16);
6914 for (int i = 0; i < 1000; i++) {
6915 for (ggml_type type_a : all_types) {
6916 for (ggml_type type_b : {GGML_TYPE_F32}) {
6917 int m = dist_m(rng);
6918 int n = dist_n(rng);
6919 int k = dist_k(rng) * ggml_blck_size(type_a);
6920 test_cases.emplace_back(new test_mul_mat(type_a, type_b, m, n, k, { 1, 1}, {1, 1}));
6921 }
6922 }
6923 }
6924#endif
6925
6926 test_cases.emplace_back(args: new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 2, 128, { 8, 1}, {1, 1}));
6927 test_cases.emplace_back(args: new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 83, 2, 128, { 8, 1}, {4, 1}));
6928 test_cases.emplace_back(args: new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 2, 64, { 8, 1}, {4, 1}));
6929 test_cases.emplace_back(args: new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 83, 2, 64, { 8, 1}, {4, 1}));
6930 test_cases.emplace_back(args: new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 45, 128, { 8, 1}, {4, 1}));
6931 test_cases.emplace_back(args: new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 128, 45, 64, { 8, 1}, {4, 1}));
6932 test_cases.emplace_back(args: new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1056, 1, 193, {1, 1}, {4, 1}, {0, 2, 1, 3}));
6933 test_cases.emplace_back(args: new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1056, 1, 67, {1, 1}, {4, 1}, {0, 2, 1, 3}));
6934 test_cases.emplace_back(args: new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 16, 32, 32, { 1, 1}, {1, 1}, {0, 1, 2, 3}, 64, 3));
6935 test_cases.emplace_back(args: new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 77, 77, {12,1}, {1,1}));
6936
6937#if 0
6938 // test the mat-mat path for Metal
6939 for (int k = 1; k < 512; ++k) {
6940 test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 127, k, {12,1}, {1,1}));
6941 test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 127, k, {12,1}, {1,1}));
6942 test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 77, k, {12,1}, {1,1}));
6943 test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 77, k, {12,1}, {1,1}));
6944 test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 64, 128, k, {12,1}, {1,1}));
6945 test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 128, k, {12,1}, {1,1}));
6946 test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, false, 50, 200, k));
6947 test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, true, 50, 200, k));
6948 test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 16, 16, false, 50, 200, k));
6949 test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 16, 16, true, 50, 200, k));
6950 }
6951#endif
6952
6953 for (auto bs2 : {1,3}) {
6954 for (auto bs : {1,2,4,8}) {
6955 for (auto nr : {1,4}) {
6956 for (uint32_t m = 0; m < 2; ++m) {
6957 for (uint32_t k = 0; k < 2; ++k) {
6958 for (ggml_type type: {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_F32}) {
6959 test_cases.emplace_back(args: new test_mul_mat(type, GGML_TYPE_F32, 1056 + m, 1, 128 + k, {bs, bs2}, {nr, 1}, {0, 2, 1, 3}));
6960 test_cases.emplace_back(args: new test_mul_mat(type, GGML_TYPE_F32, 128 + m, 1, 1056 + k, {bs, bs2}, {nr, 1}, {0, 1, 2, 3}, 2*1056 + k));
6961 }
6962 }
6963 }
6964 }
6965 }
6966 }
6967
6968 // sycl backend will limit task global_range < MAX_INT
6969 // test case for f16-type-convert-to-fp32 kernel with large k under fp32 compute dtype (occurs in stable-diffusion)
6970 // however this case needs to alloc more memory which may fail in some devices (Intel Arc770, etc.)
6971 // this case is verified (pass) in Intel(R) Data Center GPU Max 1100 (sycl backend) and NV A30 (cuda backend)
6972 // test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 512, 262144, 9216, {1, 1}, {1, 1}));
6973
6974 // test large experts*tokens
6975 for (bool b : {false, true}) {
6976 test_cases.emplace_back(args: new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, b, 32, 1024, 16));
6977 test_cases.emplace_back(args: new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 2, 2, b, 32, 8192, 64));
6978 test_cases.emplace_back(args: new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, b, 50, 200, 64));
6979 }
6980
6981 test_cases.emplace_back(args: new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 1, 1, false, 8, 16, 1));
6982 test_cases.emplace_back(args: new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, false, 32, 32, 32, 3));
6983
6984 // gpt-oss issue with Vulkan mmq_id
6985 test_cases.emplace_back(args: new test_mul_mat_id(GGML_TYPE_MXFP4, GGML_TYPE_F32, 32, 2, false, 2880, 32, 2880));
6986
6987 for (ggml_type type_a : base_types) {
6988 for (ggml_type type_b : {GGML_TYPE_F32 /*, GGML_TYPE_F16 */}) {
6989 for (int n_mats : {4, 8}) {
6990 for (int n_used : {1, 2, 4}) {
6991 for (bool b : {false, true}) {
6992 for (int n : {1, 4, 5, 17, 32, 129}) {
6993 int m = 512;
6994 int k = 256;
6995 test_cases.emplace_back(args: new test_mul_mat_id(type_a, type_b, n_mats, n_used, b, m, n, k));
6996 }
6997 }
6998 }
6999 }
7000 }
7001 }
7002
7003 for (ggml_type type_a : other_types) {
7004 for (ggml_type type_b : {GGML_TYPE_F32 /*, GGML_TYPE_F16 */}) {
7005 for (int n_mats : {4}) {
7006 for (int n_used : {2}) {
7007 for (bool b : {false}) {
7008 for (int n : {1, 32}) {
7009 int m = 512;
7010 int k = 256;
7011 test_cases.emplace_back(args: new test_mul_mat_id(type_a, type_b, n_mats, n_used, b, m, n, k));
7012 }
7013 }
7014 }
7015 }
7016 }
7017 }
7018
7019 for (ggml_type type_a : base_types) {
7020 for (ggml_type type_b : {GGML_TYPE_F32, GGML_TYPE_F16}) {
7021 for (int n : {1, 16}) {
7022 for (int k : {1, 16}) {
7023 for (int bs2 : {1, 3}) {
7024 for (int bs3 : {1, 3}) {
7025 for (int nr2 : {1, 2}) {
7026 for (int nr3 : {1, 2}) {
7027 test_cases.emplace_back(args: new test_out_prod(type_a, type_b, 256, n, k, {bs2, bs3}, {nr2, nr3}));
7028 }
7029 }
7030 }
7031 }
7032 }
7033 }
7034 }
7035 }
7036
7037 // add_id
7038 for (ggml_type type_a : {GGML_TYPE_F32}) {
7039 for (ggml_type type_b : {GGML_TYPE_F32}) {
7040 for (int n_mats : {4, 8}) {
7041 for (int n_used : {1, 2, 4}) {
7042 for (int n_embd : {32, 129}) {
7043 for (int n_token : {1, 32, 129}) {
7044 test_cases.emplace_back(args: new test_add_id(type_a, type_b, n_embd, n_mats, n_used, n_token));
7045 }
7046 }
7047 }
7048 }
7049 }
7050 }
7051
7052 for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
7053 test_cases.emplace_back(args: new test_sqr (type));
7054 test_cases.emplace_back(args: new test_sqrt (type));
7055 test_cases.emplace_back(args: new test_log (type));
7056 test_cases.emplace_back(args: new test_sin (type));
7057 test_cases.emplace_back(args: new test_cos (type));
7058 test_cases.emplace_back(args: new test_clamp (type));
7059 test_cases.emplace_back(args: new test_leaky_relu(type));
7060 test_cases.emplace_back(args: new test_floor (type));
7061 test_cases.emplace_back(args: new test_ceil (type));
7062 test_cases.emplace_back(args: new test_round (type));
7063 test_cases.emplace_back(args: new test_trunc (type));
7064 test_cases.emplace_back(args: new test_sqr (type, {7, 1, 5, 3}));
7065 test_cases.emplace_back(args: new test_sqrt (type, {7, 1, 5, 3}));
7066 test_cases.emplace_back(args: new test_log (type, {7, 1, 5, 3}));
7067 test_cases.emplace_back(args: new test_sin (type, {7, 1, 5, 3}));
7068 test_cases.emplace_back(args: new test_cos (type, {7, 1, 5, 3}));
7069 test_cases.emplace_back(args: new test_clamp (type, {7, 1, 5, 3}));
7070 test_cases.emplace_back(args: new test_leaky_relu(type, {7, 1, 5, 3}));
7071 test_cases.emplace_back(args: new test_floor (type, {7, 1, 5, 3}));
7072 test_cases.emplace_back(args: new test_ceil (type, {7, 1, 5, 3}));
7073 test_cases.emplace_back(args: new test_round (type, {7, 1, 5, 3}));
7074 test_cases.emplace_back(args: new test_trunc (type, {7, 1, 5, 3}));
7075 }
7076
7077 test_cases.emplace_back(args: new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 1, 1}, 5));
7078 test_cases.emplace_back(args: new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 3, 1}, 5));
7079 test_cases.emplace_back(args: new test_diag_mask_inf(GGML_TYPE_F32, {10, 10, 3, 2}, 5));
7080
7081#if 0
7082 std::uniform_int_distribution<> dist_ne1(1, 50);
7083 int exponent = 1;
7084 while (exponent < (1 << 17)) {
7085 std::uniform_int_distribution<> dist_ne0(exponent, 2*exponent);
7086
7087 for (int n = 0; n < 10; ++n) {
7088 int64_t ne0 = dist_ne0(rng);
7089 int64_t ne1 = dist_ne1(rng);
7090 test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, GGML_TYPE_F32, {ne0, ne1, 1, 1}, n/2 == 0, 0.1f, ne0 < 1000 ? 4.0f : 0.0f));
7091 }
7092
7093 exponent <<= 1;
7094 }
7095#endif
7096 for (bool mask : {false, true}) {
7097 for (bool sinks : {false, true}) {
7098 for (float max_bias : {0.0f, 8.0f}) {
7099 if (!mask && max_bias > 0.0f) continue;
7100 for (float scale : {1.0f, 0.1f}) {
7101 for (int64_t ne0 : {16, 1024}) {
7102 for (int64_t ne1 : {16, 1024}) {
7103 if (mask) {
7104 for (ggml_type m_prec : {GGML_TYPE_F32, GGML_TYPE_F16}) {
7105 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {ne0, ne1, 1, 1}, mask, sinks, m_prec, {1, 1}, scale, max_bias));
7106 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {ne0-1, ne1-1, 1, 1}, mask, sinks, m_prec, {1, 1}, scale, max_bias));
7107
7108 if (ne0 <= 32 && ne1 <= 32) {
7109 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {ne0, ne1, 1, 3}, mask, sinks, m_prec, {3, 1}, scale, max_bias));
7110 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {ne0-1, ne1-1, 1, 1}, mask, sinks, m_prec, {2, 3}, scale, max_bias));
7111 }
7112 }
7113 } else {
7114 /* The precision of mask here doesn't matter as boolean mask is false */
7115 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {ne0, ne1, 1, 1}, mask, sinks, GGML_TYPE_F32, {1, 1}, scale, max_bias));
7116 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {ne0-1, ne1-1, 1, 1}, mask, sinks, GGML_TYPE_F32, {1, 1}, scale, max_bias));
7117 }
7118 }
7119 }
7120 }
7121 }
7122 // inplace tests
7123 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, mask, sinks, GGML_TYPE_F32, {1, 1}, 0.1f, 0.0f, true));
7124 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, mask, sinks, GGML_TYPE_F16, {1, 1}, 0.1f, 0.0f, true));
7125 }
7126 }
7127 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, true, true, GGML_TYPE_F32, {1, 1}, 0.1f, 0.0f));
7128 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, true, false, GGML_TYPE_F16, {1, 1}, 0.1f, 0.0f));
7129 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {16, 2, 32, 1}, false, true, GGML_TYPE_F32, {1, 1}, 0.1f, 0.0f));
7130 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {32, 2, 32, 1}, true, true, GGML_TYPE_F32, {1, 1}, 0.1f, 0.0f));
7131 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {32, 2, 32, 1}, true, false, GGML_TYPE_F16, {1, 1}, 0.1f, 0.0f));
7132 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {32, 2, 32, 1}, true, true, GGML_TYPE_F32, {1, 1}, 0.1f, 8.0f));
7133 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {32, 2, 32, 1}, true, true, GGML_TYPE_F16, {1, 1}, 0.1f, 8.0f));
7134
7135 for (float max_bias : {0.0f, 8.0f}) {
7136 for (float scale : {1.0f, 0.1f}) {
7137 for (int64_t ne0 : {16, 1024}) {
7138 for (int64_t ne1 : {16, 1024}) {
7139 test_cases.emplace_back(args: new test_soft_max_back(GGML_TYPE_F32, {ne0, ne1, 1, 1}, scale, max_bias));
7140 test_cases.emplace_back(args: new test_soft_max_back(GGML_TYPE_F32, {ne0-1, ne1-1, 1, 1}, scale, max_bias));
7141 test_cases.emplace_back(args: new test_soft_max_back(GGML_TYPE_F32, {ne0, ne1, 2, 3}, scale, max_bias));
7142 }
7143 }
7144 }
7145 }
7146
7147 for (bool fw : {true, false}) { // fw == forward
7148 bool all = true;
7149
7150 for (float fs : { 1.0f, 1.4245f }) {
7151 for (float ef : { 0.0f, 0.7465f }) {
7152 for (float af : { 1.0f, 1.4245f }) {
7153 for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
7154 for (bool ff : {false, true}) { // freq_factors
7155 for (float v : { 0, 1 }) {
7156 test_cases.emplace_back(args: new test_rope(type, {128, 32, 2, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); // llama 7B
7157
7158 if (all) {
7159 test_cases.emplace_back(args: new test_rope(type, {128, 40, 2, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); // llama 13B
7160 test_cases.emplace_back(args: new test_rope(type, {128, 52, 2, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); // llama 30B
7161 test_cases.emplace_back(args: new test_rope(type, {128, 64, 2, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw)); // llama 65B
7162 }
7163
7164 if (all) {
7165 test_cases.emplace_back(args: new test_rope(type, { 64, 1, 2, 1}, 64, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (falcon 7B)
7166 test_cases.emplace_back(args: new test_rope(type, { 64, 71, 2, 1}, 64, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (falcon 7B)
7167 test_cases.emplace_back(args: new test_rope(type, { 64, 8, 2, 1}, 64, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (falcon 40B)
7168
7169 test_cases.emplace_back(args: new test_rope(type, { 80, 32, 2, 1}, 20, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw));
7170 test_cases.emplace_back(args: new test_rope(type, { 80, 32, 2, 1}, 32, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw));
7171 test_cases.emplace_back(args: new test_rope(type, { 80, 32, 4, 1}, 32, GGML_ROPE_TYPE_NORMAL, 512, fs, ef, af, ff, v, fw));
7172
7173 test_cases.emplace_back(args: new test_rope(type, { 80, 32, 2, 1}, 20, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (stablelm)
7174 test_cases.emplace_back(args: new test_rope(type, { 80, 32, 2, 1}, 32, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (phi-2)
7175 test_cases.emplace_back(args: new test_rope(type, { 80, 32, 4, 1}, 32, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (phi-2)
7176 }
7177
7178 if (all) {
7179 test_cases.emplace_back(args: new test_rope(type, {128, 12, 2, 1}, 128, GGML_ROPE_TYPE_MROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,m-rope (qwen2vl 2B)
7180 test_cases.emplace_back(args: new test_rope(type, {128, 28, 2, 1}, 128, GGML_ROPE_TYPE_MROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,m-rope (qwen2vl 7B)
7181 test_cases.emplace_back(args: new test_rope(type, {128, 12, 2, 1}, 20, GGML_ROPE_TYPE_MROPE, 512, fs, ef, af, ff, v, fw));
7182 test_cases.emplace_back(args: new test_rope(type, {128, 28, 2, 1}, 32, GGML_ROPE_TYPE_MROPE, 512, fs, ef, af, ff, v, fw));
7183 test_cases.emplace_back(args: new test_rope(type, {128, 12, 2, 1}, 128, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,imrope (qwen3vl 2B)
7184 test_cases.emplace_back(args: new test_rope(type, {128, 28, 2, 1}, 128, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,imrope (qwen3vl 7B)
7185 test_cases.emplace_back(args: new test_rope(type, {128, 12, 2, 1}, 20, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw));
7186 test_cases.emplace_back(args: new test_rope(type, {128, 28, 2, 1}, 32, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw));
7187 test_cases.emplace_back(args: new test_rope(type, { 80, 16, 2, 1}, 80, GGML_ROPE_TYPE_VISION, 512, fs, ef, af, ff, v, fw)); // rope_multi,m-rope (qwen2vl ViT)
7188 test_cases.emplace_back(args: new test_rope(type, {128, 16, 2, 1}, 128, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,m-rope (qwen3vl)
7189 }
7190
7191 test_cases.emplace_back(args: new test_rope(type, { 64, 128, 2, 1}, 64, GGML_ROPE_TYPE_NEOX, 512, fs, ef, af, ff, v, fw)); // neox (falcon 40B)
7192 }
7193 }
7194
7195 all = false;
7196 }
7197 }
7198 }
7199 }
7200 }
7201
7202 // single inplace test per type/mode/ff
7203 for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
7204 for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_IMROPE, GGML_ROPE_TYPE_VISION}) {
7205 for (bool ff : {false, true}) {
7206 test_cases.emplace_back(args: new test_rope(type, {128, 32, 2, 1}, 128, mode, 512, 1.4245f, 0.7465f, 1.4245f, ff, 0, true, true));
7207 }
7208 }
7209 }
7210
7211 for (int v : { 0, 1, 2, 3 }) {
7212 for (int dim : { 0, 1, 2, 3, }) {
7213 test_cases.emplace_back(args: new test_concat(GGML_TYPE_F32, {11, 12, 13, 14}, 7, dim, v));
7214 test_cases.emplace_back(args: new test_concat(GGML_TYPE_I32, {11, 12, 13, 14}, 7, dim, v));
7215 }
7216 }
7217
7218 for (ggml_sort_order order : {GGML_SORT_ORDER_ASC, GGML_SORT_ORDER_DESC}) {
7219 test_cases.emplace_back(args: new test_argsort(GGML_TYPE_F32, {8, 1, 1, 1}, order));
7220 test_cases.emplace_back(args: new test_argsort(GGML_TYPE_F32, {16, 10, 10, 10}, order));
7221 test_cases.emplace_back(args: new test_argsort(GGML_TYPE_F32, {60, 10, 10, 10}, order)); // qwen
7222 test_cases.emplace_back(args: new test_argsort(GGML_TYPE_F32, {1024, 1, 1, 1}, order));
7223 test_cases.emplace_back(args: new test_argsort(GGML_TYPE_F32, {16384, 1, 1, 1}, order)); // many backends only handle up to 1024
7224 test_cases.emplace_back(args: new test_argsort(GGML_TYPE_F32, {2, 8, 8192, 1}, order)); // bailingmoe2 (group selection)
7225 }
7226
7227 for (ggml_scale_mode mode : {GGML_SCALE_MODE_NEAREST, GGML_SCALE_MODE_BILINEAR}) {
7228 test_cases.emplace_back(args: new test_upscale(GGML_TYPE_F32, {512, 512, 3, 2}, 2, mode));
7229 test_cases.emplace_back(args: new test_upscale(GGML_TYPE_F32, {512, 512, 3, 2}, 2, mode, true));
7230 test_cases.emplace_back(args: new test_interpolate(GGML_TYPE_F32, {2, 5, 7, 11}, {5, 7, 11, 13}, mode));
7231 test_cases.emplace_back(args: new test_interpolate(GGML_TYPE_F32, {5, 7, 11, 13}, {2, 5, 7, 11}, mode));
7232 }
7233 test_cases.emplace_back(args: new test_interpolate(GGML_TYPE_F32, {2, 5, 7, 11}, {5, 7, 11, 13}, GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS));
7234 test_cases.emplace_back(args: new test_interpolate(GGML_TYPE_F32, {1, 4, 3, 2}, {2, 8, 3, 2}, GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS));
7235 test_cases.emplace_back(args: new test_interpolate(GGML_TYPE_F32, {4, 1, 3, 2}, {1, 1, 3, 2}, GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS));
7236
7237 test_cases.emplace_back(args: new test_sum());
7238 test_cases.emplace_back(args: new test_sum_rows());
7239 test_cases.emplace_back(args: new test_sum(GGML_TYPE_F32, {11, 5, 6, 3}, {0, 2, 1, 3})); // row-contiguous but non-contiguous
7240 test_cases.emplace_back(args: new test_sum(GGML_TYPE_F32, {11, 5, 6, 3}, {0, 3, 2, 1}));
7241 test_cases.emplace_back(args: new test_sum(GGML_TYPE_F32, {11, 5, 6, 3}, {0, 1, 3, 2}));
7242 test_cases.emplace_back(args: new test_sum_rows(GGML_TYPE_F32, { 11, 5, 6, 3 }, true, false));
7243 test_cases.emplace_back(args: new test_sum_rows(GGML_TYPE_F32, { 11, 5, 6, 3 }, false, true));
7244 test_cases.emplace_back(args: new test_sum_rows(GGML_TYPE_F32, { 11, 5, 6, 3 }, true, true));
7245 test_cases.emplace_back(args: new test_mean());
7246 test_cases.emplace_back(args: new test_sum(GGML_TYPE_F32, { 33, 1, 1, 1 }));
7247 test_cases.emplace_back(args: new test_sum_rows(GGML_TYPE_F32, { 33, 1, 1, 1 }));
7248 test_cases.emplace_back(args: new test_mean(GGML_TYPE_F32, { 33, 1, 1, 1 }));
7249 test_cases.emplace_back(args: new test_sum(GGML_TYPE_F32, { 33, 1024, 1, 1 }));
7250 test_cases.emplace_back(args: new test_sum_rows(GGML_TYPE_F32, { 33, 1024, 1, 1 }));
7251 test_cases.emplace_back(args: new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 }));
7252 test_cases.emplace_back(args: new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 }, { 1, 0, 2, 3 })); // sum dst not-contiguous
7253 test_cases.emplace_back(args: new test_sum_rows(GGML_TYPE_F32, { 33, 256, 1, 1 }));
7254 test_cases.emplace_back(args: new test_mean(GGML_TYPE_F32, { 33, 256, 1, 1 }));
7255 test_cases.emplace_back(args: new test_mean(GGML_TYPE_F32, { 32769, 1, 1, 1 }));
7256 test_cases.emplace_back(args: new test_group_norm(GGML_TYPE_F32, {64, 64, 320, 1}));
7257 test_cases.emplace_back(args: new test_group_norm(GGML_TYPE_F32, {9, 9, 1280, 1}));
7258 test_cases.emplace_back(args: new test_group_norm_mul_add(GGML_TYPE_F32, {64, 64, 320, 1}));
7259 test_cases.emplace_back(args: new test_group_norm_mul_add(GGML_TYPE_F32, {9, 9, 1280, 1}));
7260 test_cases.emplace_back(args: new test_acc());
7261 test_cases.emplace_back(args: new test_pad());
7262 test_cases.emplace_back(args: new test_pad_ext());
7263 test_cases.emplace_back(args: new test_pad_reflect_1d());
7264 test_cases.emplace_back(args: new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 384, 4, 1}));
7265 test_cases.emplace_back(args: new test_roll());
7266 test_cases.emplace_back(args: new test_arange());
7267 test_cases.emplace_back(args: new test_timestep_embedding());
7268 test_cases.emplace_back(args: new test_leaky_relu());
7269
7270 for (bool v : {false, true}) {
7271 test_cases.emplace_back(args: new test_pad_ext(GGML_TYPE_F32, {512, 512, 1, 1}, 0, 1, 0, 1, 0, 0, 0, 0, v));
7272 test_cases.emplace_back(args: new test_pad_ext(GGML_TYPE_F32, {11, 22, 33, 44}, 1, 2, 3, 4, 5, 6, 7, 8, v));
7273 }
7274
7275 for (int hsk : { 40, 64, 72, 80, 96, 128, 192, 256, 576 }) {
7276 for (int hsv : { 40, 64, 72, 80, 96, 128, 192, 256, 512 }) {
7277 if (hsk != 192 && hsk != 576 && hsk != hsv) continue;
7278 if (hsk == 192 && (hsv != 128 && hsv != 192)) continue;
7279 if (hsk == 576 && hsv != 512) continue; // DeepSeek MLA
7280
7281 for (bool mask : { true, false } ) {
7282 for (bool sinks : { true, false } ) {
7283 for (float max_bias : { 0.0f, 8.0f }) {
7284 if (!mask && max_bias > 0.0f) continue;
7285 for (float logit_softcap : {0.0f, 10.0f}) {
7286 if (hsk != 128 && logit_softcap != 0.0f) continue;
7287 for (int nh : { 4, }) {
7288 for (int nr3 : { 1, 3, }) {
7289 if (hsk > 64 && nr3 > 1) continue; // skip broadcast for large head sizes
7290 for (int nr2 : { 1, 4, 16 }) {
7291 if (nr2 == 16 && hsk != 128) continue;
7292 //for (int kv : { 1, 17, 31, 33, 61, 113, 65, 127, 129, 130, 255, 260, 371, 380, 407, 512, 1024, }) {
7293 for (int kv : { 113, 512, 1024, }) {
7294 if (nr2 != 1 && kv != 512) continue;
7295 for (int nb : { 1, 3, 32, 35, }) {
7296 for (ggml_prec prec : {GGML_PREC_F32, GGML_PREC_DEFAULT}) {
7297 if (hsk != 128 && prec == GGML_PREC_DEFAULT) continue;
7298 for (ggml_type type_KV : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0}) {
7299 test_cases.emplace_back(args: new test_flash_attn_ext(
7300 hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV));
7301 // run fewer test cases permuted
7302 if (mask == true && max_bias == 0.0f && logit_softcap == 0 && kv == 512) {
7303 test_cases.emplace_back(args: new test_flash_attn_ext(
7304 hsk, hsv, nh, {nr2, nr3}, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_KV, {0, 2, 1, 3}));
7305 }
7306 }
7307 }
7308 }
7309 }
7310 }
7311 }
7312 }
7313 }
7314 }
7315 }
7316 }
7317 }
7318 }
7319
7320 test_cases.emplace_back(args: new test_cross_entropy_loss (GGML_TYPE_F32, { 10, 5, 4, 3}));
7321 test_cases.emplace_back(args: new test_cross_entropy_loss (GGML_TYPE_F32, {30000, 1, 1, 1}));
7322 test_cases.emplace_back(args: new test_cross_entropy_loss_back(GGML_TYPE_F32, { 10, 5, 4, 3}));
7323 test_cases.emplace_back(args: new test_cross_entropy_loss_back(GGML_TYPE_F32, {30000, 1, 1, 1}));
7324
7325 test_cases.emplace_back(args: new test_opt_step_adamw(GGML_TYPE_F32, {10, 5, 4, 3}));
7326 test_cases.emplace_back(args: new test_opt_step_sgd(GGML_TYPE_F32, {10, 5, 4, 3}));
7327
7328 for (ggml_type type : base_types) {
7329 for (bool with_gate : {false, true}) {
7330 for (bool use_id : {false, true}) {
7331 for (bool b : {false, true}) {
7332 if (!use_id && b) {
7333 continue;
7334 }
7335 for (bool with_bias : {false, true}) {
7336 if (!with_gate && !with_bias) {
7337 continue;
7338 }
7339 for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) {
7340 if (!with_bias && glu_op == GGML_GLU_OP_SWIGLU_OAI) {
7341 continue;
7342 }
7343 if (!with_gate && glu_op != GGML_GLU_OP_SWIGLU) {
7344 continue;
7345 }
7346 test_cases.emplace_back(args: new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256,
7347 use_id, 16, 8, b, with_bias, with_gate));
7348 }
7349 }
7350 }
7351 }
7352 }
7353 }
7354
7355 for (bool with_norm : {false, true}) {
7356 test_cases.emplace_back(args: new test_topk_moe({8, 22, 1, 1}, 4, with_norm));
7357 test_cases.emplace_back(args: new test_topk_moe({32, 22, 1, 1}, 8, with_norm));
7358 test_cases.emplace_back(args: new test_topk_moe({128, 1, 1, 1}, 128, with_norm));
7359 }
7360
7361 test_cases.emplace_back(args: new test_topk_moe({ 8, 22, 1, 1 }, 4, /*with_norm*/ false, /*delayed_softmax*/ true));
7362 test_cases.emplace_back(args: new test_topk_moe({ 32, 22, 1, 1 }, 8, /*with_norm*/ false, /*delayed_softmax*/ true));
7363
7364#if 0
7365 // these tests are disabled to save execution time, sbut they can be handy for debugging
7366 test_cases.emplace_back(new test_llama(2, true));
7367 test_cases.emplace_back(new test_llama(1));
7368 test_cases.emplace_back(new test_llama(2));
7369 test_cases.emplace_back(new test_falcon(1));
7370 test_cases.emplace_back(new test_falcon(2));
7371#endif
7372
7373 return test_cases;
7374}
7375
7376// Test cases for performance evaluation: should be representative of real-world use cases
7377static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
7378 std::vector<std::unique_ptr<test_case>> test_cases;
7379
7380 // Conv2d: K=CRS=NPQ=4096 matmul performance
7381 uint32_t iwh_idx = 0;
7382 uint32_t kwh_idx = 1;
7383 uint32_t Cout_idx = 2;
7384 uint32_t Cin_idx = 3;
7385 uint32_t B_idx = 4;
7386 std::vector<std::array<int, 5>> cases = {
7387 //{IWH, KWH, Cout, Cin, B}
7388 // K=CRS=NPQ=4096 conv2d matmul performance
7389 {19, 4, 4096, 256, 16},
7390 // K=128, CRS=128, NPQ=4096
7391 { 19, 4, 128, 8, 16},
7392 // K=130, CRS=128, NPQ=4096
7393 { 19, 4, 130, 8, 16},
7394 // Edge case: K x CRS is small
7395 { 19, 2, 4, 4, 16},
7396 // A ConvNet's first layer
7397 { 224, 3, 8, 3, 1 },
7398 // A ConvNet's first layer with 2x2 convolution, and 1 channel
7399 { 224, 2, 8, 1, 1 },
7400 // A ConvNet's first layer with 2x2 convolution, and 1 channel, several images in the batch
7401 { 224, 2, 8, 1, 8 },
7402 // A middle layer of a ConvNet
7403 { 58, 3, 64, 32, 1 },
7404 // A middle layer of a ConvNet, several images in the batch
7405 { 58, 3, 64, 32, 8 },
7406 // A deep layer of a ConvNet, several images in the batch
7407 { 16, 3, 512, 128, 8 },
7408 };
7409
7410 for (auto kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
7411 for (auto act_case : cases) {
7412 // Direct CONV_2D
7413 test_cases.emplace_back(args: new test_conv_2d(
7414 { act_case[iwh_idx], act_case[iwh_idx], act_case[Cin_idx], act_case[B_idx] },
7415 { act_case[kwh_idx], act_case[kwh_idx], act_case[Cin_idx], act_case[Cout_idx] },
7416 kernel_type, 1, 1, 0, 0, 1, 1, false));
7417 }
7418 }
7419
7420 test_cases.emplace_back(args: new test_bin_bcast(ggml_add, GGML_TYPE_F32, {4096, 1, 1, 1}, {1, 1, 1, 1}));
7421 test_cases.emplace_back(args: new test_bin_bcast(ggml_add, GGML_TYPE_F32, {4096, 1, 1, 1}, {1, 512, 1, 1}));
7422
7423 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F32, GGML_TYPE_F16, {512, 3072, 1, 1}));
7424 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {8192, 512, 2, 1}, {0, 2, 1, 3}));
7425 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {3072, 512, 2, 1}, {0, 2, 1, 3}));
7426 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F32, GGML_TYPE_Q4_0, {8192, 512, 2, 1}));
7427 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_Q4_0, GGML_TYPE_F32, {8192, 512, 2, 1}));
7428
7429 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {768*1024, 256, 1, 1}, {1, 0, 2, 3}, {0, 0, 0, 0}));
7430 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768*1024, 256, 1, 1}, {1, 0, 2, 3}, {0, 0, 0, 0}));
7431 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768, 1024, 256, 1}, {1, 0, 2, 3}, {0, 0, 0, 0}));
7432 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {768, 1024, 256, 1}, {1, 0, 2, 3}, {0, 0, 0, 0}));
7433
7434 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {768*1024, 256, 1, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
7435 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {768, 1024, 256, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
7436 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768*1024, 256, 1, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
7437 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768, 1024, 256, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
7438 test_cases.emplace_back(args: new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {768, 1024, 256, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
7439
7440
7441 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {4096, 4096, 5, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f));
7442 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {12888, 256, 5, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f));
7443 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {77, 4096, 5, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f));
7444 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {1024, 1024, 10, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f));
7445 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {77, 1024, 10, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f));
7446 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {256, 256, 20, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f));
7447 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {64, 64, 20, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f));
7448 test_cases.emplace_back(args: new test_soft_max(GGML_TYPE_F32, {77, 64, 20, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f));
7449
7450 test_cases.emplace_back(args: new test_argmax(GGML_TYPE_F32, {32, 10, 1, 1}));
7451 test_cases.emplace_back(args: new test_argmax(GGML_TYPE_F32, {1024, 10, 1, 1}));
7452 test_cases.emplace_back(args: new test_argmax(GGML_TYPE_F32, {32000, 512, 1, 1}));
7453
7454 test_cases.emplace_back(args: new test_pad_reflect_1d(GGML_TYPE_F32, {512, 34, 2, 1}));
7455 test_cases.emplace_back(args: new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 80, 1, 1}));
7456 test_cases.emplace_back(args: new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 80, 4, 1}));
7457 test_cases.emplace_back(args: new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 384, 1, 1}));
7458 test_cases.emplace_back(args: new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 384, 4, 1}));
7459
7460 test_cases.emplace_back(args: new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 16416, 1, 128, {8, 1}, {4, 1}, {0, 2, 1, 3}));
7461 test_cases.emplace_back(args: new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 128, 1, 16416, {8, 1}, {4, 1}, {0, 1, 2, 3}, 2*16416));
7462
7463 for (int bs : {1, 2, 3, 4, 5, 8, 512}) {
7464 for (ggml_type type_a : all_types) {
7465 for (ggml_type type_b : {GGML_TYPE_F32}) {
7466 test_cases.emplace_back(args: new test_mul_mat(type_a, type_b, 4096, bs, 14336, {1, 1}, {1, 1}));
7467 }
7468 }
7469 }
7470
7471 // qwen3-30b-a3b
7472 for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) {
7473 for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS}) {
7474 for (ggml_type type_b : {GGML_TYPE_F32}) {
7475 test_cases.emplace_back(args: new test_mul_mat_id(type_a, type_b, 128, 8, false, 768, bs, 2048, 1));
7476 }
7477 }
7478 }
7479
7480 for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) {
7481 for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS}) {
7482 for (ggml_type type_b : {GGML_TYPE_F32}) {
7483 test_cases.emplace_back(args: new test_mul_mat_id(type_a, type_b, 32, 4, false, 1792, bs, 2048, 1));
7484 }
7485 }
7486 }
7487
7488
7489 // gpt-oss-20b
7490 for (int bs : {1, 4, 8, 512}) {
7491 for (ggml_type type_a : {GGML_TYPE_MXFP4}) {
7492 for (ggml_type type_b : {GGML_TYPE_F32}) {
7493 test_cases.emplace_back(args: new test_mul_mat_id(type_a, type_b, 32, 4, false, 2880, bs, 2880, 1));
7494 }
7495 }
7496 }
7497
7498 for (int K : {3, 5}) {
7499 for (int IC : {256, 2560}) {
7500 for (int IW_IH : {32, 64, 256}) {
7501 if (IC == 2560 && IW_IH == 256) {
7502 // too big
7503 continue;
7504 }
7505 test_cases.emplace_back(args: new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32, {IW_IH, IW_IH, IC, 1}, {K, K, IC, 1}, 1, 1, 1, 1, 1, 1, true));
7506 }
7507 }
7508 }
7509
7510 for (int kv : { 4096, 8192, 16384, }) {
7511 for (int hs : { 64, 128, }) {
7512 for (int nr : { 1, 4, }) {
7513 test_cases.emplace_back(args: new test_flash_attn_ext(hs, hs, 8, {nr, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16));
7514 }
7515 }
7516 }
7517
7518 test_cases.emplace_back(args: new test_conv_2d_dw({512, 512, 256, 1}, {3, 3, 1, 256}, 1, 1, 1, false));
7519 test_cases.emplace_back(args: new test_conv_2d_dw({512, 512, 256, 1}, {3, 3, 1, 256}, 1, 1, 1, true));
7520
7521 test_cases.emplace_back(args: new test_conv_transpose_2d({256, 256, 256, 1}, {3, 3, 16, 256}, 1));
7522 test_cases.emplace_back(args: new test_conv_transpose_2d({16, 16, 16, 1}, {3, 3, 8, 16}, 1));
7523 test_cases.emplace_back(args: new test_conv_transpose_2d({10, 10, 9, 1}, {3, 3, 1, 9}, 2));
7524
7525 test_cases.emplace_back(args: new test_mean(GGML_TYPE_F32, {256, 256, 3, 1}));
7526
7527
7528 for (int n_token : {1, 512}) {
7529 test_cases.emplace_back(args: new test_add_id(GGML_TYPE_F32, GGML_TYPE_F32, 2880, 128, 4, n_token));
7530 test_cases.emplace_back(args: new test_add_id(GGML_TYPE_F32, GGML_TYPE_F32, 2880, 32, 4, n_token));
7531 }
7532
7533 std::vector<std::array<int64_t, 4>> reduce_rows_cases = {
7534 { 8192, 1, 1, 1 },
7535 { 8192, 8192, 1, 1 },
7536 { 128, 8192, 1, 1 },
7537 };
7538
7539 for (auto it: reduce_rows_cases){
7540 test_cases.emplace_back(args: new test_mean(GGML_TYPE_F32, it));
7541 test_cases.emplace_back(args: new test_sum_rows(GGML_TYPE_F32, it));
7542 test_cases.emplace_back(args: new test_sum(GGML_TYPE_F32, it));
7543 }
7544
7545 return test_cases;
7546}
7547
7548static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op_names_filter, const char * params_filter,
7549 printer * output_printer) {
7550 auto filter_test_cases = [](std::vector<std::unique_ptr<test_case>> & test_cases, const char * params_filter) {
7551 if (params_filter == nullptr) {
7552 return;
7553 }
7554
7555 std::regex params_filter_regex(params_filter);
7556
7557 for (auto it = test_cases.begin(); it != test_cases.end();) {
7558 if (!std::regex_search(s: (*it)->vars(), e: params_filter_regex)) {
7559 it = test_cases.erase(position: it);
7560 continue;
7561 }
7562
7563 it++;
7564 }
7565 };
7566
7567 if (mode == MODE_TEST) {
7568 auto test_cases = make_test_cases_eval();
7569 filter_test_cases(test_cases, params_filter);
7570 ggml_backend_t backend_cpu = ggml_backend_init_by_type(type: GGML_BACKEND_DEVICE_TYPE_CPU, NULL);
7571 if (backend_cpu == NULL) {
7572 test_operation_info info("", "", "CPU");
7573 info.set_error(component: "backend", details: "Failed to initialize CPU backend");
7574 output_printer->print_operation(info);
7575 return false;
7576 }
7577
7578 size_t n_ok = 0;
7579 size_t tests_run = 0;
7580 std::vector<std::string> failed_tests;
7581 for (auto & test : test_cases) {
7582 test_status_t status = test->eval(backend1: backend, backend2: backend_cpu, op_names_filter, output_printer);
7583 if (status == test_status_t::SKIPPED || status == test_status_t::NOT_SUPPORTED) {
7584 continue;
7585 }
7586 tests_run++;
7587 if (status == test_status_t::OK) {
7588 n_ok++;
7589 } else if (status == test_status_t::FAIL) {
7590 failed_tests.push_back(x: test->current_op_name + "(" + test->vars() + ")");
7591 }
7592 }
7593 output_printer->print_summary(info: test_summary_info(n_ok, tests_run, false));
7594 output_printer->print_failed_tests(failed_tests);
7595
7596 ggml_backend_free(backend: backend_cpu);
7597
7598 return n_ok == tests_run;
7599 }
7600
7601 if (mode == MODE_GRAD) {
7602 auto test_cases = make_test_cases_eval();
7603 filter_test_cases(test_cases, params_filter);
7604 size_t n_ok = 0;
7605 for (auto & test : test_cases) {
7606 if (test->eval_grad(backend, op_names_filter, output_printer)) {
7607 n_ok++;
7608 }
7609 }
7610 output_printer->print_summary(info: test_summary_info(n_ok, test_cases.size(), false));
7611
7612 return n_ok == test_cases.size();
7613 }
7614
7615 if (mode == MODE_PERF) {
7616 auto test_cases = make_test_cases_perf();
7617 filter_test_cases(test_cases, params_filter);
7618 for (auto & test : test_cases) {
7619 test->eval_perf(backend, op_names_filter, output_printer);
7620 }
7621 return true;
7622 }
7623
7624 if (mode == MODE_SUPPORT) {
7625 auto test_cases = make_test_cases_eval();
7626 filter_test_cases(test_cases, params_filter);
7627
7628 // Filter out fusion cases
7629 test_cases.erase(
7630 first: std::remove_if(first: test_cases.begin(), last: test_cases.end(), pred: [](const std::unique_ptr<test_case> & tc) {
7631 return tc->run_whole_graph();
7632 }),
7633 last: test_cases.end()
7634 );
7635
7636 for (auto & test : test_cases) {
7637 test->eval_support(backend, op_names_filter, output_printer);
7638 }
7639 return true;
7640 }
7641
7642 GGML_ABORT("fatal error");
7643}
7644
7645static void list_all_ops() {
7646 printf(format: "GGML operations:\n");
7647 std::set<std::string> all_ops;
7648
7649 for (int i = 1; i < GGML_OP_COUNT; i++) {
7650 all_ops.insert(x: ggml_op_name(op: (enum ggml_op)i));
7651 }
7652 for (int i = 0; i < GGML_UNARY_OP_COUNT; i++) {
7653 all_ops.insert(x: ggml_unary_op_name(op: (enum ggml_unary_op)i));
7654 }
7655 for (int i = 0; i < GGML_GLU_OP_COUNT; i++) {
7656 all_ops.insert(x: ggml_glu_op_name(op: (enum ggml_glu_op)i));
7657 }
7658 for (const auto & op : all_ops) {
7659 printf(format: " %s\n", op.c_str());
7660 }
7661 printf(format: "\nTotal: %zu operations\n", all_ops.size());
7662}
7663
7664static void show_test_coverage() {
7665 std::set<std::string> all_ops;
7666 for (int i = 1; i < GGML_OP_COUNT; i++) {
7667 auto op = (enum ggml_op)i;
7668 if (op == GGML_OP_VIEW ||
7669 op == GGML_OP_RESHAPE ||
7670 op == GGML_OP_PERMUTE ||
7671 op == GGML_OP_TRANSPOSE ||
7672 op == GGML_OP_CONT ||
7673 op == GGML_OP_GLU ||
7674 op == GGML_OP_UNARY) {
7675 continue;
7676 }
7677 all_ops.insert(x: ggml_op_name(op));
7678 }
7679 for (int i = 0; i < GGML_UNARY_OP_COUNT; i++) {
7680 all_ops.insert(x: ggml_unary_op_name(op: (enum ggml_unary_op)i));
7681 }
7682 for (int i = 0; i < GGML_GLU_OP_COUNT; i++) {
7683 all_ops.insert(x: ggml_glu_op_name(op: (enum ggml_glu_op)i));
7684 }
7685 auto test_cases = make_test_cases_eval();
7686 // Filter out fusion cases
7687 test_cases.erase(
7688 first: std::remove_if(first: test_cases.begin(), last: test_cases.end(), pred: [](const std::unique_ptr<test_case> & tc) {
7689 return tc->run_whole_graph();
7690 }),
7691 last: test_cases.end()
7692 );
7693
7694 std::set<std::string> tested_ops;
7695
7696 ggml_init_params params = {
7697 /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead(),
7698 /* .mem_base = */ NULL,
7699 /* .no_alloc = */ true,
7700 };
7701
7702 for (auto & test_case : test_cases) {
7703 ggml_context * ctx = ggml_init(params);
7704 if (ctx) {
7705 test_case->mode = MODE_TEST;
7706 ggml_tensor * out = test_case->build_graph(ctx);
7707 if (out && out->op != GGML_OP_NONE) {
7708 if (out->op == GGML_OP_UNARY) {
7709 tested_ops.insert(x: ggml_unary_op_name(op: ggml_get_unary_op(tensor: out)));
7710 } else if (out->op == GGML_OP_GLU) {
7711 tested_ops.insert(x: ggml_glu_op_name(op: ggml_get_glu_op(tensor: out)));
7712 } else {
7713 tested_ops.insert(x: ggml_op_name(op: out->op));
7714 }
7715 }
7716 ggml_free(ctx);
7717 }
7718 }
7719 std::set<std::string> covered_ops;
7720 std::set<std::string> uncovered_ops;
7721 for (const auto & op : all_ops) {
7722 if (tested_ops.count(x: op) > 0) {
7723 covered_ops.insert(x: op);
7724 } else {
7725 uncovered_ops.insert(x: op);
7726 }
7727 }
7728
7729 printf(format: "Operations covered by tests (%zu):\n", covered_ops.size());
7730 for (const auto & op : covered_ops) {
7731 printf(format: " ✓ %s\n", op.c_str());
7732 }
7733 printf(format: "\nOperations without tests (%zu):\n", uncovered_ops.size());
7734 for (const auto & op : uncovered_ops) {
7735 printf(format: " ✗ %s\n", op.c_str());
7736 }
7737
7738 printf(format: "\nCoverage Summary:\n");
7739 printf(format: " Total operations: %zu\n", all_ops.size());
7740 printf(format: " Tested operations: %zu\n", covered_ops.size());
7741 printf(format: " Untested operations: %zu\n", uncovered_ops.size());
7742 printf(format: " Coverage: %.1f%%\n", (double)covered_ops.size() / all_ops.size() * 100.0);
7743}
7744
7745static void usage(char ** argv) {
7746 printf(format: "Usage: %s [mode] [-o <op,..>] [-b <backend>] [-p <params regex>] [--output <console|sql|csv>] [--list-ops] [--show-coverage]\n", argv[0]);
7747 printf(format: " valid modes:\n");
7748 printf(format: " - test (default, compare with CPU backend for correctness)\n");
7749 printf(format: " - grad (compare gradients from backpropagation with method of finite differences)\n");
7750 printf(format: " - perf (performance evaluation)\n");
7751 printf(format: " - support (probe backend operation support)\n");
7752 printf(format: " op names for -o are as given by ggml_op_desc() (e.g. ADD, MUL_MAT, etc),\n");
7753 printf(format: " optionally including the full test case string (e.g. \"ADD(type=f16,ne=[1,1,8,1],nr=[1,1,1,1],nf=1)\")\n");
7754 printf(format: " --output specifies output format (default: console, options: console, sql, csv)\n");
7755 printf(format: " --list-ops lists all available GGML operations\n");
7756 printf(format: " --show-coverage shows test coverage\n");
7757}
7758
7759int main(int argc, char ** argv) {
7760 test_mode mode = MODE_TEST;
7761 output_formats output_format = CONSOLE;
7762 const char * op_names_filter = nullptr;
7763 const char * backend_filter = nullptr;
7764 const char * params_filter = nullptr;
7765
7766 for (int i = 1; i < argc; i++) {
7767 if (strcmp(s1: argv[i], s2: "test") == 0) {
7768 mode = MODE_TEST;
7769 } else if (strcmp(s1: argv[i], s2: "perf") == 0) {
7770 mode = MODE_PERF;
7771 } else if (strcmp(s1: argv[i], s2: "grad") == 0) {
7772 mode = MODE_GRAD;
7773 } else if (strcmp(s1: argv[i], s2: "support") == 0) {
7774 mode = MODE_SUPPORT;
7775 } else if (strcmp(s1: argv[i], s2: "-o") == 0) {
7776 if (i + 1 < argc) {
7777 op_names_filter = argv[++i];
7778 } else {
7779 usage(argv);
7780 return 1;
7781 }
7782 } else if (strcmp(s1: argv[i], s2: "-b") == 0) {
7783 if (i + 1 < argc) {
7784 backend_filter = argv[++i];
7785 } else {
7786 usage(argv);
7787 return 1;
7788 }
7789 } else if (strcmp(s1: argv[i], s2: "-p") == 0) {
7790 if (i + 1 < argc) {
7791 params_filter = argv[++i];
7792 } else {
7793 usage(argv);
7794 return 1;
7795 }
7796 } else if (strcmp(s1: argv[i], s2: "--output") == 0) {
7797 if (i + 1 < argc) {
7798 if (!output_format_from_str(s: argv[++i], format&: output_format)) {
7799 usage(argv);
7800 return 1;
7801 }
7802 } else {
7803 usage(argv);
7804 return 1;
7805 }
7806 } else if (strcmp(s1: argv[i], s2: "--list-ops") == 0) {
7807 list_all_ops();
7808 return 0;
7809 } else if (strcmp(s1: argv[i], s2: "--show-coverage") == 0) {
7810 show_test_coverage();
7811 return 0;
7812 } else {
7813 usage(argv);
7814 return 1;
7815 }
7816 }
7817
7818 // load and enumerate backends
7819 ggml_backend_load_all();
7820
7821 // Create printer for output format
7822 std::unique_ptr<printer> output_printer = create_printer(format: output_format);
7823 if (output_printer) {
7824 output_printer->print_header();
7825 }
7826
7827 output_printer->print_testing_start(info: testing_start_info(ggml_backend_dev_count()));
7828
7829 size_t n_ok = 0;
7830
7831 for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
7832 ggml_backend_dev_t dev = ggml_backend_dev_get(index: i);
7833
7834 if (backend_filter != NULL && strcmp(s1: backend_filter, s2: ggml_backend_dev_name(device: dev)) != 0) {
7835 output_printer->print_backend_init(
7836 info: backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(device: dev), true, "Skipping"));
7837 n_ok++;
7838 continue;
7839 }
7840
7841 if (backend_filter == NULL && ggml_backend_dev_type(device: dev) == GGML_BACKEND_DEVICE_TYPE_CPU && mode != MODE_GRAD) {
7842 output_printer->print_backend_init(info: backend_init_info(
7843 i, ggml_backend_dev_count(), ggml_backend_dev_name(device: dev), true, "Skipping CPU backend"));
7844 n_ok++;
7845 continue;
7846 }
7847
7848 ggml_backend_t backend = ggml_backend_dev_init(device: dev, NULL);
7849 GGML_ASSERT(backend != NULL);
7850
7851 ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(device: dev);
7852 auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, name: "ggml_backend_set_n_threads");
7853 if (ggml_backend_set_n_threads_fn) {
7854 // TODO: better value for n_threads
7855 ggml_backend_set_n_threads_fn(backend, std::thread::hardware_concurrency());
7856 }
7857
7858 size_t free, total; // NOLINT
7859 ggml_backend_dev_memory(device: dev, free: &free, total: &total);
7860 output_printer->print_backend_init(info: backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(device: dev),
7861 false, "", ggml_backend_dev_description(device: dev),
7862 total / 1024 / 1024, free / 1024 / 1024, true));
7863
7864 bool ok = test_backend(backend, mode, op_names_filter, params_filter, output_printer: output_printer.get());
7865
7866 if (ok) {
7867 n_ok++;
7868 }
7869 output_printer->print_backend_status(
7870 info: backend_status_info(ggml_backend_name(backend), ok ? test_status_t::OK : test_status_t::FAIL));
7871
7872 ggml_backend_free(backend);
7873 }
7874
7875 ggml_quantize_free();
7876
7877 if (output_printer) {
7878 output_printer->print_footer();
7879 }
7880
7881 output_printer->print_overall_summary(
7882 info: overall_summary_info(n_ok, ggml_backend_dev_count(), n_ok == ggml_backend_dev_count()));
7883
7884 if (n_ok != ggml_backend_dev_count()) {
7885 return 1;
7886 }
7887
7888 return 0;
7889}
7890