1#define _CRT_SECURE_NO_DEPRECATE // Disables "unsafe" warnings on Windows
2#define _USE_MATH_DEFINES // For M_PI on MSVC
3
4#include "ggml-backend-impl.h"
5#include "ggml-backend.h"
6#include "traits.h"
7#include "ggml-cpu-impl.h"
8#include "ggml-cpu.h"
9#include "ggml-impl.h"
10#include "quants.h"
11#include "ggml-threading.h"
12#include "unary-ops.h"
13#include "binary-ops.h"
14#include "vec.h"
15#include "ops.h"
16#include "ggml.h"
17
18#if defined(_MSC_VER) || defined(__MINGW32__)
19#include <malloc.h> // using malloc.h with MSC/MINGW
20#elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)
21#include <alloca.h>
22#endif
23
24#include <assert.h>
25#include <errno.h>
26#include <time.h>
27#include <math.h>
28#include <stdlib.h>
29#include <string.h>
30#include <stdint.h>
31#include <inttypes.h>
32#include <stdio.h>
33#include <float.h>
34#include <limits.h>
35#include <stdarg.h>
36#include <signal.h>
37#if defined(__gnu_linux__)
38#include <syscall.h>
39#endif
40
41#ifdef GGML_USE_OPENMP
42#include <omp.h>
43#endif
44
45#if defined(__ARM_FEATURE_SVE) || defined(__ARM_FEATURE_MATMUL_INT8)
46#undef GGML_USE_LLAMAFILE
47#endif
48
49#ifdef GGML_USE_LLAMAFILE
50#include "llamafile/sgemm.h"
51#endif
52
53// Note: once we move threading into a separate C++ file
54// will use std::hardware_destructive_interference_size instead of hardcoding it here
55// and we'll use C++ attribute syntax.
56#define GGML_CACHE_LINE 64
57
58#if defined(__clang__) || defined(__GNUC__)
59#define GGML_CACHE_ALIGN __attribute__((aligned(GGML_CACHE_LINE)))
60#endif
61
62#if defined(__has_feature)
63#if __has_feature(thread_sanitizer)
64#define GGML_TSAN_ENABLED 1
65#endif
66#else // __has_feature
67#if defined(__SANITIZE_THREAD__)
68#define GGML_TSAN_ENABLED 1
69#endif
70#endif // __has_feature
71
72#define UNUSED GGML_UNUSED
73#define SWAP(x, y, T) do { T SWAP = x; (x) = y; (y) = SWAP; } while (0)
74
75// precomputed f32 table for f16 (256 KB) (simd-mappings.h)
76float ggml_table_f32_f16[1 << 16];
77
78#if defined(__ARM_ARCH)
79struct ggml_arm_arch_features_type {
80 int sve_cnt;
81} ggml_arm_arch_features = { 0 };
82#endif
83
84
85#if defined(_WIN32)
86
87#define WIN32_LEAN_AND_MEAN
88#ifndef NOMINMAX
89 #define NOMINMAX
90#endif
91#include <windows.h>
92
93#if defined(_MSC_VER) && !defined(__clang__)
94#define GGML_CACHE_ALIGN __declspec(align(GGML_CACHE_LINE))
95
96typedef volatile LONG atomic_int;
97typedef atomic_int atomic_bool;
98typedef atomic_int atomic_flag;
99
100#define ATOMIC_FLAG_INIT 0
101
102typedef enum {
103 memory_order_relaxed,
104 memory_order_consume,
105 memory_order_acquire,
106 memory_order_release,
107 memory_order_acq_rel,
108 memory_order_seq_cst
109} memory_order;
110
111static void atomic_store(atomic_int * ptr, LONG val) {
112 InterlockedExchange(ptr, val);
113}
114static void atomic_store_explicit(atomic_int * ptr, LONG val, memory_order mo) {
115 // TODO: add support for explicit memory order
116 InterlockedExchange(ptr, val);
117}
118static LONG atomic_load(atomic_int * ptr) {
119 return InterlockedCompareExchange(ptr, 0, 0);
120}
121static LONG atomic_load_explicit(atomic_int * ptr, memory_order mo) {
122 // TODO: add support for explicit memory order
123 return InterlockedCompareExchange(ptr, 0, 0);
124}
125static LONG atomic_fetch_add(atomic_int * ptr, LONG inc) {
126 return InterlockedExchangeAdd(ptr, inc);
127}
128static LONG atomic_fetch_add_explicit(atomic_int * ptr, LONG inc, memory_order mo) {
129 // TODO: add support for explicit memory order
130 return InterlockedExchangeAdd(ptr, inc);
131}
132static atomic_bool atomic_flag_test_and_set(atomic_flag * ptr) {
133 return InterlockedExchange(ptr, 1);
134}
135static void atomic_flag_clear(atomic_flag * ptr) {
136 InterlockedExchange(ptr, 0);
137}
138static void atomic_thread_fence(memory_order mo) {
139 MemoryBarrier();
140}
141#else // clang
142#include <stdatomic.h>
143#endif
144
145typedef HANDLE pthread_t;
146
147typedef DWORD thread_ret_t;
148static int pthread_create(pthread_t * out, void * unused, thread_ret_t(*func)(void *), void * arg) {
149 (void) unused;
150 HANDLE handle = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE) func, arg, 0, NULL);
151 if (handle == NULL)
152 {
153 return EAGAIN;
154 }
155
156 *out = handle;
157 return 0;
158}
159
160static int pthread_join(pthread_t thread, void * unused) {
161 (void) unused;
162 int ret = (int) WaitForSingleObject(thread, INFINITE);
163 CloseHandle(thread);
164 return ret;
165}
166
167static int sched_yield (void) {
168 Sleep (0);
169 return 0;
170}
171#else
172
173#include <pthread.h>
174#include <stdatomic.h>
175#include <sched.h>
176#if defined(__FreeBSD__)
177#include <pthread_np.h>
178#endif
179
180typedef void * thread_ret_t;
181
182#include <sys/types.h>
183#include <sys/stat.h>
184#include <unistd.h>
185
186#endif
187
188typedef pthread_t ggml_thread_t;
189
190#if defined(__APPLE__)
191#include <unistd.h>
192#include <mach/mach.h>
193#include <TargetConditionals.h>
194#endif
195
196static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {
197 [GGML_TYPE_F32] = {
198 .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_fp32,
199 .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f32,
200 .vec_dot_type = GGML_TYPE_F32,
201 .nrows = 1,
202 },
203 [GGML_TYPE_F16] = {
204 .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_fp16,
205 .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f16,
206 .vec_dot_type = GGML_TYPE_F16,
207 .nrows = 1,
208 },
209 [GGML_TYPE_Q4_0] = {
210 .from_float = quantize_row_q4_0,
211 .vec_dot = ggml_vec_dot_q4_0_q8_0,
212 .vec_dot_type = GGML_TYPE_Q8_0,
213#if defined (__ARM_FEATURE_MATMUL_INT8)
214 .nrows = 2,
215#else
216 .nrows = 1,
217#endif
218 },
219 [GGML_TYPE_Q4_1] = {
220 .from_float = quantize_row_q4_1,
221 .vec_dot = ggml_vec_dot_q4_1_q8_1,
222 .vec_dot_type = GGML_TYPE_Q8_1,
223#if defined (__ARM_FEATURE_MATMUL_INT8)
224 .nrows = 2,
225#else
226 .nrows = 1,
227#endif
228 },
229 [GGML_TYPE_Q5_0] = {
230 .from_float = quantize_row_q5_0,
231 .vec_dot = ggml_vec_dot_q5_0_q8_0,
232 .vec_dot_type = GGML_TYPE_Q8_0,
233 .nrows = 1,
234 },
235 [GGML_TYPE_Q5_1] = {
236 .from_float = quantize_row_q5_1,
237 .vec_dot = ggml_vec_dot_q5_1_q8_1,
238 .vec_dot_type = GGML_TYPE_Q8_1,
239 .nrows = 1,
240 },
241 [GGML_TYPE_Q8_0] = {
242 .from_float = quantize_row_q8_0,
243 .vec_dot = ggml_vec_dot_q8_0_q8_0,
244 .vec_dot_type = GGML_TYPE_Q8_0,
245#if defined (__ARM_FEATURE_MATMUL_INT8)
246 .nrows = 2,
247#else
248 .nrows = 1,
249#endif
250 },
251 [GGML_TYPE_Q8_1] = {
252 .from_float = quantize_row_q8_1,
253 .vec_dot_type = GGML_TYPE_Q8_1,
254 .nrows = 1,
255 },
256 [GGML_TYPE_MXFP4] = {
257 .from_float = quantize_row_mxfp4,
258 .vec_dot = ggml_vec_dot_mxfp4_q8_0,
259 .vec_dot_type = GGML_TYPE_Q8_0,
260 .nrows = 1,
261 },
262 [GGML_TYPE_Q2_K] = {
263 .from_float = quantize_row_q2_K,
264 .vec_dot = ggml_vec_dot_q2_K_q8_K,
265 .vec_dot_type = GGML_TYPE_Q8_K,
266 .nrows = 1,
267 },
268 [GGML_TYPE_Q3_K] = {
269 .from_float = quantize_row_q3_K,
270 .vec_dot = ggml_vec_dot_q3_K_q8_K,
271 .vec_dot_type = GGML_TYPE_Q8_K,
272 .nrows = 1,
273 },
274 [GGML_TYPE_Q4_K] = {
275 .from_float = quantize_row_q4_K,
276 .vec_dot = ggml_vec_dot_q4_K_q8_K,
277 .vec_dot_type = GGML_TYPE_Q8_K,
278#if defined (__ARM_FEATURE_MATMUL_INT8)
279 .nrows = 2,
280#else
281 .nrows = 1,
282#endif
283 },
284 [GGML_TYPE_Q5_K] = {
285 .from_float = quantize_row_q5_K,
286 .vec_dot = ggml_vec_dot_q5_K_q8_K,
287 .vec_dot_type = GGML_TYPE_Q8_K,
288 .nrows = 1,
289 },
290 [GGML_TYPE_Q6_K] = {
291 .from_float = quantize_row_q6_K,
292 .vec_dot = ggml_vec_dot_q6_K_q8_K,
293 .vec_dot_type = GGML_TYPE_Q8_K,
294#if defined (__ARM_FEATURE_MATMUL_INT8)
295 .nrows = 2,
296#else
297 .nrows = 1,
298#endif
299 },
300 [GGML_TYPE_IQ2_XXS] = {
301 .from_float = NULL,
302 .vec_dot = ggml_vec_dot_iq2_xxs_q8_K,
303 .vec_dot_type = GGML_TYPE_Q8_K,
304 .nrows = 1,
305 },
306 [GGML_TYPE_IQ2_XS] = {
307 .from_float = NULL,
308 .vec_dot = ggml_vec_dot_iq2_xs_q8_K,
309 .vec_dot_type = GGML_TYPE_Q8_K,
310 .nrows = 1,
311 },
312 [GGML_TYPE_IQ3_XXS] = {
313 // NOTE: from_float for iq3 and iq2_s was removed because these quants require initialization in ggml_quantize_init
314 //.from_float = quantize_row_iq3_xxs,
315 .vec_dot = ggml_vec_dot_iq3_xxs_q8_K,
316 .vec_dot_type = GGML_TYPE_Q8_K,
317 .nrows = 1,
318 },
319 [GGML_TYPE_IQ3_S] = {
320 //.from_float = quantize_row_iq3_s,
321 .vec_dot = ggml_vec_dot_iq3_s_q8_K,
322 .vec_dot_type = GGML_TYPE_Q8_K,
323 .nrows = 1,
324 },
325 [GGML_TYPE_IQ2_S] = {
326 //.from_float = quantize_row_iq2_s,
327 .vec_dot = ggml_vec_dot_iq2_s_q8_K,
328 .vec_dot_type = GGML_TYPE_Q8_K,
329 .nrows = 1,
330 },
331 [GGML_TYPE_IQ1_S] = {
332 .from_float = NULL,
333 .vec_dot = ggml_vec_dot_iq1_s_q8_K,
334 .vec_dot_type = GGML_TYPE_Q8_K,
335 .nrows = 1,
336 },
337 [GGML_TYPE_IQ1_M] = {
338 .from_float = NULL,
339 .vec_dot = ggml_vec_dot_iq1_m_q8_K,
340 .vec_dot_type = GGML_TYPE_Q8_K,
341 .nrows = 1,
342 },
343 [GGML_TYPE_IQ4_NL] = {
344 .from_float = quantize_row_iq4_nl,
345 .vec_dot = ggml_vec_dot_iq4_nl_q8_0,
346 .vec_dot_type = GGML_TYPE_Q8_0,
347 .nrows = 1,
348 },
349 [GGML_TYPE_IQ4_XS] = {
350 .from_float = quantize_row_iq4_xs,
351 .vec_dot = ggml_vec_dot_iq4_xs_q8_K,
352 .vec_dot_type = GGML_TYPE_Q8_K,
353 .nrows = 1,
354 },
355 [GGML_TYPE_Q8_K] = {
356 .from_float = quantize_row_q8_K,
357 },
358 [GGML_TYPE_BF16] = {
359 .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_bf16,
360 .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_bf16,
361 .vec_dot_type = GGML_TYPE_BF16,
362 .nrows = 1,
363 },
364 [GGML_TYPE_TQ1_0] = {
365 .from_float = quantize_row_tq1_0,
366 .vec_dot = ggml_vec_dot_tq1_0_q8_K,
367 .vec_dot_type = GGML_TYPE_Q8_K,
368 .nrows = 1,
369 },
370 [GGML_TYPE_TQ2_0] = {
371 .from_float = quantize_row_tq2_0,
372 .vec_dot = ggml_vec_dot_tq2_0_q8_K,
373 .vec_dot_type = GGML_TYPE_Q8_K,
374 .nrows = 1,
375 },
376 [GGML_TYPE_I32] = {
377 .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_i32,
378 },
379};
380
381const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type) {
382 return &type_traits_cpu[type];
383}
384
385//
386// Threading defs
387//
388
389typedef pthread_t ggml_thread_t;
390
391#if defined(_WIN32)
392
393typedef CONDITION_VARIABLE ggml_cond_t;
394typedef SRWLOCK ggml_mutex_t;
395
396#define ggml_mutex_init(m) InitializeSRWLock(m)
397#define ggml_mutex_destroy(m)
398#define ggml_mutex_lock(m) AcquireSRWLockExclusive(m)
399#define ggml_mutex_unlock(m) ReleaseSRWLockExclusive(m)
400#define ggml_mutex_lock_shared(m) AcquireSRWLockShared(m)
401#define ggml_mutex_unlock_shared(m) ReleaseSRWLockShared(m)
402
403#define ggml_cond_init(c) InitializeConditionVariable(c)
404#define ggml_cond_destroy(c)
405#define ggml_cond_wait(c, m) SleepConditionVariableSRW(c, m, INFINITE, CONDITION_VARIABLE_LOCKMODE_SHARED)
406#define ggml_cond_broadcast(c) WakeAllConditionVariable(c)
407
408#define ggml_thread_create pthread_create
409#define ggml_thread_join pthread_join
410
411#else
412
413typedef pthread_cond_t ggml_cond_t;
414typedef pthread_mutex_t ggml_mutex_t;
415
416#define ggml_mutex_init(m) pthread_mutex_init(m, NULL)
417#define ggml_mutex_destroy(m) pthread_mutex_destroy(m)
418#define ggml_mutex_lock(m) pthread_mutex_lock(m)
419#define ggml_mutex_unlock(m) pthread_mutex_unlock(m)
420#define ggml_mutex_lock_shared(m) pthread_mutex_lock(m)
421#define ggml_mutex_unlock_shared(m) pthread_mutex_unlock(m)
422
423#define ggml_lock_init(x) UNUSED(x)
424#define ggml_lock_destroy(x) UNUSED(x)
425#if defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64))
426#define ggml_lock_lock(x) _mm_pause()
427#else
428#define ggml_lock_lock(x) UNUSED(x)
429#endif
430#define ggml_lock_unlock(x) UNUSED(x)
431
432#define GGML_LOCK_INITIALIZER 0
433#define ggml_cond_init(c) pthread_cond_init(c, NULL)
434#define ggml_cond_destroy(c) pthread_cond_destroy(c)
435#define ggml_cond_wait(c, m) pthread_cond_wait(c, m)
436#define ggml_cond_broadcast(c) pthread_cond_broadcast(c)
437
438#define ggml_thread_create pthread_create
439#define ggml_thread_join pthread_join
440
441#endif
442
443// Threadpool def
444struct ggml_threadpool {
445 ggml_mutex_t mutex; // mutex for cond.var
446 ggml_cond_t cond; // cond.var for waiting for new work
447
448 struct ggml_cgraph * cgraph;
449 struct ggml_cplan * cplan;
450
451 // synchronization primitives
452 atomic_int n_graph; // incremented when there is work to be done (i.e each graph)
453 atomic_int GGML_CACHE_ALIGN n_barrier;
454 atomic_int GGML_CACHE_ALIGN n_barrier_passed;
455 atomic_int GGML_CACHE_ALIGN current_chunk; // currently processing chunk during Mat_Mul, shared between all the threads.
456
457 // these are atomic as an annotation for thread-sanitizer
458 atomic_bool stop; // Used for stopping the threadpool altogether
459 atomic_bool pause; // Used for pausing the threadpool or individual threads
460 atomic_int abort; // Used for aborting processing of a graph
461
462 struct ggml_compute_state * workers; // per thread state
463 int n_threads_max; // number of threads in the pool
464 atomic_int n_threads_cur; // number of threads used in the current graph
465
466 int32_t prio; // Scheduling priority
467 uint32_t poll; // Polling level (0 - no polling)
468
469 enum ggml_status ec;
470};
471
472// Per-thread state
473struct ggml_compute_state {
474#ifndef GGML_USE_OPENMP
475 ggml_thread_t thrd;
476 int last_graph;
477 bool pending;
478#endif
479 bool cpumask[GGML_MAX_N_THREADS];
480 struct ggml_threadpool * threadpool;
481 int ith;
482};
483
484// Helpers for polling loops
485#if defined(__aarch64__) && ( defined(__clang__) || defined(__GNUC__) )
486static inline void ggml_thread_cpu_relax(void) {
487 __asm__ volatile("yield" ::: "memory");
488}
489#elif defined(__x86_64__)
490static inline void ggml_thread_cpu_relax(void) {
491 _mm_pause();
492}
493#else
494static inline void ggml_thread_cpu_relax(void) {;}
495#endif
496
497//
498// NUMA support
499//
500
501#define GGML_NUMA_MAX_NODES 8
502#define GGML_NUMA_MAX_CPUS 512
503
504struct ggml_numa_node {
505 uint32_t cpus[GGML_NUMA_MAX_CPUS]; // hardware threads on this node
506 uint32_t n_cpus;
507};
508
509struct ggml_numa_nodes {
510 enum ggml_numa_strategy numa_strategy;
511 struct ggml_numa_node nodes[GGML_NUMA_MAX_NODES];
512 uint32_t n_nodes;
513 uint32_t total_cpus; // hardware threads on system
514 uint32_t current_node; // node on which main process is execting
515#if defined(__gnu_linux__)
516 cpu_set_t cpuset; // cpuset from numactl
517#else
518 uint32_t cpuset; // no NUMA support outside of Linux at this time. Use a portable datatype
519#endif
520};
521
522//
523// ggml state
524//
525
526struct ggml_state {
527 struct ggml_numa_nodes numa;
528};
529
530static struct ggml_state g_state = {0};
531
532void ggml_barrier(struct ggml_threadpool * tp) {
533 int n_threads = atomic_load_explicit(&tp->n_threads_cur, memory_order_relaxed);
534 if (n_threads == 1) {
535 return;
536 }
537
538#ifdef GGML_USE_OPENMP
539 #pragma omp barrier
540#else
541 int n_passed = atomic_load_explicit(&tp->n_barrier_passed, memory_order_relaxed);
542
543 // enter barrier (full seq-cst fence)
544 int n_barrier = atomic_fetch_add_explicit(&tp->n_barrier, 1, memory_order_seq_cst);
545
546 if (n_barrier == (n_threads - 1)) {
547 // last thread
548 atomic_store_explicit(&tp->n_barrier, 0, memory_order_relaxed);
549
550 // exit barrier (fill seq-cst fence)
551 atomic_fetch_add_explicit(&tp->n_barrier_passed, 1, memory_order_seq_cst);
552 return;
553 }
554
555 // wait for other threads
556 while (atomic_load_explicit(&tp->n_barrier_passed, memory_order_relaxed) == n_passed) {
557 ggml_thread_cpu_relax();
558 }
559
560 // exit barrier (full seq-cst fence)
561 // TSAN doesn't support standalone fence yet, we use a dummy read-modify-write instead
562 #ifdef GGML_TSAN_ENABLED
563 atomic_fetch_add_explicit(&tp->n_barrier_passed, 0, memory_order_seq_cst);
564 #else
565 atomic_thread_fence(memory_order_seq_cst);
566 #endif
567#endif
568}
569
570void ggml_threadpool_chunk_set(struct ggml_threadpool * tp, int value) {
571 atomic_store_explicit(&tp->current_chunk, value, memory_order_relaxed);
572}
573
574int ggml_threadpool_chunk_add(struct ggml_threadpool * tp, int value) {
575 return atomic_fetch_add_explicit(&tp->current_chunk, value, memory_order_relaxed);
576}
577
578#if defined(__gnu_linux__)
579static cpu_set_t ggml_get_numa_affinity(void) {
580 cpu_set_t cpuset;
581 pthread_t thread;
582 thread = pthread_self();
583 CPU_ZERO(&cpuset);
584 pthread_getaffinity_np(th: thread, cpusetsize: sizeof(cpu_set_t), cpuset: &cpuset);
585 return cpuset;
586}
587#else
588static uint32_t ggml_get_numa_affinity(void) {
589 return 0; // no NUMA support
590}
591#endif
592
593void ggml_numa_init(enum ggml_numa_strategy numa_flag) {
594 if (g_state.numa.n_nodes > 0) {
595 fprintf(stderr, format: "ggml_numa_init: NUMA already initialized\n");
596
597 return;
598 }
599
600#if defined(__gnu_linux__)
601 struct stat st;
602 char path[256];
603 int rv;
604
605 // set numa scheme
606 g_state.numa.numa_strategy = numa_flag;
607
608 GGML_PRINT_DEBUG("numa strategy %u\n",g_state.numa.numa_strategy);
609
610 g_state.numa.cpuset = ggml_get_numa_affinity();
611
612 // enumerate nodes
613 while (g_state.numa.n_nodes < GGML_NUMA_MAX_NODES) {
614 rv = snprintf(s: path, maxlen: sizeof(path), format: "/sys/devices/system/node/node%u", g_state.numa.n_nodes);
615 GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path));
616 if (stat(file: path, buf: &st) != 0) { break; }
617 ++g_state.numa.n_nodes;
618 }
619
620 // enumerate CPUs
621 while (g_state.numa.total_cpus < GGML_NUMA_MAX_CPUS) {
622 rv = snprintf(s: path, maxlen: sizeof(path), format: "/sys/devices/system/cpu/cpu%u", g_state.numa.total_cpus);
623 GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path));
624 if (stat(file: path, buf: &st) != 0) { break; }
625 ++g_state.numa.total_cpus;
626 }
627
628 GGML_PRINT_DEBUG("found %u numa nodes, %u CPUs\n", g_state.numa.n_nodes, g_state.numa.total_cpus);
629
630 // figure out which node we're on
631 uint current_cpu;
632 int getcpu_ret = 0;
633#if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ > 33) || defined(__COSMOPOLITAN__)
634 getcpu_ret = getcpu(&current_cpu, &g_state.numa.current_node);
635#else
636 // old glibc doesn't have a wrapper for this call. Fall back on direct syscall
637# if !defined(SYS_getcpu) && defined(SYS_get_cpu)
638# define SYS_getcpu SYS_get_cpu // some older glibc versions use this name
639# endif
640 getcpu_ret = syscall(SYS_getcpu, &current_cpu, &g_state.numa.current_node);
641#endif
642
643 if (g_state.numa.n_nodes < 1 || g_state.numa.total_cpus < 1 || getcpu_ret != 0) {
644 g_state.numa.n_nodes = 0;
645 return;
646 }
647
648 GGML_PRINT_DEBUG("found our process on numa node %u, CPU %u\n", g_state.numa.current_node, current_cpu);
649
650 for (uint32_t n = 0; n < g_state.numa.n_nodes; ++n) {
651 struct ggml_numa_node * node = &g_state.numa.nodes[n];
652 GGML_PRINT_DEBUG("CPUs on node %u:", n);
653 node->n_cpus = 0;
654 for (uint32_t c = 0; c < g_state.numa.total_cpus; ++c) {
655 rv = snprintf(s: path, maxlen: sizeof(path), format: "/sys/devices/system/node/node%u/cpu%u", n, c);
656 GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path));
657 if (stat(file: path, buf: &st) == 0) {
658 node->cpus[node->n_cpus++] = c;
659 GGML_PRINT_DEBUG(" %u", c);
660 }
661 }
662 GGML_PRINT_DEBUG("\n");
663 }
664
665 if (ggml_is_numa()) {
666 FILE *fptr = fopen(filename: "/proc/sys/kernel/numa_balancing", modes: "r");
667 if (fptr != NULL) {
668 char buf[42];
669 if (fgets(s: buf, n: sizeof(buf), stream: fptr) && strncmp(s1: buf, s2: "0\n", n: sizeof(buf)) != 0) {
670 GGML_LOG_WARN("/proc/sys/kernel/numa_balancing is enabled, this has been observed to impair performance\n");
671 }
672 fclose(stream: fptr);
673 }
674 }
675#else
676 UNUSED(numa_flag);
677 // TODO
678#endif
679}
680
681bool ggml_is_numa(void) {
682 return g_state.numa.n_nodes > 1;
683}
684
685#if defined(__ARM_ARCH)
686
687#if defined(__linux__) && defined(__aarch64__)
688#include <sys/auxv.h>
689#endif
690
691static void ggml_init_arm_arch_features(void) {
692#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE)
693#if defined(__linux__)
694 ggml_arm_arch_features.sve_cnt = PR_SVE_VL_LEN_MASK & prctl(PR_SVE_GET_VL);
695#else
696 // TODO: add support of SVE for non-linux systems
697#error "TODO: SVE is not supported on this platform. To use SVE, sve_cnt needs to be initialized here."
698#endif
699#endif
700}
701
702#endif // __ARM_ARCH
703
704struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value) {
705 GGML_ASSERT(!ggml_get_no_alloc(ctx));
706
707 struct ggml_tensor * result = ggml_new_tensor_1d(ctx, type: GGML_TYPE_I32, ne0: 1);
708
709 ggml_set_i32(tensor: result, value);
710
711 return result;
712}
713
714struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value) {
715 GGML_ASSERT(!ggml_get_no_alloc(ctx));
716
717 struct ggml_tensor * result = ggml_new_tensor_1d(ctx, type: GGML_TYPE_F32, ne0: 1);
718
719 ggml_set_f32(tensor: result, value);
720
721 return result;
722}
723
724struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value) {
725 const int n = ggml_nrows(tensor);
726 const int nc = tensor->ne[0];
727 const size_t n1 = tensor->nb[1];
728
729 char * const data = tensor->data;
730
731 switch (tensor->type) {
732 case GGML_TYPE_I8:
733 {
734 assert(tensor->nb[0] == sizeof(int8_t));
735 for (int i = 0; i < n; i++) {
736 ggml_vec_set_i8(n: nc, x: (int8_t *)(data + i*n1), v: value);
737 }
738 } break;
739 case GGML_TYPE_I16:
740 {
741 assert(tensor->nb[0] == sizeof(int16_t));
742 for (int i = 0; i < n; i++) {
743 ggml_vec_set_i16(n: nc, x: (int16_t *)(data + i*n1), v: value);
744 }
745 } break;
746 case GGML_TYPE_I32:
747 {
748 assert(tensor->nb[0] == sizeof(int32_t));
749 for (int i = 0; i < n; i++) {
750 ggml_vec_set_i32(n: nc, x: (int32_t *)(data + i*n1), v: value);
751 }
752 } break;
753 case GGML_TYPE_F16:
754 {
755 assert(tensor->nb[0] == sizeof(ggml_fp16_t));
756 for (int i = 0; i < n; i++) {
757 ggml_vec_set_f16(n: nc, x: (ggml_fp16_t *)(data + i*n1), GGML_CPU_FP32_TO_FP16(value));
758 }
759 } break;
760 case GGML_TYPE_BF16:
761 {
762 assert(tensor->nb[0] == sizeof(ggml_fp16_t));
763 for (int i = 0; i < n; i++) {
764 ggml_vec_set_bf16(n: nc, x: (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value));
765 }
766 } break;
767 case GGML_TYPE_F32:
768 {
769 assert(tensor->nb[0] == sizeof(float));
770 for (int i = 0; i < n; i++) {
771 ggml_vec_set_f32(n: nc, x: (float *)(data + i*n1), v: value);
772 }
773 } break;
774 default:
775 {
776 GGML_ABORT("fatal error");
777 }
778 }
779
780 return tensor;
781}
782
783struct ggml_tensor * ggml_set_f32(struct ggml_tensor * tensor, float value) {
784 const int n = ggml_nrows(tensor);
785 const int nc = tensor->ne[0];
786 const size_t n1 = tensor->nb[1];
787
788 char * const data = tensor->data;
789
790 switch (tensor->type) {
791 case GGML_TYPE_I8:
792 {
793 assert(tensor->nb[0] == sizeof(int8_t));
794 for (int i = 0; i < n; i++) {
795 ggml_vec_set_i8(n: nc, x: (int8_t *)(data + i*n1), v: value);
796 }
797 } break;
798 case GGML_TYPE_I16:
799 {
800 assert(tensor->nb[0] == sizeof(int16_t));
801 for (int i = 0; i < n; i++) {
802 ggml_vec_set_i16(n: nc, x: (int16_t *)(data + i*n1), v: value);
803 }
804 } break;
805 case GGML_TYPE_I32:
806 {
807 assert(tensor->nb[0] == sizeof(int32_t));
808 for (int i = 0; i < n; i++) {
809 ggml_vec_set_i32(n: nc, x: (int32_t *)(data + i*n1), v: value);
810 }
811 } break;
812 case GGML_TYPE_F16:
813 {
814 assert(tensor->nb[0] == sizeof(ggml_fp16_t));
815 for (int i = 0; i < n; i++) {
816 ggml_vec_set_f16(n: nc, x: (ggml_fp16_t *)(data + i*n1), GGML_CPU_FP32_TO_FP16(value));
817 }
818 } break;
819 case GGML_TYPE_BF16:
820 {
821 assert(tensor->nb[0] == sizeof(ggml_bf16_t));
822 for (int i = 0; i < n; i++) {
823 ggml_vec_set_bf16(n: nc, x: (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value));
824 }
825 } break;
826 case GGML_TYPE_F32:
827 {
828 assert(tensor->nb[0] == sizeof(float));
829 for (int i = 0; i < n; i++) {
830 ggml_vec_set_f32(n: nc, x: (float *)(data + i*n1), v: value);
831 }
832 } break;
833 default:
834 {
835 GGML_ABORT("fatal error");
836 }
837 }
838
839 return tensor;
840}
841
842int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i) {
843 if (!ggml_is_contiguous(tensor)) {
844 int64_t id[4] = { 0, 0, 0, 0 };
845 ggml_unravel_index(tensor, i, i0: &id[0], i1: &id[1], i2: &id[2], i3: &id[3]);
846 return ggml_get_i32_nd(tensor, i0: id[0], i1: id[1], i2: id[2], i3: id[3]);
847 }
848 switch (tensor->type) {
849 case GGML_TYPE_I8:
850 {
851 GGML_ASSERT(tensor->nb[0] == sizeof(int8_t));
852 return ((int8_t *)(tensor->data))[i];
853 }
854 case GGML_TYPE_I16:
855 {
856 GGML_ASSERT(tensor->nb[0] == sizeof(int16_t));
857 return ((int16_t *)(tensor->data))[i];
858 }
859 case GGML_TYPE_I32:
860 {
861 GGML_ASSERT(tensor->nb[0] == sizeof(int32_t));
862 return ((int32_t *)(tensor->data))[i];
863 }
864 case GGML_TYPE_F16:
865 {
866 GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));
867 return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i]);
868 }
869 case GGML_TYPE_BF16:
870 {
871 GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));
872 return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i]);
873 }
874 case GGML_TYPE_F32:
875 {
876 GGML_ASSERT(tensor->nb[0] == sizeof(float));
877 return ((float *)(tensor->data))[i];
878 }
879 default:
880 {
881 GGML_ABORT("fatal error");
882 }
883 }
884}
885
886void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value) {
887 if (!ggml_is_contiguous(tensor)) {
888 int64_t id[4] = { 0, 0, 0, 0 };
889 ggml_unravel_index(tensor, i, i0: &id[0], i1: &id[1], i2: &id[2], i3: &id[3]);
890 ggml_set_i32_nd(tensor, i0: id[0], i1: id[1], i2: id[2], i3: id[3], value);
891 return;
892 }
893 switch (tensor->type) {
894 case GGML_TYPE_I8:
895 {
896 GGML_ASSERT(tensor->nb[0] == sizeof(int8_t));
897 ((int8_t *)(tensor->data))[i] = value;
898 } break;
899 case GGML_TYPE_I16:
900 {
901 GGML_ASSERT(tensor->nb[0] == sizeof(int16_t));
902 ((int16_t *)(tensor->data))[i] = value;
903 } break;
904 case GGML_TYPE_I32:
905 {
906 GGML_ASSERT(tensor->nb[0] == sizeof(int32_t));
907 ((int32_t *)(tensor->data))[i] = value;
908 } break;
909 case GGML_TYPE_F16:
910 {
911 GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));
912 ((ggml_fp16_t *)(tensor->data))[i] = GGML_CPU_FP32_TO_FP16(value);
913 } break;
914 case GGML_TYPE_BF16:
915 {
916 GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));
917 ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value);
918 } break;
919 case GGML_TYPE_F32:
920 {
921 GGML_ASSERT(tensor->nb[0] == sizeof(float));
922 ((float *)(tensor->data))[i] = value;
923 } break;
924 default:
925 {
926 GGML_ABORT("fatal error");
927 }
928 }
929}
930
931int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3) {
932 void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];
933 switch (tensor->type) {
934 case GGML_TYPE_I8:
935 return ((int8_t *) data)[0];
936 case GGML_TYPE_I16:
937 return ((int16_t *) data)[0];
938 case GGML_TYPE_I32:
939 return ((int32_t *) data)[0];
940 case GGML_TYPE_F16:
941 return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *) data)[0]);
942 case GGML_TYPE_BF16:
943 return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0]);
944 case GGML_TYPE_F32:
945 return ((float *) data)[0];
946 default:
947 GGML_ABORT("fatal error");
948 }
949}
950
951void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value) {
952 void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];
953 switch (tensor->type) {
954 case GGML_TYPE_I8:
955 {
956 ((int8_t *)(data))[0] = value;
957 } break;
958 case GGML_TYPE_I16:
959 {
960 ((int16_t *)(data))[0] = value;
961 } break;
962 case GGML_TYPE_I32:
963 {
964 ((int32_t *)(data))[0] = value;
965 } break;
966 case GGML_TYPE_F16:
967 {
968 ((ggml_fp16_t *)(data))[0] = GGML_CPU_FP32_TO_FP16(value);
969 } break;
970 case GGML_TYPE_BF16:
971 {
972 ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value);
973 } break;
974 case GGML_TYPE_F32:
975 {
976 ((float *)(data))[0] = value;
977 } break;
978 default:
979 {
980 GGML_ABORT("fatal error");
981 }
982 }
983}
984
985float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i) {
986 if (!ggml_is_contiguous(tensor)) {
987 int64_t id[4] = { 0, 0, 0, 0 };
988 ggml_unravel_index(tensor, i, i0: &id[0], i1: &id[1], i2: &id[2], i3: &id[3]);
989 return ggml_get_f32_nd(tensor, i0: id[0], i1: id[1], i2: id[2], i3: id[3]);
990 }
991 switch (tensor->type) {
992 case GGML_TYPE_I8:
993 {
994 return ((int8_t *)(tensor->data))[i];
995 }
996 case GGML_TYPE_I16:
997 {
998 return ((int16_t *)(tensor->data))[i];
999 }
1000 case GGML_TYPE_I32:
1001 {
1002 return ((int32_t *)(tensor->data))[i];
1003 }
1004 case GGML_TYPE_F16:
1005 {
1006 return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i]);
1007 }
1008 case GGML_TYPE_BF16:
1009 {
1010 return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i]);
1011 }
1012 case GGML_TYPE_F32:
1013 {
1014 return ((float *)(tensor->data))[i];
1015 }
1016 default:
1017 {
1018 GGML_ABORT("fatal error");
1019 }
1020 }
1021}
1022
1023void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value) {
1024 if (!ggml_is_contiguous(tensor)) {
1025 int64_t id[4] = { 0, 0, 0, 0 };
1026 ggml_unravel_index(tensor, i, i0: &id[0], i1: &id[1], i2: &id[2], i3: &id[3]);
1027 ggml_set_f32_nd(tensor, i0: id[0], i1: id[1], i2: id[2], i3: id[3], value);
1028 return;
1029 }
1030 switch (tensor->type) {
1031 case GGML_TYPE_I8:
1032 {
1033 ((int8_t *)(tensor->data))[i] = value;
1034 } break;
1035 case GGML_TYPE_I16:
1036 {
1037 ((int16_t *)(tensor->data))[i] = value;
1038 } break;
1039 case GGML_TYPE_I32:
1040 {
1041 ((int32_t *)(tensor->data))[i] = value;
1042 } break;
1043 case GGML_TYPE_F16:
1044 {
1045 ((ggml_fp16_t *)(tensor->data))[i] = GGML_CPU_FP32_TO_FP16(value);
1046 } break;
1047 case GGML_TYPE_BF16:
1048 {
1049 ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value);
1050 } break;
1051 case GGML_TYPE_F32:
1052 {
1053 ((float *)(tensor->data))[i] = value;
1054 } break;
1055 default:
1056 {
1057 GGML_ABORT("fatal error");
1058 }
1059 }
1060}
1061
1062float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3) {
1063 void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];
1064 switch (tensor->type) {
1065 case GGML_TYPE_I8:
1066 return ((int8_t *) data)[0];
1067 case GGML_TYPE_I16:
1068 return ((int16_t *) data)[0];
1069 case GGML_TYPE_I32:
1070 return ((int32_t *) data)[0];
1071 case GGML_TYPE_F16:
1072 return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *) data)[0]);
1073 case GGML_TYPE_BF16:
1074 return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0]);
1075 case GGML_TYPE_F32:
1076 return ((float *) data)[0];
1077 default:
1078 GGML_ABORT("fatal error");
1079 }
1080}
1081
1082void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value) {
1083 void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];
1084 switch (tensor->type) {
1085 case GGML_TYPE_I8:
1086 {
1087 ((int8_t *)(data))[0] = value;
1088 } break;
1089 case GGML_TYPE_I16:
1090 {
1091 ((int16_t *)(data))[0] = value;
1092 } break;
1093 case GGML_TYPE_I32:
1094 {
1095 ((int32_t *)(data))[0] = value;
1096 } break;
1097 case GGML_TYPE_F16:
1098 {
1099 ((ggml_fp16_t *)(data))[0] = GGML_CPU_FP32_TO_FP16(value);
1100 } break;
1101 case GGML_TYPE_BF16:
1102 {
1103 ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value);
1104 } break;
1105 case GGML_TYPE_F32:
1106 {
1107 ((float *)(data))[0] = value;
1108 } break;
1109 default:
1110 {
1111 GGML_ABORT("fatal error");
1112 }
1113 }
1114}
1115
1116////////////////////////////////////////////////////////////////////////////////
1117
1118// ggml_compute_forward_mul_mat
1119
1120static void ggml_compute_forward_mul_mat_one_chunk(
1121 const struct ggml_compute_params * params,
1122 struct ggml_tensor * dst,
1123 const enum ggml_type type,
1124 const int64_t num_rows_per_vec_dot,
1125 const int64_t ir0_start,
1126 const int64_t ir0_end,
1127 const int64_t ir1_start,
1128 const int64_t ir1_end) {
1129
1130 const struct ggml_tensor * src0 = dst->src[0];
1131 const struct ggml_tensor * src1 = dst->src[1];
1132
1133 GGML_TENSOR_BINARY_OP_LOCALS
1134
1135 const bool src1_cont = ggml_is_contiguous(tensor: src1);
1136
1137 ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot;
1138 enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;
1139
1140 // broadcast factors
1141 const int64_t r2 = ne12 / ne02;
1142 const int64_t r3 = ne13 / ne03;
1143
1144 //printf("ir0_start = %6lld, ir0_end = %6lld, ir1_start = %6lld, ir1_end = %6lld\n", ir0_start, ir0_end, ir1_start, ir1_end);
1145
1146 // threads with no work simply yield (not sure if it helps)
1147 if (ir0_start >= ir0_end || ir1_start >= ir1_end) {
1148 return;
1149 }
1150
1151 const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
1152 const size_t row_size = ggml_row_size(type: vec_dot_type, ne: ne10);
1153
1154 assert(ne12 % ne02 == 0);
1155 assert(ne13 % ne03 == 0);
1156
1157 // block-tiling attempt
1158 const int64_t blck_0 = 16;
1159 const int64_t blck_1 = 16;
1160
1161 const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11;
1162
1163 // attempt to reduce false-sharing (does not seem to make a difference)
1164 // 16 * 2, accounting for mmla kernels
1165 float tmp[32];
1166
1167 for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) {
1168 for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) {
1169 for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) {
1170 const int64_t i13 = (ir1 / (ne12 * ne1));
1171 const int64_t i12 = (ir1 - i13 * ne12 * ne1) / ne1;
1172 const int64_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1);
1173
1174 // broadcast src0 into src1
1175 const int64_t i03 = i13 / r3;
1176 const int64_t i02 = i12 / r2;
1177
1178 const int64_t i1 = i11;
1179 const int64_t i2 = i12;
1180 const int64_t i3 = i13;
1181
1182 const char * src0_row = (const char*)src0->data + (0 + i02 * nb02 + i03 * nb03);
1183
1184 // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides
1185 // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using
1186 // the original src1 data pointer, so we should index using the indices directly
1187 // TODO: this is a bit of a hack, we should probably have a better way to handle this
1188 const char * src1_col = (const char*)wdata +
1189 (src1_cont || src1->type != vec_dot_type
1190 ? (i11 + i12 * ne11 + i13 * ne12 * ne11) * row_size
1191 : (i11 * nb11 + i12 * nb12 + i13 * nb13));
1192 float * dst_col = (float*)((char*)dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3));
1193
1194 //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) {
1195 // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col);
1196 //}
1197
1198 for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) {
1199 vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot);
1200 }
1201
1202 for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) {
1203 memcpy(dest: &dst_col[iir0 + cn * nb1 / nb0], src: tmp + (cn * 16), n: (MIN(iir0 + blck_0, ir0_end) - iir0) * sizeof(float));
1204 }
1205 }
1206 }
1207 }
1208}
1209
1210void ggml_compute_forward_mul_mat(
1211 const struct ggml_compute_params * params,
1212 struct ggml_tensor * dst) {
1213
1214 const struct ggml_tensor * src0 = dst->src[0];
1215 const struct ggml_tensor * src1 = dst->src[1];
1216
1217 GGML_TENSOR_BINARY_OP_LOCALS
1218
1219 const int ith = params->ith;
1220 const int nth = params->nth;
1221
1222 enum ggml_type const vec_dot_type = type_traits_cpu[src0->type].vec_dot_type;
1223 ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float;
1224 int64_t const vec_dot_num_rows = type_traits_cpu[src0->type].nrows;
1225
1226 GGML_ASSERT(ne0 == ne01);
1227 GGML_ASSERT(ne1 == ne11);
1228 GGML_ASSERT(ne2 == ne12);
1229 GGML_ASSERT(ne3 == ne13);
1230
1231 // we don't support permuted src0 or src1
1232 GGML_ASSERT(nb00 == ggml_type_size(src0->type));
1233 GGML_ASSERT(nb10 == ggml_type_size(src1->type));
1234
1235 // dst cannot be transposed or permuted
1236 GGML_ASSERT(nb0 == sizeof(float));
1237 GGML_ASSERT(nb0 <= nb1);
1238 GGML_ASSERT(nb1 <= nb2);
1239 GGML_ASSERT(nb2 <= nb3);
1240
1241 // nb01 >= nb00 - src0 is not transposed
1242 // compute by src0 rows
1243
1244 // TODO: extract to "extra_op"
1245#if GGML_USE_LLAMAFILE
1246 // broadcast factors
1247 const int64_t r2 = ne12 / ne02;
1248 const int64_t r3 = ne13 / ne03;
1249
1250 const bool src1_cont = ggml_is_contiguous(tensor: src1);
1251
1252 if (src1_cont) {
1253 for (int64_t i13 = 0; i13 < ne13; i13++)
1254 for (int64_t i12 = 0; i12 < ne12; i12++)
1255 if (!llamafile_sgemm(params,
1256 ne01, ne11, ne00/ggml_blck_size(type: src0->type),
1257 (const char *)src0->data + i12/r2*nb02 + i13/r3*nb03,
1258 nb01/ggml_type_size(type: src0->type),
1259 (const char *)src1->data + i12*nb12 + i13*nb13,
1260 nb11/ggml_type_size(type: src1->type),
1261 (char *)dst->data + i12*nb2 + i13*nb3,
1262 nb1/ggml_type_size(type: dst->type),
1263 src0->type,
1264 src1->type,
1265 dst->type))
1266 goto UseGgmlGemm1;
1267 return;
1268 }
1269UseGgmlGemm1:;
1270#endif
1271
1272 if (src1->type != vec_dot_type) {
1273 char * wdata = params->wdata;
1274
1275 const size_t nbw0 = ggml_type_size(type: vec_dot_type);
1276 const size_t nbw1 = ggml_row_size(type: vec_dot_type, ne: ne10);
1277 const size_t nbw2 = nbw1*ne11;
1278 const size_t nbw3 = nbw2*ne12;
1279
1280 assert(params->wsize >= ne13*nbw3);
1281 GGML_ASSERT(src1->type == GGML_TYPE_F32);
1282
1283 #if 0
1284 for (int64_t i13 = 0; i13 < ne13; ++i13) {
1285 for (int64_t i12 = 0; i12 < ne12; ++i12) {
1286 for (int64_t i11 = ith; i11 < ne11; i11 += nth) {
1287 from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11),
1288 (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1),
1289 ne10);
1290 }
1291 }
1292 }
1293 #else
1294 for (int64_t i13 = 0; i13 < ne13; ++i13) {
1295 for (int64_t i12 = 0; i12 < ne12; ++i12) {
1296 for (int64_t i11 = 0; i11 < ne11; ++i11) {
1297 size_t bs = ggml_blck_size(type: vec_dot_type);
1298 int64_t ne10_block_start = (ith * ne10/bs) / nth;
1299 int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth;
1300 from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10),
1301 (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0),
1302 (ne10_block_end - ne10_block_start) * bs);
1303 }
1304 }
1305 }
1306 #endif
1307 }
1308
1309 if (ith == 0) {
1310 // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start.
1311 atomic_store_explicit(&params->threadpool->current_chunk, nth, memory_order_relaxed);
1312 }
1313
1314 ggml_barrier(tp: params->threadpool);
1315
1316#if GGML_USE_LLAMAFILE
1317 if (src1->type != vec_dot_type) {
1318 const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
1319 const size_t row_size = ggml_row_size(type: vec_dot_type, ne: ne10);
1320
1321 for (int64_t i13 = 0; i13 < ne13; i13++)
1322 for (int64_t i12 = 0; i12 < ne12; i12++)
1323 if (!llamafile_sgemm(params,
1324 ne01, ne11, ne00/ggml_blck_size(type: src0->type),
1325 (const char *)src0->data + i12/r2*nb02 + i13/r3*nb03,
1326 nb01/ggml_type_size(type: src0->type),
1327 (const char *)wdata + (i12*ne11 + i13*ne12*ne11)*row_size,
1328 row_size/ggml_type_size(type: vec_dot_type),
1329 (char *)dst->data + i12*nb2 + i13*nb3,
1330 nb1/ggml_type_size(type: dst->type),
1331 src0->type,
1332 vec_dot_type,
1333 dst->type))
1334 goto UseGgmlGemm2;
1335 return;
1336 }
1337UseGgmlGemm2:;
1338#endif
1339
1340 // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers)
1341 const int64_t nr0 = ne0;
1342
1343 // This is the size of the rest of the dimensions of the result
1344 const int64_t nr1 = ne1 * ne2 * ne3;
1345
1346 // Now select a reasonable chunk size.
1347 int chunk_size = 16;
1348
1349 // We need to step up the size if it's small
1350 if (nr0 == 1 || nr1 == 1) {
1351 chunk_size = 64;
1352 }
1353
1354 // distribute the work across the inner or outer loop based on which one is larger
1355 // The number of chunks in the 0/1 dim.
1356 // CEIL(nr0/chunk_size)
1357 int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size;
1358 int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size;
1359
1360 // If the chunking is poor for the number of threads on this setup, scrap the whole plan. Re-chunk it by thread.
1361 // Also, chunking by thread was measured to have perform better on NUMA systems. See https://github.com/ggml-org/llama.cpp/pull/6915
1362 // In theory, chunking should be just as useful on NUMA and non NUMA systems, but testing disagreed with that.
1363 if (nchunk0 * nchunk1 < nth * 4 || ggml_is_numa()) {
1364 // distribute the thread work across the inner or outer loop based on which one is larger
1365 nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows
1366 nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows
1367 }
1368
1369 // The number of elements in each chunk
1370 const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0;
1371 const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1;
1372
1373 // The first chunk comes from our thread_id, the rest will get auto-assigned.
1374 int current_chunk = ith;
1375
1376 while (current_chunk < nchunk0 * nchunk1) {
1377 const int64_t ith0 = current_chunk % nchunk0;
1378 const int64_t ith1 = current_chunk / nchunk0;
1379
1380 const int64_t ir0_start = dr0 * ith0;
1381 const int64_t ir0_end = MIN(ir0_start + dr0, nr0);
1382
1383 const int64_t ir1_start = dr1 * ith1;
1384 const int64_t ir1_end = MIN(ir1_start + dr1, nr1);
1385
1386 // dot kernels can handle 1 row and col at a time, but mmla kernels can process 2 rows and cols
1387 int64_t num_rows_per_vec_dot = vec_dot_num_rows;
1388
1389 // these checks are needed to avoid crossing dim1 boundaries
1390 // can be optimized, but the logic would become more complicated, so keeping it like this for simplicity
1391 if ((nr0 % 2 != 0) || (ne11 % 2 != 0) || ((ir0_end - ir0_start) % 2 != 0) || ((ir1_end - ir1_start) % 2 != 0)) {
1392 num_rows_per_vec_dot = 1;
1393 }
1394 ggml_compute_forward_mul_mat_one_chunk(params, dst, type: src0->type, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end);
1395
1396 if (nth >= nchunk0 * nchunk1) {
1397 break;
1398 }
1399
1400 current_chunk = atomic_fetch_add_explicit(&params->threadpool->current_chunk, 1, memory_order_relaxed);
1401 }
1402}
1403
1404// ggml_compute_forward_mul_mat_id
1405
1406#define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id)*ids->ne[0]*ids->ne[1] + (i1)]
1407
1408struct mmid_row_mapping {
1409 int32_t i1;
1410 int32_t i2;
1411};
1412
1413static void ggml_compute_forward_mul_mat_id_one_chunk(
1414 struct ggml_tensor * dst,
1415 const struct ggml_tensor * src0,
1416 const struct ggml_tensor * src1,
1417 const struct ggml_tensor * ids,
1418 const int64_t cur_a,
1419 const int64_t ir0_start,
1420 const int64_t ir0_end,
1421 const int64_t ir1_start,
1422 const int64_t ir1_end,
1423 const char * src0_cur,
1424 const struct mmid_row_mapping * matrix_rows,
1425 const size_t row_size,
1426 const bool src1_cont,
1427 const void * wdata) {
1428
1429 GGML_TENSOR_BINARY_OP_LOCALS
1430
1431 const enum ggml_type type = src0->type;
1432
1433 ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot;
1434 enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;
1435
1436 const int64_t blck_0 = 16;
1437 const int64_t blck_1 = 16;
1438
1439 float tmp[16];
1440
1441 for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) {
1442 for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) {
1443 for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ++ir1) {
1444 const int64_t _i12 = ir1; // logical row index for this expert
1445
1446 struct mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, _i12);
1447 const int id = row_mapping.i1; // selected expert index
1448
1449 const int64_t i11 = id % ne11;
1450 const int64_t i12 = row_mapping.i2; // row index in src1
1451
1452 const int64_t i1 = id; // selected expert index
1453 const int64_t i2 = i12; // row
1454
1455 // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides
1456 // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using
1457 // the original src1 data pointer, so we should index using the indices directly
1458 // TODO: this is a bit of a hack, we should probably have a better way to handle this
1459 const char * src1_col = (const char *) wdata +
1460 (src1_cont || src1->type != vec_dot_type
1461 ? (i11 + i12*ne11)*row_size
1462 : (i11*nb11 + i12*nb12));
1463
1464 float * dst_col = (float *) ((char *) dst->data + (i1*nb1 + i2*nb2));
1465
1466 for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) {
1467 vec_dot(ne00, &tmp[ir0 - iir0], 0, src0_cur + ir0*nb01, 0, src1_col, 0, 1);
1468 }
1469
1470 memcpy(dest: &dst_col[iir0], src: tmp, n: (MIN(iir0 + blck_0, ir0_end) - iir0)*sizeof(float));
1471 }
1472 }
1473 }
1474}
1475
1476static void * incr_ptr_aligned(void ** p, size_t size, size_t align) {
1477
1478 void * ptr = *p;
1479 ptr = (void *) GGML_PAD((uintptr_t) ptr, align);
1480 *p = (void *) ((char *) ptr + size);
1481 return ptr;
1482}
1483
1484static void ggml_compute_forward_mul_mat_id(
1485 const struct ggml_compute_params * params,
1486 struct ggml_tensor * dst) {
1487
1488 const struct ggml_tensor * src0 = dst->src[0];
1489 const struct ggml_tensor * src1 = dst->src[1];
1490 const struct ggml_tensor * ids = dst->src[2];
1491
1492 GGML_TENSOR_BINARY_OP_LOCALS
1493
1494 const int ith = params->ith;
1495 const int nth = params->nth;
1496
1497 const enum ggml_type type = src0->type;
1498
1499 const bool src1_cont = ggml_is_contiguous(tensor: src1);
1500
1501 enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;
1502 ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float;
1503
1504 // we don't support permuted src0 or src1
1505 GGML_ASSERT(nb00 == ggml_type_size(type));
1506 GGML_ASSERT(nb10 == ggml_type_size(src1->type));
1507
1508 // dst cannot be transposed or permuted
1509 GGML_ASSERT(nb0 == sizeof(float));
1510 GGML_ASSERT(nb0 <= nb1);
1511 GGML_ASSERT(nb1 <= nb2);
1512 GGML_ASSERT(nb2 <= nb3);
1513
1514 // row groups
1515 const int n_ids = ids->ne[0]; // n_expert_used
1516 const int n_as = ne02; // n_expert
1517
1518 void * wdata_cur = params->wdata;
1519
1520 if (src1->type != vec_dot_type) {
1521 incr_ptr_aligned(p: &wdata_cur, size: ggml_row_size(type: vec_dot_type, ne: ggml_nelements(tensor: src1)), align: sizeof(int64_t));
1522 }
1523
1524 int64_t * matrix_row_counts = // [n_as]
1525 incr_ptr_aligned(p: &wdata_cur, size: n_as*sizeof(int64_t), align: sizeof(int64_t));
1526
1527 struct mmid_row_mapping * matrix_rows = // [n_as][ids->ne[0]*ids->ne[1]]
1528 incr_ptr_aligned(p: &wdata_cur, size: n_as*ids->ne[0]*ids->ne[1]*sizeof(struct mmid_row_mapping), align: sizeof(int64_t));
1529
1530 char (*atomic_current_chunk)[CACHE_LINE_SIZE] = // [n_as]
1531 incr_ptr_aligned(p: &wdata_cur, CACHE_LINE_SIZE * n_as, CACHE_LINE_SIZE);
1532
1533 GGML_ASSERT(params->wsize >= (size_t)((char *) wdata_cur - (char *) params->wdata));
1534
1535 if (src1->type != vec_dot_type) {
1536 char * wdata = params->wdata;
1537
1538 const size_t nbw0 = ggml_type_size(type: vec_dot_type);
1539 const size_t nbw1 = ggml_row_size(type: vec_dot_type, ne: ne10);
1540 const size_t nbw2 = nbw1*ne11;
1541 const size_t nbw3 = nbw2*ne12;
1542
1543 assert(params->wsize >= ne13*nbw3);
1544 GGML_ASSERT(src1->type == GGML_TYPE_F32);
1545
1546#if 0
1547 for (int64_t i13 = 0; i13 < ne13; ++i13) {
1548 for (int64_t i12 = ith; i12 < ne12; i12 += nth) {
1549 for (int64_t i11 = 0; i11 < ne11; ++i11) {
1550 from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11),
1551 (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1),
1552 ne10);
1553 }
1554 }
1555 }
1556#else
1557 for (int64_t i13 = 0; i13 < ne13; ++i13) {
1558 for (int64_t i12 = 0; i12 < ne12; ++i12) {
1559 for (int64_t i11 = 0; i11 < ne11; ++i11) {
1560 size_t bs = ggml_blck_size(type: vec_dot_type);
1561 int64_t ne10_block_start = (ith * ne10/bs) / nth;
1562 int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth;
1563 from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10),
1564 (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0),
1565 (ne10_block_end - ne10_block_start) * bs);
1566 }
1567 }
1568 }
1569#endif
1570 }
1571
1572 if (ith == 0) {
1573 // initialize matrix_row_counts
1574 memset(s: matrix_row_counts, c: 0, n: n_as*sizeof(int64_t));
1575
1576 // group rows by src0 matrix
1577 for (int64_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
1578 for (int id = 0; id < n_ids; ++id) {
1579 const int32_t i02 = *(const int32_t *) ((const char *) ids->data + iid1*ids->nb[1] + id*ids->nb[0]);
1580
1581 assert(i02 >= 0 && i02 < n_as);
1582
1583 MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = (struct mmid_row_mapping) {id, iid1};
1584 matrix_row_counts[i02] += 1;
1585 }
1586 }
1587 }
1588
1589 // reset current_chunk
1590 for (int cur_a = ith; cur_a < n_as; cur_a += nth) {
1591 atomic_int * current_chunk_ctr = (atomic_int *)(atomic_current_chunk + cur_a);
1592 *current_chunk_ctr = nth;
1593 }
1594
1595 ggml_barrier(tp: params->threadpool);
1596
1597 for (int cur_a = 0; cur_a < n_as; ++cur_a) {
1598 const int64_t cne1 = matrix_row_counts[cur_a];
1599
1600 if (cne1 == 0) {
1601 continue;
1602 }
1603
1604 const char * src0_cur = (const char *) src0->data + cur_a * nb02;
1605 const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
1606 const size_t row_size = ggml_row_size(type: vec_dot_type, ne: ne10);
1607
1608 const int64_t nr0 = ne01;
1609 const int64_t nr1 = cne1;
1610
1611 int chunk_size = 16;
1612 if (nr0 == 1 || nr1 == 1) {
1613 chunk_size = 64;
1614 }
1615
1616 // disable for NUMA
1617 const bool disable_chunking = ggml_is_numa();
1618
1619 int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size;
1620 int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size;
1621
1622 if (nchunk0 * nchunk1 < nth * 4 || disable_chunking) {
1623 nchunk0 = nr0 > nr1 ? nth : 1;
1624 nchunk1 = nr0 > nr1 ? 1 : nth;
1625 }
1626
1627 const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0;
1628 const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1;
1629
1630 int current_chunk = ith;
1631
1632 atomic_int * current_chunk_ctr = (atomic_int *)(atomic_current_chunk + cur_a);
1633
1634 while (current_chunk < nchunk0 * nchunk1) {
1635 const int64_t ith0 = current_chunk % nchunk0;
1636 const int64_t ith1 = current_chunk / nchunk0;
1637
1638 const int64_t ir0_start = dr0 * ith0;
1639 const int64_t ir0_end = MIN(ir0_start + dr0, nr0);
1640
1641 const int64_t ir1_start = dr1 * ith1;
1642 const int64_t ir1_end = MIN(ir1_start + dr1, nr1);
1643
1644 ggml_compute_forward_mul_mat_id_one_chunk(
1645 dst, src0, src1, ids, cur_a,
1646 ir0_start, ir0_end, ir1_start, ir1_end,
1647 src0_cur, matrix_rows, row_size, src1_cont, wdata
1648 );
1649
1650 if (nth >= nchunk0 * nchunk1) {
1651 break;
1652 }
1653
1654 current_chunk = atomic_fetch_add_explicit(current_chunk_ctr, 1, memory_order_relaxed);
1655 }
1656 }
1657}
1658
1659/////////////////////////////////
1660
1661static void ggml_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) {
1662 GGML_ASSERT(params);
1663
1664 if (tensor->op == GGML_OP_NONE || ggml_is_empty(tensor)) {
1665 return;
1666 }
1667
1668 // extra_buffer op?
1669 if (ggml_cpu_extra_compute_forward(params, op: tensor)) {
1670 return;
1671 }
1672
1673 switch (tensor->op) {
1674 case GGML_OP_DUP:
1675 {
1676 ggml_compute_forward_dup(params, dst: tensor);
1677 } break;
1678 case GGML_OP_ADD:
1679 {
1680 ggml_compute_forward_add(params, dst: tensor);
1681 } break;
1682 case GGML_OP_ADD_ID:
1683 {
1684 ggml_compute_forward_add_id(params, dst: tensor);
1685 } break;
1686 case GGML_OP_ADD1:
1687 {
1688 ggml_compute_forward_add1(params, dst: tensor);
1689 } break;
1690 case GGML_OP_ACC:
1691 {
1692 ggml_compute_forward_acc(params, dst: tensor);
1693 } break;
1694 case GGML_OP_SUB:
1695 {
1696 ggml_compute_forward_sub(params, dst: tensor);
1697 } break;
1698 case GGML_OP_MUL:
1699 {
1700 ggml_compute_forward_mul(params, dst: tensor);
1701 } break;
1702 case GGML_OP_DIV:
1703 {
1704 ggml_compute_forward_div(params, dst: tensor);
1705 } break;
1706 case GGML_OP_SQR:
1707 {
1708 ggml_compute_forward_sqr(params, dst: tensor);
1709 } break;
1710 case GGML_OP_SQRT:
1711 {
1712 ggml_compute_forward_sqrt(params, dst: tensor);
1713 } break;
1714 case GGML_OP_LOG:
1715 {
1716 ggml_compute_forward_log(params, dst: tensor);
1717 } break;
1718 case GGML_OP_SIN:
1719 {
1720 ggml_compute_forward_sin(params, dst: tensor);
1721 } break;
1722 case GGML_OP_COS:
1723 {
1724 ggml_compute_forward_cos(params, dst: tensor);
1725 } break;
1726 case GGML_OP_SUM:
1727 {
1728 ggml_compute_forward_sum(params, dst: tensor);
1729 } break;
1730 case GGML_OP_SUM_ROWS:
1731 {
1732 ggml_compute_forward_sum_rows(params, dst: tensor);
1733 } break;
1734 case GGML_OP_MEAN:
1735 {
1736 ggml_compute_forward_mean(params, dst: tensor);
1737 } break;
1738 case GGML_OP_ARGMAX:
1739 {
1740 ggml_compute_forward_argmax(params, dst: tensor);
1741 } break;
1742 case GGML_OP_COUNT_EQUAL:
1743 {
1744 ggml_compute_forward_count_equal(params, dst: tensor);
1745 } break;
1746 case GGML_OP_REPEAT:
1747 {
1748 ggml_compute_forward_repeat(params, dst: tensor);
1749 } break;
1750 case GGML_OP_REPEAT_BACK:
1751 {
1752 ggml_compute_forward_repeat_back(params, dst: tensor);
1753 } break;
1754 case GGML_OP_CONCAT:
1755 {
1756 ggml_compute_forward_concat(params, dst: tensor);
1757 } break;
1758 case GGML_OP_SILU_BACK:
1759 {
1760 ggml_compute_forward_silu_back(params, dst: tensor);
1761 } break;
1762 case GGML_OP_NORM:
1763 {
1764 ggml_compute_forward_norm(params, dst: tensor);
1765 } break;
1766 case GGML_OP_RMS_NORM:
1767 {
1768 ggml_compute_forward_rms_norm(params, dst: tensor);
1769 } break;
1770 case GGML_OP_RMS_NORM_BACK:
1771 {
1772 ggml_compute_forward_rms_norm_back(params, dst: tensor);
1773 } break;
1774 case GGML_OP_GROUP_NORM:
1775 {
1776 ggml_compute_forward_group_norm(params, dst: tensor);
1777 } break;
1778 case GGML_OP_L2_NORM:
1779 {
1780 ggml_compute_forward_l2_norm(params, dst: tensor);
1781 } break;
1782 case GGML_OP_MUL_MAT:
1783 {
1784 ggml_compute_forward_mul_mat(params, dst: tensor);
1785 } break;
1786 case GGML_OP_MUL_MAT_ID:
1787 {
1788 ggml_compute_forward_mul_mat_id(params, dst: tensor);
1789 } break;
1790 case GGML_OP_OUT_PROD:
1791 {
1792 ggml_compute_forward_out_prod(params, dst: tensor);
1793 } break;
1794 case GGML_OP_SCALE:
1795 {
1796 ggml_compute_forward_scale(params, dst: tensor);
1797 } break;
1798 case GGML_OP_SET:
1799 {
1800 ggml_compute_forward_set(params, dst: tensor);
1801 } break;
1802 case GGML_OP_CPY:
1803 {
1804 ggml_compute_forward_cpy(params, dst: tensor);
1805 } break;
1806 case GGML_OP_CONT:
1807 {
1808 ggml_compute_forward_cont(params, dst: tensor);
1809 } break;
1810 case GGML_OP_RESHAPE:
1811 {
1812 ggml_compute_forward_reshape(params, dst: tensor);
1813 } break;
1814 case GGML_OP_VIEW:
1815 {
1816 ggml_compute_forward_view(params, dst: tensor);
1817 } break;
1818 case GGML_OP_PERMUTE:
1819 {
1820 ggml_compute_forward_permute(params, dst: tensor);
1821 } break;
1822 case GGML_OP_TRANSPOSE:
1823 {
1824 ggml_compute_forward_transpose(params, dst: tensor);
1825 } break;
1826 case GGML_OP_GET_ROWS:
1827 {
1828 ggml_compute_forward_get_rows(params, dst: tensor);
1829 } break;
1830 case GGML_OP_GET_ROWS_BACK:
1831 {
1832 ggml_compute_forward_get_rows_back(params, dst: tensor);
1833 } break;
1834 case GGML_OP_SET_ROWS:
1835 {
1836 ggml_compute_forward_set_rows(params, dst: tensor);
1837 } break;
1838 case GGML_OP_DIAG:
1839 {
1840 ggml_compute_forward_diag(params, dst: tensor);
1841 } break;
1842 case GGML_OP_DIAG_MASK_INF:
1843 {
1844 ggml_compute_forward_diag_mask_inf(params, dst: tensor);
1845 } break;
1846 case GGML_OP_DIAG_MASK_ZERO:
1847 {
1848 ggml_compute_forward_diag_mask_zero(params, dst: tensor);
1849 } break;
1850 case GGML_OP_SOFT_MAX:
1851 {
1852 ggml_compute_forward_soft_max(params, dst: tensor);
1853 } break;
1854 case GGML_OP_SOFT_MAX_BACK:
1855 {
1856 ggml_compute_forward_soft_max_ext_back(params, dst: tensor);
1857 } break;
1858 case GGML_OP_ROPE:
1859 {
1860 ggml_compute_forward_rope(params, dst: tensor);
1861 } break;
1862 case GGML_OP_ROPE_BACK:
1863 {
1864 ggml_compute_forward_rope_back(params, dst: tensor);
1865 } break;
1866 case GGML_OP_CLAMP:
1867 {
1868 ggml_compute_forward_clamp(params, dst: tensor);
1869 } break;
1870 case GGML_OP_CONV_TRANSPOSE_1D:
1871 {
1872 ggml_compute_forward_conv_transpose_1d(params, dst: tensor);
1873 } break;
1874 case GGML_OP_IM2COL:
1875 {
1876 ggml_compute_forward_im2col(params, dst: tensor);
1877 } break;
1878 case GGML_OP_IM2COL_BACK:
1879 {
1880 ggml_compute_forward_im2col_back_f32(params, dst: tensor);
1881 } break;
1882 case GGML_OP_IM2COL_3D:
1883 {
1884 ggml_compute_forward_im2col_3d(params, dst: tensor);
1885 } break;
1886 case GGML_OP_CONV_2D:
1887 {
1888 ggml_compute_forward_conv_2d(params, dst: tensor);
1889 } break;
1890 case GGML_OP_CONV_3D:
1891 {
1892 ggml_compute_forward_conv_3d(params, dst: tensor);
1893 } break;
1894 case GGML_OP_CONV_2D_DW:
1895 {
1896 ggml_compute_forward_conv_2d_dw(params, dst: tensor);
1897 } break;
1898 case GGML_OP_CONV_TRANSPOSE_2D:
1899 {
1900 ggml_compute_forward_conv_transpose_2d(params, dst: tensor);
1901 } break;
1902 case GGML_OP_POOL_1D:
1903 {
1904 ggml_compute_forward_pool_1d(params, dst: tensor);
1905 } break;
1906 case GGML_OP_POOL_2D:
1907 {
1908 ggml_compute_forward_pool_2d(params, dst: tensor);
1909 } break;
1910 case GGML_OP_POOL_2D_BACK:
1911 {
1912 ggml_compute_forward_pool_2d_back(params, dst: tensor);
1913 } break;
1914 case GGML_OP_UPSCALE:
1915 {
1916 ggml_compute_forward_upscale(params, dst: tensor);
1917 } break;
1918 case GGML_OP_PAD:
1919 {
1920 ggml_compute_forward_pad(params, dst: tensor);
1921 } break;
1922 case GGML_OP_PAD_REFLECT_1D:
1923 {
1924 ggml_compute_forward_pad_reflect_1d(params, dst: tensor);
1925 } break;
1926 case GGML_OP_ROLL:
1927 {
1928 ggml_compute_forward_roll(params, dst: tensor);
1929 } break;
1930 case GGML_OP_ARANGE:
1931 {
1932 ggml_compute_forward_arange(params, dst: tensor);
1933 } break;
1934 case GGML_OP_TIMESTEP_EMBEDDING:
1935 {
1936 ggml_compute_forward_timestep_embedding(params, dst: tensor);
1937 } break;
1938 case GGML_OP_ARGSORT:
1939 {
1940 ggml_compute_forward_argsort(params, dst: tensor);
1941 } break;
1942 case GGML_OP_LEAKY_RELU:
1943 {
1944 ggml_compute_forward_leaky_relu(params, dst: tensor);
1945 } break;
1946 case GGML_OP_FLASH_ATTN_EXT:
1947 {
1948 ggml_compute_forward_flash_attn_ext(params, dst: tensor);
1949 } break;
1950 case GGML_OP_FLASH_ATTN_BACK:
1951 {
1952 int32_t t = ggml_get_op_params_i32(tensor, i: 0);
1953 GGML_ASSERT(t == 0 || t == 1);
1954 bool masked = t != 0;
1955 ggml_compute_forward_flash_attn_back(params, masked, dst: tensor);
1956 } break;
1957 case GGML_OP_SSM_CONV:
1958 {
1959 ggml_compute_forward_ssm_conv(params, dst: tensor);
1960 } break;
1961 case GGML_OP_SSM_SCAN:
1962 {
1963 ggml_compute_forward_ssm_scan(params, dst: tensor);
1964 } break;
1965 case GGML_OP_WIN_PART:
1966 {
1967 ggml_compute_forward_win_part(params, dst: tensor);
1968 } break;
1969 case GGML_OP_WIN_UNPART:
1970 {
1971 ggml_compute_forward_win_unpart(params, dst: tensor);
1972 } break;
1973 case GGML_OP_UNARY:
1974 {
1975 ggml_compute_forward_unary(params, dst: tensor);
1976 } break;
1977 case GGML_OP_GLU:
1978 {
1979 ggml_compute_forward_glu(params, dst: tensor);
1980 } break;
1981 case GGML_OP_GET_REL_POS:
1982 {
1983 ggml_compute_forward_get_rel_pos(params, dst: tensor);
1984 } break;
1985 case GGML_OP_ADD_REL_POS:
1986 {
1987 ggml_compute_forward_add_rel_pos(params, dst: tensor);
1988 } break;
1989 case GGML_OP_RWKV_WKV6:
1990 {
1991 ggml_compute_forward_rwkv_wkv6(params, dst: tensor);
1992 } break;
1993 case GGML_OP_GATED_LINEAR_ATTN:
1994 {
1995 ggml_compute_forward_gla(params, dst: tensor);
1996 } break;
1997 case GGML_OP_RWKV_WKV7:
1998 {
1999 ggml_compute_forward_rwkv_wkv7(params, dst: tensor);
2000 } break;
2001 case GGML_OP_MAP_CUSTOM1:
2002 {
2003 ggml_compute_forward_map_custom1(params, dst: tensor);
2004 }
2005 break;
2006 case GGML_OP_MAP_CUSTOM2:
2007 {
2008 ggml_compute_forward_map_custom2(params, dst: tensor);
2009 }
2010 break;
2011 case GGML_OP_MAP_CUSTOM3:
2012 {
2013 ggml_compute_forward_map_custom3(params, dst: tensor);
2014 }
2015 break;
2016 case GGML_OP_CUSTOM:
2017 {
2018 ggml_compute_forward_custom(params, dst: tensor);
2019 }
2020 break;
2021 case GGML_OP_CROSS_ENTROPY_LOSS:
2022 {
2023 ggml_compute_forward_cross_entropy_loss(params, dst: tensor);
2024 }
2025 break;
2026 case GGML_OP_CROSS_ENTROPY_LOSS_BACK:
2027 {
2028 ggml_compute_forward_cross_entropy_loss_back(params, dst: tensor);
2029 }
2030 break;
2031 case GGML_OP_OPT_STEP_ADAMW:
2032 {
2033 ggml_compute_forward_opt_step_adamw(params, dst: tensor);
2034 }
2035 break;
2036 case GGML_OP_OPT_STEP_SGD:
2037 {
2038 ggml_compute_forward_opt_step_sgd(params, dst: tensor);
2039 }
2040 break;
2041 case GGML_OP_NONE:
2042 {
2043 // nop
2044 } break;
2045 case GGML_OP_COUNT:
2046 {
2047 GGML_ABORT("fatal error");
2048 }
2049 }
2050}
2051
2052// Android's libc implementation "bionic" does not support setting affinity
2053#if defined(__gnu_linux__)
2054static void set_numa_thread_affinity(int thread_n) {
2055 if (!ggml_is_numa()) {
2056 return;
2057 }
2058
2059 int node_num;
2060 int rv;
2061 size_t setsize = CPU_ALLOC_SIZE(g_state.numa.total_cpus);
2062
2063 switch(g_state.numa.numa_strategy) {
2064 case GGML_NUMA_STRATEGY_DISTRIBUTE:
2065 // run thread on node_num thread_n / (threads per node)
2066 node_num = thread_n % g_state.numa.n_nodes;
2067 break;
2068 case GGML_NUMA_STRATEGY_ISOLATE:
2069 // run thread on current_node
2070 node_num = g_state.numa.current_node;
2071 break;
2072 case GGML_NUMA_STRATEGY_NUMACTL:
2073 // use the cpuset that numactl gave us
2074 rv = pthread_setaffinity_np(th: pthread_self(), cpusetsize: setsize, cpuset: &g_state.numa.cpuset);
2075 if (rv) {
2076 fprintf(stderr, format: "warning: pthread_setaffinity_np() failed: %s\n",strerror(errnum: rv));
2077 }
2078 return;
2079 default:
2080 return;
2081 }
2082
2083 struct ggml_numa_node * node = &g_state.numa.nodes[node_num];
2084
2085 cpu_set_t * cpus = CPU_ALLOC(g_state.numa.total_cpus);
2086 CPU_ZERO_S(setsize, cpus);
2087 for (size_t i = 0; i < node->n_cpus; ++i) {
2088 CPU_SET_S(node->cpus[i], setsize, cpus);
2089 }
2090
2091 rv = pthread_setaffinity_np(th: pthread_self(), cpusetsize: setsize, cpuset: cpus);
2092 if (rv) {
2093 fprintf(stderr, format: "warning: pthread_setaffinity_np() failed: %s\n", strerror(errnum: rv));
2094 }
2095
2096 CPU_FREE(cpus);
2097}
2098
2099static void clear_numa_thread_affinity(void) {
2100 if (!ggml_is_numa()) {
2101 return;
2102 }
2103
2104 size_t setsize = CPU_ALLOC_SIZE(g_state.numa.total_cpus);
2105
2106 cpu_set_t * cpus = CPU_ALLOC(g_state.numa.total_cpus);
2107 CPU_ZERO_S(setsize, cpus);
2108 for (unsigned i = 0; i < g_state.numa.total_cpus; ++i) {
2109 CPU_SET_S(i, setsize, cpus);
2110 }
2111
2112 int rv = pthread_setaffinity_np(th: pthread_self(), cpusetsize: setsize, cpuset: cpus);
2113 if (rv) {
2114 fprintf(stderr, format: "warning: pthread_setaffinity_np() failed: %s\n", strerror(errnum: rv));
2115 }
2116
2117 CPU_FREE(cpus);
2118}
2119#else
2120// TODO: Windows etc.
2121// (the linux implementation may also work on BSD, someone should test)
2122static void set_numa_thread_affinity(int thread_n) { UNUSED(thread_n); }
2123static void clear_numa_thread_affinity(void) {}
2124#endif
2125
2126static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
2127 int n_tasks = 0;
2128
2129 if (ggml_is_empty(tensor: node)) {
2130 // no need to multi-thread a no-op
2131 n_tasks = 1;
2132 return n_tasks;
2133 }
2134
2135 switch (node->op) {
2136 case GGML_OP_CPY:
2137 case GGML_OP_DUP:
2138 case GGML_OP_CONT:
2139 case GGML_OP_ADD:
2140 case GGML_OP_ADD_ID:
2141 case GGML_OP_ADD1:
2142 case GGML_OP_ACC:
2143 {
2144 n_tasks = n_threads;
2145 } break;
2146 case GGML_OP_SUB:
2147 case GGML_OP_SQR:
2148 case GGML_OP_SQRT:
2149 case GGML_OP_LOG:
2150 case GGML_OP_SIN:
2151 case GGML_OP_COS:
2152 case GGML_OP_SUM:
2153 case GGML_OP_SUM_ROWS:
2154 case GGML_OP_MEAN:
2155 case GGML_OP_ARGMAX:
2156 {
2157 n_tasks = 1;
2158 } break;
2159 case GGML_OP_COUNT_EQUAL:
2160 {
2161 n_tasks = n_threads;
2162 } break;
2163 case GGML_OP_REPEAT:
2164 case GGML_OP_REPEAT_BACK:
2165 case GGML_OP_LEAKY_RELU:
2166 {
2167 n_tasks = 1;
2168 } break;
2169 case GGML_OP_UNARY:
2170 switch (ggml_get_unary_op(tensor: node)) {
2171 case GGML_UNARY_OP_ABS:
2172 case GGML_UNARY_OP_SGN:
2173 case GGML_UNARY_OP_NEG:
2174 case GGML_UNARY_OP_STEP:
2175 case GGML_UNARY_OP_TANH:
2176 case GGML_UNARY_OP_ELU:
2177 case GGML_UNARY_OP_RELU:
2178 case GGML_UNARY_OP_SIGMOID:
2179 case GGML_UNARY_OP_HARDSWISH:
2180 case GGML_UNARY_OP_HARDSIGMOID:
2181 case GGML_UNARY_OP_EXP:
2182 case GGML_UNARY_OP_FLOOR:
2183 case GGML_UNARY_OP_CEIL:
2184 case GGML_UNARY_OP_ROUND:
2185 case GGML_UNARY_OP_TRUNC:
2186 {
2187 n_tasks = 1;
2188 } break;
2189
2190 case GGML_UNARY_OP_GELU:
2191 case GGML_UNARY_OP_GELU_ERF:
2192 case GGML_UNARY_OP_GELU_QUICK:
2193 case GGML_UNARY_OP_SILU:
2194 case GGML_UNARY_OP_XIELU:
2195 {
2196 n_tasks = n_threads;
2197 } break;
2198 default:
2199 GGML_ABORT("fatal error");
2200 }
2201 break;
2202 case GGML_OP_GLU:
2203 switch (ggml_get_glu_op(tensor: node)) {
2204 case GGML_GLU_OP_REGLU:
2205 case GGML_GLU_OP_GEGLU:
2206 case GGML_GLU_OP_SWIGLU:
2207 case GGML_GLU_OP_SWIGLU_OAI:
2208 case GGML_GLU_OP_GEGLU_ERF:
2209 case GGML_GLU_OP_GEGLU_QUICK:
2210 {
2211 n_tasks = n_threads;
2212 } break;
2213 default:
2214 GGML_ABORT("fatal error");
2215 }
2216 break;
2217 case GGML_OP_SILU_BACK:
2218 case GGML_OP_MUL:
2219 case GGML_OP_DIV:
2220 case GGML_OP_NORM:
2221 case GGML_OP_RMS_NORM:
2222 case GGML_OP_RMS_NORM_BACK:
2223 case GGML_OP_L2_NORM:
2224 case GGML_OP_GROUP_NORM:
2225 case GGML_OP_CONCAT:
2226 case GGML_OP_MUL_MAT:
2227 case GGML_OP_MUL_MAT_ID:
2228 case GGML_OP_OUT_PROD:
2229 {
2230 n_tasks = n_threads;
2231 } break;
2232 case GGML_OP_GET_ROWS:
2233 case GGML_OP_SET_ROWS:
2234 {
2235 // FIXME: get_rows can use additional threads, but the cost of launching additional threads
2236 // decreases performance with GPU offloading
2237 //n_tasks = n_threads;
2238 n_tasks = 1;
2239 } break;
2240 case GGML_OP_SCALE:
2241 case GGML_OP_SET:
2242 case GGML_OP_RESHAPE:
2243 case GGML_OP_VIEW:
2244 case GGML_OP_PERMUTE:
2245 case GGML_OP_TRANSPOSE:
2246 case GGML_OP_GET_ROWS_BACK:
2247 case GGML_OP_DIAG:
2248 {
2249 n_tasks = 1;
2250 } break;
2251 case GGML_OP_DIAG_MASK_ZERO:
2252 case GGML_OP_DIAG_MASK_INF:
2253 case GGML_OP_SOFT_MAX_BACK:
2254 case GGML_OP_ROPE:
2255 case GGML_OP_ROPE_BACK:
2256 case GGML_OP_ADD_REL_POS:
2257 {
2258 n_tasks = n_threads;
2259 } break;
2260 case GGML_OP_CLAMP:
2261 {
2262 n_tasks = 1; //TODO
2263 } break;
2264 case GGML_OP_SOFT_MAX:
2265 {
2266 n_tasks = MIN(n_threads, ggml_nrows(node->src[0]));
2267 } break;
2268 case GGML_OP_IM2COL:
2269 case GGML_OP_IM2COL_BACK:
2270 case GGML_OP_IM2COL_3D:
2271 case GGML_OP_CONV_2D:
2272 case GGML_OP_CONV_3D:
2273 case GGML_OP_CONV_2D_DW:
2274 case GGML_OP_CONV_TRANSPOSE_1D:
2275 case GGML_OP_CONV_TRANSPOSE_2D:
2276 {
2277 n_tasks = n_threads;
2278 } break;
2279 case GGML_OP_POOL_1D:
2280 case GGML_OP_POOL_2D:
2281 case GGML_OP_POOL_2D_BACK:
2282 {
2283 n_tasks = 1;
2284 } break;
2285 case GGML_OP_UPSCALE:
2286 case GGML_OP_PAD:
2287 case GGML_OP_PAD_REFLECT_1D:
2288 case GGML_OP_ROLL:
2289 case GGML_OP_ARANGE:
2290 case GGML_OP_TIMESTEP_EMBEDDING:
2291 case GGML_OP_ARGSORT:
2292 case GGML_OP_FLASH_ATTN_EXT:
2293 case GGML_OP_FLASH_ATTN_BACK:
2294 case GGML_OP_SSM_CONV:
2295 case GGML_OP_SSM_SCAN:
2296 case GGML_OP_RWKV_WKV6:
2297 case GGML_OP_GATED_LINEAR_ATTN:
2298 case GGML_OP_RWKV_WKV7:
2299 {
2300 n_tasks = n_threads;
2301 } break;
2302 case GGML_OP_WIN_PART:
2303 case GGML_OP_WIN_UNPART:
2304 case GGML_OP_GET_REL_POS:
2305 {
2306 n_tasks = 1;
2307 } break;
2308 case GGML_OP_MAP_CUSTOM1:
2309 {
2310 struct ggml_map_custom1_op_params p;
2311 memcpy(dest: &p, src: node->op_params, n: sizeof(p));
2312 if (p.n_tasks == GGML_N_TASKS_MAX) {
2313 n_tasks = n_threads;
2314 } else {
2315 n_tasks = MIN(p.n_tasks, n_threads);
2316 }
2317 } break;
2318 case GGML_OP_MAP_CUSTOM2:
2319 {
2320 struct ggml_map_custom2_op_params p;
2321 memcpy(dest: &p, src: node->op_params, n: sizeof(p));
2322 if (p.n_tasks == GGML_N_TASKS_MAX) {
2323 n_tasks = n_threads;
2324 } else {
2325 n_tasks = MIN(p.n_tasks, n_threads);
2326 }
2327 } break;
2328 case GGML_OP_MAP_CUSTOM3:
2329 {
2330 struct ggml_map_custom3_op_params p;
2331 memcpy(dest: &p, src: node->op_params, n: sizeof(p));
2332 if (p.n_tasks == GGML_N_TASKS_MAX) {
2333 n_tasks = n_threads;
2334 } else {
2335 n_tasks = MIN(p.n_tasks, n_threads);
2336 }
2337 } break;
2338 case GGML_OP_CUSTOM:
2339 {
2340 struct ggml_custom_op_params p;
2341 memcpy(dest: &p, src: node->op_params, n: sizeof(p));
2342 if (p.n_tasks == GGML_N_TASKS_MAX) {
2343 n_tasks = n_threads;
2344 } else {
2345 n_tasks = MIN(p.n_tasks, n_threads);
2346 }
2347 } break;
2348 case GGML_OP_CROSS_ENTROPY_LOSS:
2349 case GGML_OP_CROSS_ENTROPY_LOSS_BACK:
2350 case GGML_OP_OPT_STEP_ADAMW:
2351 case GGML_OP_OPT_STEP_SGD:
2352 {
2353 n_tasks = n_threads;
2354 } break;
2355 case GGML_OP_NONE:
2356 {
2357 n_tasks = 1;
2358 } break;
2359 case GGML_OP_COUNT:
2360 {
2361 GGML_ABORT("fatal error");
2362 }
2363 default:
2364 {
2365 fprintf(stderr, format: "%s: op not implemented: ", __func__);
2366 if (node->op < GGML_OP_COUNT) {
2367 fprintf(stderr, format: "%s\n", ggml_op_name(op: node->op));
2368 } else {
2369 fprintf(stderr, format: "%d\n", node->op);
2370 }
2371 GGML_ABORT("fatal error");
2372 }
2373 }
2374
2375 assert(n_tasks > 0);
2376
2377 return n_tasks;
2378}
2379
2380static thread_ret_t ggml_graph_compute_secondary_thread(void* data);
2381
2382#if defined(_WIN32)
2383#include "windows.h"
2384
2385// TODO: support > 64 CPUs
2386static bool ggml_thread_apply_affinity(bool * mask) {
2387 HANDLE h = GetCurrentThread();
2388 uint64_t bitmask = 0ULL;
2389
2390 assert(GGML_MAX_N_THREADS >= 64);
2391
2392 for (int32_t i = 0; i < 8; i++) {
2393 int32_t idx = i * 8;
2394 uint8_t val = 0;
2395 val |= mask[idx + 0] << 0;
2396 val |= mask[idx + 1] << 1;
2397 val |= mask[idx + 2] << 2;
2398 val |= mask[idx + 3] << 3;
2399 val |= mask[idx + 4] << 4;
2400 val |= mask[idx + 5] << 5;
2401 val |= mask[idx + 6] << 6;
2402 val |= mask[idx + 7] << 7;
2403 bitmask |= (uint64_t)val << idx;
2404 }
2405
2406 for (int32_t i = 64; i < GGML_MAX_N_THREADS; i++) {
2407 if (mask[i]) {
2408 fprintf(stderr, "warn: setting thread-affinity for > 64 CPUs isn't supported on windows!\n");
2409 break;
2410 }
2411 }
2412
2413 DWORD_PTR m = (DWORD_PTR)bitmask;
2414
2415 m = SetThreadAffinityMask(h, m);
2416
2417 return m != 0;
2418}
2419
2420static bool ggml_thread_apply_priority(int32_t prio) {
2421 // Note that on Windows the Process Priority Class must be updated in order to set Thread priority.
2422 // This is up to the applications.
2423 DWORD p = THREAD_PRIORITY_NORMAL;
2424 switch (prio) {
2425 case GGML_SCHED_PRIO_LOW: p = THREAD_PRIORITY_BELOW_NORMAL; break;
2426 case GGML_SCHED_PRIO_NORMAL: p = THREAD_PRIORITY_NORMAL; break;
2427 case GGML_SCHED_PRIO_MEDIUM: p = THREAD_PRIORITY_ABOVE_NORMAL; break;
2428 case GGML_SCHED_PRIO_HIGH: p = THREAD_PRIORITY_HIGHEST; break;
2429 case GGML_SCHED_PRIO_REALTIME: p = THREAD_PRIORITY_TIME_CRITICAL; break;
2430 }
2431
2432 if (prio != GGML_SCHED_PRIO_LOW) {
2433 // Tell Windows that this thread should not be throttled (needs its own CPU core).
2434 // Newer Windows 11 versions aggresively park (offline) CPU cores and often place
2435 // all our threads onto the first 4 cores which results in terrible performance with
2436 // n_threads > 4
2437 #if _WIN32_WINNT >= 0x0602
2438 THREAD_POWER_THROTTLING_STATE t;
2439 ZeroMemory(&t, sizeof(t));
2440 t.Version = THREAD_POWER_THROTTLING_CURRENT_VERSION;
2441 t.ControlMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED;
2442 t.StateMask = 0;
2443
2444 if (!SetThreadInformation(GetCurrentThread(), ThreadPowerThrottling, &t, sizeof(t))) {
2445 GGML_LOG_DEBUG("failed to disable thread power throttling %d : (%d)\n", prio, (int) GetLastError());
2446 return false;
2447 }
2448 #endif
2449 }
2450
2451 if (prio == GGML_SCHED_PRIO_NORMAL) {
2452 // Keep inherited policy/priority
2453 return true;
2454 }
2455
2456 if (!SetThreadPriority(GetCurrentThread(), p)) {
2457 fprintf(stderr, "warn: failed to set thread priority %d : (%d)\n", prio, (int) GetLastError());
2458 return false;
2459 }
2460
2461 return true;
2462}
2463
2464#elif defined(__APPLE__)
2465#include <sys/types.h>
2466#include <sys/resource.h>
2467
2468static bool ggml_thread_apply_affinity(const bool * mask) {
2469 // Not supported on Apple platforms
2470 UNUSED(mask);
2471 return true;
2472}
2473
2474static bool ggml_thread_apply_priority(int32_t prio) {
2475 struct sched_param p;
2476 int32_t policy = SCHED_OTHER;
2477 switch (prio) {
2478 // TODO: there seems to be no way to set lower prio on Apple platforms
2479 case GGML_SCHED_PRIO_LOW: policy = SCHED_OTHER; p.sched_priority = 0; break;
2480 case GGML_SCHED_PRIO_NORMAL: policy = SCHED_OTHER; p.sched_priority = 0; break;
2481 case GGML_SCHED_PRIO_MEDIUM: policy = SCHED_FIFO; p.sched_priority = 40; break;
2482 case GGML_SCHED_PRIO_HIGH: policy = SCHED_FIFO; p.sched_priority = 80; break;
2483 case GGML_SCHED_PRIO_REALTIME: policy = SCHED_FIFO; p.sched_priority = 90; break;
2484 }
2485
2486 if (prio == GGML_SCHED_PRIO_NORMAL) {
2487 // Keep inherited policy/priority
2488 return true;
2489 }
2490
2491 int32_t err = pthread_setschedparam(pthread_self(), policy, &p);
2492 if (err != 0) {
2493 fprintf(stderr, "warn: failed to set thread priority %d : %s (%d)\n", prio, strerror(err), err);
2494 return false;
2495 }
2496
2497 return true;
2498}
2499
2500#elif defined(__gnu_linux__)
2501// TODO: this may not work on BSD, to be verified
2502
2503static bool ggml_thread_apply_affinity(const bool * mask) {
2504 cpu_set_t cpuset;
2505 int err;
2506
2507 CPU_ZERO(&cpuset);
2508
2509 for (uint32_t i = 0; i < GGML_MAX_N_THREADS; i++) {
2510 if (mask[i]) {
2511 GGML_PRINT_DEBUG("Thread %lx: adding %d to cpuset\n", pthread_self(), i);
2512 CPU_SET(i, &cpuset);
2513 }
2514 }
2515
2516#ifdef __ANDROID__
2517 err = sched_setaffinity(0, sizeof(cpuset), &cpuset);
2518 if (err < 0) {
2519 err = errno;
2520 }
2521#else
2522 err = pthread_setaffinity_np(th: pthread_self(), cpusetsize: sizeof(cpuset), cpuset: &cpuset);
2523#endif
2524 if (err != 0) {
2525 fprintf(stderr, format: "warn: failed to set affinity mask 0x%llx : %s (%d)\n", (unsigned long long)mask, strerror(errnum: err), err);
2526 return false;
2527 }
2528
2529 return true;
2530}
2531
2532static bool ggml_thread_apply_priority(int32_t prio) {
2533 struct sched_param p;
2534 int32_t policy = SCHED_OTHER;
2535 switch (prio) {
2536 case GGML_SCHED_PRIO_LOW: policy = SCHED_BATCH; p.sched_priority = 0; break;
2537 case GGML_SCHED_PRIO_NORMAL: policy = SCHED_OTHER; p.sched_priority = 0; break;
2538 case GGML_SCHED_PRIO_MEDIUM: policy = SCHED_FIFO; p.sched_priority = 40; break;
2539 case GGML_SCHED_PRIO_HIGH: policy = SCHED_FIFO; p.sched_priority = 80; break;
2540 case GGML_SCHED_PRIO_REALTIME: policy = SCHED_FIFO; p.sched_priority = 90; break;
2541 }
2542
2543 if (prio == GGML_SCHED_PRIO_NORMAL) {
2544 // Keep inherited policy/priority
2545 return true;
2546 }
2547
2548 int32_t err = pthread_setschedparam(target_thread: pthread_self(), policy: policy, param: &p);
2549 if (err != 0) {
2550 fprintf(stderr, format: "warn: failed to set thread priority %d : %s (%d)\n", prio, strerror(errnum: err), err);
2551 return false;
2552 }
2553
2554 return true;
2555}
2556
2557#else // unsupported platforms
2558
2559static bool ggml_thread_apply_affinity(const bool * mask) {
2560 UNUSED(mask);
2561 return true;
2562}
2563
2564static bool ggml_thread_apply_priority(int32_t prio) {
2565 UNUSED(prio);
2566 return true;
2567}
2568
2569#endif
2570
2571static bool ggml_thread_cpumask_is_valid(const bool * mask) {
2572 for (int i = 0; i < GGML_MAX_N_THREADS; i++) {
2573 if (mask[i]) { return true; }
2574 }
2575 return false;
2576}
2577
2578static void ggml_thread_cpumask_next(const bool * global_mask, bool * local_mask, bool strict, int32_t* iter) {
2579 if (!strict) {
2580 memcpy(dest: local_mask, src: global_mask, GGML_MAX_N_THREADS);
2581 return;
2582 } else {
2583 memset(s: local_mask, c: 0, GGML_MAX_N_THREADS);
2584 int32_t base_idx = *iter;
2585 for (int32_t i = 0; i < GGML_MAX_N_THREADS; i++) {
2586 int32_t idx = base_idx + i;
2587 if (idx >= GGML_MAX_N_THREADS) {
2588 // Just a cheaper modulo
2589 idx -= GGML_MAX_N_THREADS;
2590 }
2591 if (global_mask[idx]) {
2592 local_mask[idx] = 1;
2593 *iter = idx + 1;
2594 return;
2595 }
2596 }
2597 }
2598}
2599
2600void ggml_threadpool_free(struct ggml_threadpool* threadpool) {
2601 if (!threadpool) return;
2602
2603 const int n_threads = threadpool->n_threads_max;
2604
2605#ifndef GGML_USE_OPENMP
2606 struct ggml_compute_state* workers = threadpool->workers;
2607
2608 ggml_mutex_lock(&threadpool->mutex);
2609
2610 threadpool->stop = true;
2611 threadpool->pause = false;
2612
2613 ggml_cond_broadcast(&threadpool->cond);
2614 ggml_mutex_unlock(&threadpool->mutex);
2615
2616 for (int j = 1; j < n_threads; j++) {
2617 int32_t rc = ggml_thread_join(workers[j].thrd, NULL);
2618 GGML_ASSERT(rc == GGML_EXIT_SUCCESS || rc == GGML_EXIT_ABORTED);
2619 UNUSED(rc);
2620 }
2621
2622 ggml_mutex_destroy(&threadpool->mutex);
2623 ggml_cond_destroy(&threadpool->cond);
2624#endif // GGML_USE_OPENMP
2625
2626 const size_t workers_size = sizeof(struct ggml_compute_state) * n_threads;
2627 ggml_aligned_free(ptr: threadpool->workers, size: workers_size);
2628 ggml_aligned_free(ptr: threadpool, size: sizeof(struct ggml_threadpool));
2629}
2630
2631#ifndef GGML_USE_OPENMP
2632// pause/resume must be called under mutex
2633static void ggml_threadpool_pause_locked(struct ggml_threadpool * threadpool) {
2634 GGML_PRINT_DEBUG("Pausing threadpool\n");
2635 threadpool->pause = true;
2636 ggml_cond_broadcast(&threadpool->cond);
2637}
2638
2639static void ggml_threadpool_resume_locked(struct ggml_threadpool * threadpool) {
2640 GGML_PRINT_DEBUG("Resuming threadpool\n");
2641 threadpool->pause = false;
2642 ggml_cond_broadcast(&threadpool->cond);
2643}
2644#endif
2645
2646void ggml_threadpool_pause(struct ggml_threadpool * threadpool) {
2647#ifndef GGML_USE_OPENMP
2648 ggml_mutex_lock(&threadpool->mutex);
2649 if (!threadpool->pause) {
2650 ggml_threadpool_pause_locked(threadpool);
2651 }
2652 ggml_mutex_unlock(&threadpool->mutex);
2653#else
2654 UNUSED(threadpool);
2655#endif
2656}
2657
2658void ggml_threadpool_resume(struct ggml_threadpool * threadpool) {
2659#ifndef GGML_USE_OPENMP
2660 ggml_mutex_lock(&threadpool->mutex);
2661 if (threadpool->pause) {
2662 ggml_threadpool_resume_locked(threadpool);
2663 }
2664 ggml_mutex_unlock(&threadpool->mutex);
2665#else
2666 UNUSED(threadpool);
2667#endif
2668}
2669
2670struct ggml_cplan ggml_graph_plan(
2671 const struct ggml_cgraph * cgraph,
2672 int n_threads,
2673 struct ggml_threadpool * threadpool) {
2674
2675 if (threadpool == NULL) {
2676 //GGML_PRINT_DEBUG("Threadpool is not specified. Will create a disposable threadpool : n_threads %d\n", n_threads);
2677 }
2678 if (n_threads <= 0) {
2679 n_threads = threadpool ? threadpool->n_threads_max : GGML_DEFAULT_N_THREADS;
2680 }
2681
2682 size_t work_size = 0;
2683
2684 struct ggml_cplan cplan;
2685 memset(s: &cplan, c: 0, n: sizeof(struct ggml_cplan));
2686
2687 int max_tasks = 1;
2688
2689 // thread scheduling for the different operations + work buffer size estimation
2690 for (int i = 0; i < cgraph->n_nodes; i++) {
2691 struct ggml_tensor * node = cgraph->nodes[i];
2692
2693 const int n_tasks = ggml_get_n_tasks(node, n_threads);
2694
2695 max_tasks = MAX(max_tasks, n_tasks);
2696
2697 size_t cur = 0;
2698
2699 if (!ggml_cpu_extra_work_size(n_threads, op: node, size: &cur)) {
2700 switch (node->op) {
2701 case GGML_OP_CPY:
2702 case GGML_OP_DUP:
2703 {
2704 if (ggml_is_quantized(type: node->type) ||
2705 // F16 -> BF16 and BF16 -> F16 copies go through intermediate F32
2706 (node->src[0]->type == GGML_TYPE_F16 && node->src[1] && node->src[1]->type == GGML_TYPE_BF16) ||
2707 (node->src[0]->type == GGML_TYPE_BF16 && node->src[1] && node->src[1]->type == GGML_TYPE_F16) ||
2708 // conversion between F32 and I32
2709 (node->src[0]->type == GGML_TYPE_F32 && node->src[1] && node->src[1]->type == GGML_TYPE_I32) ||
2710 (node->src[0]->type == GGML_TYPE_I32 && node->src[1] && node->src[1]->type == GGML_TYPE_F32)) {
2711 cur = ggml_type_size(type: GGML_TYPE_F32) * node->ne[0] * n_tasks;
2712 }
2713 } break;
2714 case GGML_OP_ADD:
2715 case GGML_OP_ADD_ID:
2716 case GGML_OP_ADD1:
2717 {
2718 if (ggml_is_quantized(type: node->src[0]->type)) {
2719 cur = ggml_type_size(type: GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
2720 }
2721 } break;
2722 case GGML_OP_ACC:
2723 {
2724 if (ggml_is_quantized(type: node->src[0]->type)) {
2725 cur = ggml_type_size(type: GGML_TYPE_F32) * node->src[1]->ne[0] * n_tasks;
2726 }
2727 } break;
2728 case GGML_OP_COUNT_EQUAL:
2729 {
2730 cur = ggml_type_size(type: node->type)*n_tasks;
2731 } break;
2732 case GGML_OP_MUL_MAT:
2733 {
2734 const enum ggml_type vec_dot_type = type_traits_cpu[node->src[0]->type].vec_dot_type;
2735
2736 if (node->src[1]->type != vec_dot_type) {
2737 cur = ggml_row_size(type: vec_dot_type, ne: ggml_nelements(tensor: node->src[1]));
2738 }
2739 } break;
2740 case GGML_OP_MUL_MAT_ID:
2741 {
2742 cur = 0;
2743 const struct ggml_tensor * src0 = node->src[0];
2744 const struct ggml_tensor * src1 = node->src[1];
2745 const struct ggml_tensor * ids = node->src[2];
2746 const enum ggml_type vec_dot_type = type_traits_cpu[src0->type].vec_dot_type;
2747 const int n_as = src0->ne[2];
2748 // src1
2749 if (src1->type != vec_dot_type) {
2750 cur += ggml_row_size(type: vec_dot_type, ne: ggml_nelements(tensor: src1)) + sizeof(int64_t);
2751 }
2752 // matrix_row_counts
2753 cur += n_as * sizeof(int64_t) + sizeof(int64_t);
2754 // matrix_rows
2755 cur += n_as*ids->ne[0]*ids->ne[1]*sizeof(struct mmid_row_mapping) + sizeof(int64_t);
2756 // atomic_current_chunk
2757 cur += CACHE_LINE_SIZE*n_as + CACHE_LINE_SIZE;
2758 } break;
2759 case GGML_OP_OUT_PROD:
2760 {
2761 if (ggml_is_quantized(type: node->src[0]->type)) {
2762 cur = ggml_type_size(type: GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
2763 }
2764 } break;
2765 case GGML_OP_SOFT_MAX:
2766 case GGML_OP_ROPE:
2767 case GGML_OP_ROPE_BACK:
2768 {
2769 cur = ggml_type_size(type: GGML_TYPE_F32) * node->ne[0] * n_tasks;
2770 } break;
2771 case GGML_OP_CONV_TRANSPOSE_1D:
2772 {
2773 GGML_ASSERT(node->src[0]->ne[3] == 1);
2774 GGML_ASSERT(node->src[1]->ne[2] == 1);
2775 GGML_ASSERT(node->src[1]->ne[3] == 1);
2776
2777 const int64_t ne00 = node->src[0]->ne[0]; // K
2778 const int64_t ne01 = node->src[0]->ne[1]; // Cout
2779 const int64_t ne02 = node->src[0]->ne[2]; // Cin
2780 const int64_t ne10 = node->src[1]->ne[0]; // L
2781 const int64_t ne11 = node->src[1]->ne[1]; // Cin
2782
2783 if ((node->src[0]->type == GGML_TYPE_F16 ||
2784 node->src[0]->type == GGML_TYPE_BF16) &&
2785 node->src[1]->type == GGML_TYPE_F32) {
2786 cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02;
2787 cur += sizeof(ggml_fp16_t)*ne10*ne11;
2788 } else if (node->src[0]->type == GGML_TYPE_F32 &&
2789 node->src[1]->type == GGML_TYPE_F32) {
2790 cur += sizeof(float)*ne00*ne01*ne02;
2791 cur += sizeof(float)*ne10*ne11;
2792 } else {
2793 GGML_ABORT("fatal error");
2794 }
2795 } break;
2796 case GGML_OP_CONV_2D:
2797 case GGML_OP_CONV_3D:
2798 {
2799 cur = GGML_IM2COL_WORK_SIZE;
2800 } break;
2801 case GGML_OP_CONV_TRANSPOSE_2D:
2802 {
2803 const int64_t ne00 = node->src[0]->ne[0]; // W
2804 const int64_t ne01 = node->src[0]->ne[1]; // H
2805 const int64_t ne02 = node->src[0]->ne[2]; // Channels Out
2806 const int64_t ne03 = node->src[0]->ne[3]; // Channels In
2807
2808 const int64_t ne10 = node->src[1]->ne[0]; // W
2809 const int64_t ne11 = node->src[1]->ne[1]; // H
2810 const int64_t ne12 = node->src[1]->ne[2]; // Channels In
2811
2812 cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02*ne03;
2813 cur += sizeof(ggml_fp16_t)*ne10*ne11*ne12;
2814 } break;
2815 case GGML_OP_FLASH_ATTN_EXT:
2816 {
2817 const int64_t ne10 = node->src[1]->ne[0]; // DK
2818 const int64_t ne20 = node->src[2]->ne[0]; // DV
2819
2820 cur = sizeof(float)*(1*ne10 + 2*ne20)*n_tasks; // 1x head size K + 2x head size V (per thread)
2821 } break;
2822 case GGML_OP_FLASH_ATTN_BACK:
2823 {
2824 const int64_t D = node->src[0]->ne[0];
2825 const int64_t ne11 = ggml_up(n: node->src[1]->ne[1], GGML_SOFT_MAX_UNROLL);
2826 const int64_t mxDn = MAX(D, ne11) * 2; // *2 because of S and SM in ggml_compute_forward_flash_attn_back
2827 if (node->src[1]->type == GGML_TYPE_F32) {
2828 cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)
2829 cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2
2830 } else if (node->src[1]->type == GGML_TYPE_F16) {
2831 cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)
2832 cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2
2833 } else if (node->src[1]->type == GGML_TYPE_BF16) {
2834 cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)
2835 cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2
2836 }
2837 } break;
2838
2839 case GGML_OP_CROSS_ENTROPY_LOSS:
2840 {
2841 cur = ggml_type_size(type: node->type)*(n_tasks + node->src[0]->ne[0]*n_tasks);
2842 } break;
2843 case GGML_OP_COUNT:
2844 {
2845 GGML_ABORT("fatal error");
2846 }
2847 default:
2848 break;
2849 }
2850 }
2851
2852 work_size = MAX(work_size, cur);
2853 }
2854
2855 if (work_size > 0) {
2856 work_size += CACHE_LINE_SIZE*(n_threads);
2857 }
2858
2859 cplan.threadpool = threadpool;
2860 cplan.n_threads = MIN(max_tasks, n_threads);
2861 cplan.work_size = work_size;
2862 cplan.work_data = NULL;
2863
2864 return cplan;
2865}
2866
2867static thread_ret_t ggml_graph_compute_thread(void * data) {
2868 struct ggml_compute_state * state = (struct ggml_compute_state *) data;
2869 struct ggml_threadpool * tp = state->threadpool;
2870
2871 const struct ggml_cgraph * cgraph = tp->cgraph;
2872 const struct ggml_cplan * cplan = tp->cplan;
2873
2874 set_numa_thread_affinity(state->ith);
2875
2876 struct ggml_compute_params params = {
2877 /*.ith =*/ state->ith,
2878 /*.nth =*/ atomic_load_explicit(&tp->n_threads_cur, memory_order_relaxed),
2879 /*.wsize =*/ cplan->work_size,
2880 /*.wdata =*/ cplan->work_data,
2881 /*.threadpool=*/ tp,
2882 };
2883
2884 for (int node_n = 0; node_n < cgraph->n_nodes && atomic_load_explicit(&tp->abort, memory_order_relaxed) != node_n; node_n++) {
2885 struct ggml_tensor * node = cgraph->nodes[node_n];
2886
2887 ggml_compute_forward(params: &params, tensor: node);
2888
2889 if (state->ith == 0 && cplan->abort_callback &&
2890 cplan->abort_callback(cplan->abort_callback_data)) {
2891 atomic_store_explicit(&tp->abort, node_n + 1, memory_order_relaxed);
2892 tp->ec = GGML_STATUS_ABORTED;
2893 }
2894
2895 if (node_n + 1 < cgraph->n_nodes) {
2896 ggml_barrier(tp: state->threadpool);
2897 }
2898 }
2899
2900 ggml_barrier(tp: state->threadpool);
2901
2902 return 0;
2903}
2904
2905#ifndef GGML_USE_OPENMP
2906
2907// check if thread is active
2908static inline bool ggml_graph_compute_thread_active(struct ggml_compute_state * state) {
2909 struct ggml_threadpool * threadpool = state->threadpool;
2910 int n_threads = atomic_load_explicit(&threadpool->n_threads_cur, memory_order_relaxed);
2911 return (state->ith < n_threads);
2912}
2913
2914// check if thread is ready to proceed (exit from polling or sleeping)
2915static inline bool ggml_graph_compute_thread_ready(struct ggml_compute_state * state) {
2916 struct ggml_threadpool * threadpool = state->threadpool;
2917
2918 if (state->pending || threadpool->stop || threadpool->pause) { return true; }
2919
2920 // check for new graph/work
2921 int new_graph = atomic_load_explicit(&threadpool->n_graph, memory_order_relaxed);
2922 if (new_graph != state->last_graph) {
2923 state->pending = ggml_graph_compute_thread_active(state);
2924 state->last_graph = new_graph;
2925 }
2926
2927 return state->pending;
2928}
2929
2930// sync thread state after polling
2931static inline void ggml_graph_compute_thread_sync(struct ggml_compute_state * state) {
2932 // TSAN doesn't support standalone fence yet, we use a dummy read-modify-write instead
2933 #ifdef GGML_TSAN_ENABLED
2934 atomic_fetch_add_explicit(&state->threadpool->n_graph, 0, memory_order_seq_cst);
2935 #else
2936 atomic_thread_fence(memory_order_seq_cst);
2937 #endif
2938 UNUSED(state);
2939}
2940
2941static inline bool ggml_graph_compute_poll_for_work(struct ggml_compute_state * state) {
2942 struct ggml_threadpool * threadpool = state->threadpool;
2943
2944 // Skip polling for unused threads
2945 if (!ggml_graph_compute_thread_active(state)) {
2946 return state->pending;
2947 }
2948
2949 // This seems to make 0 ... 100 a decent range for polling level across modern processors.
2950 // Perhaps, we can adjust it dynamically based on load and things.
2951 const uint64_t n_rounds = 1024UL * 128 * threadpool->poll;
2952
2953 for (uint64_t i=0; !ggml_graph_compute_thread_ready(state) && i < n_rounds; i++) {
2954 // No new work. Keep polling.
2955 ggml_thread_cpu_relax();
2956 }
2957
2958 return state->pending;
2959}
2960
2961static inline bool ggml_graph_compute_check_for_work(struct ggml_compute_state * state) {
2962 struct ggml_threadpool * threadpool = state->threadpool;
2963
2964 if (ggml_graph_compute_poll_for_work(state)) {
2965 ggml_graph_compute_thread_sync(state);
2966 return state->pending;
2967 }
2968
2969 ggml_mutex_lock_shared(&threadpool->mutex);
2970 while (!ggml_graph_compute_thread_ready(state)) {
2971 // No new work. Wait for the signal.
2972 GGML_PRINT_DEBUG("thread #%d waiting for work (sleeping)\n", state->ith);
2973 ggml_cond_wait(&threadpool->cond, &threadpool->mutex);
2974 }
2975 ggml_mutex_unlock_shared(&threadpool->mutex);
2976
2977 return state->pending;
2978}
2979
2980static thread_ret_t ggml_graph_compute_secondary_thread(void* data) {
2981 struct ggml_compute_state * state = (struct ggml_compute_state *) data;
2982 struct ggml_threadpool * threadpool = state->threadpool;
2983
2984 ggml_thread_apply_priority(threadpool->prio);
2985 if (ggml_thread_cpumask_is_valid(state->cpumask)) {
2986 ggml_thread_apply_affinity(state->cpumask);
2987 }
2988
2989 while (true) {
2990 // Check if we need to sleep
2991 while (threadpool->pause) {
2992 GGML_PRINT_DEBUG("thread #%d inside pause loop\n", state->ith);
2993 ggml_mutex_lock_shared(&threadpool->mutex);
2994 if (threadpool->pause) {
2995 ggml_cond_wait(&threadpool->cond, &threadpool->mutex);
2996 }
2997 GGML_PRINT_DEBUG("thread #%d resuming after wait\n", state->ith);
2998 ggml_mutex_unlock_shared(&threadpool->mutex);
2999 }
3000
3001 // This needs to be checked for after the cond_wait
3002 if (threadpool->stop) break;
3003
3004 // Check if there is new work
3005 // The main thread is the only one that can dispatch new work
3006
3007 ggml_graph_compute_check_for_work(state);
3008 if (state->pending) {
3009 state->pending = false;
3010
3011 ggml_graph_compute_thread(state);
3012 }
3013 }
3014
3015 return (thread_ret_t) 0;
3016}
3017
3018// Start processing new graph
3019static void ggml_graph_compute_kickoff(struct ggml_threadpool * threadpool, int n_threads)
3020{
3021 // Always take the mutex here because the worker threads are doing hybrid poll/wait
3022
3023 ggml_mutex_lock(&threadpool->mutex);
3024
3025 GGML_PRINT_DEBUG("threadpool: n_threads_cur %d n_threads %d\n", threadpool->n_threads_cur, n_threads);
3026
3027 // Update the number of active threads
3028 atomic_store_explicit(&threadpool->n_threads_cur, n_threads, memory_order_relaxed);
3029
3030 // Indicate the graph is ready to be processed
3031 // We need the full seq-cst fence here because of the polling threads (used in thread_sync)
3032 atomic_fetch_add_explicit(&threadpool->n_graph, 1, memory_order_seq_cst);
3033
3034 if (threadpool->pause) {
3035 // Update main thread prio and affinity to match the threadpool settings
3036 ggml_thread_apply_priority(threadpool->prio);
3037 if (ggml_thread_cpumask_is_valid(threadpool->workers[0].cpumask)) {
3038 ggml_thread_apply_affinity(threadpool->workers[0].cpumask);
3039 }
3040
3041 // resume does cond broadcast
3042 ggml_threadpool_resume_locked(threadpool);
3043 } else {
3044 ggml_cond_broadcast(&threadpool->cond);
3045 }
3046
3047 ggml_mutex_unlock(&threadpool->mutex);
3048}
3049
3050#endif // GGML_USE_OPENMP
3051
3052static struct ggml_threadpool * ggml_threadpool_new_impl(
3053 struct ggml_threadpool_params * tpp,
3054 struct ggml_cgraph * cgraph,
3055 struct ggml_cplan * cplan) {
3056
3057 struct ggml_threadpool * threadpool =
3058 ggml_aligned_malloc(size: sizeof(struct ggml_threadpool));
3059 {
3060 threadpool->cgraph = cgraph;
3061 threadpool->cplan = cplan;
3062 threadpool->n_graph = 0;
3063 threadpool->n_barrier = 0;
3064 threadpool->n_barrier_passed = 0;
3065 threadpool->current_chunk = 0;
3066 threadpool->stop = false;
3067 threadpool->pause = tpp->paused;
3068 threadpool->abort = -1;
3069 threadpool->workers = NULL;
3070 threadpool->n_threads_max = tpp->n_threads;
3071 threadpool->n_threads_cur = tpp->n_threads;
3072 threadpool->poll = tpp->poll;
3073 threadpool->prio = tpp->prio;
3074 threadpool->ec = GGML_STATUS_SUCCESS;
3075 }
3076
3077 // Allocate and init workers state
3078 const size_t workers_size = sizeof(struct ggml_compute_state) * tpp->n_threads;
3079 struct ggml_compute_state * workers = ggml_aligned_malloc(size: workers_size);
3080
3081 memset(s: workers, c: 0, n: workers_size);
3082 for (int j = 0; j < tpp->n_threads; j++) {
3083 workers[j].threadpool = threadpool;
3084 workers[j].ith = j;
3085 }
3086
3087 threadpool->workers = workers;
3088
3089#ifdef GGML_USE_OPENMP
3090 int32_t cpumask_iter = 0;
3091
3092 // Compute CPU masks for each thread
3093 for (int j = 0; j < tpp->n_threads; j++) {
3094 ggml_thread_cpumask_next(global_mask: tpp->cpumask, local_mask: workers[j].cpumask, strict: tpp->strict_cpu, iter: &cpumask_iter);
3095 }
3096#else // GGML_USE_OPENMP
3097 ggml_mutex_init(&threadpool->mutex);
3098 ggml_cond_init(&threadpool->cond);
3099
3100 // Spin the threads for all workers, and update CPU placements.
3101 // Place the main thread last (towards the higher numbered CPU cores).
3102
3103 int32_t cpumask_iter = 0;
3104
3105 for (int j = 1; j < tpp->n_threads; j++) {
3106 ggml_thread_cpumask_next(tpp->cpumask, workers[j].cpumask, tpp->strict_cpu, &cpumask_iter);
3107
3108 int32_t rc = ggml_thread_create(&workers[j].thrd, NULL, ggml_graph_compute_secondary_thread, &workers[j]);
3109 GGML_ASSERT(rc == 0);
3110 }
3111
3112 ggml_thread_cpumask_next(tpp->cpumask, workers[0].cpumask, tpp->strict_cpu, &cpumask_iter);
3113
3114 if (!threadpool->pause) {
3115 // Update main thread prio and affinity at the start, otherwise we'll do it in resume
3116 ggml_thread_apply_priority(threadpool->prio);
3117 if (ggml_thread_cpumask_is_valid(threadpool->workers[0].cpumask)) {
3118 ggml_thread_apply_affinity(threadpool->workers[0].cpumask);
3119 }
3120 }
3121#endif // GGML_USE_OPENMP
3122
3123 return threadpool;
3124}
3125
3126struct ggml_threadpool * ggml_threadpool_new(struct ggml_threadpool_params * tpp) {
3127 return ggml_threadpool_new_impl(tpp, NULL, NULL);
3128}
3129
3130enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {
3131 ggml_cpu_init();
3132
3133 GGML_ASSERT(cplan);
3134 GGML_ASSERT(cplan->n_threads > 0);
3135 GGML_ASSERT(cplan->work_size == 0 || cplan->work_data != NULL);
3136
3137 int n_threads = cplan->n_threads;
3138 struct ggml_threadpool * threadpool = cplan->threadpool;
3139
3140 bool disposable_threadpool = false;
3141
3142 if (threadpool == NULL) {
3143 //GGML_PRINT_DEBUG("Threadpool is not specified. Will create a disposable threadpool : n_threads %d\n", n_threads);
3144 disposable_threadpool = true;
3145
3146 struct ggml_threadpool_params ttp = ggml_threadpool_params_default(n_threads);
3147 threadpool = ggml_threadpool_new_impl(tpp: &ttp, cgraph, cplan);
3148 } else {
3149 // Reset some of the parameters that need resetting
3150 // No worker threads should be accessing the parameters below at this stage
3151 threadpool->cgraph = cgraph;
3152 threadpool->cplan = cplan;
3153 threadpool->current_chunk = 0;
3154 threadpool->abort = -1;
3155 threadpool->ec = GGML_STATUS_SUCCESS;
3156 }
3157
3158#ifdef GGML_USE_OPENMP
3159 if (n_threads > 1) {
3160 #pragma omp parallel num_threads(n_threads)
3161 {
3162 #pragma omp single
3163 {
3164 // update the number of threads from the actual number of threads that we got from OpenMP
3165 n_threads = omp_get_num_threads();
3166 atomic_store_explicit(&threadpool->n_threads_cur, n_threads, memory_order_relaxed);
3167 }
3168
3169 // Apply thread CPU mask and priority
3170 int ith = omp_get_thread_num();
3171
3172 ggml_thread_apply_priority(prio: threadpool->prio);
3173 if (ggml_thread_cpumask_is_valid(mask: threadpool->workers[ith].cpumask)) {
3174 ggml_thread_apply_affinity(mask: threadpool->workers[ith].cpumask);
3175 }
3176 ggml_graph_compute_thread(data: &threadpool->workers[ith]);
3177 }
3178 } else {
3179 atomic_store_explicit(&threadpool->n_threads_cur, 1, memory_order_relaxed);
3180 ggml_graph_compute_thread(data: &threadpool->workers[0]);
3181 }
3182#else
3183 if (n_threads > threadpool->n_threads_max) {
3184 GGML_LOG_WARN("cplan requested more threads (%d) than available (%d)\n", n_threads, threadpool->n_threads_max);
3185 n_threads = threadpool->n_threads_max;
3186 }
3187
3188 // Kick all threads to start the new graph
3189 ggml_graph_compute_kickoff(threadpool, n_threads);
3190
3191 // This is a work thread too
3192 ggml_graph_compute_thread(&threadpool->workers[0]);
3193#endif
3194
3195 // don't leave affinity set on the main thread
3196 clear_numa_thread_affinity();
3197
3198 enum ggml_status ret = threadpool->ec;
3199
3200 if (disposable_threadpool) {
3201 ggml_threadpool_free(threadpool);
3202 }
3203
3204 return ret;
3205}
3206
3207enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads) {
3208 struct ggml_cplan cplan = ggml_graph_plan(cgraph, n_threads, NULL);
3209
3210 cplan.work_data = (uint8_t *)ggml_new_buffer(ctx, nbytes: cplan.work_size);
3211
3212 return ggml_graph_compute(cgraph, cplan: &cplan);
3213}
3214
3215void ggml_cpu_fp32_to_fp32(const float * x, float * y, int64_t n) {
3216 memcpy(dest: y, src: x, n: n * sizeof(float));
3217}
3218
3219void ggml_cpu_fp32_to_fp16(const float * x, ggml_fp16_t * y, int64_t n) {
3220 int64_t i = 0;
3221#if defined(__F16C__)
3222#if defined(__AVX512F__)
3223 for (; i + 15 < n; i += 16) {
3224 __m512 x_vec = _mm512_loadu_ps(x + i);
3225 __m256i y_vec = _mm512_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);
3226 _mm256_storeu_si256((__m256i *)(y + i), y_vec);
3227 }
3228#endif
3229 for (; i + 7 < n; i += 8) {
3230 __m256 x_vec = _mm256_loadu_ps(p: x + i);
3231 __m128i y_vec = _mm256_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);
3232 _mm_storeu_si128(p: (__m128i *)(y + i), b: y_vec);
3233 }
3234 for (; i + 3 < n; i += 4) {
3235 __m128 x_vec = _mm_loadu_ps(p: x + i);
3236 __m128i y_vec = _mm_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);
3237 _mm_storel_epi64(p: (__m128i *)(y + i), a: y_vec);
3238 }
3239#elif defined(__riscv_zvfh)
3240 for (int vl; i < n; i += vl) {
3241 vl = __riscv_vsetvl_e32m2(n - i);
3242 vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl);
3243 vfloat16m1_t vy = __riscv_vfncvt_f_f_w_f16m1(vx, vl);
3244 __riscv_vse16_v_f16m1((_Float16 *)&y[i], vy, vl);
3245 }
3246#endif
3247 for (; i < n; ++i) {
3248 y[i] = GGML_CPU_FP32_TO_FP16(x[i]);
3249 }
3250}
3251
3252void ggml_cpu_fp16_to_fp32(const ggml_fp16_t * x, float * y, int64_t n) {
3253 int64_t i = 0;
3254#if defined(__F16C__)
3255#if defined(__AVX512F__)
3256 for (; i + 15 < n; i += 16) {
3257 __m256i x_vec = _mm256_loadu_si256((const __m256i *)(x + i));
3258 __m512 y_vec = _mm512_cvtph_ps(x_vec);
3259 _mm512_storeu_ps(y + i, y_vec);
3260 }
3261#endif
3262 for (; i + 7 < n; i += 8) {
3263 __m128i x_vec = _mm_loadu_si128(p: (const __m128i *)(x + i));
3264 __m256 y_vec = _mm256_cvtph_ps(a: x_vec);
3265 _mm256_storeu_ps(p: y + i, a: y_vec);
3266 }
3267 for (; i + 3 < n; i += 4) {
3268 __m128i x_vec = _mm_loadl_epi64(p: (const __m128i *)(x + i));
3269 __m128 y_vec = _mm_cvtph_ps(a: x_vec);
3270 _mm_storeu_ps(p: y + i, a: y_vec);
3271 }
3272#endif
3273
3274 for (; i < n; ++i) {
3275 y[i] = GGML_CPU_FP16_TO_FP32(x[i]);
3276 }
3277}
3278
3279void ggml_cpu_fp32_to_bf16(const float * x, ggml_bf16_t * y, int64_t n) {
3280 int64_t i = 0;
3281 for (; i < n; ++i) {
3282 y[i] = GGML_FP32_TO_BF16(x[i]);
3283 }
3284}
3285
3286void ggml_cpu_fp32_to_i32(const float * x, int32_t * y, int64_t n) {
3287 int64_t i = 0;
3288 for (; i < n; ++i) {
3289 y[i] = x[i];
3290 }
3291}
3292
3293void ggml_cpu_bf16_to_fp32(const ggml_bf16_t * x, float * y, int64_t n) {
3294 int64_t i = 0;
3295#if defined(__AVX2__)
3296#if defined(__AVX512F__)
3297 for (; i + 15 < n; i += 16) {
3298 _mm512_storeu_ps(y + i,
3299 _mm512_castsi512_ps(
3300 _mm512_slli_epi32(
3301 _mm512_cvtepu16_epi32(
3302 _mm256_loadu_si256(
3303 (const __m256i *)(x + i))),
3304 16)));
3305 }
3306#endif
3307 for (; i + 7 < n; i += 8) {
3308 _mm256_storeu_ps(p: y + i,
3309 a: _mm256_castsi256_ps(
3310 a: _mm256_slli_epi32(
3311 a: _mm256_cvtepu16_epi32(
3312 V: _mm_loadu_si128(
3313 p: (const __m128i *)(x + i))),
3314 count: 16)));
3315 }
3316#endif
3317 for (; i < n; i++) {
3318 y[i] = GGML_BF16_TO_FP32(x[i]);
3319 }
3320}
3321
3322int ggml_cpu_has_avx(void) {
3323#if defined(__AVX__)
3324 return 1;
3325#else
3326 return 0;
3327#endif
3328}
3329
3330int ggml_cpu_has_avx_vnni(void) {
3331#if defined(__AVXVNNI__)
3332 return 1;
3333#else
3334 return 0;
3335#endif
3336}
3337
3338int ggml_cpu_has_avx2(void) {
3339#if defined(__AVX2__)
3340 return 1;
3341#else
3342 return 0;
3343#endif
3344}
3345
3346int ggml_cpu_has_avx512(void) {
3347#if defined(__AVX512F__)
3348 return 1;
3349#else
3350 return 0;
3351#endif
3352}
3353
3354int ggml_cpu_has_avx512_vbmi(void) {
3355#if defined(__AVX512VBMI__)
3356 return 1;
3357#else
3358 return 0;
3359#endif
3360}
3361
3362int ggml_cpu_has_avx512_vnni(void) {
3363#if defined(__AVX512VNNI__)
3364 return 1;
3365#else
3366 return 0;
3367#endif
3368}
3369
3370int ggml_cpu_has_avx512_bf16(void) {
3371#if defined(__AVX512BF16__)
3372 return 1;
3373#else
3374 return 0;
3375#endif
3376}
3377
3378int ggml_cpu_has_amx_int8(void) {
3379#if defined(__AMX_INT8__)
3380 return 1;
3381#else
3382 return 0;
3383#endif
3384}
3385
3386int ggml_cpu_has_bmi2(void) {
3387#if defined(__BMI2__)
3388 return 1;
3389#else
3390 return 0;
3391#endif
3392}
3393
3394int ggml_cpu_has_fma(void) {
3395#if defined(__FMA__)
3396 return 1;
3397#else
3398 return 0;
3399#endif
3400}
3401
3402int ggml_cpu_has_arm_fma(void) {
3403#if defined(__ARM_FEATURE_FMA)
3404 return 1;
3405#else
3406 return 0;
3407#endif
3408}
3409
3410int ggml_cpu_has_riscv_v(void) {
3411#if defined(__riscv_v_intrinsic)
3412 return 1;
3413#else
3414 return 0;
3415#endif
3416}
3417
3418int ggml_cpu_has_f16c(void) {
3419#if defined(__F16C__)
3420 return 1;
3421#else
3422 return 0;
3423#endif
3424}
3425
3426int ggml_cpu_has_fp16_va(void) {
3427#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
3428 return 1;
3429#else
3430 return 0;
3431#endif
3432}
3433
3434int ggml_cpu_has_wasm_simd(void) {
3435#if defined(__wasm_simd128__)
3436 return 1;
3437#else
3438 return 0;
3439#endif
3440}
3441
3442int ggml_cpu_has_llamafile(void) {
3443#if defined(GGML_USE_LLAMAFILE)
3444 return 1;
3445#else
3446 return 0;
3447#endif
3448}
3449
3450int ggml_cpu_has_sse3(void) {
3451#if defined(__SSE3__)
3452 return 1;
3453#else
3454 return 0;
3455#endif
3456}
3457
3458int ggml_cpu_has_ssse3(void) {
3459#if defined(__SSSE3__)
3460 return 1;
3461#else
3462 return 0;
3463#endif
3464}
3465
3466int ggml_cpu_has_vsx(void) {
3467#if defined(__POWER9_VECTOR__)
3468 return 1;
3469#else
3470 return 0;
3471#endif
3472}
3473
3474int ggml_cpu_has_vxe(void) {
3475#if defined(__VXE__) || defined(__VXE2__)
3476 return 1;
3477#else
3478 return 0;
3479#endif
3480}
3481
3482int ggml_cpu_has_neon(void) {
3483#if defined(__ARM_ARCH) && defined(__ARM_NEON)
3484 return 1;
3485#else
3486 return 0;
3487#endif
3488}
3489
3490int ggml_cpu_has_dotprod(void) {
3491#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_DOTPROD)
3492 return 1;
3493#else
3494 return 0;
3495#endif
3496}
3497
3498int ggml_cpu_has_sve(void) {
3499#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SVE)
3500 return 1;
3501#else
3502 return 0;
3503#endif
3504}
3505
3506int ggml_cpu_has_matmul_int8(void) {
3507#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_MATMUL_INT8)
3508 return 1;
3509#else
3510 return 0;
3511#endif
3512}
3513
3514int ggml_cpu_get_sve_cnt(void) {
3515#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SVE)
3516 return ggml_arm_arch_features.sve_cnt;
3517#else
3518 return 0;
3519#endif
3520}
3521
3522int ggml_cpu_has_sme(void) {
3523#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SME)
3524 return 1;
3525#else
3526 return 0;
3527#endif
3528}
3529
3530void ggml_cpu_init(void) {
3531 // needed to initialize ggml_time
3532 {
3533 struct ggml_init_params params = { 0, NULL, false };
3534 struct ggml_context * ctx = ggml_init(params);
3535 ggml_free(ctx);
3536 }
3537
3538 ggml_critical_section_start();
3539
3540 static bool is_first_call = true;
3541
3542 if (is_first_call) {
3543 // initialize GELU, Quick GELU, SILU and EXP F32 tables
3544 {
3545 const uint64_t t_start = ggml_time_us(); UNUSED(t_start);
3546
3547 for (int i = 0; i < (1 << 16); ++i) {
3548 union {
3549 uint16_t u16;
3550 ggml_fp16_t fp16;
3551 } u = {i};
3552 float f = GGML_COMPUTE_FP16_TO_FP32(u.fp16);
3553 ggml_table_f32_f16[i] = f;
3554 ggml_table_gelu_f16[i] = GGML_CPU_FP32_TO_FP16(ggml_gelu_f32(f));
3555 ggml_table_gelu_quick_f16[i] = GGML_CPU_FP32_TO_FP16(ggml_gelu_quick_f32(f));
3556 }
3557
3558 const uint64_t t_end = ggml_time_us(); UNUSED(t_end);
3559
3560 GGML_PRINT_DEBUG("%s: GELU, Quick GELU, SILU and EXP tables initialized in %f ms\n", __func__, (t_end - t_start)/1000.0);
3561
3562#ifdef GGML_USE_OPENMP
3563 //if (!getenv("OMP_WAIT_POLICY")) {
3564 // // set the wait policy to active, so that OpenMP threads don't sleep
3565 // setenv("OMP_WAIT_POLICY", "active", 0)
3566 //}
3567
3568 if (!getenv(name: "KMP_BLOCKTIME")) {
3569 // set the time to wait before sleeping a thread
3570 // this is less aggressive than setting the wait policy to active, but should achieve similar results in most cases
3571#ifdef _WIN32
3572 _putenv_s("KMP_BLOCKTIME", "200"); // 200ms
3573#else
3574 setenv(name: "KMP_BLOCKTIME", value: "200", replace: 0); // 200ms
3575#endif
3576 }
3577#endif
3578 }
3579
3580#if defined(__ARM_ARCH)
3581 ggml_init_arm_arch_features();
3582#endif
3583
3584 is_first_call = false;
3585 }
3586
3587 ggml_critical_section_end();
3588}
3589