1#pragma once
2
3#include "ggml.h"
4
5#include <stddef.h>
6#include <stdint.h>
7
8// !!! Internal header, to be used by mtmd only !!!
9
10struct clip_ctx;
11
12struct clip_image_size {
13 int width;
14 int height;
15};
16
17struct clip_image_f32;
18struct clip_image_u8_batch;
19struct clip_image_f32_batch;
20
21enum clip_modality {
22 CLIP_MODALITY_VISION,
23 CLIP_MODALITY_AUDIO,
24};
25
26enum clip_flash_attn_type {
27 CLIP_FLASH_ATTN_TYPE_AUTO = -1,
28 CLIP_FLASH_ATTN_TYPE_DISABLED = 0,
29 CLIP_FLASH_ATTN_TYPE_ENABLED = 1,
30};
31
32struct clip_context_params {
33 bool use_gpu;
34 enum ggml_log_level verbosity;
35 enum clip_flash_attn_type flash_attn_type;
36 int image_min_tokens;
37 int image_max_tokens;
38};
39
40struct clip_init_result {
41 struct clip_ctx * ctx_v; // vision context
42 struct clip_ctx * ctx_a; // audio context
43};
44
45struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params);
46
47void clip_free(struct clip_ctx * ctx);
48
49size_t clip_embd_nbytes(const struct clip_ctx * ctx);
50size_t clip_embd_nbytes_by_img(const struct clip_ctx * ctx, int img_w, int img_h);
51
52int32_t clip_get_image_size (const struct clip_ctx * ctx);
53int32_t clip_get_patch_size (const struct clip_ctx * ctx);
54int32_t clip_get_hidden_size(const struct clip_ctx * ctx);
55
56// TODO: should be enum, not string
57const char * clip_patch_merge_type(const struct clip_ctx * ctx);
58
59int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * img);
60
61// for M-RoPE, this will be the number of token positions in X and Y directions
62// for other models, X will be the total number of tokens and Y will be 1
63int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 * img);
64int clip_n_output_tokens_y(const struct clip_ctx * ctx, struct clip_image_f32 * img);
65
66// this should be equal to the embedding dimension of the text model
67int clip_n_mmproj_embd(const struct clip_ctx * ctx);
68
69struct clip_image_size * clip_image_size_init(void);
70struct clip_image_u8 * clip_image_u8_init (void);
71struct clip_image_f32 * clip_image_f32_init(void);
72struct clip_image_f32_batch * clip_image_f32_batch_init(void); // only used by libllava
73
74// nx, ny are the output image dimensions
75unsigned char * clip_image_u8_get_data(struct clip_image_u8 * img, uint32_t * nx, uint32_t * ny);
76
77void clip_image_size_free (struct clip_image_size * img_size);
78void clip_image_u8_free (struct clip_image_u8 * img);
79void clip_image_f32_free(struct clip_image_f32 * img);
80void clip_image_u8_batch_free (struct clip_image_u8_batch * batch);
81void clip_image_f32_batch_free(struct clip_image_f32_batch * batch);
82
83// use for accessing underlay data of clip_image_f32_batch
84size_t clip_image_f32_batch_n_images(const struct clip_image_f32_batch * batch); // equivalent to batch->size()
85size_t clip_image_f32_batch_nx(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->nx
86size_t clip_image_f32_batch_ny(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->ny
87struct clip_image_f32 * clip_image_f32_get_img(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->data
88
89/**
90 * Build image from pixels decoded by other libraries instead of stb_image.h for better performance.
91 * The memory layout is RGBRGBRGB..., input buffer length must be 3*nx*ny bytes
92 */
93void clip_build_img_from_pixels(const unsigned char * rgb_pixels, int nx, int ny, struct clip_image_u8 * img);
94
95/** preprocess img and store the result in res_imgs, pad_to_square may be overridden to false depending on model configuration */
96bool clip_image_preprocess(struct clip_ctx * ctx, const struct clip_image_u8 * img, struct clip_image_f32_batch * res_imgs );
97
98struct ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx);
99
100bool clip_image_encode (struct clip_ctx * ctx, int n_threads, struct clip_image_f32 * img, float * vec);
101bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, float * vec);
102
103int clip_is_minicpmv(const struct clip_ctx * ctx);
104bool clip_is_glm(const struct clip_ctx * ctx);
105bool clip_is_qwen2vl(const struct clip_ctx * ctx);
106bool clip_is_llava(const struct clip_ctx * ctx);
107bool clip_is_gemma3(const struct clip_ctx * ctx);
108
109bool clip_encode_float_image (struct clip_ctx * ctx, int n_threads, float * img, int h, int w, float * vec);
110
111// use by audio input
112void clip_image_f32_batch_add_mel(struct clip_image_f32_batch * batch, int n_mel, int n_frames, float * mel);
113
114bool clip_has_vision_encoder(const struct clip_ctx * ctx);
115bool clip_has_audio_encoder(const struct clip_ctx * ctx);
116bool clip_has_whisper_encoder(const struct clip_ctx * ctx);
117