1/*
2** FFI C call handling.
3** Copyright (C) 2005-2021 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#include "lj_obj.h"
7
8#if LJ_HASFFI
9
10#include "lj_gc.h"
11#include "lj_err.h"
12#include "lj_tab.h"
13#include "lj_ctype.h"
14#include "lj_cconv.h"
15#include "lj_cdata.h"
16#include "lj_ccall.h"
17#include "lj_trace.h"
18
19/* Target-specific handling of register arguments. */
20#if LJ_TARGET_X86
21/* -- x86 calling conventions --------------------------------------------- */
22
23#if LJ_ABI_WIN
24
25#define CCALL_HANDLE_STRUCTRET \
26 /* Return structs bigger than 8 by reference (on stack only). */ \
27 cc->retref = (sz > 8); \
28 if (cc->retref) cc->stack[nsp++] = (GPRArg)dp;
29
30#define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
31
32#else
33
34#if LJ_TARGET_OSX
35
36#define CCALL_HANDLE_STRUCTRET \
37 /* Return structs of size 1, 2, 4 or 8 in registers. */ \
38 cc->retref = !(sz == 1 || sz == 2 || sz == 4 || sz == 8); \
39 if (cc->retref) { \
40 if (ngpr < maxgpr) \
41 cc->gpr[ngpr++] = (GPRArg)dp; \
42 else \
43 cc->stack[nsp++] = (GPRArg)dp; \
44 } else { /* Struct with single FP field ends up in FPR. */ \
45 cc->resx87 = ccall_classify_struct(cts, ctr); \
46 }
47
48#define CCALL_HANDLE_STRUCTRET2 \
49 if (cc->resx87) sp = (uint8_t *)&cc->fpr[0]; \
50 memcpy(dp, sp, ctr->size);
51
52#else
53
54#define CCALL_HANDLE_STRUCTRET \
55 cc->retref = 1; /* Return all structs by reference (in reg or on stack). */ \
56 if (ngpr < maxgpr) \
57 cc->gpr[ngpr++] = (GPRArg)dp; \
58 else \
59 cc->stack[nsp++] = (GPRArg)dp;
60
61#endif
62
63#define CCALL_HANDLE_COMPLEXRET \
64 /* Return complex float in GPRs and complex double by reference. */ \
65 cc->retref = (sz > 8); \
66 if (cc->retref) { \
67 if (ngpr < maxgpr) \
68 cc->gpr[ngpr++] = (GPRArg)dp; \
69 else \
70 cc->stack[nsp++] = (GPRArg)dp; \
71 }
72
73#endif
74
75#define CCALL_HANDLE_COMPLEXRET2 \
76 if (!cc->retref) \
77 *(int64_t *)dp = *(int64_t *)sp; /* Copy complex float from GPRs. */
78
79#define CCALL_HANDLE_STRUCTARG \
80 ngpr = maxgpr; /* Pass all structs by value on the stack. */
81
82#define CCALL_HANDLE_COMPLEXARG \
83 isfp = 1; /* Pass complex by value on stack. */
84
85#define CCALL_HANDLE_REGARG \
86 if (!isfp) { /* Only non-FP values may be passed in registers. */ \
87 if (n > 1) { /* Anything > 32 bit is passed on the stack. */ \
88 if (!LJ_ABI_WIN) ngpr = maxgpr; /* Prevent reordering. */ \
89 } else if (ngpr + 1 <= maxgpr) { \
90 dp = &cc->gpr[ngpr]; \
91 ngpr += n; \
92 goto done; \
93 } \
94 }
95
96#elif LJ_TARGET_X64 && LJ_ABI_WIN
97/* -- Windows/x64 calling conventions ------------------------------------- */
98
99#define CCALL_HANDLE_STRUCTRET \
100 /* Return structs of size 1, 2, 4 or 8 in a GPR. */ \
101 cc->retref = !(sz == 1 || sz == 2 || sz == 4 || sz == 8); \
102 if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
103
104#define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
105
106#define CCALL_HANDLE_COMPLEXRET2 \
107 if (!cc->retref) \
108 *(int64_t *)dp = *(int64_t *)sp; /* Copy complex float from GPRs. */
109
110#define CCALL_HANDLE_STRUCTARG \
111 /* Pass structs of size 1, 2, 4 or 8 in a GPR by value. */ \
112 if (!(sz == 1 || sz == 2 || sz == 4 || sz == 8)) { \
113 rp = cdataptr(lj_cdata_new(cts, did, sz)); \
114 sz = CTSIZE_PTR; /* Pass all other structs by reference. */ \
115 }
116
117#define CCALL_HANDLE_COMPLEXARG \
118 /* Pass complex float in a GPR and complex double by reference. */ \
119 if (sz != 2*sizeof(float)) { \
120 rp = cdataptr(lj_cdata_new(cts, did, sz)); \
121 sz = CTSIZE_PTR; \
122 }
123
124/* Windows/x64 argument registers are strictly positional (use ngpr). */
125#define CCALL_HANDLE_REGARG \
126 if (isfp) { \
127 if (ngpr < maxgpr) { dp = &cc->fpr[ngpr++]; nfpr = ngpr; goto done; } \
128 } else { \
129 if (ngpr < maxgpr) { dp = &cc->gpr[ngpr++]; goto done; } \
130 }
131
132#elif LJ_TARGET_X64
133/* -- POSIX/x64 calling conventions --------------------------------------- */
134
135#define CCALL_HANDLE_STRUCTRET \
136 int rcl[2]; rcl[0] = rcl[1] = 0; \
137 if (ccall_classify_struct(cts, ctr, rcl, 0)) { \
138 cc->retref = 1; /* Return struct by reference. */ \
139 cc->gpr[ngpr++] = (GPRArg)dp; \
140 } else { \
141 cc->retref = 0; /* Return small structs in registers. */ \
142 }
143
144#define CCALL_HANDLE_STRUCTRET2 \
145 int rcl[2]; rcl[0] = rcl[1] = 0; \
146 ccall_classify_struct(cts, ctr, rcl, 0); \
147 ccall_struct_ret(cc, rcl, dp, ctr->size);
148
149#define CCALL_HANDLE_COMPLEXRET \
150 /* Complex values are returned in one or two FPRs. */ \
151 cc->retref = 0;
152
153#define CCALL_HANDLE_COMPLEXRET2 \
154 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPR. */ \
155 *(int64_t *)dp = cc->fpr[0].l[0]; \
156 } else { /* Copy non-contiguous complex double from FPRs. */ \
157 ((int64_t *)dp)[0] = cc->fpr[0].l[0]; \
158 ((int64_t *)dp)[1] = cc->fpr[1].l[0]; \
159 }
160
161#define CCALL_HANDLE_STRUCTARG \
162 int rcl[2]; rcl[0] = rcl[1] = 0; \
163 if (!ccall_classify_struct(cts, d, rcl, 0)) { \
164 cc->nsp = nsp; cc->ngpr = ngpr; cc->nfpr = nfpr; \
165 if (ccall_struct_arg(cc, cts, d, rcl, o, narg)) goto err_nyi; \
166 nsp = cc->nsp; ngpr = cc->ngpr; nfpr = cc->nfpr; \
167 continue; \
168 } /* Pass all other structs by value on stack. */
169
170#define CCALL_HANDLE_COMPLEXARG \
171 isfp = 2; /* Pass complex in FPRs or on stack. Needs postprocessing. */
172
173#define CCALL_HANDLE_REGARG \
174 if (isfp) { /* Try to pass argument in FPRs. */ \
175 int n2 = ctype_isvector(d->info) ? 1 : n; \
176 if (nfpr + n2 <= CCALL_NARG_FPR) { \
177 dp = &cc->fpr[nfpr]; \
178 nfpr += n2; \
179 goto done; \
180 } \
181 } else { /* Try to pass argument in GPRs. */ \
182 /* Note that reordering is explicitly allowed in the x64 ABI. */ \
183 if (n <= 2 && ngpr + n <= maxgpr) { \
184 dp = &cc->gpr[ngpr]; \
185 ngpr += n; \
186 goto done; \
187 } \
188 }
189
190#elif LJ_TARGET_ARM
191/* -- ARM calling conventions --------------------------------------------- */
192
193#if LJ_ABI_SOFTFP
194
195#define CCALL_HANDLE_STRUCTRET \
196 /* Return structs of size <= 4 in a GPR. */ \
197 cc->retref = !(sz <= 4); \
198 if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
199
200#define CCALL_HANDLE_COMPLEXRET \
201 cc->retref = 1; /* Return all complex values by reference. */ \
202 cc->gpr[ngpr++] = (GPRArg)dp;
203
204#define CCALL_HANDLE_COMPLEXRET2 \
205 UNUSED(dp); /* Nothing to do. */
206
207#define CCALL_HANDLE_STRUCTARG \
208 /* Pass all structs by value in registers and/or on the stack. */
209
210#define CCALL_HANDLE_COMPLEXARG \
211 /* Pass complex by value in 2 or 4 GPRs. */
212
213#define CCALL_HANDLE_REGARG_FP1
214#define CCALL_HANDLE_REGARG_FP2
215
216#else
217
218#define CCALL_HANDLE_STRUCTRET \
219 cc->retref = !ccall_classify_struct(cts, ctr, ct); \
220 if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
221
222#define CCALL_HANDLE_STRUCTRET2 \
223 if (ccall_classify_struct(cts, ctr, ct) > 1) sp = (uint8_t *)&cc->fpr[0]; \
224 memcpy(dp, sp, ctr->size);
225
226#define CCALL_HANDLE_COMPLEXRET \
227 if (!(ct->info & CTF_VARARG)) cc->retref = 0; /* Return complex in FPRs. */
228
229#define CCALL_HANDLE_COMPLEXRET2 \
230 if (!(ct->info & CTF_VARARG)) memcpy(dp, &cc->fpr[0], ctr->size);
231
232#define CCALL_HANDLE_STRUCTARG \
233 isfp = (ccall_classify_struct(cts, d, ct) > 1);
234 /* Pass all structs by value in registers and/or on the stack. */
235
236#define CCALL_HANDLE_COMPLEXARG \
237 isfp = 1; /* Pass complex by value in FPRs or on stack. */
238
239#define CCALL_HANDLE_REGARG_FP1 \
240 if (isfp && !(ct->info & CTF_VARARG)) { \
241 if ((d->info & CTF_ALIGN) > CTALIGN_PTR) { \
242 if (nfpr + (n >> 1) <= CCALL_NARG_FPR) { \
243 dp = &cc->fpr[nfpr]; \
244 nfpr += (n >> 1); \
245 goto done; \
246 } \
247 } else { \
248 if (sz > 1 && fprodd != nfpr) fprodd = 0; \
249 if (fprodd) { \
250 if (2*nfpr+n <= 2*CCALL_NARG_FPR+1) { \
251 dp = (void *)&cc->fpr[fprodd-1].f[1]; \
252 nfpr += (n >> 1); \
253 if ((n & 1)) fprodd = 0; else fprodd = nfpr-1; \
254 goto done; \
255 } \
256 } else { \
257 if (2*nfpr+n <= 2*CCALL_NARG_FPR) { \
258 dp = (void *)&cc->fpr[nfpr]; \
259 nfpr += (n >> 1); \
260 if ((n & 1)) fprodd = ++nfpr; else fprodd = 0; \
261 goto done; \
262 } \
263 } \
264 } \
265 fprodd = 0; /* No reordering after the first FP value is on stack. */ \
266 } else {
267
268#define CCALL_HANDLE_REGARG_FP2 }
269
270#endif
271
272#define CCALL_HANDLE_REGARG \
273 CCALL_HANDLE_REGARG_FP1 \
274 if ((d->info & CTF_ALIGN) > CTALIGN_PTR) { \
275 if (ngpr < maxgpr) \
276 ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
277 } \
278 if (ngpr < maxgpr) { \
279 dp = &cc->gpr[ngpr]; \
280 if (ngpr + n > maxgpr) { \
281 nsp += ngpr + n - maxgpr; /* Assumes contiguous gpr/stack fields. */ \
282 if (nsp > CCALL_MAXSTACK) goto err_nyi; /* Too many arguments. */ \
283 ngpr = maxgpr; \
284 } else { \
285 ngpr += n; \
286 } \
287 goto done; \
288 } CCALL_HANDLE_REGARG_FP2
289
290#define CCALL_HANDLE_RET \
291 if ((ct->info & CTF_VARARG)) sp = (uint8_t *)&cc->gpr[0];
292
293#elif LJ_TARGET_ARM64
294/* -- ARM64 calling conventions ------------------------------------------- */
295
296#define CCALL_HANDLE_STRUCTRET \
297 cc->retref = !ccall_classify_struct(cts, ctr); \
298 if (cc->retref) cc->retp = dp;
299
300#define CCALL_HANDLE_STRUCTRET2 \
301 unsigned int cl = ccall_classify_struct(cts, ctr); \
302 if ((cl & 4)) { /* Combine float HFA from separate registers. */ \
303 CTSize i = (cl >> 8) - 1; \
304 do { ((uint32_t *)dp)[i] = cc->fpr[i].lo; } while (i--); \
305 } else { \
306 if (cl > 1) sp = (uint8_t *)&cc->fpr[0]; \
307 memcpy(dp, sp, ctr->size); \
308 }
309
310#define CCALL_HANDLE_COMPLEXRET \
311 /* Complex values are returned in one or two FPRs. */ \
312 cc->retref = 0;
313
314#define CCALL_HANDLE_COMPLEXRET2 \
315 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPRs. */ \
316 ((float *)dp)[0] = cc->fpr[0].f; \
317 ((float *)dp)[1] = cc->fpr[1].f; \
318 } else { /* Copy complex double from FPRs. */ \
319 ((double *)dp)[0] = cc->fpr[0].d; \
320 ((double *)dp)[1] = cc->fpr[1].d; \
321 }
322
323#define CCALL_HANDLE_STRUCTARG \
324 unsigned int cl = ccall_classify_struct(cts, d); \
325 if (cl == 0) { /* Pass struct by reference. */ \
326 rp = cdataptr(lj_cdata_new(cts, did, sz)); \
327 sz = CTSIZE_PTR; \
328 } else if (cl > 1) { /* Pass struct in FPRs or on stack. */ \
329 isfp = (cl & 4) ? 2 : 1; \
330 } /* else: Pass struct in GPRs or on stack. */
331
332#define CCALL_HANDLE_COMPLEXARG \
333 /* Pass complex by value in separate (!) FPRs or on stack. */ \
334 isfp = sz == 2*sizeof(float) ? 2 : 1;
335
336#define CCALL_HANDLE_REGARG \
337 if (LJ_TARGET_IOS && isva) { \
338 /* IOS: All variadic arguments are on the stack. */ \
339 } else if (isfp) { /* Try to pass argument in FPRs. */ \
340 int n2 = ctype_isvector(d->info) ? 1 : \
341 isfp == 1 ? n : (d->size >> (4-isfp)); \
342 if (nfpr + n2 <= CCALL_NARG_FPR) { \
343 dp = &cc->fpr[nfpr]; \
344 nfpr += n2; \
345 goto done; \
346 } else { \
347 nfpr = CCALL_NARG_FPR; /* Prevent reordering. */ \
348 if (LJ_TARGET_IOS && d->size < 8) goto err_nyi; \
349 } \
350 } else { /* Try to pass argument in GPRs. */ \
351 if (!LJ_TARGET_IOS && (d->info & CTF_ALIGN) > CTALIGN_PTR) \
352 ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
353 if (ngpr + n <= maxgpr) { \
354 dp = &cc->gpr[ngpr]; \
355 ngpr += n; \
356 goto done; \
357 } else { \
358 ngpr = maxgpr; /* Prevent reordering. */ \
359 if (LJ_TARGET_IOS && d->size < 8) goto err_nyi; \
360 } \
361 }
362
363#if LJ_BE
364#define CCALL_HANDLE_RET \
365 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
366 sp = (uint8_t *)&cc->fpr[0].f;
367#endif
368
369
370#elif LJ_TARGET_PPC
371/* -- PPC calling conventions --------------------------------------------- */
372
373#define CCALL_HANDLE_STRUCTRET \
374 cc->retref = 1; /* Return all structs by reference. */ \
375 cc->gpr[ngpr++] = (GPRArg)dp;
376
377#define CCALL_HANDLE_COMPLEXRET \
378 /* Complex values are returned in 2 or 4 GPRs. */ \
379 cc->retref = 0;
380
381#define CCALL_HANDLE_COMPLEXRET2 \
382 memcpy(dp, sp, ctr->size); /* Copy complex from GPRs. */
383
384#define CCALL_HANDLE_STRUCTARG \
385 rp = cdataptr(lj_cdata_new(cts, did, sz)); \
386 sz = CTSIZE_PTR; /* Pass all structs by reference. */
387
388#define CCALL_HANDLE_COMPLEXARG \
389 /* Pass complex by value in 2 or 4 GPRs. */
390
391#define CCALL_HANDLE_GPR \
392 /* Try to pass argument in GPRs. */ \
393 if (n > 1) { \
394 /* int64_t or complex (float). */ \
395 lj_assertL(n == 2 || n == 4, "bad GPR size %d", n); \
396 if (ctype_isinteger(d->info) || ctype_isfp(d->info)) \
397 ngpr = (ngpr + 1u) & ~1u; /* Align int64_t to regpair. */ \
398 else if (ngpr + n > maxgpr) \
399 ngpr = maxgpr; /* Prevent reordering. */ \
400 } \
401 if (ngpr + n <= maxgpr) { \
402 dp = &cc->gpr[ngpr]; \
403 ngpr += n; \
404 goto done; \
405 } \
406
407#if LJ_ABI_SOFTFP
408#define CCALL_HANDLE_REGARG CCALL_HANDLE_GPR
409#else
410#define CCALL_HANDLE_REGARG \
411 if (isfp) { /* Try to pass argument in FPRs. */ \
412 if (nfpr + 1 <= CCALL_NARG_FPR) { \
413 dp = &cc->fpr[nfpr]; \
414 nfpr += 1; \
415 d = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */ \
416 goto done; \
417 } \
418 } else { \
419 CCALL_HANDLE_GPR \
420 }
421#endif
422
423#if !LJ_ABI_SOFTFP
424#define CCALL_HANDLE_RET \
425 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
426 ctr = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */
427#endif
428
429#elif LJ_TARGET_MIPS32
430/* -- MIPS o32 calling conventions ---------------------------------------- */
431
432#define CCALL_HANDLE_STRUCTRET \
433 cc->retref = 1; /* Return all structs by reference. */ \
434 cc->gpr[ngpr++] = (GPRArg)dp;
435
436#define CCALL_HANDLE_COMPLEXRET \
437 /* Complex values are returned in 1 or 2 FPRs. */ \
438 cc->retref = 0;
439
440#if LJ_ABI_SOFTFP
441#define CCALL_HANDLE_COMPLEXRET2 \
442 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from GPRs. */ \
443 ((intptr_t *)dp)[0] = cc->gpr[0]; \
444 ((intptr_t *)dp)[1] = cc->gpr[1]; \
445 } else { /* Copy complex double from GPRs. */ \
446 ((intptr_t *)dp)[0] = cc->gpr[0]; \
447 ((intptr_t *)dp)[1] = cc->gpr[1]; \
448 ((intptr_t *)dp)[2] = cc->gpr[2]; \
449 ((intptr_t *)dp)[3] = cc->gpr[3]; \
450 }
451#else
452#define CCALL_HANDLE_COMPLEXRET2 \
453 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPRs. */ \
454 ((float *)dp)[0] = cc->fpr[0].f; \
455 ((float *)dp)[1] = cc->fpr[1].f; \
456 } else { /* Copy complex double from FPRs. */ \
457 ((double *)dp)[0] = cc->fpr[0].d; \
458 ((double *)dp)[1] = cc->fpr[1].d; \
459 }
460#endif
461
462#define CCALL_HANDLE_STRUCTARG \
463 /* Pass all structs by value in registers and/or on the stack. */
464
465#define CCALL_HANDLE_COMPLEXARG \
466 /* Pass complex by value in 2 or 4 GPRs. */
467
468#define CCALL_HANDLE_GPR \
469 if ((d->info & CTF_ALIGN) > CTALIGN_PTR) \
470 ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
471 if (ngpr < maxgpr) { \
472 dp = &cc->gpr[ngpr]; \
473 if (ngpr + n > maxgpr) { \
474 nsp += ngpr + n - maxgpr; /* Assumes contiguous gpr/stack fields. */ \
475 if (nsp > CCALL_MAXSTACK) goto err_nyi; /* Too many arguments. */ \
476 ngpr = maxgpr; \
477 } else { \
478 ngpr += n; \
479 } \
480 goto done; \
481 }
482
483#if !LJ_ABI_SOFTFP /* MIPS32 hard-float */
484#define CCALL_HANDLE_REGARG \
485 if (isfp && nfpr < CCALL_NARG_FPR && !(ct->info & CTF_VARARG)) { \
486 /* Try to pass argument in FPRs. */ \
487 dp = n == 1 ? (void *)&cc->fpr[nfpr].f : (void *)&cc->fpr[nfpr].d; \
488 nfpr++; ngpr += n; \
489 goto done; \
490 } else { /* Try to pass argument in GPRs. */ \
491 nfpr = CCALL_NARG_FPR; \
492 CCALL_HANDLE_GPR \
493 }
494#else /* MIPS32 soft-float */
495#define CCALL_HANDLE_REGARG CCALL_HANDLE_GPR
496#endif
497
498#if !LJ_ABI_SOFTFP
499/* On MIPS64 soft-float, position of float return values is endian-dependant. */
500#define CCALL_HANDLE_RET \
501 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
502 sp = (uint8_t *)&cc->fpr[0].f;
503#endif
504
505#elif LJ_TARGET_MIPS64
506/* -- MIPS n64 calling conventions ---------------------------------------- */
507
508#define CCALL_HANDLE_STRUCTRET \
509 cc->retref = !(sz <= 16); \
510 if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
511
512#define CCALL_HANDLE_STRUCTRET2 \
513 ccall_copy_struct(cc, ctr, dp, sp, ccall_classify_struct(cts, ctr, ct));
514
515#define CCALL_HANDLE_COMPLEXRET \
516 /* Complex values are returned in 1 or 2 FPRs. */ \
517 cc->retref = 0;
518
519#if LJ_ABI_SOFTFP /* MIPS64 soft-float */
520
521#define CCALL_HANDLE_COMPLEXRET2 \
522 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from GPRs. */ \
523 ((intptr_t *)dp)[0] = cc->gpr[0]; \
524 } else { /* Copy complex double from GPRs. */ \
525 ((intptr_t *)dp)[0] = cc->gpr[0]; \
526 ((intptr_t *)dp)[1] = cc->gpr[1]; \
527 }
528
529#define CCALL_HANDLE_COMPLEXARG \
530 /* Pass complex by value in 2 or 4 GPRs. */
531
532/* Position of soft-float 'float' return value depends on endianess. */
533#define CCALL_HANDLE_RET \
534 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
535 sp = (uint8_t *)cc->gpr + LJ_ENDIAN_SELECT(0, 4);
536
537#else /* MIPS64 hard-float */
538
539#define CCALL_HANDLE_COMPLEXRET2 \
540 if (ctr->size == 2*sizeof(float)) { /* Copy complex float from FPRs. */ \
541 ((float *)dp)[0] = cc->fpr[0].f; \
542 ((float *)dp)[1] = cc->fpr[1].f; \
543 } else { /* Copy complex double from FPRs. */ \
544 ((double *)dp)[0] = cc->fpr[0].d; \
545 ((double *)dp)[1] = cc->fpr[1].d; \
546 }
547
548#define CCALL_HANDLE_COMPLEXARG \
549 if (sz == 2*sizeof(float)) { \
550 isfp = 2; \
551 if (ngpr < maxgpr) \
552 sz *= 2; \
553 }
554
555#define CCALL_HANDLE_RET \
556 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
557 sp = (uint8_t *)&cc->fpr[0].f;
558
559#endif
560
561#define CCALL_HANDLE_STRUCTARG \
562 /* Pass all structs by value in registers and/or on the stack. */
563
564#define CCALL_HANDLE_REGARG \
565 if (ngpr < maxgpr) { \
566 dp = &cc->gpr[ngpr]; \
567 if (ngpr + n > maxgpr) { \
568 nsp += ngpr + n - maxgpr; /* Assumes contiguous gpr/stack fields. */ \
569 if (nsp > CCALL_MAXSTACK) goto err_nyi; /* Too many arguments. */ \
570 ngpr = maxgpr; \
571 } else { \
572 ngpr += n; \
573 } \
574 goto done; \
575 }
576
577#else
578#error "Missing calling convention definitions for this architecture"
579#endif
580
581#ifndef CCALL_HANDLE_STRUCTRET2
582#define CCALL_HANDLE_STRUCTRET2 \
583 memcpy(dp, sp, ctr->size); /* Copy struct return value from GPRs. */
584#endif
585
586/* -- x86 OSX ABI struct classification ----------------------------------- */
587
588#if LJ_TARGET_X86 && LJ_TARGET_OSX
589
590/* Check for struct with single FP field. */
591static int ccall_classify_struct(CTState *cts, CType *ct)
592{
593 CTSize sz = ct->size;
594 if (!(sz == sizeof(float) || sz == sizeof(double))) return 0;
595 if ((ct->info & CTF_UNION)) return 0;
596 while (ct->sib) {
597 ct = ctype_get(cts, ct->sib);
598 if (ctype_isfield(ct->info)) {
599 CType *sct = ctype_rawchild(cts, ct);
600 if (ctype_isfp(sct->info)) {
601 if (sct->size == sz)
602 return (sz >> 2); /* Return 1 for float or 2 for double. */
603 } else if (ctype_isstruct(sct->info)) {
604 if (sct->size)
605 return ccall_classify_struct(cts, sct);
606 } else {
607 break;
608 }
609 } else if (ctype_isbitfield(ct->info)) {
610 break;
611 } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
612 CType *sct = ctype_rawchild(cts, ct);
613 if (sct->size)
614 return ccall_classify_struct(cts, sct);
615 }
616 }
617 return 0;
618}
619
620#endif
621
622/* -- x64 struct classification ------------------------------------------- */
623
624#if LJ_TARGET_X64 && !LJ_ABI_WIN
625
626/* Register classes for x64 struct classification. */
627#define CCALL_RCL_INT 1
628#define CCALL_RCL_SSE 2
629#define CCALL_RCL_MEM 4
630/* NYI: classify vectors. */
631
632static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs);
633
634/* Classify a C type. */
635static void ccall_classify_ct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
636{
637 if (ctype_isarray(ct->info)) {
638 CType *cct = ctype_rawchild(cts, ct);
639 CTSize eofs, esz = cct->size, asz = ct->size;
640 for (eofs = 0; eofs < asz; eofs += esz)
641 ccall_classify_ct(cts, cct, rcl, ofs+eofs);
642 } else if (ctype_isstruct(ct->info)) {
643 ccall_classify_struct(cts, ct, rcl, ofs);
644 } else {
645 int cl = ctype_isfp(ct->info) ? CCALL_RCL_SSE : CCALL_RCL_INT;
646 lj_assertCTS(ctype_hassize(ct->info),
647 "classify ctype %08x without size", ct->info);
648 if ((ofs & (ct->size-1))) cl = CCALL_RCL_MEM; /* Unaligned. */
649 rcl[(ofs >= 8)] |= cl;
650 }
651}
652
653/* Recursively classify a struct based on its fields. */
654static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
655{
656 if (ct->size > 16) return CCALL_RCL_MEM; /* Too big, gets memory class. */
657 while (ct->sib) {
658 CTSize fofs;
659 ct = ctype_get(cts, ct->sib);
660 fofs = ofs+ct->size;
661 if (ctype_isfield(ct->info))
662 ccall_classify_ct(cts, ctype_rawchild(cts, ct), rcl, fofs);
663 else if (ctype_isbitfield(ct->info))
664 rcl[(fofs >= 8)] |= CCALL_RCL_INT; /* NYI: unaligned bitfields? */
665 else if (ctype_isxattrib(ct->info, CTA_SUBTYPE))
666 ccall_classify_struct(cts, ctype_rawchild(cts, ct), rcl, fofs);
667 }
668 return ((rcl[0]|rcl[1]) & CCALL_RCL_MEM); /* Memory class? */
669}
670
671/* Try to split up a small struct into registers. */
672static int ccall_struct_reg(CCallState *cc, CTState *cts, GPRArg *dp, int *rcl)
673{
674 MSize ngpr = cc->ngpr, nfpr = cc->nfpr;
675 uint32_t i;
676 UNUSED(cts);
677 for (i = 0; i < 2; i++) {
678 lj_assertCTS(!(rcl[i] & CCALL_RCL_MEM), "pass mem struct in reg");
679 if ((rcl[i] & CCALL_RCL_INT)) { /* Integer class takes precedence. */
680 if (ngpr >= CCALL_NARG_GPR) return 1; /* Register overflow. */
681 cc->gpr[ngpr++] = dp[i];
682 } else if ((rcl[i] & CCALL_RCL_SSE)) {
683 if (nfpr >= CCALL_NARG_FPR) return 1; /* Register overflow. */
684 cc->fpr[nfpr++].l[0] = dp[i];
685 }
686 }
687 cc->ngpr = ngpr; cc->nfpr = nfpr;
688 return 0; /* Ok. */
689}
690
691/* Pass a small struct argument. */
692static int ccall_struct_arg(CCallState *cc, CTState *cts, CType *d, int *rcl,
693 TValue *o, int narg)
694{
695 GPRArg dp[2];
696 dp[0] = dp[1] = 0;
697 /* Convert to temp. struct. */
698 lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
699 if (ccall_struct_reg(cc, cts, dp, rcl)) {
700 /* Register overflow? Pass on stack. */
701 MSize nsp = cc->nsp, n = rcl[1] ? 2 : 1;
702 if (nsp + n > CCALL_MAXSTACK) return 1; /* Too many arguments. */
703 cc->nsp = nsp + n;
704 memcpy(&cc->stack[nsp], dp, n*CTSIZE_PTR);
705 }
706 return 0; /* Ok. */
707}
708
709/* Combine returned small struct. */
710static void ccall_struct_ret(CCallState *cc, int *rcl, uint8_t *dp, CTSize sz)
711{
712 GPRArg sp[2];
713 MSize ngpr = 0, nfpr = 0;
714 uint32_t i;
715 for (i = 0; i < 2; i++) {
716 if ((rcl[i] & CCALL_RCL_INT)) { /* Integer class takes precedence. */
717 sp[i] = cc->gpr[ngpr++];
718 } else if ((rcl[i] & CCALL_RCL_SSE)) {
719 sp[i] = cc->fpr[nfpr++].l[0];
720 }
721 }
722 memcpy(dp, sp, sz);
723}
724#endif
725
726/* -- ARM hard-float ABI struct classification ---------------------------- */
727
728#if LJ_TARGET_ARM && !LJ_ABI_SOFTFP
729
730/* Classify a struct based on its fields. */
731static unsigned int ccall_classify_struct(CTState *cts, CType *ct, CType *ctf)
732{
733 CTSize sz = ct->size;
734 unsigned int r = 0, n = 0, isu = (ct->info & CTF_UNION);
735 if ((ctf->info & CTF_VARARG)) goto noth;
736 while (ct->sib) {
737 CType *sct;
738 ct = ctype_get(cts, ct->sib);
739 if (ctype_isfield(ct->info)) {
740 sct = ctype_rawchild(cts, ct);
741 if (ctype_isfp(sct->info)) {
742 r |= sct->size;
743 if (!isu) n++; else if (n == 0) n = 1;
744 } else if (ctype_iscomplex(sct->info)) {
745 r |= (sct->size >> 1);
746 if (!isu) n += 2; else if (n < 2) n = 2;
747 } else if (ctype_isstruct(sct->info)) {
748 goto substruct;
749 } else {
750 goto noth;
751 }
752 } else if (ctype_isbitfield(ct->info)) {
753 goto noth;
754 } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
755 sct = ctype_rawchild(cts, ct);
756 substruct:
757 if (sct->size > 0) {
758 unsigned int s = ccall_classify_struct(cts, sct, ctf);
759 if (s <= 1) goto noth;
760 r |= (s & 255);
761 if (!isu) n += (s >> 8); else if (n < (s >>8)) n = (s >> 8);
762 }
763 }
764 }
765 if ((r == 4 || r == 8) && n <= 4)
766 return r + (n << 8);
767noth: /* Not a homogeneous float/double aggregate. */
768 return (sz <= 4); /* Return structs of size <= 4 in a GPR. */
769}
770
771#endif
772
773/* -- ARM64 ABI struct classification ------------------------------------- */
774
775#if LJ_TARGET_ARM64
776
777/* Classify a struct based on its fields. */
778static unsigned int ccall_classify_struct(CTState *cts, CType *ct)
779{
780 CTSize sz = ct->size;
781 unsigned int r = 0, n = 0, isu = (ct->info & CTF_UNION);
782 while (ct->sib) {
783 CType *sct;
784 ct = ctype_get(cts, ct->sib);
785 if (ctype_isfield(ct->info)) {
786 sct = ctype_rawchild(cts, ct);
787 if (ctype_isfp(sct->info)) {
788 r |= sct->size;
789 if (!isu) n++; else if (n == 0) n = 1;
790 } else if (ctype_iscomplex(sct->info)) {
791 r |= (sct->size >> 1);
792 if (!isu) n += 2; else if (n < 2) n = 2;
793 } else if (ctype_isstruct(sct->info)) {
794 goto substruct;
795 } else {
796 goto noth;
797 }
798 } else if (ctype_isbitfield(ct->info)) {
799 goto noth;
800 } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
801 sct = ctype_rawchild(cts, ct);
802 substruct:
803 if (sct->size > 0) {
804 unsigned int s = ccall_classify_struct(cts, sct);
805 if (s <= 1) goto noth;
806 r |= (s & 255);
807 if (!isu) n += (s >> 8); else if (n < (s >>8)) n = (s >> 8);
808 }
809 }
810 }
811 if ((r == 4 || r == 8) && n <= 4)
812 return r + (n << 8);
813noth: /* Not a homogeneous float/double aggregate. */
814 return (sz <= 16); /* Return structs of size <= 16 in GPRs. */
815}
816
817#endif
818
819/* -- MIPS64 ABI struct classification ---------------------------- */
820
821#if LJ_TARGET_MIPS64
822
823#define FTYPE_FLOAT 1
824#define FTYPE_DOUBLE 2
825
826/* Classify FP fields (max. 2) and their types. */
827static unsigned int ccall_classify_struct(CTState *cts, CType *ct, CType *ctf)
828{
829 int n = 0, ft = 0;
830 if ((ctf->info & CTF_VARARG) || (ct->info & CTF_UNION))
831 goto noth;
832 while (ct->sib) {
833 CType *sct;
834 ct = ctype_get(cts, ct->sib);
835 if (n == 2) {
836 goto noth;
837 } else if (ctype_isfield(ct->info)) {
838 sct = ctype_rawchild(cts, ct);
839 if (ctype_isfp(sct->info)) {
840 ft |= (sct->size == 4 ? FTYPE_FLOAT : FTYPE_DOUBLE) << 2*n;
841 n++;
842 } else {
843 goto noth;
844 }
845 } else if (ctype_isbitfield(ct->info) ||
846 ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
847 goto noth;
848 }
849 }
850 if (n <= 2)
851 return ft;
852noth: /* Not a homogeneous float/double aggregate. */
853 return 0; /* Struct is in GPRs. */
854}
855
856static void ccall_copy_struct(CCallState *cc, CType *ctr, void *dp, void *sp,
857 int ft)
858{
859 if (LJ_ABI_SOFTFP ? ft :
860 ((ft & 3) == FTYPE_FLOAT || (ft >> 2) == FTYPE_FLOAT)) {
861 int i, ofs = 0;
862 for (i = 0; ft != 0; i++, ft >>= 2) {
863 if ((ft & 3) == FTYPE_FLOAT) {
864#if LJ_ABI_SOFTFP
865 /* The 2nd FP struct result is in CARG1 (gpr[2]) and not CRET2. */
866 memcpy((uint8_t *)dp + ofs,
867 (uint8_t *)&cc->gpr[2*i] + LJ_ENDIAN_SELECT(0, 4), 4);
868#else
869 *(float *)((uint8_t *)dp + ofs) = cc->fpr[i].f;
870#endif
871 ofs += 4;
872 } else {
873 ofs = (ofs + 7) & ~7; /* 64 bit alignment. */
874#if LJ_ABI_SOFTFP
875 *(intptr_t *)((uint8_t *)dp + ofs) = cc->gpr[2*i];
876#else
877 *(double *)((uint8_t *)dp + ofs) = cc->fpr[i].d;
878#endif
879 ofs += 8;
880 }
881 }
882 } else {
883#if !LJ_ABI_SOFTFP
884 if (ft) sp = (uint8_t *)&cc->fpr[0];
885#endif
886 memcpy(dp, sp, ctr->size);
887 }
888}
889
890#endif
891
892/* -- Common C call handling ---------------------------------------------- */
893
894/* Infer the destination CTypeID for a vararg argument. */
895CTypeID lj_ccall_ctid_vararg(CTState *cts, cTValue *o)
896{
897 if (tvisnumber(o)) {
898 return CTID_DOUBLE;
899 } else if (tviscdata(o)) {
900 CTypeID id = cdataV(o)->ctypeid;
901 CType *s = ctype_get(cts, id);
902 if (ctype_isrefarray(s->info)) {
903 return lj_ctype_intern(cts,
904 CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(s->info)), CTSIZE_PTR);
905 } else if (ctype_isstruct(s->info) || ctype_isfunc(s->info)) {
906 /* NYI: how to pass a struct by value in a vararg argument? */
907 return lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR);
908 } else if (ctype_isfp(s->info) && s->size == sizeof(float)) {
909 return CTID_DOUBLE;
910 } else {
911 return id;
912 }
913 } else if (tvisstr(o)) {
914 return CTID_P_CCHAR;
915 } else if (tvisbool(o)) {
916 return CTID_BOOL;
917 } else {
918 return CTID_P_VOID;
919 }
920}
921
922/* Setup arguments for C call. */
923static int ccall_set_args(lua_State *L, CTState *cts, CType *ct,
924 CCallState *cc)
925{
926 int gcsteps = 0;
927 TValue *o, *top = L->top;
928 CTypeID fid;
929 CType *ctr;
930 MSize maxgpr, ngpr = 0, nsp = 0, narg;
931#if CCALL_NARG_FPR
932 MSize nfpr = 0;
933#if LJ_TARGET_ARM
934 MSize fprodd = 0;
935#endif
936#endif
937
938 /* Clear unused regs to get some determinism in case of misdeclaration. */
939 memset(cc->gpr, 0, sizeof(cc->gpr));
940#if CCALL_NUM_FPR
941 memset(cc->fpr, 0, sizeof(cc->fpr));
942#endif
943
944#if LJ_TARGET_X86
945 /* x86 has several different calling conventions. */
946 cc->resx87 = 0;
947 switch (ctype_cconv(ct->info)) {
948 case CTCC_FASTCALL: maxgpr = 2; break;
949 case CTCC_THISCALL: maxgpr = 1; break;
950 default: maxgpr = 0; break;
951 }
952#else
953 maxgpr = CCALL_NARG_GPR;
954#endif
955
956 /* Perform required setup for some result types. */
957 ctr = ctype_rawchild(cts, ct);
958 if (ctype_isvector(ctr->info)) {
959 if (!(CCALL_VECTOR_REG && (ctr->size == 8 || ctr->size == 16)))
960 goto err_nyi;
961 } else if (ctype_iscomplex(ctr->info) || ctype_isstruct(ctr->info)) {
962 /* Preallocate cdata object and anchor it after arguments. */
963 CTSize sz = ctr->size;
964 GCcdata *cd = lj_cdata_new(cts, ctype_cid(ct->info), sz);
965 void *dp = cdataptr(cd);
966 setcdataV(L, L->top++, cd);
967 if (ctype_isstruct(ctr->info)) {
968 CCALL_HANDLE_STRUCTRET
969 } else {
970 CCALL_HANDLE_COMPLEXRET
971 }
972#if LJ_TARGET_X86
973 } else if (ctype_isfp(ctr->info)) {
974 cc->resx87 = ctr->size == sizeof(float) ? 1 : 2;
975#endif
976 }
977
978 /* Skip initial attributes. */
979 fid = ct->sib;
980 while (fid) {
981 CType *ctf = ctype_get(cts, fid);
982 if (!ctype_isattrib(ctf->info)) break;
983 fid = ctf->sib;
984 }
985
986 /* Walk through all passed arguments. */
987 for (o = L->base+1, narg = 1; o < top; o++, narg++) {
988 CTypeID did;
989 CType *d;
990 CTSize sz;
991 MSize n, isfp = 0, isva = 0;
992 void *dp, *rp = NULL;
993
994 if (fid) { /* Get argument type from field. */
995 CType *ctf = ctype_get(cts, fid);
996 fid = ctf->sib;
997 lj_assertL(ctype_isfield(ctf->info), "field expected");
998 did = ctype_cid(ctf->info);
999 } else {
1000 if (!(ct->info & CTF_VARARG))
1001 lj_err_caller(L, LJ_ERR_FFI_NUMARG); /* Too many arguments. */
1002 did = lj_ccall_ctid_vararg(cts, o); /* Infer vararg type. */
1003 isva = 1;
1004 }
1005 d = ctype_raw(cts, did);
1006 sz = d->size;
1007
1008 /* Find out how (by value/ref) and where (GPR/FPR) to pass an argument. */
1009 if (ctype_isnum(d->info)) {
1010 if (sz > 8) goto err_nyi;
1011 if ((d->info & CTF_FP))
1012 isfp = 1;
1013 } else if (ctype_isvector(d->info)) {
1014 if (CCALL_VECTOR_REG && (sz == 8 || sz == 16))
1015 isfp = 1;
1016 else
1017 goto err_nyi;
1018 } else if (ctype_isstruct(d->info)) {
1019 CCALL_HANDLE_STRUCTARG
1020 } else if (ctype_iscomplex(d->info)) {
1021 CCALL_HANDLE_COMPLEXARG
1022 } else {
1023 sz = CTSIZE_PTR;
1024 }
1025 sz = (sz + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
1026 n = sz / CTSIZE_PTR; /* Number of GPRs or stack slots needed. */
1027
1028 CCALL_HANDLE_REGARG /* Handle register arguments. */
1029
1030 /* Otherwise pass argument on stack. */
1031 if (CCALL_ALIGN_STACKARG && !rp && (d->info & CTF_ALIGN) > CTALIGN_PTR) {
1032 MSize align = (1u << ctype_align(d->info-CTALIGN_PTR)) -1;
1033 nsp = (nsp + align) & ~align; /* Align argument on stack. */
1034 }
1035 if (nsp + n > CCALL_MAXSTACK) { /* Too many arguments. */
1036 err_nyi:
1037 lj_err_caller(L, LJ_ERR_FFI_NYICALL);
1038 }
1039 dp = &cc->stack[nsp];
1040 nsp += n;
1041 isva = 0;
1042
1043 done:
1044 if (rp) { /* Pass by reference. */
1045 gcsteps++;
1046 *(void **)dp = rp;
1047 dp = rp;
1048 }
1049 lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
1050 /* Extend passed integers to 32 bits at least. */
1051 if (ctype_isinteger_or_bool(d->info) && d->size < 4) {
1052 if (d->info & CTF_UNSIGNED)
1053 *(uint32_t *)dp = d->size == 1 ? (uint32_t)*(uint8_t *)dp :
1054 (uint32_t)*(uint16_t *)dp;
1055 else
1056 *(int32_t *)dp = d->size == 1 ? (int32_t)*(int8_t *)dp :
1057 (int32_t)*(int16_t *)dp;
1058 }
1059#if LJ_TARGET_ARM64 && LJ_BE
1060 if (isfp && d->size == sizeof(float))
1061 ((float *)dp)[1] = ((float *)dp)[0]; /* Floats occupy high slot. */
1062#endif
1063#if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE)
1064 if ((ctype_isinteger_or_bool(d->info) || ctype_isenum(d->info)
1065#if LJ_TARGET_MIPS64
1066 || (isfp && nsp == 0)
1067#endif
1068 ) && d->size <= 4) {
1069 *(int64_t *)dp = (int64_t)*(int32_t *)dp; /* Sign-extend to 64 bit. */
1070 }
1071#endif
1072#if LJ_TARGET_X64 && LJ_ABI_WIN
1073 if (isva) { /* Windows/x64 mirrors varargs in both register sets. */
1074 if (nfpr == ngpr)
1075 cc->gpr[ngpr-1] = cc->fpr[ngpr-1].l[0];
1076 else
1077 cc->fpr[ngpr-1].l[0] = cc->gpr[ngpr-1];
1078 }
1079#else
1080 UNUSED(isva);
1081#endif
1082#if LJ_TARGET_X64 && !LJ_ABI_WIN
1083 if (isfp == 2 && n == 2 && (uint8_t *)dp == (uint8_t *)&cc->fpr[nfpr-2]) {
1084 cc->fpr[nfpr-1].d[0] = cc->fpr[nfpr-2].d[1]; /* Split complex double. */
1085 cc->fpr[nfpr-2].d[1] = 0;
1086 }
1087#elif LJ_TARGET_ARM64 || (LJ_TARGET_MIPS64 && !LJ_ABI_SOFTFP)
1088 if (isfp == 2 && (uint8_t *)dp < (uint8_t *)cc->stack) {
1089 /* Split float HFA or complex float into separate registers. */
1090 CTSize i = (sz >> 2) - 1;
1091 do { ((uint64_t *)dp)[i] = ((uint32_t *)dp)[i]; } while (i--);
1092 }
1093#else
1094 UNUSED(isfp);
1095#endif
1096 }
1097 if (fid) lj_err_caller(L, LJ_ERR_FFI_NUMARG); /* Too few arguments. */
1098
1099#if LJ_TARGET_X64 || (LJ_TARGET_PPC && !LJ_ABI_SOFTFP)
1100 cc->nfpr = nfpr; /* Required for vararg functions. */
1101#endif
1102 cc->nsp = nsp;
1103 cc->spadj = (CCALL_SPS_FREE + CCALL_SPS_EXTRA)*CTSIZE_PTR;
1104 if (nsp > CCALL_SPS_FREE)
1105 cc->spadj += (((nsp-CCALL_SPS_FREE)*CTSIZE_PTR + 15u) & ~15u);
1106 return gcsteps;
1107}
1108
1109/* Get results from C call. */
1110static int ccall_get_results(lua_State *L, CTState *cts, CType *ct,
1111 CCallState *cc, int *ret)
1112{
1113 CType *ctr = ctype_rawchild(cts, ct);
1114 uint8_t *sp = (uint8_t *)&cc->gpr[0];
1115 if (ctype_isvoid(ctr->info)) {
1116 *ret = 0; /* Zero results. */
1117 return 0; /* No additional GC step. */
1118 }
1119 *ret = 1; /* One result. */
1120 if (ctype_isstruct(ctr->info)) {
1121 /* Return cdata object which is already on top of stack. */
1122 if (!cc->retref) {
1123 void *dp = cdataptr(cdataV(L->top-1)); /* Use preallocated object. */
1124 CCALL_HANDLE_STRUCTRET2
1125 }
1126 return 1; /* One GC step. */
1127 }
1128 if (ctype_iscomplex(ctr->info)) {
1129 /* Return cdata object which is already on top of stack. */
1130 void *dp = cdataptr(cdataV(L->top-1)); /* Use preallocated object. */
1131 CCALL_HANDLE_COMPLEXRET2
1132 return 1; /* One GC step. */
1133 }
1134 if (LJ_BE && ctr->size < CTSIZE_PTR &&
1135 (ctype_isinteger_or_bool(ctr->info) || ctype_isenum(ctr->info)))
1136 sp += (CTSIZE_PTR - ctr->size);
1137#if CCALL_NUM_FPR
1138 if (ctype_isfp(ctr->info) || ctype_isvector(ctr->info))
1139 sp = (uint8_t *)&cc->fpr[0];
1140#endif
1141#ifdef CCALL_HANDLE_RET
1142 CCALL_HANDLE_RET
1143#endif
1144 /* No reference types end up here, so there's no need for the CTypeID. */
1145 lj_assertL(!(ctype_isrefarray(ctr->info) || ctype_isstruct(ctr->info)),
1146 "unexpected reference ctype");
1147 return lj_cconv_tv_ct(cts, ctr, 0, L->top-1, sp);
1148}
1149
1150/* Call C function. */
1151int lj_ccall_func(lua_State *L, GCcdata *cd)
1152{
1153 CTState *cts = ctype_cts(L);
1154 CType *ct = ctype_raw(cts, cd->ctypeid);
1155 CTSize sz = CTSIZE_PTR;
1156 if (ctype_isptr(ct->info)) {
1157 sz = ct->size;
1158 ct = ctype_rawchild(cts, ct);
1159 }
1160 if (ctype_isfunc(ct->info)) {
1161 CCallState cc;
1162 int gcsteps, ret;
1163 cc.func = (void (*)(void))cdata_getptr(cdataptr(cd), sz);
1164 gcsteps = ccall_set_args(L, cts, ct, &cc);
1165 ct = (CType *)((intptr_t)ct-(intptr_t)cts->tab);
1166 cts->cb.slot = ~0u;
1167 lj_vm_ffi_call(&cc);
1168 if (cts->cb.slot != ~0u) { /* Blacklist function that called a callback. */
1169 TValue tv;
1170 tv.u64 = ((uintptr_t)(void *)cc.func >> 2) | U64x(800000000, 00000000);
1171 setboolV(lj_tab_set(L, cts->miscmap, &tv), 1);
1172 }
1173 ct = (CType *)((intptr_t)ct+(intptr_t)cts->tab); /* May be reallocated. */
1174 gcsteps += ccall_get_results(L, cts, ct, &cc, &ret);
1175#if LJ_TARGET_X86 && LJ_ABI_WIN
1176 /* Automatically detect __stdcall and fix up C function declaration. */
1177 if (cc.spadj && ctype_cconv(ct->info) == CTCC_CDECL) {
1178 CTF_INSERT(ct->info, CCONV, CTCC_STDCALL);
1179 lj_trace_abort(G(L));
1180 }
1181#endif
1182 while (gcsteps-- > 0)
1183 lj_gc_check(L);
1184 return ret;
1185 }
1186 return -1; /* Not a function. */
1187}
1188
1189#endif
1190