1/*
2** Trace recorder for C data operations.
3** Copyright (C) 2005-2014 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#define lj_ffrecord_c
7#define LUA_CORE
8
9#include "lj_obj.h"
10
11#if LJ_HASJIT && LJ_HASFFI
12
13#include "lj_err.h"
14#include "lj_str.h"
15#include "lj_tab.h"
16#include "lj_frame.h"
17#include "lj_ctype.h"
18#include "lj_cdata.h"
19#include "lj_cparse.h"
20#include "lj_cconv.h"
21#include "lj_clib.h"
22#include "lj_ccall.h"
23#include "lj_ff.h"
24#include "lj_ir.h"
25#include "lj_jit.h"
26#include "lj_ircall.h"
27#include "lj_iropt.h"
28#include "lj_trace.h"
29#include "lj_record.h"
30#include "lj_ffrecord.h"
31#include "lj_snap.h"
32#include "lj_crecord.h"
33#include "lj_dispatch.h"
34
35/* Some local macros to save typing. Undef'd at the end. */
36#define IR(ref) (&J->cur.ir[(ref)])
37
38/* Pass IR on to next optimization in chain (FOLD). */
39#define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
40
41#define emitconv(a, dt, st, flags) \
42 emitir(IRT(IR_CONV, (dt)), (a), (st)|((dt) << 5)|(flags))
43
44/* -- C type checks ------------------------------------------------------- */
45
46static GCcdata *argv2cdata(jit_State *J, TRef tr, cTValue *o)
47{
48 GCcdata *cd;
49 TRef trtypeid;
50 if (!tref_iscdata(tr))
51 lj_trace_err(J, LJ_TRERR_BADTYPE);
52 cd = cdataV(o);
53 /* Specialize to the CTypeID. */
54 trtypeid = emitir(IRT(IR_FLOAD, IRT_U16), tr, IRFL_CDATA_CTYPEID);
55 emitir(IRTG(IR_EQ, IRT_INT), trtypeid, lj_ir_kint(J, (int32_t)cd->ctypeid));
56 return cd;
57}
58
59/* Specialize to the CTypeID held by a cdata constructor. */
60static CTypeID crec_constructor(jit_State *J, GCcdata *cd, TRef tr)
61{
62 CTypeID id;
63 lua_assert(tref_iscdata(tr) && cd->ctypeid == CTID_CTYPEID);
64 id = *(CTypeID *)cdataptr(cd);
65 tr = emitir(IRT(IR_FLOAD, IRT_INT), tr, IRFL_CDATA_INT);
66 emitir(IRTG(IR_EQ, IRT_INT), tr, lj_ir_kint(J, (int32_t)id));
67 return id;
68}
69
70static CTypeID argv2ctype(jit_State *J, TRef tr, cTValue *o)
71{
72 if (tref_isstr(tr)) {
73 GCstr *s = strV(o);
74 CPState cp;
75 CTypeID oldtop;
76 /* Specialize to the string containing the C type declaration. */
77 emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, s));
78 cp.L = J->L;
79 cp.cts = ctype_ctsG(J2G(J));
80 oldtop = cp.cts->top;
81 cp.srcname = strdata(s);
82 cp.p = strdata(s);
83 cp.param = NULL;
84 cp.mode = CPARSE_MODE_ABSTRACT|CPARSE_MODE_NOIMPLICIT;
85 if (lj_cparse(&cp) || cp.cts->top > oldtop) /* Avoid new struct defs. */
86 lj_trace_err(J, LJ_TRERR_BADTYPE);
87 return cp.val.id;
88 } else {
89 GCcdata *cd = argv2cdata(J, tr, o);
90 return cd->ctypeid == CTID_CTYPEID ? crec_constructor(J, cd, tr) :
91 cd->ctypeid;
92 }
93}
94
95/* Convert CType to IRType (if possible). */
96static IRType crec_ct2irt(CTState *cts, CType *ct)
97{
98 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
99 if (LJ_LIKELY(ctype_isnum(ct->info))) {
100 if ((ct->info & CTF_FP)) {
101 if (ct->size == sizeof(double))
102 return IRT_NUM;
103 else if (ct->size == sizeof(float))
104 return IRT_FLOAT;
105 } else {
106 uint32_t b = lj_fls(ct->size);
107 if (b <= 3)
108 return IRT_I8 + 2*b + ((ct->info & CTF_UNSIGNED) ? 1 : 0);
109 }
110 } else if (ctype_isptr(ct->info)) {
111 return (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
112 } else if (ctype_iscomplex(ct->info)) {
113 if (ct->size == 2*sizeof(double))
114 return IRT_NUM;
115 else if (ct->size == 2*sizeof(float))
116 return IRT_FLOAT;
117 }
118 return IRT_CDATA;
119}
120
121/* -- Optimized memory fill and copy -------------------------------------- */
122
123/* Maximum length and unroll of inlined copy/fill. */
124#define CREC_COPY_MAXUNROLL 16
125#define CREC_COPY_MAXLEN 128
126
127#define CREC_FILL_MAXUNROLL 16
128
129/* Number of windowed registers used for optimized memory copy. */
130#if LJ_TARGET_X86
131#define CREC_COPY_REGWIN 2
132#elif LJ_TARGET_PPC || LJ_TARGET_MIPS
133#define CREC_COPY_REGWIN 8
134#else
135#define CREC_COPY_REGWIN 4
136#endif
137
138/* List of memory offsets for copy/fill. */
139typedef struct CRecMemList {
140 CTSize ofs; /* Offset in bytes. */
141 IRType tp; /* Type of load/store. */
142 TRef trofs; /* TRef of interned offset. */
143 TRef trval; /* TRef of load value. */
144} CRecMemList;
145
146/* Generate copy list for element-wise struct copy. */
147static MSize crec_copy_struct(CRecMemList *ml, CTState *cts, CType *ct)
148{
149 CTypeID fid = ct->sib;
150 MSize mlp = 0;
151 while (fid) {
152 CType *df = ctype_get(cts, fid);
153 fid = df->sib;
154 if (ctype_isfield(df->info)) {
155 CType *cct;
156 IRType tp;
157 if (!gcref(df->name)) continue; /* Ignore unnamed fields. */
158 cct = ctype_rawchild(cts, df); /* Field type. */
159 tp = crec_ct2irt(cts, cct);
160 if (tp == IRT_CDATA) return 0; /* NYI: aggregates. */
161 if (mlp >= CREC_COPY_MAXUNROLL) return 0;
162 ml[mlp].ofs = df->size;
163 ml[mlp].tp = tp;
164 mlp++;
165 if (ctype_iscomplex(cct->info)) {
166 if (mlp >= CREC_COPY_MAXUNROLL) return 0;
167 ml[mlp].ofs = df->size + (cct->size >> 1);
168 ml[mlp].tp = tp;
169 mlp++;
170 }
171 } else if (!ctype_isconstval(df->info)) {
172 /* NYI: bitfields and sub-structures. */
173 return 0;
174 }
175 }
176 return mlp;
177}
178
179/* Generate unrolled copy list, from highest to lowest step size/alignment. */
180static MSize crec_copy_unroll(CRecMemList *ml, CTSize len, CTSize step,
181 IRType tp)
182{
183 CTSize ofs = 0;
184 MSize mlp = 0;
185 if (tp == IRT_CDATA) tp = IRT_U8 + 2*lj_fls(step);
186 do {
187 while (ofs + step <= len) {
188 if (mlp >= CREC_COPY_MAXUNROLL) return 0;
189 ml[mlp].ofs = ofs;
190 ml[mlp].tp = tp;
191 mlp++;
192 ofs += step;
193 }
194 step >>= 1;
195 tp -= 2;
196 } while (ofs < len);
197 return mlp;
198}
199
200/*
201** Emit copy list with windowed loads/stores.
202** LJ_TARGET_UNALIGNED: may emit unaligned loads/stores (not marked as such).
203*/
204static void crec_copy_emit(jit_State *J, CRecMemList *ml, MSize mlp,
205 TRef trdst, TRef trsrc)
206{
207 MSize i, j, rwin = 0;
208 for (i = 0, j = 0; i < mlp; ) {
209 TRef trofs = lj_ir_kintp(J, ml[i].ofs);
210 TRef trsptr = emitir(IRT(IR_ADD, IRT_PTR), trsrc, trofs);
211 ml[i].trval = emitir(IRT(IR_XLOAD, ml[i].tp), trsptr, 0);
212 ml[i].trofs = trofs;
213 i++;
214 rwin += (LJ_SOFTFP && ml[i].tp == IRT_NUM) ? 2 : 1;
215 if (rwin >= CREC_COPY_REGWIN || i >= mlp) { /* Flush buffered stores. */
216 rwin = 0;
217 for ( ; j < i; j++) {
218 TRef trdptr = emitir(IRT(IR_ADD, IRT_PTR), trdst, ml[j].trofs);
219 emitir(IRT(IR_XSTORE, ml[j].tp), trdptr, ml[j].trval);
220 }
221 }
222 }
223}
224
225/* Optimized memory copy. */
226static void crec_copy(jit_State *J, TRef trdst, TRef trsrc, TRef trlen,
227 CType *ct)
228{
229 if (tref_isk(trlen)) { /* Length must be constant. */
230 CRecMemList ml[CREC_COPY_MAXUNROLL];
231 MSize mlp = 0;
232 CTSize step = 1, len = (CTSize)IR(tref_ref(trlen))->i;
233 IRType tp = IRT_CDATA;
234 int needxbar = 0;
235 if (len == 0) return; /* Shortcut. */
236 if (len > CREC_COPY_MAXLEN) goto fallback;
237 if (ct) {
238 CTState *cts = ctype_ctsG(J2G(J));
239 lua_assert(ctype_isarray(ct->info) || ctype_isstruct(ct->info));
240 if (ctype_isarray(ct->info)) {
241 CType *cct = ctype_rawchild(cts, ct);
242 tp = crec_ct2irt(cts, cct);
243 if (tp == IRT_CDATA) goto rawcopy;
244 step = lj_ir_type_size[tp];
245 lua_assert((len & (step-1)) == 0);
246 } else if ((ct->info & CTF_UNION)) {
247 step = (1u << ctype_align(ct->info));
248 goto rawcopy;
249 } else {
250 mlp = crec_copy_struct(ml, cts, ct);
251 goto emitcopy;
252 }
253 } else {
254 rawcopy:
255 needxbar = 1;
256 if (LJ_TARGET_UNALIGNED || step >= CTSIZE_PTR)
257 step = CTSIZE_PTR;
258 }
259 mlp = crec_copy_unroll(ml, len, step, tp);
260 emitcopy:
261 if (mlp) {
262 crec_copy_emit(J, ml, mlp, trdst, trsrc);
263 if (needxbar)
264 emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
265 return;
266 }
267 }
268fallback:
269 /* Call memcpy. Always needs a barrier to disable alias analysis. */
270 lj_ir_call(J, IRCALL_memcpy, trdst, trsrc, trlen);
271 emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
272}
273
274/* Generate unrolled fill list, from highest to lowest step size/alignment. */
275static MSize crec_fill_unroll(CRecMemList *ml, CTSize len, CTSize step)
276{
277 CTSize ofs = 0;
278 MSize mlp = 0;
279 IRType tp = IRT_U8 + 2*lj_fls(step);
280 do {
281 while (ofs + step <= len) {
282 if (mlp >= CREC_COPY_MAXUNROLL) return 0;
283 ml[mlp].ofs = ofs;
284 ml[mlp].tp = tp;
285 mlp++;
286 ofs += step;
287 }
288 step >>= 1;
289 tp -= 2;
290 } while (ofs < len);
291 return mlp;
292}
293
294/*
295** Emit stores for fill list.
296** LJ_TARGET_UNALIGNED: may emit unaligned stores (not marked as such).
297*/
298static void crec_fill_emit(jit_State *J, CRecMemList *ml, MSize mlp,
299 TRef trdst, TRef trfill)
300{
301 MSize i;
302 for (i = 0; i < mlp; i++) {
303 TRef trofs = lj_ir_kintp(J, ml[i].ofs);
304 TRef trdptr = emitir(IRT(IR_ADD, IRT_PTR), trdst, trofs);
305 emitir(IRT(IR_XSTORE, ml[i].tp), trdptr, trfill);
306 }
307}
308
309/* Optimized memory fill. */
310static void crec_fill(jit_State *J, TRef trdst, TRef trlen, TRef trfill,
311 CTSize step)
312{
313 if (tref_isk(trlen)) { /* Length must be constant. */
314 CRecMemList ml[CREC_FILL_MAXUNROLL];
315 MSize mlp;
316 CTSize len = (CTSize)IR(tref_ref(trlen))->i;
317 if (len == 0) return; /* Shortcut. */
318 if (LJ_TARGET_UNALIGNED || step >= CTSIZE_PTR)
319 step = CTSIZE_PTR;
320 if (step * CREC_FILL_MAXUNROLL < len) goto fallback;
321 mlp = crec_fill_unroll(ml, len, step);
322 if (!mlp) goto fallback;
323 if (tref_isk(trfill) || ml[0].tp != IRT_U8)
324 trfill = emitconv(trfill, IRT_INT, IRT_U8, 0);
325 if (ml[0].tp != IRT_U8) { /* Scatter U8 to U16/U32/U64. */
326 if (CTSIZE_PTR == 8 && ml[0].tp == IRT_U64) {
327 if (tref_isk(trfill)) /* Pointless on x64 with zero-extended regs. */
328 trfill = emitconv(trfill, IRT_U64, IRT_U32, 0);
329 trfill = emitir(IRT(IR_MUL, IRT_U64), trfill,
330 lj_ir_kint64(J, U64x(01010101,01010101)));
331 } else {
332 trfill = emitir(IRTI(IR_MUL), trfill,
333 lj_ir_kint(J, ml[0].tp == IRT_U16 ? 0x0101 : 0x01010101));
334 }
335 }
336 crec_fill_emit(J, ml, mlp, trdst, trfill);
337 } else {
338fallback:
339 /* Call memset. Always needs a barrier to disable alias analysis. */
340 lj_ir_call(J, IRCALL_memset, trdst, trfill, trlen); /* Note: arg order! */
341 }
342 emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
343}
344
345/* -- Convert C type to C type -------------------------------------------- */
346
347/*
348** This code mirrors the code in lj_cconv.c. It performs the same steps
349** for the trace recorder that lj_cconv.c does for the interpreter.
350**
351** One major difference is that we can get away with much fewer checks
352** here. E.g. checks for casts, constness or correct types can often be
353** omitted, even if they might fail. The interpreter subsequently throws
354** an error, which aborts the trace.
355**
356** All operations are specialized to their C types, so the on-trace
357** outcome must be the same as the outcome in the interpreter. If the
358** interpreter doesn't throw an error, then the trace is correct, too.
359** Care must be taken not to generate invalid (temporary) IR or to
360** trigger asserts.
361*/
362
363/* Determine whether a passed number or cdata number is non-zero. */
364static int crec_isnonzero(CType *s, void *p)
365{
366 if (p == (void *)0)
367 return 0;
368 if (p == (void *)1)
369 return 1;
370 if ((s->info & CTF_FP)) {
371 if (s->size == sizeof(float))
372 return (*(float *)p != 0);
373 else
374 return (*(double *)p != 0);
375 } else {
376 if (s->size == 1)
377 return (*(uint8_t *)p != 0);
378 else if (s->size == 2)
379 return (*(uint16_t *)p != 0);
380 else if (s->size == 4)
381 return (*(uint32_t *)p != 0);
382 else
383 return (*(uint64_t *)p != 0);
384 }
385}
386
387static TRef crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp,
388 void *svisnz)
389{
390 IRType dt = crec_ct2irt(ctype_ctsG(J2G(J)), d);
391 IRType st = crec_ct2irt(ctype_ctsG(J2G(J)), s);
392 CTSize dsize = d->size, ssize = s->size;
393 CTInfo dinfo = d->info, sinfo = s->info;
394
395 if (ctype_type(dinfo) > CT_MAYCONVERT || ctype_type(sinfo) > CT_MAYCONVERT)
396 goto err_conv;
397
398 /*
399 ** Note: Unlike lj_cconv_ct_ct(), sp holds the _value_ of pointers and
400 ** numbers up to 8 bytes. Otherwise sp holds a pointer.
401 */
402
403 switch (cconv_idx2(dinfo, sinfo)) {
404 /* Destination is a bool. */
405 case CCX(B, B):
406 goto xstore; /* Source operand is already normalized. */
407 case CCX(B, I):
408 case CCX(B, F):
409 if (st != IRT_CDATA) {
410 /* Specialize to the result of a comparison against 0. */
411 TRef zero = (st == IRT_NUM || st == IRT_FLOAT) ? lj_ir_knum(J, 0) :
412 (st == IRT_I64 || st == IRT_U64) ? lj_ir_kint64(J, 0) :
413 lj_ir_kint(J, 0);
414 int isnz = crec_isnonzero(s, svisnz);
415 emitir(IRTG(isnz ? IR_NE : IR_EQ, st), sp, zero);
416 sp = lj_ir_kint(J, isnz);
417 goto xstore;
418 }
419 goto err_nyi;
420
421 /* Destination is an integer. */
422 case CCX(I, B):
423 case CCX(I, I):
424 conv_I_I:
425 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
426 /* Extend 32 to 64 bit integer. */
427 if (dsize == 8 && ssize < 8 && !(LJ_64 && (sinfo & CTF_UNSIGNED)))
428 sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st,
429 (sinfo & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
430 else if (dsize < 8 && ssize == 8) /* Truncate from 64 bit integer. */
431 sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, 0);
432 else if (st == IRT_INT)
433 sp = lj_opt_narrow_toint(J, sp);
434 xstore:
435 if (dt == IRT_I64 || dt == IRT_U64) lj_needsplit(J);
436 if (dp == 0) return sp;
437 emitir(IRT(IR_XSTORE, dt), dp, sp);
438 break;
439 case CCX(I, C):
440 sp = emitir(IRT(IR_XLOAD, st), sp, 0); /* Load re. */
441 /* fallthrough */
442 case CCX(I, F):
443 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
444 sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, IRCONV_TRUNC|IRCONV_ANY);
445 goto xstore;
446 case CCX(I, P):
447 case CCX(I, A):
448 sinfo = CTINFO(CT_NUM, CTF_UNSIGNED);
449 ssize = CTSIZE_PTR;
450 st = IRT_UINTP;
451 if (((dsize ^ ssize) & 8) == 0) { /* Must insert no-op type conversion. */
452 sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, IRT_PTR, 0);
453 goto xstore;
454 }
455 goto conv_I_I;
456
457 /* Destination is a floating-point number. */
458 case CCX(F, B):
459 case CCX(F, I):
460 conv_F_I:
461 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
462 sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st, 0);
463 goto xstore;
464 case CCX(F, C):
465 sp = emitir(IRT(IR_XLOAD, st), sp, 0); /* Load re. */
466 /* fallthrough */
467 case CCX(F, F):
468 conv_F_F:
469 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
470 if (dt != st) sp = emitconv(sp, dt, st, 0);
471 goto xstore;
472
473 /* Destination is a complex number. */
474 case CCX(C, I):
475 case CCX(C, F):
476 { /* Clear im. */
477 TRef ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
478 emitir(IRT(IR_XSTORE, dt), ptr, lj_ir_knum(J, 0));
479 }
480 /* Convert to re. */
481 if ((sinfo & CTF_FP)) goto conv_F_F; else goto conv_F_I;
482
483 case CCX(C, C):
484 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
485 {
486 TRef re, im, ptr;
487 re = emitir(IRT(IR_XLOAD, st), sp, 0);
488 ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, (ssize >> 1)));
489 im = emitir(IRT(IR_XLOAD, st), ptr, 0);
490 if (dt != st) {
491 re = emitconv(re, dt, st, 0);
492 im = emitconv(im, dt, st, 0);
493 }
494 emitir(IRT(IR_XSTORE, dt), dp, re);
495 ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
496 emitir(IRT(IR_XSTORE, dt), ptr, im);
497 }
498 break;
499
500 /* Destination is a vector. */
501 case CCX(V, I):
502 case CCX(V, F):
503 case CCX(V, C):
504 case CCX(V, V):
505 goto err_nyi;
506
507 /* Destination is a pointer. */
508 case CCX(P, P):
509 case CCX(P, A):
510 case CCX(P, S):
511 /* There are only 32 bit pointers/addresses on 32 bit machines.
512 ** Also ok on x64, since all 32 bit ops clear the upper part of the reg.
513 */
514 goto xstore;
515 case CCX(P, I):
516 if (st == IRT_CDATA) goto err_nyi;
517 if (!LJ_64 && ssize == 8) /* Truncate from 64 bit integer. */
518 sp = emitconv(sp, IRT_U32, st, 0);
519 goto xstore;
520 case CCX(P, F):
521 if (st == IRT_CDATA) goto err_nyi;
522 /* The signed conversion is cheaper. x64 really has 47 bit pointers. */
523 sp = emitconv(sp, (LJ_64 && dsize == 8) ? IRT_I64 : IRT_U32,
524 st, IRCONV_TRUNC|IRCONV_ANY);
525 goto xstore;
526
527 /* Destination is an array. */
528 case CCX(A, A):
529 /* Destination is a struct/union. */
530 case CCX(S, S):
531 if (dp == 0) goto err_conv;
532 crec_copy(J, dp, sp, lj_ir_kint(J, dsize), d);
533 break;
534
535 default:
536 err_conv:
537 err_nyi:
538 lj_trace_err(J, LJ_TRERR_NYICONV);
539 break;
540 }
541 return 0;
542}
543
544/* -- Convert C type to TValue (load) ------------------------------------- */
545
546static TRef crec_tv_ct(jit_State *J, CType *s, CTypeID sid, TRef sp)
547{
548 CTState *cts = ctype_ctsG(J2G(J));
549 IRType t = crec_ct2irt(cts, s);
550 CTInfo sinfo = s->info;
551 if (ctype_isnum(sinfo)) {
552 TRef tr;
553 if (t == IRT_CDATA)
554 goto err_nyi; /* NYI: copyval of >64 bit integers. */
555 tr = emitir(IRT(IR_XLOAD, t), sp, 0);
556 if (t == IRT_FLOAT || t == IRT_U32) { /* Keep uint32_t/float as numbers. */
557 return emitconv(tr, IRT_NUM, t, 0);
558 } else if (t == IRT_I64 || t == IRT_U64) { /* Box 64 bit integer. */
559 sp = tr;
560 lj_needsplit(J);
561 } else if ((sinfo & CTF_BOOL)) {
562 /* Assume not equal to zero. Fixup and emit pending guard later. */
563 lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
564 J->postproc = LJ_POST_FIXGUARD;
565 return TREF_TRUE;
566 } else {
567 return tr;
568 }
569 } else if (ctype_isptr(sinfo) || ctype_isenum(sinfo)) {
570 sp = emitir(IRT(IR_XLOAD, t), sp, 0); /* Box pointers and enums. */
571 } else if (ctype_isrefarray(sinfo) || ctype_isstruct(sinfo)) {
572 cts->L = J->L;
573 sid = lj_ctype_intern(cts, CTINFO_REF(sid), CTSIZE_PTR); /* Create ref. */
574 } else if (ctype_iscomplex(sinfo)) { /* Unbox/box complex. */
575 ptrdiff_t esz = (ptrdiff_t)(s->size >> 1);
576 TRef ptr, tr1, tr2, dp;
577 dp = emitir(IRTG(IR_CNEW, IRT_CDATA), lj_ir_kint(J, sid), TREF_NIL);
578 tr1 = emitir(IRT(IR_XLOAD, t), sp, 0);
579 ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, esz));
580 tr2 = emitir(IRT(IR_XLOAD, t), ptr, 0);
581 ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)));
582 emitir(IRT(IR_XSTORE, t), ptr, tr1);
583 ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)+esz));
584 emitir(IRT(IR_XSTORE, t), ptr, tr2);
585 return dp;
586 } else {
587 /* NYI: copyval of vectors. */
588 err_nyi:
589 lj_trace_err(J, LJ_TRERR_NYICONV);
590 }
591 /* Box pointer, ref, enum or 64 bit integer. */
592 return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, sid), sp);
593}
594
595/* -- Convert TValue to C type (store) ------------------------------------ */
596
597static TRef crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, cTValue *sval)
598{
599 CTState *cts = ctype_ctsG(J2G(J));
600 CTypeID sid = CTID_P_VOID;
601 void *svisnz = 0;
602 CType *s;
603 if (LJ_LIKELY(tref_isinteger(sp))) {
604 sid = CTID_INT32;
605 svisnz = (void *)(intptr_t)(tvisint(sval)?(intV(sval)!=0):!tviszero(sval));
606 } else if (tref_isnum(sp)) {
607 sid = CTID_DOUBLE;
608 svisnz = (void *)(intptr_t)(tvisint(sval)?(intV(sval)!=0):!tviszero(sval));
609 } else if (tref_isbool(sp)) {
610 sp = lj_ir_kint(J, tref_istrue(sp) ? 1 : 0);
611 sid = CTID_BOOL;
612 } else if (tref_isnil(sp)) {
613 sp = lj_ir_kptr(J, NULL);
614 } else if (tref_isudata(sp)) {
615 GCudata *ud = udataV(sval);
616 if (ud->udtype == UDTYPE_IO_FILE) {
617 TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), sp, IRFL_UDATA_UDTYPE);
618 emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, UDTYPE_IO_FILE));
619 sp = emitir(IRT(IR_FLOAD, IRT_PTR), sp, IRFL_UDATA_FILE);
620 } else {
621 sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCudata)));
622 }
623 } else if (tref_isstr(sp)) {
624 if (ctype_isenum(d->info)) { /* Match string against enum constant. */
625 GCstr *str = strV(sval);
626 CTSize ofs;
627 CType *cct = lj_ctype_getfield(cts, d, str, &ofs);
628 /* Specialize to the name of the enum constant. */
629 emitir(IRTG(IR_EQ, IRT_STR), sp, lj_ir_kstr(J, str));
630 if (cct && ctype_isconstval(cct->info)) {
631 lua_assert(ctype_child(cts, cct)->size == 4);
632 svisnz = (void *)(intptr_t)(ofs != 0);
633 sp = lj_ir_kint(J, (int32_t)ofs);
634 sid = ctype_cid(cct->info);
635 } /* else: interpreter will throw. */
636 } else if (ctype_isrefarray(d->info)) { /* Copy string to array. */
637 lj_trace_err(J, LJ_TRERR_BADTYPE); /* NYI */
638 } else { /* Otherwise pass the string data as a const char[]. */
639 /* Don't use STRREF. It folds with SNEW, which loses the trailing NUL. */
640 sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCstr)));
641 sid = CTID_A_CCHAR;
642 }
643 } else { /* NYI: tref_istab(sp), tref_islightud(sp). */
644 IRType t;
645 sid = argv2cdata(J, sp, sval)->ctypeid;
646 s = ctype_raw(cts, sid);
647 svisnz = cdataptr(cdataV(sval));
648 t = crec_ct2irt(cts, s);
649 if (ctype_isptr(s->info)) {
650 sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_PTR);
651 if (ctype_isref(s->info)) {
652 svisnz = *(void **)svisnz;
653 s = ctype_rawchild(cts, s);
654 if (ctype_isenum(s->info)) s = ctype_child(cts, s);
655 t = crec_ct2irt(cts, s);
656 } else {
657 goto doconv;
658 }
659 } else if (t == IRT_I64 || t == IRT_U64) {
660 sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INT64);
661 lj_needsplit(J);
662 goto doconv;
663 } else if (t == IRT_INT || t == IRT_U32) {
664 if (ctype_isenum(s->info)) s = ctype_child(cts, s);
665 sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INT);
666 goto doconv;
667 } else {
668 sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCcdata)));
669 }
670 if (ctype_isnum(s->info) && t != IRT_CDATA)
671 sp = emitir(IRT(IR_XLOAD, t), sp, 0); /* Load number value. */
672 goto doconv;
673 }
674 s = ctype_get(cts, sid);
675doconv:
676 if (ctype_isenum(d->info)) d = ctype_child(cts, d);
677 return crec_ct_ct(J, d, s, dp, sp, svisnz);
678}
679
680/* -- C data metamethods -------------------------------------------------- */
681
682/* This would be rather difficult in FOLD, so do it here:
683** (base+k)+(idx*sz)+ofs ==> (base+idx*sz)+(ofs+k)
684** (base+(idx+k)*sz)+ofs ==> (base+idx*sz)+(ofs+k*sz)
685*/
686static TRef crec_reassoc_ofs(jit_State *J, TRef tr, ptrdiff_t *ofsp, MSize sz)
687{
688 IRIns *ir = IR(tref_ref(tr));
689 if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD) && irref_isk(ir->op2) &&
690 (ir->o == IR_ADD || ir->o == IR_ADDOV || ir->o == IR_SUBOV)) {
691 IRIns *irk = IR(ir->op2);
692 ptrdiff_t k;
693 if (LJ_64 && irk->o == IR_KINT64)
694 k = (ptrdiff_t)ir_kint64(irk)->u64 * sz;
695 else
696 k = (ptrdiff_t)irk->i * sz;
697 if (ir->o == IR_SUBOV) *ofsp -= k; else *ofsp += k;
698 tr = ir->op1; /* Not a TRef, but the caller doesn't care. */
699 }
700 return tr;
701}
702
703/* Record ctype __index/__newindex metamethods. */
704static void crec_index_meta(jit_State *J, CTState *cts, CType *ct,
705 RecordFFData *rd)
706{
707 CTypeID id = ctype_typeid(cts, ct);
708 cTValue *tv = lj_ctype_meta(cts, id, rd->data ? MM_newindex : MM_index);
709 if (!tv)
710 lj_trace_err(J, LJ_TRERR_BADTYPE);
711 if (tvisfunc(tv)) {
712 J->base[-1] = lj_ir_kfunc(J, funcV(tv)) | TREF_FRAME;
713 rd->nres = -1; /* Pending tailcall. */
714 } else if (rd->data == 0 && tvistab(tv) && tref_isstr(J->base[1])) {
715 /* Specialize to result of __index lookup. */
716 cTValue *o = lj_tab_get(J->L, tabV(tv), &rd->argv[1]);
717 J->base[0] = lj_record_constify(J, o);
718 if (!J->base[0])
719 lj_trace_err(J, LJ_TRERR_BADTYPE);
720 /* Always specialize to the key. */
721 emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
722 } else {
723 /* NYI: resolving of non-function metamethods. */
724 /* NYI: non-string keys for __index table. */
725 /* NYI: stores to __newindex table. */
726 lj_trace_err(J, LJ_TRERR_BADTYPE);
727 }
728}
729
730void LJ_FASTCALL recff_cdata_index(jit_State *J, RecordFFData *rd)
731{
732 TRef idx, ptr = J->base[0];
733 ptrdiff_t ofs = sizeof(GCcdata);
734 GCcdata *cd = argv2cdata(J, ptr, &rd->argv[0]);
735 CTState *cts = ctype_ctsG(J2G(J));
736 CType *ct = ctype_raw(cts, cd->ctypeid);
737 CTypeID sid = 0;
738
739 /* Resolve pointer or reference for cdata object. */
740 if (ctype_isptr(ct->info)) {
741 IRType t = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
742 if (ctype_isref(ct->info)) ct = ctype_rawchild(cts, ct);
743 ptr = emitir(IRT(IR_FLOAD, t), ptr, IRFL_CDATA_PTR);
744 ofs = 0;
745 ptr = crec_reassoc_ofs(J, ptr, &ofs, 1);
746 }
747
748again:
749 idx = J->base[1];
750 if (tref_isnumber(idx)) {
751 idx = lj_opt_narrow_cindex(J, idx);
752 if (ctype_ispointer(ct->info)) {
753 CTSize sz;
754 integer_key:
755 if ((ct->info & CTF_COMPLEX))
756 idx = emitir(IRT(IR_BAND, IRT_INTP), idx, lj_ir_kintp(J, 1));
757 sz = lj_ctype_size(cts, (sid = ctype_cid(ct->info)));
758 idx = crec_reassoc_ofs(J, idx, &ofs, sz);
759#if LJ_TARGET_ARM || LJ_TARGET_PPC
760 /* Hoist base add to allow fusion of index/shift into operands. */
761 if (LJ_LIKELY(J->flags & JIT_F_OPT_LOOP) && ofs
762#if LJ_TARGET_ARM
763 && (sz == 1 || sz == 4)
764#endif
765 ) {
766 ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
767 ofs = 0;
768 }
769#endif
770 idx = emitir(IRT(IR_MUL, IRT_INTP), idx, lj_ir_kintp(J, sz));
771 ptr = emitir(IRT(IR_ADD, IRT_PTR), idx, ptr);
772 }
773 } else if (tref_iscdata(idx)) {
774 GCcdata *cdk = cdataV(&rd->argv[1]);
775 CType *ctk = ctype_raw(cts, cdk->ctypeid);
776 IRType t = crec_ct2irt(cts, ctk);
777 if (ctype_ispointer(ct->info) && t >= IRT_I8 && t <= IRT_U64) {
778 if (ctk->size == 8) {
779 idx = emitir(IRT(IR_FLOAD, t), idx, IRFL_CDATA_INT64);
780 } else if (ctk->size == 4) {
781 idx = emitir(IRT(IR_FLOAD, t), idx, IRFL_CDATA_INT);
782 } else {
783 idx = emitir(IRT(IR_ADD, IRT_PTR), idx,
784 lj_ir_kintp(J, sizeof(GCcdata)));
785 idx = emitir(IRT(IR_XLOAD, t), idx, 0);
786 }
787 if (LJ_64 && ctk->size < sizeof(intptr_t) && !(ctk->info & CTF_UNSIGNED))
788 idx = emitconv(idx, IRT_INTP, IRT_INT, IRCONV_SEXT);
789 if (!LJ_64 && ctk->size > sizeof(intptr_t)) {
790 idx = emitconv(idx, IRT_INTP, t, 0);
791 lj_needsplit(J);
792 }
793 goto integer_key;
794 }
795 } else if (tref_isstr(idx)) {
796 GCstr *name = strV(&rd->argv[1]);
797 if (cd->ctypeid == CTID_CTYPEID)
798 ct = ctype_raw(cts, crec_constructor(J, cd, ptr));
799 if (ctype_isstruct(ct->info)) {
800 CTSize fofs;
801 CType *fct;
802 fct = lj_ctype_getfield(cts, ct, name, &fofs);
803 if (fct) {
804 /* Always specialize to the field name. */
805 emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
806 if (ctype_isconstval(fct->info)) {
807 if (fct->size >= 0x80000000u &&
808 (ctype_child(cts, fct)->info & CTF_UNSIGNED)) {
809 J->base[0] = lj_ir_knum(J, (lua_Number)(uint32_t)fct->size);
810 return;
811 }
812 J->base[0] = lj_ir_kint(J, (int32_t)fct->size);
813 return; /* Interpreter will throw for newindex. */
814 } else if (ctype_isbitfield(fct->info)) {
815 lj_trace_err(J, LJ_TRERR_NYICONV);
816 } else {
817 lua_assert(ctype_isfield(fct->info));
818 sid = ctype_cid(fct->info);
819 }
820 ofs += (ptrdiff_t)fofs;
821 }
822 } else if (ctype_iscomplex(ct->info)) {
823 if (name->len == 2 &&
824 ((strdata(name)[0] == 'r' && strdata(name)[1] == 'e') ||
825 (strdata(name)[0] == 'i' && strdata(name)[1] == 'm'))) {
826 /* Always specialize to the field name. */
827 emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
828 if (strdata(name)[0] == 'i') ofs += (ct->size >> 1);
829 sid = ctype_cid(ct->info);
830 }
831 }
832 }
833 if (!sid) {
834 if (ctype_isptr(ct->info)) { /* Automatically perform '->'. */
835 CType *cct = ctype_rawchild(cts, ct);
836 if (ctype_isstruct(cct->info)) {
837 ct = cct;
838 if (tref_isstr(idx)) goto again;
839 }
840 }
841 crec_index_meta(J, cts, ct, rd);
842 return;
843 }
844
845 if (ofs)
846 ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
847
848 /* Resolve reference for field. */
849 ct = ctype_get(cts, sid);
850 if (ctype_isref(ct->info))
851 ptr = emitir(IRT(IR_XLOAD, IRT_PTR), ptr, 0);
852
853 while (ctype_isattrib(ct->info))
854 ct = ctype_child(cts, ct); /* Skip attributes. */
855
856 if (rd->data == 0) { /* __index metamethod. */
857 J->base[0] = crec_tv_ct(J, ct, sid, ptr);
858 } else { /* __newindex metamethod. */
859 rd->nres = 0;
860 J->needsnap = 1;
861 crec_ct_tv(J, ct, ptr, J->base[2], &rd->argv[2]);
862 }
863}
864
865/* Record setting a finalizer. */
866static void crec_finalizer(jit_State *J, TRef trcd, cTValue *fin)
867{
868 TRef trlo = lj_ir_call(J, IRCALL_lj_cdata_setfin, trcd);
869 TRef trhi = emitir(IRT(IR_ADD, IRT_P32), trlo, lj_ir_kint(J, 4));
870 if (LJ_BE) { TRef tmp = trlo; trlo = trhi; trhi = tmp; }
871 if (tvisfunc(fin)) {
872 emitir(IRT(IR_XSTORE, IRT_P32), trlo, lj_ir_kfunc(J, funcV(fin)));
873 emitir(IRTI(IR_XSTORE), trhi, lj_ir_kint(J, LJ_TFUNC));
874 } else if (tviscdata(fin)) {
875 emitir(IRT(IR_XSTORE, IRT_P32), trlo,
876 lj_ir_kgc(J, obj2gco(cdataV(fin)), IRT_CDATA));
877 emitir(IRTI(IR_XSTORE), trhi, lj_ir_kint(J, LJ_TCDATA));
878 } else {
879 lj_trace_err(J, LJ_TRERR_BADTYPE);
880 }
881 J->needsnap = 1;
882}
883
884/* Record cdata allocation. */
885static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
886{
887 CTState *cts = ctype_ctsG(J2G(J));
888 CTSize sz;
889 CTInfo info = lj_ctype_info(cts, id, &sz);
890 CType *d = ctype_raw(cts, id);
891 TRef trid;
892 if (!sz || sz > 128 || (info & CTF_VLA) || ctype_align(info) > CT_MEMALIGN)
893 lj_trace_err(J, LJ_TRERR_NYICONV); /* NYI: large/special allocations. */
894 trid = lj_ir_kint(J, id);
895 /* Use special instruction to box pointer or 32/64 bit integer. */
896 if (ctype_isptr(info) || (ctype_isinteger(info) && (sz == 4 || sz == 8))) {
897 TRef sp = J->base[1] ? crec_ct_tv(J, d, 0, J->base[1], &rd->argv[1]) :
898 ctype_isptr(info) ? lj_ir_kptr(J, NULL) :
899 sz == 4 ? lj_ir_kint(J, 0) :
900 (lj_needsplit(J), lj_ir_kint64(J, 0));
901 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, sp);
902 } else {
903 TRef trcd = emitir(IRTG(IR_CNEW, IRT_CDATA), trid, TREF_NIL);
904 cTValue *fin;
905 J->base[0] = trcd;
906 if (J->base[1] && !J->base[2] &&
907 !lj_cconv_multi_init(cts, d, &rd->argv[1])) {
908 goto single_init;
909 } else if (ctype_isarray(d->info)) {
910 CType *dc = ctype_rawchild(cts, d); /* Array element type. */
911 CTSize ofs, esize = dc->size;
912 TRef sp = 0;
913 TValue tv;
914 TValue *sval = &tv;
915 MSize i;
916 tv.u64 = 0;
917 if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info)))
918 lj_trace_err(J, LJ_TRERR_NYICONV); /* NYI: init array of aggregates. */
919 for (i = 1, ofs = 0; ofs < sz; ofs += esize) {
920 TRef dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
921 lj_ir_kintp(J, ofs + sizeof(GCcdata)));
922 if (J->base[i]) {
923 sp = J->base[i];
924 sval = &rd->argv[i];
925 i++;
926 } else if (i != 2) {
927 sp = ctype_isnum(dc->info) ? lj_ir_kint(J, 0) : TREF_NIL;
928 }
929 crec_ct_tv(J, dc, dp, sp, sval);
930 }
931 } else if (ctype_isstruct(d->info)) {
932 CTypeID fid = d->sib;
933 MSize i = 1;
934 while (fid) {
935 CType *df = ctype_get(cts, fid);
936 fid = df->sib;
937 if (ctype_isfield(df->info)) {
938 CType *dc;
939 TRef sp, dp;
940 TValue tv;
941 TValue *sval = &tv;
942 setintV(&tv, 0);
943 if (!gcref(df->name)) continue; /* Ignore unnamed fields. */
944 dc = ctype_rawchild(cts, df); /* Field type. */
945 if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info) ||
946 ctype_isenum(dc->info)))
947 lj_trace_err(J, LJ_TRERR_NYICONV); /* NYI: init aggregates. */
948 if (J->base[i]) {
949 sp = J->base[i];
950 sval = &rd->argv[i];
951 i++;
952 } else {
953 sp = ctype_isptr(dc->info) ? TREF_NIL : lj_ir_kint(J, 0);
954 }
955 dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
956 lj_ir_kintp(J, df->size + sizeof(GCcdata)));
957 crec_ct_tv(J, dc, dp, sp, sval);
958 } else if (!ctype_isconstval(df->info)) {
959 /* NYI: init bitfields and sub-structures. */
960 lj_trace_err(J, LJ_TRERR_NYICONV);
961 }
962 }
963 } else {
964 TRef dp;
965 single_init:
966 dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, lj_ir_kintp(J, sizeof(GCcdata)));
967 if (J->base[1]) {
968 crec_ct_tv(J, d, dp, J->base[1], &rd->argv[1]);
969 } else {
970 TValue tv;
971 tv.u64 = 0;
972 crec_ct_tv(J, d, dp, lj_ir_kint(J, 0), &tv);
973 }
974 }
975 /* Handle __gc metamethod. */
976 fin = lj_ctype_meta(cts, id, MM_gc);
977 if (fin)
978 crec_finalizer(J, trcd, fin);
979 }
980}
981
982/* Record argument conversions. */
983static TRef crec_call_args(jit_State *J, RecordFFData *rd,
984 CTState *cts, CType *ct)
985{
986 TRef args[CCI_NARGS_MAX];
987 CTypeID fid;
988 MSize i, n;
989 TRef tr, *base;
990 cTValue *o;
991#if LJ_TARGET_X86
992#if LJ_ABI_WIN
993 TRef *arg0 = NULL, *arg1 = NULL;
994#endif
995 int ngpr = 0;
996 if (ctype_cconv(ct->info) == CTCC_THISCALL)
997 ngpr = 1;
998 else if (ctype_cconv(ct->info) == CTCC_FASTCALL)
999 ngpr = 2;
1000#endif
1001
1002 /* Skip initial attributes. */
1003 fid = ct->sib;
1004 while (fid) {
1005 CType *ctf = ctype_get(cts, fid);
1006 if (!ctype_isattrib(ctf->info)) break;
1007 fid = ctf->sib;
1008 }
1009 args[0] = TREF_NIL;
1010 for (n = 0, base = J->base+1, o = rd->argv+1; *base; n++, base++, o++) {
1011 CTypeID did;
1012 CType *d;
1013
1014 if (n >= CCI_NARGS_MAX)
1015 lj_trace_err(J, LJ_TRERR_NYICALL);
1016
1017 if (fid) { /* Get argument type from field. */
1018 CType *ctf = ctype_get(cts, fid);
1019 fid = ctf->sib;
1020 lua_assert(ctype_isfield(ctf->info));
1021 did = ctype_cid(ctf->info);
1022 } else {
1023 if (!(ct->info & CTF_VARARG))
1024 lj_trace_err(J, LJ_TRERR_NYICALL); /* Too many arguments. */
1025 did = lj_ccall_ctid_vararg(cts, o); /* Infer vararg type. */
1026 }
1027 d = ctype_raw(cts, did);
1028 if (!(ctype_isnum(d->info) || ctype_isptr(d->info) ||
1029 ctype_isenum(d->info)))
1030 lj_trace_err(J, LJ_TRERR_NYICALL);
1031 tr = crec_ct_tv(J, d, 0, *base, o);
1032 if (ctype_isinteger_or_bool(d->info)) {
1033 if (d->size < 4) {
1034 if ((d->info & CTF_UNSIGNED))
1035 tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_U8 : IRT_U16, 0);
1036 else
1037 tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_I8 : IRT_I16,IRCONV_SEXT);
1038 }
1039 } else if (LJ_SOFTFP && ctype_isfp(d->info) && d->size > 4) {
1040 lj_needsplit(J);
1041 }
1042#if LJ_TARGET_X86
1043 /* 64 bit args must not end up in registers for fastcall/thiscall. */
1044#if LJ_ABI_WIN
1045 if (!ctype_isfp(d->info)) {
1046 /* Sigh, the Windows/x86 ABI allows reordering across 64 bit args. */
1047 if (tref_typerange(tr, IRT_I64, IRT_U64)) {
1048 if (ngpr) {
1049 arg0 = &args[n]; args[n++] = TREF_NIL; ngpr--;
1050 if (ngpr) {
1051 arg1 = &args[n]; args[n++] = TREF_NIL; ngpr--;
1052 }
1053 }
1054 } else {
1055 if (arg0) { *arg0 = tr; arg0 = NULL; n--; continue; }
1056 if (arg1) { *arg1 = tr; arg1 = NULL; n--; continue; }
1057 if (ngpr) ngpr--;
1058 }
1059 }
1060#else
1061 if (!ctype_isfp(d->info) && ngpr) {
1062 if (tref_typerange(tr, IRT_I64, IRT_U64)) {
1063 /* No reordering for other x86 ABIs. Simply add alignment args. */
1064 do { args[n++] = TREF_NIL; } while (--ngpr);
1065 } else {
1066 ngpr--;
1067 }
1068 }
1069#endif
1070#endif
1071 args[n] = tr;
1072 }
1073 tr = args[0];
1074 for (i = 1; i < n; i++)
1075 tr = emitir(IRT(IR_CARG, IRT_NIL), tr, args[i]);
1076 return tr;
1077}
1078
1079/* Create a snapshot for the caller, simulating a 'false' return value. */
1080static void crec_snap_caller(jit_State *J)
1081{
1082 lua_State *L = J->L;
1083 TValue *base = L->base, *top = L->top;
1084 const BCIns *pc = J->pc;
1085 TRef ftr = J->base[-1];
1086 ptrdiff_t delta;
1087 if (!frame_islua(base-1) || J->framedepth <= 0)
1088 lj_trace_err(J, LJ_TRERR_NYICALL);
1089 J->pc = frame_pc(base-1); delta = 1+bc_a(J->pc[-1]);
1090 L->top = base; L->base = base - delta;
1091 J->base[-1] = TREF_FALSE;
1092 J->base -= delta; J->baseslot -= (BCReg)delta;
1093 J->maxslot = (BCReg)delta; J->framedepth--;
1094 lj_snap_add(J);
1095 L->base = base; L->top = top;
1096 J->framedepth++; J->maxslot = 1;
1097 J->base += delta; J->baseslot += (BCReg)delta;
1098 J->base[-1] = ftr; J->pc = pc;
1099}
1100
1101/* Record function call. */
1102static int crec_call(jit_State *J, RecordFFData *rd, GCcdata *cd)
1103{
1104 CTState *cts = ctype_ctsG(J2G(J));
1105 CType *ct = ctype_raw(cts, cd->ctypeid);
1106 IRType tp = IRT_PTR;
1107 if (ctype_isptr(ct->info)) {
1108 tp = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
1109 ct = ctype_rawchild(cts, ct);
1110 }
1111 if (ctype_isfunc(ct->info)) {
1112 TRef func = emitir(IRT(IR_FLOAD, tp), J->base[0], IRFL_CDATA_PTR);
1113 CType *ctr = ctype_rawchild(cts, ct);
1114 IRType t = crec_ct2irt(cts, ctr);
1115 TRef tr;
1116 TValue tv;
1117 /* Check for blacklisted C functions that might call a callback. */
1118 setlightudV(&tv,
1119 cdata_getptr(cdataptr(cd), (LJ_64 && tp == IRT_P64) ? 8 : 4));
1120 if (tvistrue(lj_tab_get(J->L, cts->miscmap, &tv)))
1121 lj_trace_err(J, LJ_TRERR_BLACKL);
1122 if (ctype_isvoid(ctr->info)) {
1123 t = IRT_NIL;
1124 rd->nres = 0;
1125 } else if (!(ctype_isnum(ctr->info) || ctype_isptr(ctr->info) ||
1126 ctype_isenum(ctr->info)) || t == IRT_CDATA) {
1127 lj_trace_err(J, LJ_TRERR_NYICALL);
1128 }
1129 if ((ct->info & CTF_VARARG)
1130#if LJ_TARGET_X86
1131 || ctype_cconv(ct->info) != CTCC_CDECL
1132#endif
1133 )
1134 func = emitir(IRT(IR_CARG, IRT_NIL), func,
1135 lj_ir_kint(J, ctype_typeid(cts, ct)));
1136 tr = emitir(IRT(IR_CALLXS, t), crec_call_args(J, rd, cts, ct), func);
1137 if (ctype_isbool(ctr->info)) {
1138 if (frame_islua(J->L->base-1) && bc_b(frame_pc(J->L->base-1)[-1]) == 1) {
1139 /* Don't check result if ignored. */
1140 tr = TREF_NIL;
1141 } else {
1142 crec_snap_caller(J);
1143#if LJ_TARGET_X86ORX64
1144 /* Note: only the x86/x64 backend supports U8 and only for EQ(tr, 0). */
1145 lj_ir_set(J, IRTG(IR_NE, IRT_U8), tr, lj_ir_kint(J, 0));
1146#else
1147 lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
1148#endif
1149 J->postproc = LJ_POST_FIXGUARDSNAP;
1150 tr = TREF_TRUE;
1151 }
1152 } else if (t == IRT_PTR || (LJ_64 && t == IRT_P32) ||
1153 t == IRT_I64 || t == IRT_U64 || ctype_isenum(ctr->info)) {
1154 TRef trid = lj_ir_kint(J, ctype_cid(ct->info));
1155 tr = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, tr);
1156 if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
1157 } else if (t == IRT_FLOAT || t == IRT_U32) {
1158 tr = emitconv(tr, IRT_NUM, t, 0);
1159 } else if (t == IRT_I8 || t == IRT_I16) {
1160 tr = emitconv(tr, IRT_INT, t, IRCONV_SEXT);
1161 } else if (t == IRT_U8 || t == IRT_U16) {
1162 tr = emitconv(tr, IRT_INT, t, 0);
1163 }
1164 J->base[0] = tr;
1165 J->needsnap = 1;
1166 return 1;
1167 }
1168 return 0;
1169}
1170
1171void LJ_FASTCALL recff_cdata_call(jit_State *J, RecordFFData *rd)
1172{
1173 CTState *cts = ctype_ctsG(J2G(J));
1174 GCcdata *cd = argv2cdata(J, J->base[0], &rd->argv[0]);
1175 CTypeID id = cd->ctypeid;
1176 CType *ct;
1177 cTValue *tv;
1178 MMS mm = MM_call;
1179 if (id == CTID_CTYPEID) {
1180 id = crec_constructor(J, cd, J->base[0]);
1181 mm = MM_new;
1182 } else if (crec_call(J, rd, cd)) {
1183 return;
1184 }
1185 /* Record ctype __call/__new metamethod. */
1186 ct = ctype_raw(cts, id);
1187 tv = lj_ctype_meta(cts, ctype_isptr(ct->info) ? ctype_cid(ct->info) : id, mm);
1188 if (tv) {
1189 if (tvisfunc(tv)) {
1190 J->base[-1] = lj_ir_kfunc(J, funcV(tv)) | TREF_FRAME;
1191 rd->nres = -1; /* Pending tailcall. */
1192 return;
1193 }
1194 } else if (mm == MM_new) {
1195 crec_alloc(J, rd, id);
1196 return;
1197 }
1198 /* No metamethod or NYI: non-function metamethods. */
1199 lj_trace_err(J, LJ_TRERR_BADTYPE);
1200}
1201
1202static TRef crec_arith_int64(jit_State *J, TRef *sp, CType **s, MMS mm)
1203{
1204 if (ctype_isnum(s[0]->info) && ctype_isnum(s[1]->info)) {
1205 IRType dt;
1206 CTypeID id;
1207 TRef tr;
1208 MSize i;
1209 IROp op;
1210 lj_needsplit(J);
1211 if (((s[0]->info & CTF_UNSIGNED) && s[0]->size == 8) ||
1212 ((s[1]->info & CTF_UNSIGNED) && s[1]->size == 8)) {
1213 dt = IRT_U64; id = CTID_UINT64;
1214 } else {
1215 dt = IRT_I64; id = CTID_INT64;
1216 if (mm < MM_add &&
1217 !((s[0]->info | s[1]->info) & CTF_FP) &&
1218 s[0]->size == 4 && s[1]->size == 4) { /* Try to narrow comparison. */
1219 if (!((s[0]->info ^ s[1]->info) & CTF_UNSIGNED) ||
1220 (tref_isk(sp[1]) && IR(tref_ref(sp[1]))->i >= 0)) {
1221 dt = (s[0]->info & CTF_UNSIGNED) ? IRT_U32 : IRT_INT;
1222 goto comp;
1223 } else if (tref_isk(sp[0]) && IR(tref_ref(sp[0]))->i >= 0) {
1224 dt = (s[1]->info & CTF_UNSIGNED) ? IRT_U32 : IRT_INT;
1225 goto comp;
1226 }
1227 }
1228 }
1229 for (i = 0; i < 2; i++) {
1230 IRType st = tref_type(sp[i]);
1231 if (st == IRT_NUM || st == IRT_FLOAT)
1232 sp[i] = emitconv(sp[i], dt, st, IRCONV_TRUNC|IRCONV_ANY);
1233 else if (!(st == IRT_I64 || st == IRT_U64))
1234 sp[i] = emitconv(sp[i], dt, IRT_INT,
1235 (s[i]->info & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
1236 }
1237 if (mm < MM_add) {
1238 comp:
1239 /* Assume true comparison. Fixup and emit pending guard later. */
1240 if (mm == MM_eq) {
1241 op = IR_EQ;
1242 } else {
1243 op = mm == MM_lt ? IR_LT : IR_LE;
1244 if (dt == IRT_U32 || dt == IRT_U64)
1245 op += (IR_ULT-IR_LT);
1246 }
1247 lj_ir_set(J, IRTG(op, dt), sp[0], sp[1]);
1248 J->postproc = LJ_POST_FIXGUARD;
1249 return TREF_TRUE;
1250 } else {
1251 tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, dt), sp[0], sp[1]);
1252 }
1253 return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1254 }
1255 return 0;
1256}
1257
1258static TRef crec_arith_ptr(jit_State *J, TRef *sp, CType **s, MMS mm)
1259{
1260 CTState *cts = ctype_ctsG(J2G(J));
1261 CType *ctp = s[0];
1262 if (ctype_isptr(ctp->info) || ctype_isrefarray(ctp->info)) {
1263 if ((mm == MM_sub || mm == MM_eq || mm == MM_lt || mm == MM_le) &&
1264 (ctype_isptr(s[1]->info) || ctype_isrefarray(s[1]->info))) {
1265 if (mm == MM_sub) { /* Pointer difference. */
1266 TRef tr;
1267 CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
1268 if (sz == 0 || (sz & (sz-1)) != 0)
1269 return 0; /* NYI: integer division. */
1270 tr = emitir(IRT(IR_SUB, IRT_INTP), sp[0], sp[1]);
1271 tr = emitir(IRT(IR_BSAR, IRT_INTP), tr, lj_ir_kint(J, lj_fls(sz)));
1272#if LJ_64
1273 tr = emitconv(tr, IRT_NUM, IRT_INTP, 0);
1274#endif
1275 return tr;
1276 } else { /* Pointer comparison (unsigned). */
1277 /* Assume true comparison. Fixup and emit pending guard later. */
1278 IROp op = mm == MM_eq ? IR_EQ : mm == MM_lt ? IR_ULT : IR_ULE;
1279 lj_ir_set(J, IRTG(op, IRT_PTR), sp[0], sp[1]);
1280 J->postproc = LJ_POST_FIXGUARD;
1281 return TREF_TRUE;
1282 }
1283 }
1284 if (!((mm == MM_add || mm == MM_sub) && ctype_isnum(s[1]->info)))
1285 return 0;
1286 } else if (mm == MM_add && ctype_isnum(ctp->info) &&
1287 (ctype_isptr(s[1]->info) || ctype_isrefarray(s[1]->info))) {
1288 TRef tr = sp[0]; sp[0] = sp[1]; sp[1] = tr; /* Swap pointer and index. */
1289 ctp = s[1];
1290 } else {
1291 return 0;
1292 }
1293 {
1294 TRef tr = sp[1];
1295 IRType t = tref_type(tr);
1296 CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
1297 CTypeID id;
1298#if LJ_64
1299 if (t == IRT_NUM || t == IRT_FLOAT)
1300 tr = emitconv(tr, IRT_INTP, t, IRCONV_TRUNC|IRCONV_ANY);
1301 else if (!(t == IRT_I64 || t == IRT_U64))
1302 tr = emitconv(tr, IRT_INTP, IRT_INT,
1303 ((t - IRT_I8) & 1) ? 0 : IRCONV_SEXT);
1304#else
1305 if (!tref_typerange(sp[1], IRT_I8, IRT_U32)) {
1306 tr = emitconv(tr, IRT_INTP, t,
1307 (t == IRT_NUM || t == IRT_FLOAT) ?
1308 IRCONV_TRUNC|IRCONV_ANY : 0);
1309 }
1310#endif
1311 tr = emitir(IRT(IR_MUL, IRT_INTP), tr, lj_ir_kintp(J, sz));
1312 tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, IRT_PTR), sp[0], tr);
1313 id = lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(ctp->info)),
1314 CTSIZE_PTR);
1315 return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1316 }
1317}
1318
1319/* Record ctype arithmetic metamethods. */
1320static void crec_arith_meta(jit_State *J, CTState *cts, RecordFFData *rd)
1321{
1322 cTValue *tv = NULL;
1323 if (J->base[0]) {
1324 if (tviscdata(&rd->argv[0])) {
1325 CTypeID id = argv2cdata(J, J->base[0], &rd->argv[0])->ctypeid;
1326 CType *ct = ctype_raw(cts, id);
1327 if (ctype_isptr(ct->info)) id = ctype_cid(ct->info);
1328 tv = lj_ctype_meta(cts, id, (MMS)rd->data);
1329 }
1330 if (!tv && J->base[1] && tviscdata(&rd->argv[1])) {
1331 CTypeID id = argv2cdata(J, J->base[1], &rd->argv[1])->ctypeid;
1332 CType *ct = ctype_raw(cts, id);
1333 if (ctype_isptr(ct->info)) id = ctype_cid(ct->info);
1334 tv = lj_ctype_meta(cts, id, (MMS)rd->data);
1335 }
1336 }
1337 if (tv) {
1338 if (tvisfunc(tv)) {
1339 J->base[-1] = lj_ir_kfunc(J, funcV(tv)) | TREF_FRAME;
1340 rd->nres = -1; /* Pending tailcall. */
1341 return;
1342 } /* NYI: non-function metamethods. */
1343 } else if ((MMS)rd->data == MM_eq) {
1344 J->base[0] = TREF_FALSE;
1345 return;
1346 }
1347 lj_trace_err(J, LJ_TRERR_BADTYPE);
1348}
1349
1350void LJ_FASTCALL recff_cdata_arith(jit_State *J, RecordFFData *rd)
1351{
1352 CTState *cts = ctype_ctsG(J2G(J));
1353 TRef sp[2];
1354 CType *s[2];
1355 MSize i;
1356 for (i = 0; i < 2; i++) {
1357 TRef tr = J->base[i];
1358 CType *ct = ctype_get(cts, CTID_DOUBLE);
1359 if (!tr) {
1360 goto trymeta;
1361 } else if (tref_iscdata(tr)) {
1362 CTypeID id = argv2cdata(J, tr, &rd->argv[i])->ctypeid;
1363 IRType t;
1364 ct = ctype_raw(cts, id);
1365 t = crec_ct2irt(cts, ct);
1366 if (ctype_isptr(ct->info)) { /* Resolve pointer or reference. */
1367 tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_PTR);
1368 if (ctype_isref(ct->info)) {
1369 ct = ctype_rawchild(cts, ct);
1370 t = crec_ct2irt(cts, ct);
1371 }
1372 } else if (t == IRT_I64 || t == IRT_U64) {
1373 tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_INT64);
1374 lj_needsplit(J);
1375 goto ok;
1376 } else if (t == IRT_INT || t == IRT_U32) {
1377 tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_INT);
1378 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1379 goto ok;
1380 } else if (ctype_isfunc(ct->info)) {
1381 tr = emitir(IRT(IR_FLOAD, IRT_PTR), tr, IRFL_CDATA_PTR);
1382 ct = ctype_get(cts,
1383 lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR));
1384 goto ok;
1385 } else {
1386 tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCcdata)));
1387 }
1388 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1389 if (ctype_isnum(ct->info)) {
1390 if (t == IRT_CDATA) goto trymeta;
1391 if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
1392 tr = emitir(IRT(IR_XLOAD, t), tr, 0);
1393 } else if (!(ctype_isptr(ct->info) || ctype_isrefarray(ct->info))) {
1394 goto trymeta;
1395 }
1396 } else if (tref_isnil(tr)) {
1397 tr = lj_ir_kptr(J, NULL);
1398 ct = ctype_get(cts, CTID_P_VOID);
1399 } else if (tref_isinteger(tr)) {
1400 ct = ctype_get(cts, CTID_INT32);
1401 } else if (tref_isstr(tr)) {
1402 TRef tr2 = J->base[1-i];
1403 CTypeID id = argv2cdata(J, tr2, &rd->argv[1-i])->ctypeid;
1404 ct = ctype_raw(cts, id);
1405 if (ctype_isenum(ct->info)) { /* Match string against enum constant. */
1406 GCstr *str = strV(&rd->argv[i]);
1407 CTSize ofs;
1408 CType *cct = lj_ctype_getfield(cts, ct, str, &ofs);
1409 if (cct && ctype_isconstval(cct->info)) {
1410 /* Specialize to the name of the enum constant. */
1411 emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, str));
1412 ct = ctype_child(cts, cct);
1413 tr = lj_ir_kint(J, (int32_t)ofs);
1414 } /* else: interpreter will throw. */
1415 } /* else: interpreter will throw. */
1416 } else if (!tref_isnum(tr)) {
1417 goto trymeta;
1418 }
1419 ok:
1420 s[i] = ct;
1421 sp[i] = tr;
1422 }
1423 {
1424 TRef tr;
1425 if ((tr = crec_arith_int64(J, sp, s, (MMS)rd->data)) ||
1426 (tr = crec_arith_ptr(J, sp, s, (MMS)rd->data))) {
1427 J->base[0] = tr;
1428 /* Fixup cdata comparisons, too. Avoids some cdata escapes. */
1429 if (J->postproc == LJ_POST_FIXGUARD && frame_iscont(J->L->base-1) &&
1430 !irt_isguard(J->guardemit)) {
1431 const BCIns *pc = frame_contpc(J->L->base-1) - 1;
1432 if (bc_op(*pc) <= BC_ISNEP) {
1433 setframe_pc(&J2G(J)->tmptv, pc);
1434 J2G(J)->tmptv.u32.lo = ((tref_istrue(tr) ^ bc_op(*pc)) & 1);
1435 J->postproc = LJ_POST_FIXCOMP;
1436 }
1437 }
1438 } else {
1439 trymeta:
1440 crec_arith_meta(J, cts, rd);
1441 }
1442 }
1443}
1444
1445/* -- C library namespace metamethods ------------------------------------- */
1446
1447void LJ_FASTCALL recff_clib_index(jit_State *J, RecordFFData *rd)
1448{
1449 CTState *cts = ctype_ctsG(J2G(J));
1450 if (tref_isudata(J->base[0]) && tref_isstr(J->base[1]) &&
1451 udataV(&rd->argv[0])->udtype == UDTYPE_FFI_CLIB) {
1452 CLibrary *cl = (CLibrary *)uddata(udataV(&rd->argv[0]));
1453 GCstr *name = strV(&rd->argv[1]);
1454 CType *ct;
1455 CTypeID id = lj_ctype_getname(cts, &ct, name, CLNS_INDEX);
1456 cTValue *tv = lj_tab_getstr(cl->cache, name);
1457 rd->nres = rd->data;
1458 if (id && tv && !tvisnil(tv)) {
1459 /* Specialize to the symbol name and make the result a constant. */
1460 emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, name));
1461 if (ctype_isconstval(ct->info)) {
1462 if (ct->size >= 0x80000000u &&
1463 (ctype_child(cts, ct)->info & CTF_UNSIGNED))
1464 J->base[0] = lj_ir_knum(J, (lua_Number)(uint32_t)ct->size);
1465 else
1466 J->base[0] = lj_ir_kint(J, (int32_t)ct->size);
1467 } else if (ctype_isextern(ct->info)) {
1468 CTypeID sid = ctype_cid(ct->info);
1469 void *sp = *(void **)cdataptr(cdataV(tv));
1470 TRef ptr;
1471 ct = ctype_raw(cts, sid);
1472 if (LJ_64 && !checkptr32(sp))
1473 ptr = lj_ir_kintp(J, (uintptr_t)sp);
1474 else
1475 ptr = lj_ir_kptr(J, sp);
1476 if (rd->data) {
1477 J->base[0] = crec_tv_ct(J, ct, sid, ptr);
1478 } else {
1479 J->needsnap = 1;
1480 crec_ct_tv(J, ct, ptr, J->base[2], &rd->argv[2]);
1481 }
1482 } else {
1483 J->base[0] = lj_ir_kgc(J, obj2gco(cdataV(tv)), IRT_CDATA);
1484 }
1485 } else {
1486 lj_trace_err(J, LJ_TRERR_NOCACHE);
1487 }
1488 } /* else: interpreter will throw. */
1489}
1490
1491/* -- FFI library functions ----------------------------------------------- */
1492
1493static TRef crec_toint(jit_State *J, CTState *cts, TRef sp, TValue *sval)
1494{
1495 return crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0, sp, sval);
1496}
1497
1498void LJ_FASTCALL recff_ffi_new(jit_State *J, RecordFFData *rd)
1499{
1500 crec_alloc(J, rd, argv2ctype(J, J->base[0], &rd->argv[0]));
1501}
1502
1503void LJ_FASTCALL recff_ffi_errno(jit_State *J, RecordFFData *rd)
1504{
1505 UNUSED(rd);
1506 if (J->base[0])
1507 lj_trace_err(J, LJ_TRERR_NYICALL);
1508 J->base[0] = lj_ir_call(J, IRCALL_lj_vm_errno);
1509}
1510
1511void LJ_FASTCALL recff_ffi_string(jit_State *J, RecordFFData *rd)
1512{
1513 CTState *cts = ctype_ctsG(J2G(J));
1514 TRef tr = J->base[0];
1515 if (tr) {
1516 TRef trlen = J->base[1];
1517 if (!tref_isnil(trlen)) {
1518 trlen = crec_toint(J, cts, trlen, &rd->argv[1]);
1519 tr = crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, tr, &rd->argv[0]);
1520 } else {
1521 tr = crec_ct_tv(J, ctype_get(cts, CTID_P_CCHAR), 0, tr, &rd->argv[0]);
1522 trlen = lj_ir_call(J, IRCALL_strlen, tr);
1523 }
1524 J->base[0] = emitir(IRT(IR_XSNEW, IRT_STR), tr, trlen);
1525 } /* else: interpreter will throw. */
1526}
1527
1528void LJ_FASTCALL recff_ffi_copy(jit_State *J, RecordFFData *rd)
1529{
1530 CTState *cts = ctype_ctsG(J2G(J));
1531 TRef trdst = J->base[0], trsrc = J->base[1], trlen = J->base[2];
1532 if (trdst && trsrc && (trlen || tref_isstr(trsrc))) {
1533 trdst = crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, trdst, &rd->argv[0]);
1534 trsrc = crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, trsrc, &rd->argv[1]);
1535 if (trlen) {
1536 trlen = crec_toint(J, cts, trlen, &rd->argv[2]);
1537 } else {
1538 trlen = emitir(IRTI(IR_FLOAD), J->base[1], IRFL_STR_LEN);
1539 trlen = emitir(IRTI(IR_ADD), trlen, lj_ir_kint(J, 1));
1540 }
1541 rd->nres = 0;
1542 crec_copy(J, trdst, trsrc, trlen, NULL);
1543 } /* else: interpreter will throw. */
1544}
1545
1546void LJ_FASTCALL recff_ffi_fill(jit_State *J, RecordFFData *rd)
1547{
1548 CTState *cts = ctype_ctsG(J2G(J));
1549 TRef trdst = J->base[0], trlen = J->base[1], trfill = J->base[2];
1550 if (trdst && trlen) {
1551 CTSize step = 1;
1552 if (tviscdata(&rd->argv[0])) { /* Get alignment of original destination. */
1553 CTSize sz;
1554 CType *ct = ctype_raw(cts, cdataV(&rd->argv[0])->ctypeid);
1555 if (ctype_isptr(ct->info))
1556 ct = ctype_rawchild(cts, ct);
1557 step = (1u<<ctype_align(lj_ctype_info(cts, ctype_typeid(cts, ct), &sz)));
1558 }
1559 trdst = crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, trdst, &rd->argv[0]);
1560 trlen = crec_toint(J, cts, trlen, &rd->argv[1]);
1561 if (trfill)
1562 trfill = crec_toint(J, cts, trfill, &rd->argv[2]);
1563 else
1564 trfill = lj_ir_kint(J, 0);
1565 rd->nres = 0;
1566 crec_fill(J, trdst, trlen, trfill, step);
1567 } /* else: interpreter will throw. */
1568}
1569
1570void LJ_FASTCALL recff_ffi_typeof(jit_State *J, RecordFFData *rd)
1571{
1572 if (tref_iscdata(J->base[0])) {
1573 TRef trid = lj_ir_kint(J, argv2ctype(J, J->base[0], &rd->argv[0]));
1574 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA),
1575 lj_ir_kint(J, CTID_CTYPEID), trid);
1576 } else {
1577 setfuncV(J->L, &J->errinfo, J->fn);
1578 lj_trace_err_info(J, LJ_TRERR_NYIFFU);
1579 }
1580}
1581
1582void LJ_FASTCALL recff_ffi_istype(jit_State *J, RecordFFData *rd)
1583{
1584 argv2ctype(J, J->base[0], &rd->argv[0]);
1585 if (tref_iscdata(J->base[1])) {
1586 argv2ctype(J, J->base[1], &rd->argv[1]);
1587 J->postproc = LJ_POST_FIXBOOL;
1588 J->base[0] = TREF_TRUE;
1589 } else {
1590 J->base[0] = TREF_FALSE;
1591 }
1592}
1593
1594void LJ_FASTCALL recff_ffi_abi(jit_State *J, RecordFFData *rd)
1595{
1596 if (tref_isstr(J->base[0])) {
1597 /* Specialize to the ABI string to make the boolean result a constant. */
1598 emitir(IRTG(IR_EQ, IRT_STR), J->base[0], lj_ir_kstr(J, strV(&rd->argv[0])));
1599 J->postproc = LJ_POST_FIXBOOL;
1600 J->base[0] = TREF_TRUE;
1601 } else {
1602 lj_trace_err(J, LJ_TRERR_BADTYPE);
1603 }
1604}
1605
1606/* Record ffi.sizeof(), ffi.alignof(), ffi.offsetof(). */
1607void LJ_FASTCALL recff_ffi_xof(jit_State *J, RecordFFData *rd)
1608{
1609 CTypeID id = argv2ctype(J, J->base[0], &rd->argv[0]);
1610 if (rd->data == FF_ffi_sizeof) {
1611 CType *ct = lj_ctype_rawref(ctype_ctsG(J2G(J)), id);
1612 if (ctype_isvltype(ct->info))
1613 lj_trace_err(J, LJ_TRERR_BADTYPE);
1614 } else if (rd->data == FF_ffi_offsetof) { /* Specialize to the field name. */
1615 if (!tref_isstr(J->base[1]))
1616 lj_trace_err(J, LJ_TRERR_BADTYPE);
1617 emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
1618 rd->nres = 3; /* Just in case. */
1619 }
1620 J->postproc = LJ_POST_FIXCONST;
1621 J->base[0] = J->base[1] = J->base[2] = TREF_NIL;
1622}
1623
1624void LJ_FASTCALL recff_ffi_gc(jit_State *J, RecordFFData *rd)
1625{
1626 argv2cdata(J, J->base[0], &rd->argv[0]);
1627 crec_finalizer(J, J->base[0], &rd->argv[1]);
1628}
1629
1630/* -- Miscellaneous library functions ------------------------------------- */
1631
1632void LJ_FASTCALL lj_crecord_tonumber(jit_State *J, RecordFFData *rd)
1633{
1634 CTState *cts = ctype_ctsG(J2G(J));
1635 CType *d, *ct = lj_ctype_rawref(cts, cdataV(&rd->argv[0])->ctypeid);
1636 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1637 if (ctype_isnum(ct->info) || ctype_iscomplex(ct->info)) {
1638 if (ctype_isinteger_or_bool(ct->info) && ct->size <= 4 &&
1639 !(ct->size == 4 && (ct->info & CTF_UNSIGNED)))
1640 d = ctype_get(cts, CTID_INT32);
1641 else
1642 d = ctype_get(cts, CTID_DOUBLE);
1643 J->base[0] = crec_ct_tv(J, d, 0, J->base[0], &rd->argv[0]);
1644 } else {
1645 J->base[0] = TREF_NIL;
1646 }
1647}
1648
1649#undef IR
1650#undef emitir
1651#undef emitconv
1652
1653#endif
1654