1/*
2** Trace recorder for C data operations.
3** Copyright (C) 2005-2021 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#define lj_ffrecord_c
7#define LUA_CORE
8
9#include "lj_obj.h"
10
11#if LJ_HASJIT && LJ_HASFFI
12
13#include "lj_err.h"
14#include "lj_tab.h"
15#include "lj_frame.h"
16#include "lj_ctype.h"
17#include "lj_cdata.h"
18#include "lj_cparse.h"
19#include "lj_cconv.h"
20#include "lj_carith.h"
21#include "lj_clib.h"
22#include "lj_ccall.h"
23#include "lj_ff.h"
24#include "lj_ir.h"
25#include "lj_jit.h"
26#include "lj_ircall.h"
27#include "lj_iropt.h"
28#include "lj_trace.h"
29#include "lj_record.h"
30#include "lj_ffrecord.h"
31#include "lj_snap.h"
32#include "lj_crecord.h"
33#include "lj_dispatch.h"
34#include "lj_strfmt.h"
35
36/* Some local macros to save typing. Undef'd at the end. */
37#define IR(ref) (&J->cur.ir[(ref)])
38
39/* Pass IR on to next optimization in chain (FOLD). */
40#define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
41
42#define emitconv(a, dt, st, flags) \
43 emitir(IRT(IR_CONV, (dt)), (a), (st)|((dt) << 5)|(flags))
44
45/* -- C type checks ------------------------------------------------------- */
46
47static GCcdata *argv2cdata(jit_State *J, TRef tr, cTValue *o)
48{
49 GCcdata *cd;
50 TRef trtypeid;
51 if (!tref_iscdata(tr))
52 lj_trace_err(J, LJ_TRERR_BADTYPE);
53 cd = cdataV(o);
54 /* Specialize to the CTypeID. */
55 trtypeid = emitir(IRT(IR_FLOAD, IRT_U16), tr, IRFL_CDATA_CTYPEID);
56 emitir(IRTG(IR_EQ, IRT_INT), trtypeid, lj_ir_kint(J, (int32_t)cd->ctypeid));
57 return cd;
58}
59
60/* Specialize to the CTypeID held by a cdata constructor. */
61static CTypeID crec_constructor(jit_State *J, GCcdata *cd, TRef tr)
62{
63 CTypeID id;
64 lj_assertJ(tref_iscdata(tr) && cd->ctypeid == CTID_CTYPEID,
65 "expected CTypeID cdata");
66 id = *(CTypeID *)cdataptr(cd);
67 tr = emitir(IRT(IR_FLOAD, IRT_INT), tr, IRFL_CDATA_INT);
68 emitir(IRTG(IR_EQ, IRT_INT), tr, lj_ir_kint(J, (int32_t)id));
69 return id;
70}
71
72static CTypeID argv2ctype(jit_State *J, TRef tr, cTValue *o)
73{
74 if (tref_isstr(tr)) {
75 GCstr *s = strV(o);
76 CPState cp;
77 CTypeID oldtop;
78 /* Specialize to the string containing the C type declaration. */
79 emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, s));
80 cp.L = J->L;
81 cp.cts = ctype_ctsG(J2G(J));
82 oldtop = cp.cts->top;
83 cp.srcname = strdata(s);
84 cp.p = strdata(s);
85 cp.param = NULL;
86 cp.mode = CPARSE_MODE_ABSTRACT|CPARSE_MODE_NOIMPLICIT;
87 if (lj_cparse(&cp) || cp.cts->top > oldtop) /* Avoid new struct defs. */
88 lj_trace_err(J, LJ_TRERR_BADTYPE);
89 return cp.val.id;
90 } else {
91 GCcdata *cd = argv2cdata(J, tr, o);
92 return cd->ctypeid == CTID_CTYPEID ? crec_constructor(J, cd, tr) :
93 cd->ctypeid;
94 }
95}
96
97/* Convert CType to IRType (if possible). */
98static IRType crec_ct2irt(CTState *cts, CType *ct)
99{
100 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
101 if (LJ_LIKELY(ctype_isnum(ct->info))) {
102 if ((ct->info & CTF_FP)) {
103 if (ct->size == sizeof(double))
104 return IRT_NUM;
105 else if (ct->size == sizeof(float))
106 return IRT_FLOAT;
107 } else {
108 uint32_t b = lj_fls(ct->size);
109 if (b <= 3)
110 return IRT_I8 + 2*b + ((ct->info & CTF_UNSIGNED) ? 1 : 0);
111 }
112 } else if (ctype_isptr(ct->info)) {
113 return (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
114 } else if (ctype_iscomplex(ct->info)) {
115 if (ct->size == 2*sizeof(double))
116 return IRT_NUM;
117 else if (ct->size == 2*sizeof(float))
118 return IRT_FLOAT;
119 }
120 return IRT_CDATA;
121}
122
123/* -- Optimized memory fill and copy -------------------------------------- */
124
125/* Maximum length and unroll of inlined copy/fill. */
126#define CREC_COPY_MAXUNROLL 16
127#define CREC_COPY_MAXLEN 128
128
129#define CREC_FILL_MAXUNROLL 16
130
131/* Number of windowed registers used for optimized memory copy. */
132#if LJ_TARGET_X86
133#define CREC_COPY_REGWIN 2
134#elif LJ_TARGET_PPC || LJ_TARGET_MIPS
135#define CREC_COPY_REGWIN 8
136#else
137#define CREC_COPY_REGWIN 4
138#endif
139
140/* List of memory offsets for copy/fill. */
141typedef struct CRecMemList {
142 CTSize ofs; /* Offset in bytes. */
143 IRType tp; /* Type of load/store. */
144 TRef trofs; /* TRef of interned offset. */
145 TRef trval; /* TRef of load value. */
146} CRecMemList;
147
148/* Generate copy list for element-wise struct copy. */
149static MSize crec_copy_struct(CRecMemList *ml, CTState *cts, CType *ct)
150{
151 CTypeID fid = ct->sib;
152 MSize mlp = 0;
153 while (fid) {
154 CType *df = ctype_get(cts, fid);
155 fid = df->sib;
156 if (ctype_isfield(df->info)) {
157 CType *cct;
158 IRType tp;
159 if (!gcref(df->name)) continue; /* Ignore unnamed fields. */
160 cct = ctype_rawchild(cts, df); /* Field type. */
161 tp = crec_ct2irt(cts, cct);
162 if (tp == IRT_CDATA) return 0; /* NYI: aggregates. */
163 if (mlp >= CREC_COPY_MAXUNROLL) return 0;
164 ml[mlp].ofs = df->size;
165 ml[mlp].tp = tp;
166 mlp++;
167 if (ctype_iscomplex(cct->info)) {
168 if (mlp >= CREC_COPY_MAXUNROLL) return 0;
169 ml[mlp].ofs = df->size + (cct->size >> 1);
170 ml[mlp].tp = tp;
171 mlp++;
172 }
173 } else if (!ctype_isconstval(df->info)) {
174 /* NYI: bitfields and sub-structures. */
175 return 0;
176 }
177 }
178 return mlp;
179}
180
181/* Generate unrolled copy list, from highest to lowest step size/alignment. */
182static MSize crec_copy_unroll(CRecMemList *ml, CTSize len, CTSize step,
183 IRType tp)
184{
185 CTSize ofs = 0;
186 MSize mlp = 0;
187 if (tp == IRT_CDATA) tp = IRT_U8 + 2*lj_fls(step);
188 do {
189 while (ofs + step <= len) {
190 if (mlp >= CREC_COPY_MAXUNROLL) return 0;
191 ml[mlp].ofs = ofs;
192 ml[mlp].tp = tp;
193 mlp++;
194 ofs += step;
195 }
196 step >>= 1;
197 tp -= 2;
198 } while (ofs < len);
199 return mlp;
200}
201
202/*
203** Emit copy list with windowed loads/stores.
204** LJ_TARGET_UNALIGNED: may emit unaligned loads/stores (not marked as such).
205*/
206static void crec_copy_emit(jit_State *J, CRecMemList *ml, MSize mlp,
207 TRef trdst, TRef trsrc)
208{
209 MSize i, j, rwin = 0;
210 for (i = 0, j = 0; i < mlp; ) {
211 TRef trofs = lj_ir_kintp(J, ml[i].ofs);
212 TRef trsptr = emitir(IRT(IR_ADD, IRT_PTR), trsrc, trofs);
213 ml[i].trval = emitir(IRT(IR_XLOAD, ml[i].tp), trsptr, 0);
214 ml[i].trofs = trofs;
215 i++;
216 rwin += (LJ_SOFTFP32 && ml[i].tp == IRT_NUM) ? 2 : 1;
217 if (rwin >= CREC_COPY_REGWIN || i >= mlp) { /* Flush buffered stores. */
218 rwin = 0;
219 for ( ; j < i; j++) {
220 TRef trdptr = emitir(IRT(IR_ADD, IRT_PTR), trdst, ml[j].trofs);
221 emitir(IRT(IR_XSTORE, ml[j].tp), trdptr, ml[j].trval);
222 }
223 }
224 }
225}
226
227/* Optimized memory copy. */
228static void crec_copy(jit_State *J, TRef trdst, TRef trsrc, TRef trlen,
229 CType *ct)
230{
231 if (tref_isk(trlen)) { /* Length must be constant. */
232 CRecMemList ml[CREC_COPY_MAXUNROLL];
233 MSize mlp = 0;
234 CTSize step = 1, len = (CTSize)IR(tref_ref(trlen))->i;
235 IRType tp = IRT_CDATA;
236 int needxbar = 0;
237 if (len == 0) return; /* Shortcut. */
238 if (len > CREC_COPY_MAXLEN) goto fallback;
239 if (ct) {
240 CTState *cts = ctype_ctsG(J2G(J));
241 lj_assertJ(ctype_isarray(ct->info) || ctype_isstruct(ct->info),
242 "copy of non-aggregate");
243 if (ctype_isarray(ct->info)) {
244 CType *cct = ctype_rawchild(cts, ct);
245 tp = crec_ct2irt(cts, cct);
246 if (tp == IRT_CDATA) goto rawcopy;
247 step = lj_ir_type_size[tp];
248 lj_assertJ((len & (step-1)) == 0, "copy of fractional size");
249 } else if ((ct->info & CTF_UNION)) {
250 step = (1u << ctype_align(ct->info));
251 goto rawcopy;
252 } else {
253 mlp = crec_copy_struct(ml, cts, ct);
254 goto emitcopy;
255 }
256 } else {
257 rawcopy:
258 needxbar = 1;
259 if (LJ_TARGET_UNALIGNED || step >= CTSIZE_PTR)
260 step = CTSIZE_PTR;
261 }
262 mlp = crec_copy_unroll(ml, len, step, tp);
263 emitcopy:
264 if (mlp) {
265 crec_copy_emit(J, ml, mlp, trdst, trsrc);
266 if (needxbar)
267 emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
268 return;
269 }
270 }
271fallback:
272 /* Call memcpy. Always needs a barrier to disable alias analysis. */
273 lj_ir_call(J, IRCALL_memcpy, trdst, trsrc, trlen);
274 emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
275}
276
277/* Generate unrolled fill list, from highest to lowest step size/alignment. */
278static MSize crec_fill_unroll(CRecMemList *ml, CTSize len, CTSize step)
279{
280 CTSize ofs = 0;
281 MSize mlp = 0;
282 IRType tp = IRT_U8 + 2*lj_fls(step);
283 do {
284 while (ofs + step <= len) {
285 if (mlp >= CREC_COPY_MAXUNROLL) return 0;
286 ml[mlp].ofs = ofs;
287 ml[mlp].tp = tp;
288 mlp++;
289 ofs += step;
290 }
291 step >>= 1;
292 tp -= 2;
293 } while (ofs < len);
294 return mlp;
295}
296
297/*
298** Emit stores for fill list.
299** LJ_TARGET_UNALIGNED: may emit unaligned stores (not marked as such).
300*/
301static void crec_fill_emit(jit_State *J, CRecMemList *ml, MSize mlp,
302 TRef trdst, TRef trfill)
303{
304 MSize i;
305 for (i = 0; i < mlp; i++) {
306 TRef trofs = lj_ir_kintp(J, ml[i].ofs);
307 TRef trdptr = emitir(IRT(IR_ADD, IRT_PTR), trdst, trofs);
308 emitir(IRT(IR_XSTORE, ml[i].tp), trdptr, trfill);
309 }
310}
311
312/* Optimized memory fill. */
313static void crec_fill(jit_State *J, TRef trdst, TRef trlen, TRef trfill,
314 CTSize step)
315{
316 if (tref_isk(trlen)) { /* Length must be constant. */
317 CRecMemList ml[CREC_FILL_MAXUNROLL];
318 MSize mlp;
319 CTSize len = (CTSize)IR(tref_ref(trlen))->i;
320 if (len == 0) return; /* Shortcut. */
321 if (LJ_TARGET_UNALIGNED || step >= CTSIZE_PTR)
322 step = CTSIZE_PTR;
323 if (step * CREC_FILL_MAXUNROLL < len) goto fallback;
324 mlp = crec_fill_unroll(ml, len, step);
325 if (!mlp) goto fallback;
326 if (tref_isk(trfill) || ml[0].tp != IRT_U8)
327 trfill = emitconv(trfill, IRT_INT, IRT_U8, 0);
328 if (ml[0].tp != IRT_U8) { /* Scatter U8 to U16/U32/U64. */
329 if (CTSIZE_PTR == 8 && ml[0].tp == IRT_U64) {
330 if (tref_isk(trfill)) /* Pointless on x64 with zero-extended regs. */
331 trfill = emitconv(trfill, IRT_U64, IRT_U32, 0);
332 trfill = emitir(IRT(IR_MUL, IRT_U64), trfill,
333 lj_ir_kint64(J, U64x(01010101,01010101)));
334 } else {
335 trfill = emitir(IRTI(IR_MUL), trfill,
336 lj_ir_kint(J, ml[0].tp == IRT_U16 ? 0x0101 : 0x01010101));
337 }
338 }
339 crec_fill_emit(J, ml, mlp, trdst, trfill);
340 } else {
341fallback:
342 /* Call memset. Always needs a barrier to disable alias analysis. */
343 lj_ir_call(J, IRCALL_memset, trdst, trfill, trlen); /* Note: arg order! */
344 }
345 emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
346}
347
348/* -- Convert C type to C type -------------------------------------------- */
349
350/*
351** This code mirrors the code in lj_cconv.c. It performs the same steps
352** for the trace recorder that lj_cconv.c does for the interpreter.
353**
354** One major difference is that we can get away with much fewer checks
355** here. E.g. checks for casts, constness or correct types can often be
356** omitted, even if they might fail. The interpreter subsequently throws
357** an error, which aborts the trace.
358**
359** All operations are specialized to their C types, so the on-trace
360** outcome must be the same as the outcome in the interpreter. If the
361** interpreter doesn't throw an error, then the trace is correct, too.
362** Care must be taken not to generate invalid (temporary) IR or to
363** trigger asserts.
364*/
365
366/* Determine whether a passed number or cdata number is non-zero. */
367static int crec_isnonzero(CType *s, void *p)
368{
369 if (p == (void *)0)
370 return 0;
371 if (p == (void *)1)
372 return 1;
373 if ((s->info & CTF_FP)) {
374 if (s->size == sizeof(float))
375 return (*(float *)p != 0);
376 else
377 return (*(double *)p != 0);
378 } else {
379 if (s->size == 1)
380 return (*(uint8_t *)p != 0);
381 else if (s->size == 2)
382 return (*(uint16_t *)p != 0);
383 else if (s->size == 4)
384 return (*(uint32_t *)p != 0);
385 else
386 return (*(uint64_t *)p != 0);
387 }
388}
389
390static TRef crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp,
391 void *svisnz)
392{
393 IRType dt = crec_ct2irt(ctype_ctsG(J2G(J)), d);
394 IRType st = crec_ct2irt(ctype_ctsG(J2G(J)), s);
395 CTSize dsize = d->size, ssize = s->size;
396 CTInfo dinfo = d->info, sinfo = s->info;
397
398 if (ctype_type(dinfo) > CT_MAYCONVERT || ctype_type(sinfo) > CT_MAYCONVERT)
399 goto err_conv;
400
401 /*
402 ** Note: Unlike lj_cconv_ct_ct(), sp holds the _value_ of pointers and
403 ** numbers up to 8 bytes. Otherwise sp holds a pointer.
404 */
405
406 switch (cconv_idx2(dinfo, sinfo)) {
407 /* Destination is a bool. */
408 case CCX(B, B):
409 goto xstore; /* Source operand is already normalized. */
410 case CCX(B, I):
411 case CCX(B, F):
412 if (st != IRT_CDATA) {
413 /* Specialize to the result of a comparison against 0. */
414 TRef zero = (st == IRT_NUM || st == IRT_FLOAT) ? lj_ir_knum(J, 0) :
415 (st == IRT_I64 || st == IRT_U64) ? lj_ir_kint64(J, 0) :
416 lj_ir_kint(J, 0);
417 int isnz = crec_isnonzero(s, svisnz);
418 emitir(IRTG(isnz ? IR_NE : IR_EQ, st), sp, zero);
419 sp = lj_ir_kint(J, isnz);
420 goto xstore;
421 }
422 goto err_nyi;
423
424 /* Destination is an integer. */
425 case CCX(I, B):
426 case CCX(I, I):
427 conv_I_I:
428 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
429 /* Extend 32 to 64 bit integer. */
430 if (dsize == 8 && ssize < 8 && !(LJ_64 && (sinfo & CTF_UNSIGNED)))
431 sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st,
432 (sinfo & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
433 else if (dsize < 8 && ssize == 8) /* Truncate from 64 bit integer. */
434 sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, 0);
435 else if (st == IRT_INT)
436 sp = lj_opt_narrow_toint(J, sp);
437 xstore:
438 if (dt == IRT_I64 || dt == IRT_U64) lj_needsplit(J);
439 if (dp == 0) return sp;
440 emitir(IRT(IR_XSTORE, dt), dp, sp);
441 break;
442 case CCX(I, C):
443 sp = emitir(IRT(IR_XLOAD, st), sp, 0); /* Load re. */
444 /* fallthrough */
445 case CCX(I, F):
446 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
447 sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, IRCONV_ANY);
448 goto xstore;
449 case CCX(I, P):
450 case CCX(I, A):
451 sinfo = CTINFO(CT_NUM, CTF_UNSIGNED);
452 ssize = CTSIZE_PTR;
453 st = IRT_UINTP;
454 if (((dsize ^ ssize) & 8) == 0) { /* Must insert no-op type conversion. */
455 sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, IRT_PTR, 0);
456 goto xstore;
457 }
458 goto conv_I_I;
459
460 /* Destination is a floating-point number. */
461 case CCX(F, B):
462 case CCX(F, I):
463 conv_F_I:
464 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
465 sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st, 0);
466 goto xstore;
467 case CCX(F, C):
468 sp = emitir(IRT(IR_XLOAD, st), sp, 0); /* Load re. */
469 /* fallthrough */
470 case CCX(F, F):
471 conv_F_F:
472 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
473 if (dt != st) sp = emitconv(sp, dt, st, 0);
474 goto xstore;
475
476 /* Destination is a complex number. */
477 case CCX(C, I):
478 case CCX(C, F):
479 { /* Clear im. */
480 TRef ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
481 emitir(IRT(IR_XSTORE, dt), ptr, lj_ir_knum(J, 0));
482 }
483 /* Convert to re. */
484 if ((sinfo & CTF_FP)) goto conv_F_F; else goto conv_F_I;
485
486 case CCX(C, C):
487 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
488 {
489 TRef re, im, ptr;
490 re = emitir(IRT(IR_XLOAD, st), sp, 0);
491 ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, (ssize >> 1)));
492 im = emitir(IRT(IR_XLOAD, st), ptr, 0);
493 if (dt != st) {
494 re = emitconv(re, dt, st, 0);
495 im = emitconv(im, dt, st, 0);
496 }
497 emitir(IRT(IR_XSTORE, dt), dp, re);
498 ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
499 emitir(IRT(IR_XSTORE, dt), ptr, im);
500 }
501 break;
502
503 /* Destination is a vector. */
504 case CCX(V, I):
505 case CCX(V, F):
506 case CCX(V, C):
507 case CCX(V, V):
508 goto err_nyi;
509
510 /* Destination is a pointer. */
511 case CCX(P, P):
512 case CCX(P, A):
513 case CCX(P, S):
514 /* There are only 32 bit pointers/addresses on 32 bit machines.
515 ** Also ok on x64, since all 32 bit ops clear the upper part of the reg.
516 */
517 goto xstore;
518 case CCX(P, I):
519 if (st == IRT_CDATA) goto err_nyi;
520 if (!LJ_64 && ssize == 8) /* Truncate from 64 bit integer. */
521 sp = emitconv(sp, IRT_U32, st, 0);
522 goto xstore;
523 case CCX(P, F):
524 if (st == IRT_CDATA) goto err_nyi;
525 /* The signed conversion is cheaper. x64 really has 47 bit pointers. */
526 sp = emitconv(sp, (LJ_64 && dsize == 8) ? IRT_I64 : IRT_U32,
527 st, IRCONV_ANY);
528 goto xstore;
529
530 /* Destination is an array. */
531 case CCX(A, A):
532 /* Destination is a struct/union. */
533 case CCX(S, S):
534 if (dp == 0) goto err_conv;
535 crec_copy(J, dp, sp, lj_ir_kint(J, dsize), d);
536 break;
537
538 default:
539 err_conv:
540 err_nyi:
541 lj_trace_err(J, LJ_TRERR_NYICONV);
542 break;
543 }
544 return 0;
545}
546
547/* -- Convert C type to TValue (load) ------------------------------------- */
548
549static TRef crec_tv_ct(jit_State *J, CType *s, CTypeID sid, TRef sp)
550{
551 CTState *cts = ctype_ctsG(J2G(J));
552 IRType t = crec_ct2irt(cts, s);
553 CTInfo sinfo = s->info;
554 if (ctype_isnum(sinfo)) {
555 TRef tr;
556 if (t == IRT_CDATA)
557 goto err_nyi; /* NYI: copyval of >64 bit integers. */
558 tr = emitir(IRT(IR_XLOAD, t), sp, 0);
559 if (t == IRT_FLOAT || t == IRT_U32) { /* Keep uint32_t/float as numbers. */
560 return emitconv(tr, IRT_NUM, t, 0);
561 } else if (t == IRT_I64 || t == IRT_U64) { /* Box 64 bit integer. */
562 sp = tr;
563 lj_needsplit(J);
564 } else if ((sinfo & CTF_BOOL)) {
565 /* Assume not equal to zero. Fixup and emit pending guard later. */
566 lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
567 J->postproc = LJ_POST_FIXGUARD;
568 return TREF_TRUE;
569 } else {
570 return tr;
571 }
572 } else if (ctype_isptr(sinfo) || ctype_isenum(sinfo)) {
573 sp = emitir(IRT(IR_XLOAD, t), sp, 0); /* Box pointers and enums. */
574 } else if (ctype_isrefarray(sinfo) || ctype_isstruct(sinfo)) {
575 cts->L = J->L;
576 sid = lj_ctype_intern(cts, CTINFO_REF(sid), CTSIZE_PTR); /* Create ref. */
577 } else if (ctype_iscomplex(sinfo)) { /* Unbox/box complex. */
578 ptrdiff_t esz = (ptrdiff_t)(s->size >> 1);
579 TRef ptr, tr1, tr2, dp;
580 dp = emitir(IRTG(IR_CNEW, IRT_CDATA), lj_ir_kint(J, sid), TREF_NIL);
581 tr1 = emitir(IRT(IR_XLOAD, t), sp, 0);
582 ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, esz));
583 tr2 = emitir(IRT(IR_XLOAD, t), ptr, 0);
584 ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)));
585 emitir(IRT(IR_XSTORE, t), ptr, tr1);
586 ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)+esz));
587 emitir(IRT(IR_XSTORE, t), ptr, tr2);
588 return dp;
589 } else {
590 /* NYI: copyval of vectors. */
591 err_nyi:
592 lj_trace_err(J, LJ_TRERR_NYICONV);
593 }
594 /* Box pointer, ref, enum or 64 bit integer. */
595 return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, sid), sp);
596}
597
598/* -- Convert TValue to C type (store) ------------------------------------ */
599
600static TRef crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, cTValue *sval)
601{
602 CTState *cts = ctype_ctsG(J2G(J));
603 CTypeID sid = CTID_P_VOID;
604 void *svisnz = 0;
605 CType *s;
606 if (LJ_LIKELY(tref_isinteger(sp))) {
607 sid = CTID_INT32;
608 svisnz = (void *)(intptr_t)(tvisint(sval)?(intV(sval)!=0):!tviszero(sval));
609 } else if (tref_isnum(sp)) {
610 sid = CTID_DOUBLE;
611 svisnz = (void *)(intptr_t)(tvisint(sval)?(intV(sval)!=0):!tviszero(sval));
612 } else if (tref_isbool(sp)) {
613 sp = lj_ir_kint(J, tref_istrue(sp) ? 1 : 0);
614 sid = CTID_BOOL;
615 } else if (tref_isnil(sp)) {
616 sp = lj_ir_kptr(J, NULL);
617 } else if (tref_isudata(sp)) {
618 GCudata *ud = udataV(sval);
619 if (ud->udtype == UDTYPE_IO_FILE) {
620 TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), sp, IRFL_UDATA_UDTYPE);
621 emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, UDTYPE_IO_FILE));
622 sp = emitir(IRT(IR_FLOAD, IRT_PTR), sp, IRFL_UDATA_FILE);
623 } else {
624 sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCudata)));
625 }
626 } else if (tref_isstr(sp)) {
627 if (ctype_isenum(d->info)) { /* Match string against enum constant. */
628 GCstr *str = strV(sval);
629 CTSize ofs;
630 CType *cct = lj_ctype_getfield(cts, d, str, &ofs);
631 /* Specialize to the name of the enum constant. */
632 emitir(IRTG(IR_EQ, IRT_STR), sp, lj_ir_kstr(J, str));
633 if (cct && ctype_isconstval(cct->info)) {
634 lj_assertJ(ctype_child(cts, cct)->size == 4,
635 "only 32 bit const supported"); /* NYI */
636 svisnz = (void *)(intptr_t)(ofs != 0);
637 sp = lj_ir_kint(J, (int32_t)ofs);
638 sid = ctype_cid(cct->info);
639 } /* else: interpreter will throw. */
640 } else if (ctype_isrefarray(d->info)) { /* Copy string to array. */
641 lj_trace_err(J, LJ_TRERR_BADTYPE); /* NYI */
642 } else { /* Otherwise pass the string data as a const char[]. */
643 /* Don't use STRREF. It folds with SNEW, which loses the trailing NUL. */
644 sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCstr)));
645 sid = CTID_A_CCHAR;
646 }
647 } else if (tref_islightud(sp)) {
648#if LJ_64
649 lj_trace_err(J, LJ_TRERR_NYICONV);
650#endif
651 } else { /* NYI: tref_istab(sp). */
652 IRType t;
653 sid = argv2cdata(J, sp, sval)->ctypeid;
654 s = ctype_raw(cts, sid);
655 svisnz = cdataptr(cdataV(sval));
656 if (ctype_isfunc(s->info)) {
657 sid = lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|sid), CTSIZE_PTR);
658 s = ctype_get(cts, sid);
659 t = IRT_PTR;
660 } else {
661 t = crec_ct2irt(cts, s);
662 }
663 if (ctype_isptr(s->info)) {
664 sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_PTR);
665 if (ctype_isref(s->info)) {
666 svisnz = *(void **)svisnz;
667 s = ctype_rawchild(cts, s);
668 if (ctype_isenum(s->info)) s = ctype_child(cts, s);
669 t = crec_ct2irt(cts, s);
670 } else {
671 goto doconv;
672 }
673 } else if (t == IRT_I64 || t == IRT_U64) {
674 sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INT64);
675 lj_needsplit(J);
676 goto doconv;
677 } else if (t == IRT_INT || t == IRT_U32) {
678 if (ctype_isenum(s->info)) s = ctype_child(cts, s);
679 sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INT);
680 goto doconv;
681 } else {
682 sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCcdata)));
683 }
684 if (ctype_isnum(s->info) && t != IRT_CDATA)
685 sp = emitir(IRT(IR_XLOAD, t), sp, 0); /* Load number value. */
686 goto doconv;
687 }
688 s = ctype_get(cts, sid);
689doconv:
690 if (ctype_isenum(d->info)) d = ctype_child(cts, d);
691 return crec_ct_ct(J, d, s, dp, sp, svisnz);
692}
693
694/* -- C data metamethods -------------------------------------------------- */
695
696/* This would be rather difficult in FOLD, so do it here:
697** (base+k)+(idx*sz)+ofs ==> (base+idx*sz)+(ofs+k)
698** (base+(idx+k)*sz)+ofs ==> (base+idx*sz)+(ofs+k*sz)
699*/
700static TRef crec_reassoc_ofs(jit_State *J, TRef tr, ptrdiff_t *ofsp, MSize sz)
701{
702 IRIns *ir = IR(tref_ref(tr));
703 if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD) && irref_isk(ir->op2) &&
704 (ir->o == IR_ADD || ir->o == IR_ADDOV || ir->o == IR_SUBOV)) {
705 IRIns *irk = IR(ir->op2);
706 ptrdiff_t k;
707 if (LJ_64 && irk->o == IR_KINT64)
708 k = (ptrdiff_t)ir_kint64(irk)->u64 * sz;
709 else
710 k = (ptrdiff_t)irk->i * sz;
711 if (ir->o == IR_SUBOV) *ofsp -= k; else *ofsp += k;
712 tr = ir->op1; /* Not a TRef, but the caller doesn't care. */
713 }
714 return tr;
715}
716
717/* Tailcall to function. */
718static void crec_tailcall(jit_State *J, RecordFFData *rd, cTValue *tv)
719{
720 TRef kfunc = lj_ir_kfunc(J, funcV(tv));
721#if LJ_FR2
722 J->base[-2] = kfunc;
723 J->base[-1] = TREF_FRAME;
724#else
725 J->base[-1] = kfunc | TREF_FRAME;
726#endif
727 rd->nres = -1; /* Pending tailcall. */
728}
729
730/* Record ctype __index/__newindex metamethods. */
731static void crec_index_meta(jit_State *J, CTState *cts, CType *ct,
732 RecordFFData *rd)
733{
734 CTypeID id = ctype_typeid(cts, ct);
735 cTValue *tv = lj_ctype_meta(cts, id, rd->data ? MM_newindex : MM_index);
736 if (!tv)
737 lj_trace_err(J, LJ_TRERR_BADTYPE);
738 if (tvisfunc(tv)) {
739 crec_tailcall(J, rd, tv);
740 } else if (rd->data == 0 && tvistab(tv) && tref_isstr(J->base[1])) {
741 /* Specialize to result of __index lookup. */
742 cTValue *o = lj_tab_get(J->L, tabV(tv), &rd->argv[1]);
743 J->base[0] = lj_record_constify(J, o);
744 if (!J->base[0])
745 lj_trace_err(J, LJ_TRERR_BADTYPE);
746 /* Always specialize to the key. */
747 emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
748 } else {
749 /* NYI: resolving of non-function metamethods. */
750 /* NYI: non-string keys for __index table. */
751 /* NYI: stores to __newindex table. */
752 lj_trace_err(J, LJ_TRERR_BADTYPE);
753 }
754}
755
756/* Record bitfield load/store. */
757static void crec_index_bf(jit_State *J, RecordFFData *rd, TRef ptr, CTInfo info)
758{
759 IRType t = IRT_I8 + 2*lj_fls(ctype_bitcsz(info)) + ((info&CTF_UNSIGNED)?1:0);
760 TRef tr = emitir(IRT(IR_XLOAD, t), ptr, 0);
761 CTSize pos = ctype_bitpos(info), bsz = ctype_bitbsz(info), shift = 32 - bsz;
762 lj_assertJ(t <= IRT_U32, "only 32 bit bitfields supported"); /* NYI */
763 if (rd->data == 0) { /* __index metamethod. */
764 if ((info & CTF_BOOL)) {
765 tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << pos))));
766 /* Assume not equal to zero. Fixup and emit pending guard later. */
767 lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
768 J->postproc = LJ_POST_FIXGUARD;
769 tr = TREF_TRUE;
770 } else if (!(info & CTF_UNSIGNED)) {
771 tr = emitir(IRTI(IR_BSHL), tr, lj_ir_kint(J, shift - pos));
772 tr = emitir(IRTI(IR_BSAR), tr, lj_ir_kint(J, shift));
773 } else {
774 lj_assertJ(bsz < 32, "unexpected full bitfield index");
775 tr = emitir(IRTI(IR_BSHR), tr, lj_ir_kint(J, pos));
776 tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << bsz)-1)));
777 /* We can omit the U32 to NUM conversion, since bsz < 32. */
778 }
779 J->base[0] = tr;
780 } else { /* __newindex metamethod. */
781 CTState *cts = ctype_ctsG(J2G(J));
782 CType *ct = ctype_get(cts,
783 (info & CTF_BOOL) ? CTID_BOOL :
784 (info & CTF_UNSIGNED) ? CTID_UINT32 : CTID_INT32);
785 int32_t mask = (int32_t)(((1u << bsz)-1) << pos);
786 TRef sp = crec_ct_tv(J, ct, 0, J->base[2], &rd->argv[2]);
787 sp = emitir(IRTI(IR_BSHL), sp, lj_ir_kint(J, pos));
788 /* Use of the target type avoids forwarding conversions. */
789 sp = emitir(IRT(IR_BAND, t), sp, lj_ir_kint(J, mask));
790 tr = emitir(IRT(IR_BAND, t), tr, lj_ir_kint(J, (int32_t)~mask));
791 tr = emitir(IRT(IR_BOR, t), tr, sp);
792 emitir(IRT(IR_XSTORE, t), ptr, tr);
793 rd->nres = 0;
794 J->needsnap = 1;
795 }
796}
797
798void LJ_FASTCALL recff_cdata_index(jit_State *J, RecordFFData *rd)
799{
800 TRef idx, ptr = J->base[0];
801 ptrdiff_t ofs = sizeof(GCcdata);
802 GCcdata *cd = argv2cdata(J, ptr, &rd->argv[0]);
803 CTState *cts = ctype_ctsG(J2G(J));
804 CType *ct = ctype_raw(cts, cd->ctypeid);
805 CTypeID sid = 0;
806
807 /* Resolve pointer or reference for cdata object. */
808 if (ctype_isptr(ct->info)) {
809 IRType t = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
810 if (ctype_isref(ct->info)) ct = ctype_rawchild(cts, ct);
811 ptr = emitir(IRT(IR_FLOAD, t), ptr, IRFL_CDATA_PTR);
812 ofs = 0;
813 ptr = crec_reassoc_ofs(J, ptr, &ofs, 1);
814 }
815
816again:
817 idx = J->base[1];
818 if (tref_isnumber(idx)) {
819 idx = lj_opt_narrow_cindex(J, idx);
820 if (ctype_ispointer(ct->info)) {
821 CTSize sz;
822 integer_key:
823 if ((ct->info & CTF_COMPLEX))
824 idx = emitir(IRT(IR_BAND, IRT_INTP), idx, lj_ir_kintp(J, 1));
825 sz = lj_ctype_size(cts, (sid = ctype_cid(ct->info)));
826 idx = crec_reassoc_ofs(J, idx, &ofs, sz);
827#if LJ_TARGET_ARM || LJ_TARGET_PPC
828 /* Hoist base add to allow fusion of index/shift into operands. */
829 if (LJ_LIKELY(J->flags & JIT_F_OPT_LOOP) && ofs
830#if LJ_TARGET_ARM
831 && (sz == 1 || sz == 4)
832#endif
833 ) {
834 ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
835 ofs = 0;
836 }
837#endif
838 idx = emitir(IRT(IR_MUL, IRT_INTP), idx, lj_ir_kintp(J, sz));
839 ptr = emitir(IRT(IR_ADD, IRT_PTR), idx, ptr);
840 }
841 } else if (tref_iscdata(idx)) {
842 GCcdata *cdk = cdataV(&rd->argv[1]);
843 CType *ctk = ctype_raw(cts, cdk->ctypeid);
844 IRType t = crec_ct2irt(cts, ctk);
845 if (ctype_ispointer(ct->info) && t >= IRT_I8 && t <= IRT_U64) {
846 if (ctk->size == 8) {
847 idx = emitir(IRT(IR_FLOAD, t), idx, IRFL_CDATA_INT64);
848 } else if (ctk->size == 4) {
849 idx = emitir(IRT(IR_FLOAD, t), idx, IRFL_CDATA_INT);
850 } else {
851 idx = emitir(IRT(IR_ADD, IRT_PTR), idx,
852 lj_ir_kintp(J, sizeof(GCcdata)));
853 idx = emitir(IRT(IR_XLOAD, t), idx, 0);
854 }
855 if (LJ_64 && ctk->size < sizeof(intptr_t) && !(ctk->info & CTF_UNSIGNED))
856 idx = emitconv(idx, IRT_INTP, IRT_INT, IRCONV_SEXT);
857 if (!LJ_64 && ctk->size > sizeof(intptr_t)) {
858 idx = emitconv(idx, IRT_INTP, t, 0);
859 lj_needsplit(J);
860 }
861 goto integer_key;
862 }
863 } else if (tref_isstr(idx)) {
864 GCstr *name = strV(&rd->argv[1]);
865 if (cd && cd->ctypeid == CTID_CTYPEID)
866 ct = ctype_raw(cts, crec_constructor(J, cd, ptr));
867 if (ctype_isstruct(ct->info)) {
868 CTSize fofs;
869 CType *fct;
870 fct = lj_ctype_getfield(cts, ct, name, &fofs);
871 if (fct) {
872 ofs += (ptrdiff_t)fofs;
873 /* Always specialize to the field name. */
874 emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
875 if (ctype_isconstval(fct->info)) {
876 if (fct->size >= 0x80000000u &&
877 (ctype_child(cts, fct)->info & CTF_UNSIGNED)) {
878 J->base[0] = lj_ir_knum(J, (lua_Number)(uint32_t)fct->size);
879 return;
880 }
881 J->base[0] = lj_ir_kint(J, (int32_t)fct->size);
882 return; /* Interpreter will throw for newindex. */
883 } else if (ctype_isbitfield(fct->info)) {
884 if (ofs)
885 ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
886 crec_index_bf(J, rd, ptr, fct->info);
887 return;
888 } else {
889 lj_assertJ(ctype_isfield(fct->info), "field expected");
890 sid = ctype_cid(fct->info);
891 }
892 }
893 } else if (ctype_iscomplex(ct->info)) {
894 if (name->len == 2 &&
895 ((strdata(name)[0] == 'r' && strdata(name)[1] == 'e') ||
896 (strdata(name)[0] == 'i' && strdata(name)[1] == 'm'))) {
897 /* Always specialize to the field name. */
898 emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
899 if (strdata(name)[0] == 'i') ofs += (ct->size >> 1);
900 sid = ctype_cid(ct->info);
901 }
902 }
903 }
904 if (!sid) {
905 if (ctype_isptr(ct->info)) { /* Automatically perform '->'. */
906 CType *cct = ctype_rawchild(cts, ct);
907 if (ctype_isstruct(cct->info)) {
908 ct = cct;
909 cd = NULL;
910 if (tref_isstr(idx)) goto again;
911 }
912 }
913 crec_index_meta(J, cts, ct, rd);
914 return;
915 }
916
917 if (ofs)
918 ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
919
920 /* Resolve reference for field. */
921 ct = ctype_get(cts, sid);
922 if (ctype_isref(ct->info)) {
923 ptr = emitir(IRT(IR_XLOAD, IRT_PTR), ptr, 0);
924 sid = ctype_cid(ct->info);
925 ct = ctype_get(cts, sid);
926 }
927
928 while (ctype_isattrib(ct->info))
929 ct = ctype_child(cts, ct); /* Skip attributes. */
930
931 if (rd->data == 0) { /* __index metamethod. */
932 J->base[0] = crec_tv_ct(J, ct, sid, ptr);
933 } else { /* __newindex metamethod. */
934 rd->nres = 0;
935 J->needsnap = 1;
936 crec_ct_tv(J, ct, ptr, J->base[2], &rd->argv[2]);
937 }
938}
939
940/* Record setting a finalizer. */
941static void crec_finalizer(jit_State *J, TRef trcd, TRef trfin, cTValue *fin)
942{
943 if (tvisgcv(fin)) {
944 if (!trfin) trfin = lj_ir_kptr(J, gcval(fin));
945 } else if (tvisnil(fin)) {
946 trfin = lj_ir_kptr(J, NULL);
947 } else {
948 lj_trace_err(J, LJ_TRERR_BADTYPE);
949 }
950 lj_ir_call(J, IRCALL_lj_cdata_setfin, trcd,
951 trfin, lj_ir_kint(J, (int32_t)itype(fin)));
952 J->needsnap = 1;
953}
954
955/* Record cdata allocation. */
956static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
957{
958 CTState *cts = ctype_ctsG(J2G(J));
959 CTSize sz;
960 CTInfo info = lj_ctype_info(cts, id, &sz);
961 CType *d = ctype_raw(cts, id);
962 TRef trcd, trid = lj_ir_kint(J, id);
963 cTValue *fin;
964 /* Use special instruction to box pointer or 32/64 bit integer. */
965 if (ctype_isptr(info) || (ctype_isinteger(info) && (sz == 4 || sz == 8))) {
966 TRef sp = J->base[1] ? crec_ct_tv(J, d, 0, J->base[1], &rd->argv[1]) :
967 ctype_isptr(info) ? lj_ir_kptr(J, NULL) :
968 sz == 4 ? lj_ir_kint(J, 0) :
969 (lj_needsplit(J), lj_ir_kint64(J, 0));
970 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, sp);
971 return;
972 } else {
973 TRef trsz = TREF_NIL;
974 if ((info & CTF_VLA)) { /* Calculate VLA/VLS size at runtime. */
975 CTSize sz0, sz1;
976 if (!J->base[1] || J->base[2])
977 lj_trace_err(J, LJ_TRERR_NYICONV); /* NYI: init VLA/VLS. */
978 trsz = crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0,
979 J->base[1], &rd->argv[1]);
980 sz0 = lj_ctype_vlsize(cts, d, 0);
981 sz1 = lj_ctype_vlsize(cts, d, 1);
982 trsz = emitir(IRTGI(IR_MULOV), trsz, lj_ir_kint(J, (int32_t)(sz1-sz0)));
983 trsz = emitir(IRTGI(IR_ADDOV), trsz, lj_ir_kint(J, (int32_t)sz0));
984 J->base[1] = 0; /* Simplify logic below. */
985 } else if (ctype_align(info) > CT_MEMALIGN) {
986 trsz = lj_ir_kint(J, sz);
987 }
988 trcd = emitir(IRTG(IR_CNEW, IRT_CDATA), trid, trsz);
989 if (sz > 128 || (info & CTF_VLA)) {
990 TRef dp;
991 CTSize align;
992 special: /* Only handle bulk zero-fill for large/VLA/VLS types. */
993 if (J->base[1])
994 lj_trace_err(J, LJ_TRERR_NYICONV); /* NYI: init large/VLA/VLS types. */
995 dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, lj_ir_kintp(J, sizeof(GCcdata)));
996 if (trsz == TREF_NIL) trsz = lj_ir_kint(J, sz);
997 align = ctype_align(info);
998 if (align < CT_MEMALIGN) align = CT_MEMALIGN;
999 crec_fill(J, dp, trsz, lj_ir_kint(J, 0), (1u << align));
1000 } else if (J->base[1] && !J->base[2] &&
1001 !lj_cconv_multi_init(cts, d, &rd->argv[1])) {
1002 goto single_init;
1003 } else if (ctype_isarray(d->info)) {
1004 CType *dc = ctype_rawchild(cts, d); /* Array element type. */
1005 CTSize ofs, esize = dc->size;
1006 TRef sp = 0;
1007 TValue tv;
1008 TValue *sval = &tv;
1009 MSize i;
1010 tv.u64 = 0;
1011 if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info)) ||
1012 esize * CREC_FILL_MAXUNROLL < sz)
1013 goto special;
1014 for (i = 1, ofs = 0; ofs < sz; ofs += esize) {
1015 TRef dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
1016 lj_ir_kintp(J, ofs + sizeof(GCcdata)));
1017 if (J->base[i]) {
1018 sp = J->base[i];
1019 sval = &rd->argv[i];
1020 i++;
1021 } else if (i != 2) {
1022 sp = ctype_isnum(dc->info) ? lj_ir_kint(J, 0) : TREF_NIL;
1023 }
1024 crec_ct_tv(J, dc, dp, sp, sval);
1025 }
1026 } else if (ctype_isstruct(d->info)) {
1027 CTypeID fid;
1028 MSize i = 1;
1029 if (!J->base[1]) { /* Handle zero-fill of struct-of-NYI. */
1030 fid = d->sib;
1031 while (fid) {
1032 CType *df = ctype_get(cts, fid);
1033 fid = df->sib;
1034 if (ctype_isfield(df->info)) {
1035 CType *dc;
1036 if (!gcref(df->name)) continue; /* Ignore unnamed fields. */
1037 dc = ctype_rawchild(cts, df); /* Field type. */
1038 if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info) ||
1039 ctype_isenum(dc->info)))
1040 goto special;
1041 } else if (!ctype_isconstval(df->info)) {
1042 goto special;
1043 }
1044 }
1045 }
1046 fid = d->sib;
1047 while (fid) {
1048 CType *df = ctype_get(cts, fid);
1049 fid = df->sib;
1050 if (ctype_isfield(df->info)) {
1051 CType *dc;
1052 TRef sp, dp;
1053 TValue tv;
1054 TValue *sval = &tv;
1055 setintV(&tv, 0);
1056 if (!gcref(df->name)) continue; /* Ignore unnamed fields. */
1057 dc = ctype_rawchild(cts, df); /* Field type. */
1058 if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info) ||
1059 ctype_isenum(dc->info)))
1060 lj_trace_err(J, LJ_TRERR_NYICONV); /* NYI: init aggregates. */
1061 if (J->base[i]) {
1062 sp = J->base[i];
1063 sval = &rd->argv[i];
1064 i++;
1065 } else {
1066 sp = ctype_isptr(dc->info) ? TREF_NIL : lj_ir_kint(J, 0);
1067 }
1068 dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
1069 lj_ir_kintp(J, df->size + sizeof(GCcdata)));
1070 crec_ct_tv(J, dc, dp, sp, sval);
1071 if ((d->info & CTF_UNION)) {
1072 if (d->size != dc->size) /* NYI: partial init of union. */
1073 lj_trace_err(J, LJ_TRERR_NYICONV);
1074 break;
1075 }
1076 } else if (!ctype_isconstval(df->info)) {
1077 /* NYI: init bitfields and sub-structures. */
1078 lj_trace_err(J, LJ_TRERR_NYICONV);
1079 }
1080 }
1081 } else {
1082 TRef dp;
1083 single_init:
1084 dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, lj_ir_kintp(J, sizeof(GCcdata)));
1085 if (J->base[1]) {
1086 crec_ct_tv(J, d, dp, J->base[1], &rd->argv[1]);
1087 } else {
1088 TValue tv;
1089 tv.u64 = 0;
1090 crec_ct_tv(J, d, dp, lj_ir_kint(J, 0), &tv);
1091 }
1092 }
1093 }
1094 J->base[0] = trcd;
1095 /* Handle __gc metamethod. */
1096 fin = lj_ctype_meta(cts, id, MM_gc);
1097 if (fin)
1098 crec_finalizer(J, trcd, 0, fin);
1099}
1100
1101/* Record argument conversions. */
1102static TRef crec_call_args(jit_State *J, RecordFFData *rd,
1103 CTState *cts, CType *ct)
1104{
1105 TRef args[CCI_NARGS_MAX];
1106 CTypeID fid;
1107 MSize i, n;
1108 TRef tr, *base;
1109 cTValue *o;
1110#if LJ_TARGET_X86
1111#if LJ_ABI_WIN
1112 TRef *arg0 = NULL, *arg1 = NULL;
1113#endif
1114 int ngpr = 0;
1115 if (ctype_cconv(ct->info) == CTCC_THISCALL)
1116 ngpr = 1;
1117 else if (ctype_cconv(ct->info) == CTCC_FASTCALL)
1118 ngpr = 2;
1119#endif
1120
1121 /* Skip initial attributes. */
1122 fid = ct->sib;
1123 while (fid) {
1124 CType *ctf = ctype_get(cts, fid);
1125 if (!ctype_isattrib(ctf->info)) break;
1126 fid = ctf->sib;
1127 }
1128 args[0] = TREF_NIL;
1129 for (n = 0, base = J->base+1, o = rd->argv+1; *base; n++, base++, o++) {
1130 CTypeID did;
1131 CType *d;
1132
1133 if (n >= CCI_NARGS_MAX)
1134 lj_trace_err(J, LJ_TRERR_NYICALL);
1135
1136 if (fid) { /* Get argument type from field. */
1137 CType *ctf = ctype_get(cts, fid);
1138 fid = ctf->sib;
1139 lj_assertJ(ctype_isfield(ctf->info), "field expected");
1140 did = ctype_cid(ctf->info);
1141 } else {
1142 if (!(ct->info & CTF_VARARG))
1143 lj_trace_err(J, LJ_TRERR_NYICALL); /* Too many arguments. */
1144 did = lj_ccall_ctid_vararg(cts, o); /* Infer vararg type. */
1145 }
1146 d = ctype_raw(cts, did);
1147 if (!(ctype_isnum(d->info) || ctype_isptr(d->info) ||
1148 ctype_isenum(d->info)))
1149 lj_trace_err(J, LJ_TRERR_NYICALL);
1150 tr = crec_ct_tv(J, d, 0, *base, o);
1151 if (ctype_isinteger_or_bool(d->info)) {
1152 if (d->size < 4) {
1153 if ((d->info & CTF_UNSIGNED))
1154 tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_U8 : IRT_U16, 0);
1155 else
1156 tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_I8 : IRT_I16,IRCONV_SEXT);
1157 }
1158 } else if (LJ_SOFTFP32 && ctype_isfp(d->info) && d->size > 4) {
1159 lj_needsplit(J);
1160 }
1161#if LJ_TARGET_X86
1162 /* 64 bit args must not end up in registers for fastcall/thiscall. */
1163#if LJ_ABI_WIN
1164 if (!ctype_isfp(d->info)) {
1165 /* Sigh, the Windows/x86 ABI allows reordering across 64 bit args. */
1166 if (tref_typerange(tr, IRT_I64, IRT_U64)) {
1167 if (ngpr) {
1168 arg0 = &args[n]; args[n++] = TREF_NIL; ngpr--;
1169 if (ngpr) {
1170 arg1 = &args[n]; args[n++] = TREF_NIL; ngpr--;
1171 }
1172 }
1173 } else {
1174 if (arg0) { *arg0 = tr; arg0 = NULL; n--; continue; }
1175 if (arg1) { *arg1 = tr; arg1 = NULL; n--; continue; }
1176 if (ngpr) ngpr--;
1177 }
1178 }
1179#else
1180 if (!ctype_isfp(d->info) && ngpr) {
1181 if (tref_typerange(tr, IRT_I64, IRT_U64)) {
1182 /* No reordering for other x86 ABIs. Simply add alignment args. */
1183 do { args[n++] = TREF_NIL; } while (--ngpr);
1184 } else {
1185 ngpr--;
1186 }
1187 }
1188#endif
1189#endif
1190 args[n] = tr;
1191 }
1192 tr = args[0];
1193 for (i = 1; i < n; i++)
1194 tr = emitir(IRT(IR_CARG, IRT_NIL), tr, args[i]);
1195 return tr;
1196}
1197
1198/* Create a snapshot for the caller, simulating a 'false' return value. */
1199static void crec_snap_caller(jit_State *J)
1200{
1201 lua_State *L = J->L;
1202 TValue *base = L->base, *top = L->top;
1203 const BCIns *pc = J->pc;
1204 TRef ftr = J->base[-1-LJ_FR2];
1205 ptrdiff_t delta;
1206 if (!frame_islua(base-1) || J->framedepth <= 0)
1207 lj_trace_err(J, LJ_TRERR_NYICALL);
1208 J->pc = frame_pc(base-1); delta = 1+LJ_FR2+bc_a(J->pc[-1]);
1209 L->top = base; L->base = base - delta;
1210 J->base[-1-LJ_FR2] = TREF_FALSE;
1211 J->base -= delta; J->baseslot -= (BCReg)delta;
1212 J->maxslot = (BCReg)delta-LJ_FR2; J->framedepth--;
1213 lj_snap_add(J);
1214 L->base = base; L->top = top;
1215 J->framedepth++; J->maxslot = 1;
1216 J->base += delta; J->baseslot += (BCReg)delta;
1217 J->base[-1-LJ_FR2] = ftr; J->pc = pc;
1218}
1219
1220/* Record function call. */
1221static int crec_call(jit_State *J, RecordFFData *rd, GCcdata *cd)
1222{
1223 CTState *cts = ctype_ctsG(J2G(J));
1224 CType *ct = ctype_raw(cts, cd->ctypeid);
1225 IRType tp = IRT_PTR;
1226 if (ctype_isptr(ct->info)) {
1227 tp = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
1228 ct = ctype_rawchild(cts, ct);
1229 }
1230 if (ctype_isfunc(ct->info)) {
1231 TRef func = emitir(IRT(IR_FLOAD, tp), J->base[0], IRFL_CDATA_PTR);
1232 CType *ctr = ctype_rawchild(cts, ct);
1233 IRType t = crec_ct2irt(cts, ctr);
1234 TRef tr;
1235 TValue tv;
1236 /* Check for blacklisted C functions that might call a callback. */
1237 tv.u64 = ((uintptr_t)cdata_getptr(cdataptr(cd), (LJ_64 && tp == IRT_P64) ? 8 : 4) >> 2) | U64x(800000000, 00000000);
1238 if (tvistrue(lj_tab_get(J->L, cts->miscmap, &tv)))
1239 lj_trace_err(J, LJ_TRERR_BLACKL);
1240 if (ctype_isvoid(ctr->info)) {
1241 t = IRT_NIL;
1242 rd->nres = 0;
1243 } else if (!(ctype_isnum(ctr->info) || ctype_isptr(ctr->info) ||
1244 ctype_isenum(ctr->info)) || t == IRT_CDATA) {
1245 lj_trace_err(J, LJ_TRERR_NYICALL);
1246 }
1247 if ((ct->info & CTF_VARARG)
1248#if LJ_TARGET_X86
1249 || ctype_cconv(ct->info) != CTCC_CDECL
1250#endif
1251 )
1252 func = emitir(IRT(IR_CARG, IRT_NIL), func,
1253 lj_ir_kint(J, ctype_typeid(cts, ct)));
1254 tr = emitir(IRT(IR_CALLXS, t), crec_call_args(J, rd, cts, ct), func);
1255 if (ctype_isbool(ctr->info)) {
1256 if (frame_islua(J->L->base-1) && bc_b(frame_pc(J->L->base-1)[-1]) == 1) {
1257 /* Don't check result if ignored. */
1258 tr = TREF_NIL;
1259 } else {
1260 crec_snap_caller(J);
1261#if LJ_TARGET_X86ORX64
1262 /* Note: only the x86/x64 backend supports U8 and only for EQ(tr, 0). */
1263 lj_ir_set(J, IRTG(IR_NE, IRT_U8), tr, lj_ir_kint(J, 0));
1264#else
1265 lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
1266#endif
1267 J->postproc = LJ_POST_FIXGUARDSNAP;
1268 tr = TREF_TRUE;
1269 }
1270 } else if (t == IRT_PTR || (LJ_64 && t == IRT_P32) ||
1271 t == IRT_I64 || t == IRT_U64 || ctype_isenum(ctr->info)) {
1272 TRef trid = lj_ir_kint(J, ctype_cid(ct->info));
1273 tr = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, tr);
1274 if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
1275 } else if (t == IRT_FLOAT || t == IRT_U32) {
1276 tr = emitconv(tr, IRT_NUM, t, 0);
1277 } else if (t == IRT_I8 || t == IRT_I16) {
1278 tr = emitconv(tr, IRT_INT, t, IRCONV_SEXT);
1279 } else if (t == IRT_U8 || t == IRT_U16) {
1280 tr = emitconv(tr, IRT_INT, t, 0);
1281 }
1282 J->base[0] = tr;
1283 J->needsnap = 1;
1284 return 1;
1285 }
1286 return 0;
1287}
1288
1289void LJ_FASTCALL recff_cdata_call(jit_State *J, RecordFFData *rd)
1290{
1291 CTState *cts = ctype_ctsG(J2G(J));
1292 GCcdata *cd = argv2cdata(J, J->base[0], &rd->argv[0]);
1293 CTypeID id = cd->ctypeid;
1294 CType *ct;
1295 cTValue *tv;
1296 MMS mm = MM_call;
1297 if (id == CTID_CTYPEID) {
1298 id = crec_constructor(J, cd, J->base[0]);
1299 mm = MM_new;
1300 } else if (crec_call(J, rd, cd)) {
1301 return;
1302 }
1303 /* Record ctype __call/__new metamethod. */
1304 ct = ctype_raw(cts, id);
1305 tv = lj_ctype_meta(cts, ctype_isptr(ct->info) ? ctype_cid(ct->info) : id, mm);
1306 if (tv) {
1307 if (tvisfunc(tv)) {
1308 crec_tailcall(J, rd, tv);
1309 return;
1310 }
1311 } else if (mm == MM_new) {
1312 crec_alloc(J, rd, id);
1313 return;
1314 }
1315 /* No metamethod or NYI: non-function metamethods. */
1316 lj_trace_err(J, LJ_TRERR_BADTYPE);
1317}
1318
1319static TRef crec_arith_int64(jit_State *J, TRef *sp, CType **s, MMS mm)
1320{
1321 if (sp[0] && sp[1] && ctype_isnum(s[0]->info) && ctype_isnum(s[1]->info)) {
1322 IRType dt;
1323 CTypeID id;
1324 TRef tr;
1325 MSize i;
1326 IROp op;
1327 lj_needsplit(J);
1328 if (((s[0]->info & CTF_UNSIGNED) && s[0]->size == 8) ||
1329 ((s[1]->info & CTF_UNSIGNED) && s[1]->size == 8)) {
1330 dt = IRT_U64; id = CTID_UINT64;
1331 } else {
1332 dt = IRT_I64; id = CTID_INT64;
1333 if (mm < MM_add &&
1334 !((s[0]->info | s[1]->info) & CTF_FP) &&
1335 s[0]->size == 4 && s[1]->size == 4) { /* Try to narrow comparison. */
1336 if (!((s[0]->info ^ s[1]->info) & CTF_UNSIGNED) ||
1337 (tref_isk(sp[1]) && IR(tref_ref(sp[1]))->i >= 0)) {
1338 dt = (s[0]->info & CTF_UNSIGNED) ? IRT_U32 : IRT_INT;
1339 goto comp;
1340 } else if (tref_isk(sp[0]) && IR(tref_ref(sp[0]))->i >= 0) {
1341 dt = (s[1]->info & CTF_UNSIGNED) ? IRT_U32 : IRT_INT;
1342 goto comp;
1343 }
1344 }
1345 }
1346 for (i = 0; i < 2; i++) {
1347 IRType st = tref_type(sp[i]);
1348 if (st == IRT_NUM || st == IRT_FLOAT)
1349 sp[i] = emitconv(sp[i], dt, st, IRCONV_ANY);
1350 else if (!(st == IRT_I64 || st == IRT_U64))
1351 sp[i] = emitconv(sp[i], dt, IRT_INT,
1352 (s[i]->info & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
1353 }
1354 if (mm < MM_add) {
1355 comp:
1356 /* Assume true comparison. Fixup and emit pending guard later. */
1357 if (mm == MM_eq) {
1358 op = IR_EQ;
1359 } else {
1360 op = mm == MM_lt ? IR_LT : IR_LE;
1361 if (dt == IRT_U32 || dt == IRT_U64)
1362 op += (IR_ULT-IR_LT);
1363 }
1364 lj_ir_set(J, IRTG(op, dt), sp[0], sp[1]);
1365 J->postproc = LJ_POST_FIXGUARD;
1366 return TREF_TRUE;
1367 } else {
1368 tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, dt), sp[0], sp[1]);
1369 }
1370 return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1371 }
1372 return 0;
1373}
1374
1375static TRef crec_arith_ptr(jit_State *J, TRef *sp, CType **s, MMS mm)
1376{
1377 CTState *cts = ctype_ctsG(J2G(J));
1378 CType *ctp = s[0];
1379 if (!(sp[0] && sp[1])) return 0;
1380 if (ctype_isptr(ctp->info) || ctype_isrefarray(ctp->info)) {
1381 if ((mm == MM_sub || mm == MM_eq || mm == MM_lt || mm == MM_le) &&
1382 (ctype_isptr(s[1]->info) || ctype_isrefarray(s[1]->info))) {
1383 if (mm == MM_sub) { /* Pointer difference. */
1384 TRef tr;
1385 CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
1386 if (sz == 0 || (sz & (sz-1)) != 0)
1387 return 0; /* NYI: integer division. */
1388 tr = emitir(IRT(IR_SUB, IRT_INTP), sp[0], sp[1]);
1389 tr = emitir(IRT(IR_BSAR, IRT_INTP), tr, lj_ir_kint(J, lj_fls(sz)));
1390#if LJ_64
1391 tr = emitconv(tr, IRT_NUM, IRT_INTP, 0);
1392#endif
1393 return tr;
1394 } else { /* Pointer comparison (unsigned). */
1395 /* Assume true comparison. Fixup and emit pending guard later. */
1396 IROp op = mm == MM_eq ? IR_EQ : mm == MM_lt ? IR_ULT : IR_ULE;
1397 lj_ir_set(J, IRTG(op, IRT_PTR), sp[0], sp[1]);
1398 J->postproc = LJ_POST_FIXGUARD;
1399 return TREF_TRUE;
1400 }
1401 }
1402 if (!((mm == MM_add || mm == MM_sub) && ctype_isnum(s[1]->info)))
1403 return 0;
1404 } else if (mm == MM_add && ctype_isnum(ctp->info) &&
1405 (ctype_isptr(s[1]->info) || ctype_isrefarray(s[1]->info))) {
1406 TRef tr = sp[0]; sp[0] = sp[1]; sp[1] = tr; /* Swap pointer and index. */
1407 ctp = s[1];
1408 } else {
1409 return 0;
1410 }
1411 {
1412 TRef tr = sp[1];
1413 IRType t = tref_type(tr);
1414 CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
1415 CTypeID id;
1416#if LJ_64
1417 if (t == IRT_NUM || t == IRT_FLOAT)
1418 tr = emitconv(tr, IRT_INTP, t, IRCONV_ANY);
1419 else if (!(t == IRT_I64 || t == IRT_U64))
1420 tr = emitconv(tr, IRT_INTP, IRT_INT,
1421 ((t - IRT_I8) & 1) ? 0 : IRCONV_SEXT);
1422#else
1423 if (!tref_typerange(sp[1], IRT_I8, IRT_U32)) {
1424 tr = emitconv(tr, IRT_INTP, t,
1425 (t == IRT_NUM || t == IRT_FLOAT) ? IRCONV_ANY : 0);
1426 }
1427#endif
1428 tr = emitir(IRT(IR_MUL, IRT_INTP), tr, lj_ir_kintp(J, sz));
1429 tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, IRT_PTR), sp[0], tr);
1430 id = lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(ctp->info)),
1431 CTSIZE_PTR);
1432 return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1433 }
1434}
1435
1436/* Record ctype arithmetic metamethods. */
1437static TRef crec_arith_meta(jit_State *J, TRef *sp, CType **s, CTState *cts,
1438 RecordFFData *rd)
1439{
1440 cTValue *tv = NULL;
1441 if (J->base[0]) {
1442 if (tviscdata(&rd->argv[0])) {
1443 CTypeID id = argv2cdata(J, J->base[0], &rd->argv[0])->ctypeid;
1444 CType *ct = ctype_raw(cts, id);
1445 if (ctype_isptr(ct->info)) id = ctype_cid(ct->info);
1446 tv = lj_ctype_meta(cts, id, (MMS)rd->data);
1447 }
1448 if (!tv && J->base[1] && tviscdata(&rd->argv[1])) {
1449 CTypeID id = argv2cdata(J, J->base[1], &rd->argv[1])->ctypeid;
1450 CType *ct = ctype_raw(cts, id);
1451 if (ctype_isptr(ct->info)) id = ctype_cid(ct->info);
1452 tv = lj_ctype_meta(cts, id, (MMS)rd->data);
1453 }
1454 }
1455 if (tv) {
1456 if (tvisfunc(tv)) {
1457 crec_tailcall(J, rd, tv);
1458 return 0;
1459 } /* NYI: non-function metamethods. */
1460 } else if ((MMS)rd->data == MM_eq) { /* Fallback cdata pointer comparison. */
1461 if (sp[0] && sp[1] && ctype_isnum(s[0]->info) == ctype_isnum(s[1]->info)) {
1462 /* Assume true comparison. Fixup and emit pending guard later. */
1463 lj_ir_set(J, IRTG(IR_EQ, IRT_PTR), sp[0], sp[1]);
1464 J->postproc = LJ_POST_FIXGUARD;
1465 return TREF_TRUE;
1466 } else {
1467 return TREF_FALSE;
1468 }
1469 }
1470 lj_trace_err(J, LJ_TRERR_BADTYPE);
1471 return 0;
1472}
1473
1474void LJ_FASTCALL recff_cdata_arith(jit_State *J, RecordFFData *rd)
1475{
1476 CTState *cts = ctype_ctsG(J2G(J));
1477 TRef sp[2];
1478 CType *s[2];
1479 MSize i;
1480 for (i = 0; i < 2; i++) {
1481 TRef tr = J->base[i];
1482 CType *ct = ctype_get(cts, CTID_DOUBLE);
1483 if (!tr) {
1484 lj_trace_err(J, LJ_TRERR_BADTYPE);
1485 } else if (tref_iscdata(tr)) {
1486 CTypeID id = argv2cdata(J, tr, &rd->argv[i])->ctypeid;
1487 IRType t;
1488 ct = ctype_raw(cts, id);
1489 t = crec_ct2irt(cts, ct);
1490 if (ctype_isptr(ct->info)) { /* Resolve pointer or reference. */
1491 tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_PTR);
1492 if (ctype_isref(ct->info)) {
1493 ct = ctype_rawchild(cts, ct);
1494 t = crec_ct2irt(cts, ct);
1495 }
1496 } else if (t == IRT_I64 || t == IRT_U64) {
1497 tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_INT64);
1498 lj_needsplit(J);
1499 goto ok;
1500 } else if (t == IRT_INT || t == IRT_U32) {
1501 tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_INT);
1502 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1503 goto ok;
1504 } else if (ctype_isfunc(ct->info)) {
1505 tr = emitir(IRT(IR_FLOAD, IRT_PTR), tr, IRFL_CDATA_PTR);
1506 ct = ctype_get(cts,
1507 lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR));
1508 goto ok;
1509 } else {
1510 tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCcdata)));
1511 }
1512 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1513 if (ctype_isnum(ct->info)) {
1514 if (t == IRT_CDATA) {
1515 tr = 0;
1516 } else {
1517 if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
1518 tr = emitir(IRT(IR_XLOAD, t), tr, 0);
1519 }
1520 }
1521 } else if (tref_isnil(tr)) {
1522 tr = lj_ir_kptr(J, NULL);
1523 ct = ctype_get(cts, CTID_P_VOID);
1524 } else if (tref_isinteger(tr)) {
1525 ct = ctype_get(cts, CTID_INT32);
1526 } else if (tref_isstr(tr)) {
1527 TRef tr2 = J->base[1-i];
1528 CTypeID id = argv2cdata(J, tr2, &rd->argv[1-i])->ctypeid;
1529 ct = ctype_raw(cts, id);
1530 if (ctype_isenum(ct->info)) { /* Match string against enum constant. */
1531 GCstr *str = strV(&rd->argv[i]);
1532 CTSize ofs;
1533 CType *cct = lj_ctype_getfield(cts, ct, str, &ofs);
1534 if (cct && ctype_isconstval(cct->info)) {
1535 /* Specialize to the name of the enum constant. */
1536 emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, str));
1537 ct = ctype_child(cts, cct);
1538 tr = lj_ir_kint(J, (int32_t)ofs);
1539 } else { /* Interpreter will throw or return false. */
1540 ct = ctype_get(cts, CTID_P_VOID);
1541 }
1542 } else if (ctype_isptr(ct->info)) {
1543 tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCstr)));
1544 } else {
1545 ct = ctype_get(cts, CTID_P_VOID);
1546 }
1547 } else if (!tref_isnum(tr)) {
1548 tr = 0;
1549 ct = ctype_get(cts, CTID_P_VOID);
1550 }
1551 ok:
1552 s[i] = ct;
1553 sp[i] = tr;
1554 }
1555 {
1556 TRef tr;
1557 MMS mm = (MMS)rd->data;
1558 if ((mm == MM_len || mm == MM_concat ||
1559 (!(tr = crec_arith_int64(J, sp, s, mm)) &&
1560 !(tr = crec_arith_ptr(J, sp, s, mm)))) &&
1561 !(tr = crec_arith_meta(J, sp, s, cts, rd)))
1562 return;
1563 J->base[0] = tr;
1564 /* Fixup cdata comparisons, too. Avoids some cdata escapes. */
1565 if (J->postproc == LJ_POST_FIXGUARD && frame_iscont(J->L->base-1) &&
1566 !irt_isguard(J->guardemit)) {
1567 const BCIns *pc = frame_contpc(J->L->base-1) - 1;
1568 if (bc_op(*pc) <= BC_ISNEP) {
1569 J2G(J)->tmptv.u64 = (uint64_t)(uintptr_t)pc;
1570 J->postproc = LJ_POST_FIXCOMP;
1571 }
1572 }
1573 }
1574}
1575
1576/* -- C library namespace metamethods ------------------------------------- */
1577
1578void LJ_FASTCALL recff_clib_index(jit_State *J, RecordFFData *rd)
1579{
1580 CTState *cts = ctype_ctsG(J2G(J));
1581 if (tref_isudata(J->base[0]) && tref_isstr(J->base[1]) &&
1582 udataV(&rd->argv[0])->udtype == UDTYPE_FFI_CLIB) {
1583 CLibrary *cl = (CLibrary *)uddata(udataV(&rd->argv[0]));
1584 GCstr *name = strV(&rd->argv[1]);
1585 CType *ct;
1586 CTypeID id = lj_ctype_getname(cts, &ct, name, CLNS_INDEX);
1587 cTValue *tv = lj_tab_getstr(cl->cache, name);
1588 rd->nres = rd->data;
1589 if (id && tv && !tvisnil(tv)) {
1590 /* Specialize to the symbol name and make the result a constant. */
1591 emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, name));
1592 if (ctype_isconstval(ct->info)) {
1593 if (ct->size >= 0x80000000u &&
1594 (ctype_child(cts, ct)->info & CTF_UNSIGNED))
1595 J->base[0] = lj_ir_knum(J, (lua_Number)(uint32_t)ct->size);
1596 else
1597 J->base[0] = lj_ir_kint(J, (int32_t)ct->size);
1598 } else if (ctype_isextern(ct->info)) {
1599 CTypeID sid = ctype_cid(ct->info);
1600 void *sp = *(void **)cdataptr(cdataV(tv));
1601 TRef ptr;
1602 ct = ctype_raw(cts, sid);
1603 if (LJ_64 && !checkptr32(sp))
1604 ptr = lj_ir_kintp(J, (uintptr_t)sp);
1605 else
1606 ptr = lj_ir_kptr(J, sp);
1607 if (rd->data) {
1608 J->base[0] = crec_tv_ct(J, ct, sid, ptr);
1609 } else {
1610 J->needsnap = 1;
1611 crec_ct_tv(J, ct, ptr, J->base[2], &rd->argv[2]);
1612 }
1613 } else {
1614 J->base[0] = lj_ir_kgc(J, obj2gco(cdataV(tv)), IRT_CDATA);
1615 }
1616 } else {
1617 lj_trace_err(J, LJ_TRERR_NOCACHE);
1618 }
1619 } /* else: interpreter will throw. */
1620}
1621
1622/* -- FFI library functions ----------------------------------------------- */
1623
1624static TRef crec_toint(jit_State *J, CTState *cts, TRef sp, TValue *sval)
1625{
1626 return crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0, sp, sval);
1627}
1628
1629void LJ_FASTCALL recff_ffi_new(jit_State *J, RecordFFData *rd)
1630{
1631 crec_alloc(J, rd, argv2ctype(J, J->base[0], &rd->argv[0]));
1632}
1633
1634void LJ_FASTCALL recff_ffi_errno(jit_State *J, RecordFFData *rd)
1635{
1636 UNUSED(rd);
1637 if (J->base[0])
1638 lj_trace_err(J, LJ_TRERR_NYICALL);
1639 J->base[0] = lj_ir_call(J, IRCALL_lj_vm_errno);
1640}
1641
1642void LJ_FASTCALL recff_ffi_string(jit_State *J, RecordFFData *rd)
1643{
1644 CTState *cts = ctype_ctsG(J2G(J));
1645 TRef tr = J->base[0];
1646 if (tr) {
1647 TRef trlen = J->base[1];
1648 if (!tref_isnil(trlen)) {
1649 trlen = crec_toint(J, cts, trlen, &rd->argv[1]);
1650 tr = crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, tr, &rd->argv[0]);
1651 } else {
1652 tr = crec_ct_tv(J, ctype_get(cts, CTID_P_CCHAR), 0, tr, &rd->argv[0]);
1653 trlen = lj_ir_call(J, IRCALL_strlen, tr);
1654 }
1655 J->base[0] = emitir(IRT(IR_XSNEW, IRT_STR), tr, trlen);
1656 } /* else: interpreter will throw. */
1657}
1658
1659void LJ_FASTCALL recff_ffi_copy(jit_State *J, RecordFFData *rd)
1660{
1661 CTState *cts = ctype_ctsG(J2G(J));
1662 TRef trdst = J->base[0], trsrc = J->base[1], trlen = J->base[2];
1663 if (trdst && trsrc && (trlen || tref_isstr(trsrc))) {
1664 trdst = crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, trdst, &rd->argv[0]);
1665 trsrc = crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, trsrc, &rd->argv[1]);
1666 if (trlen) {
1667 trlen = crec_toint(J, cts, trlen, &rd->argv[2]);
1668 } else {
1669 trlen = emitir(IRTI(IR_FLOAD), J->base[1], IRFL_STR_LEN);
1670 trlen = emitir(IRTI(IR_ADD), trlen, lj_ir_kint(J, 1));
1671 }
1672 rd->nres = 0;
1673 crec_copy(J, trdst, trsrc, trlen, NULL);
1674 } /* else: interpreter will throw. */
1675}
1676
1677void LJ_FASTCALL recff_ffi_fill(jit_State *J, RecordFFData *rd)
1678{
1679 CTState *cts = ctype_ctsG(J2G(J));
1680 TRef trdst = J->base[0], trlen = J->base[1], trfill = J->base[2];
1681 if (trdst && trlen) {
1682 CTSize step = 1;
1683 if (tviscdata(&rd->argv[0])) { /* Get alignment of original destination. */
1684 CTSize sz;
1685 CType *ct = ctype_raw(cts, cdataV(&rd->argv[0])->ctypeid);
1686 if (ctype_isptr(ct->info))
1687 ct = ctype_rawchild(cts, ct);
1688 step = (1u<<ctype_align(lj_ctype_info(cts, ctype_typeid(cts, ct), &sz)));
1689 }
1690 trdst = crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, trdst, &rd->argv[0]);
1691 trlen = crec_toint(J, cts, trlen, &rd->argv[1]);
1692 if (trfill)
1693 trfill = crec_toint(J, cts, trfill, &rd->argv[2]);
1694 else
1695 trfill = lj_ir_kint(J, 0);
1696 rd->nres = 0;
1697 crec_fill(J, trdst, trlen, trfill, step);
1698 } /* else: interpreter will throw. */
1699}
1700
1701void LJ_FASTCALL recff_ffi_typeof(jit_State *J, RecordFFData *rd)
1702{
1703 if (tref_iscdata(J->base[0])) {
1704 TRef trid = lj_ir_kint(J, argv2ctype(J, J->base[0], &rd->argv[0]));
1705 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA),
1706 lj_ir_kint(J, CTID_CTYPEID), trid);
1707 } else {
1708 setfuncV(J->L, &J->errinfo, J->fn);
1709 lj_trace_err_info(J, LJ_TRERR_NYIFFU);
1710 }
1711}
1712
1713void LJ_FASTCALL recff_ffi_istype(jit_State *J, RecordFFData *rd)
1714{
1715 argv2ctype(J, J->base[0], &rd->argv[0]);
1716 if (tref_iscdata(J->base[1])) {
1717 argv2ctype(J, J->base[1], &rd->argv[1]);
1718 J->postproc = LJ_POST_FIXBOOL;
1719 J->base[0] = TREF_TRUE;
1720 } else {
1721 J->base[0] = TREF_FALSE;
1722 }
1723}
1724
1725void LJ_FASTCALL recff_ffi_abi(jit_State *J, RecordFFData *rd)
1726{
1727 if (tref_isstr(J->base[0])) {
1728 /* Specialize to the ABI string to make the boolean result a constant. */
1729 emitir(IRTG(IR_EQ, IRT_STR), J->base[0], lj_ir_kstr(J, strV(&rd->argv[0])));
1730 J->postproc = LJ_POST_FIXBOOL;
1731 J->base[0] = TREF_TRUE;
1732 } else {
1733 lj_trace_err(J, LJ_TRERR_BADTYPE);
1734 }
1735}
1736
1737/* Record ffi.sizeof(), ffi.alignof(), ffi.offsetof(). */
1738void LJ_FASTCALL recff_ffi_xof(jit_State *J, RecordFFData *rd)
1739{
1740 CTypeID id = argv2ctype(J, J->base[0], &rd->argv[0]);
1741 if (rd->data == FF_ffi_sizeof) {
1742 CType *ct = lj_ctype_rawref(ctype_ctsG(J2G(J)), id);
1743 if (ctype_isvltype(ct->info))
1744 lj_trace_err(J, LJ_TRERR_BADTYPE);
1745 } else if (rd->data == FF_ffi_offsetof) { /* Specialize to the field name. */
1746 if (!tref_isstr(J->base[1]))
1747 lj_trace_err(J, LJ_TRERR_BADTYPE);
1748 emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
1749 rd->nres = 3; /* Just in case. */
1750 }
1751 J->postproc = LJ_POST_FIXCONST;
1752 J->base[0] = J->base[1] = J->base[2] = TREF_NIL;
1753}
1754
1755void LJ_FASTCALL recff_ffi_gc(jit_State *J, RecordFFData *rd)
1756{
1757 argv2cdata(J, J->base[0], &rd->argv[0]);
1758 if (!J->base[1])
1759 lj_trace_err(J, LJ_TRERR_BADTYPE);
1760 crec_finalizer(J, J->base[0], J->base[1], &rd->argv[1]);
1761}
1762
1763/* -- 64 bit bit.* library functions -------------------------------------- */
1764
1765/* Determine bit operation type from argument type. */
1766static CTypeID crec_bit64_type(CTState *cts, cTValue *tv)
1767{
1768 if (tviscdata(tv)) {
1769 CType *ct = lj_ctype_rawref(cts, cdataV(tv)->ctypeid);
1770 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1771 if ((ct->info & (CTMASK_NUM|CTF_BOOL|CTF_FP|CTF_UNSIGNED)) ==
1772 CTINFO(CT_NUM, CTF_UNSIGNED) && ct->size == 8)
1773 return CTID_UINT64; /* Use uint64_t, since it has the highest rank. */
1774 return CTID_INT64; /* Otherwise use int64_t. */
1775 }
1776 return 0; /* Use regular 32 bit ops. */
1777}
1778
1779void LJ_FASTCALL recff_bit64_tobit(jit_State *J, RecordFFData *rd)
1780{
1781 CTState *cts = ctype_ctsG(J2G(J));
1782 TRef tr = crec_ct_tv(J, ctype_get(cts, CTID_INT64), 0,
1783 J->base[0], &rd->argv[0]);
1784 if (!tref_isinteger(tr))
1785 tr = emitconv(tr, IRT_INT, tref_type(tr), 0);
1786 J->base[0] = tr;
1787}
1788
1789int LJ_FASTCALL recff_bit64_unary(jit_State *J, RecordFFData *rd)
1790{
1791 CTState *cts = ctype_ctsG(J2G(J));
1792 CTypeID id = crec_bit64_type(cts, &rd->argv[0]);
1793 if (id) {
1794 TRef tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
1795 tr = emitir(IRT(rd->data, id-CTID_INT64+IRT_I64), tr, 0);
1796 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1797 return 1;
1798 }
1799 return 0;
1800}
1801
1802int LJ_FASTCALL recff_bit64_nary(jit_State *J, RecordFFData *rd)
1803{
1804 CTState *cts = ctype_ctsG(J2G(J));
1805 CTypeID id = 0;
1806 MSize i;
1807 for (i = 0; J->base[i] != 0; i++) {
1808 CTypeID aid = crec_bit64_type(cts, &rd->argv[i]);
1809 if (id < aid) id = aid; /* Determine highest type rank of all arguments. */
1810 }
1811 if (id) {
1812 CType *ct = ctype_get(cts, id);
1813 uint32_t ot = IRT(rd->data, id-CTID_INT64+IRT_I64);
1814 TRef tr = crec_ct_tv(J, ct, 0, J->base[0], &rd->argv[0]);
1815 for (i = 1; J->base[i] != 0; i++) {
1816 TRef tr2 = crec_ct_tv(J, ct, 0, J->base[i], &rd->argv[i]);
1817 tr = emitir(ot, tr, tr2);
1818 }
1819 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1820 return 1;
1821 }
1822 return 0;
1823}
1824
1825int LJ_FASTCALL recff_bit64_shift(jit_State *J, RecordFFData *rd)
1826{
1827 CTState *cts = ctype_ctsG(J2G(J));
1828 CTypeID id;
1829 TRef tsh = 0;
1830 if (J->base[0] && tref_iscdata(J->base[1])) {
1831 tsh = crec_ct_tv(J, ctype_get(cts, CTID_INT64), 0,
1832 J->base[1], &rd->argv[1]);
1833 if (!tref_isinteger(tsh))
1834 tsh = emitconv(tsh, IRT_INT, tref_type(tsh), 0);
1835 J->base[1] = tsh;
1836 }
1837 id = crec_bit64_type(cts, &rd->argv[0]);
1838 if (id) {
1839 TRef tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
1840 uint32_t op = rd->data;
1841 if (!tsh) tsh = lj_opt_narrow_tobit(J, J->base[1]);
1842 if (!(op < IR_BROL ? LJ_TARGET_MASKSHIFT : LJ_TARGET_MASKROT) &&
1843 !tref_isk(tsh))
1844 tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 63));
1845#ifdef LJ_TARGET_UNIFYROT
1846 if (op == (LJ_TARGET_UNIFYROT == 1 ? IR_BROR : IR_BROL)) {
1847 op = LJ_TARGET_UNIFYROT == 1 ? IR_BROL : IR_BROR;
1848 tsh = emitir(IRTI(IR_NEG), tsh, tsh);
1849 }
1850#endif
1851 tr = emitir(IRT(op, id-CTID_INT64+IRT_I64), tr, tsh);
1852 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1853 return 1;
1854 }
1855 return 0;
1856}
1857
1858TRef recff_bit64_tohex(jit_State *J, RecordFFData *rd, TRef hdr)
1859{
1860 CTState *cts = ctype_ctsG(J2G(J));
1861 CTypeID id = crec_bit64_type(cts, &rd->argv[0]);
1862 TRef tr, trsf = J->base[1];
1863 SFormat sf = (STRFMT_UINT|STRFMT_T_HEX);
1864 int32_t n;
1865 if (trsf) {
1866 CTypeID id2 = 0;
1867 n = (int32_t)lj_carith_check64(J->L, 2, &id2);
1868 if (id2)
1869 trsf = crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0, trsf, &rd->argv[1]);
1870 else
1871 trsf = lj_opt_narrow_tobit(J, trsf);
1872 emitir(IRTGI(IR_EQ), trsf, lj_ir_kint(J, n)); /* Specialize to n. */
1873 } else {
1874 n = id ? 16 : 8;
1875 }
1876 if (n < 0) { n = -n; sf |= STRFMT_F_UPPER; }
1877 sf |= ((SFormat)((n+1)&255) << STRFMT_SH_PREC);
1878 if (id) {
1879 tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
1880 if (n < 16)
1881 tr = emitir(IRT(IR_BAND, IRT_U64), tr,
1882 lj_ir_kint64(J, ((uint64_t)1 << 4*n)-1));
1883 } else {
1884 tr = lj_opt_narrow_tobit(J, J->base[0]);
1885 if (n < 8)
1886 tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << 4*n)-1)));
1887 tr = emitconv(tr, IRT_U64, IRT_INT, 0); /* No sign-extension. */
1888 lj_needsplit(J);
1889 }
1890 return lj_ir_call(J, IRCALL_lj_strfmt_putfxint, hdr, lj_ir_kint(J, sf), tr);
1891}
1892
1893/* -- Miscellaneous library functions ------------------------------------- */
1894
1895void LJ_FASTCALL lj_crecord_tonumber(jit_State *J, RecordFFData *rd)
1896{
1897 CTState *cts = ctype_ctsG(J2G(J));
1898 CType *d, *ct = lj_ctype_rawref(cts, cdataV(&rd->argv[0])->ctypeid);
1899 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1900 if (ctype_isnum(ct->info) || ctype_iscomplex(ct->info)) {
1901 if (ctype_isinteger_or_bool(ct->info) && ct->size <= 4 &&
1902 !(ct->size == 4 && (ct->info & CTF_UNSIGNED)))
1903 d = ctype_get(cts, CTID_INT32);
1904 else
1905 d = ctype_get(cts, CTID_DOUBLE);
1906 J->base[0] = crec_ct_tv(J, d, 0, J->base[0], &rd->argv[0]);
1907 } else {
1908 /* Specialize to the ctype that couldn't be converted. */
1909 argv2cdata(J, J->base[0], &rd->argv[0]);
1910 J->base[0] = TREF_NIL;
1911 }
1912}
1913
1914#undef IR
1915#undef emitir
1916#undef emitconv
1917
1918#endif
1919