1/*
2** Trace recorder (bytecode -> SSA IR).
3** Copyright (C) 2005-2021 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#define lj_record_c
7#define LUA_CORE
8
9#include "lj_obj.h"
10
11#if LJ_HASJIT
12
13#include "lj_err.h"
14#include "lj_str.h"
15#include "lj_tab.h"
16#include "lj_meta.h"
17#include "lj_frame.h"
18#if LJ_HASFFI
19#include "lj_ctype.h"
20#endif
21#include "lj_bc.h"
22#include "lj_ff.h"
23#if LJ_HASPROFILE
24#include "lj_debug.h"
25#endif
26#include "lj_ir.h"
27#include "lj_jit.h"
28#include "lj_ircall.h"
29#include "lj_iropt.h"
30#include "lj_trace.h"
31#include "lj_record.h"
32#include "lj_ffrecord.h"
33#include "lj_snap.h"
34#include "lj_dispatch.h"
35#include "lj_vm.h"
36#include "lj_prng.h"
37
38/* Some local macros to save typing. Undef'd at the end. */
39#define IR(ref) (&J->cur.ir[(ref)])
40
41/* Pass IR on to next optimization in chain (FOLD). */
42#define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
43
44/* Emit raw IR without passing through optimizations. */
45#define emitir_raw(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_ir_emit(J))
46
47/* -- Sanity checks ------------------------------------------------------- */
48
49#ifdef LUA_USE_ASSERT
50/* Sanity check the whole IR -- sloooow. */
51static void rec_check_ir(jit_State *J)
52{
53 IRRef i, nins = J->cur.nins, nk = J->cur.nk;
54 lj_assertJ(nk <= REF_BIAS && nins >= REF_BIAS && nins < 65536,
55 "inconsistent IR layout");
56 for (i = nk; i < nins; i++) {
57 IRIns *ir = IR(i);
58 uint32_t mode = lj_ir_mode[ir->o];
59 IRRef op1 = ir->op1;
60 IRRef op2 = ir->op2;
61 const char *err = NULL;
62 switch (irm_op1(mode)) {
63 case IRMnone:
64 if (op1 != 0) err = "IRMnone op1 used";
65 break;
66 case IRMref:
67 if (op1 < nk || (i >= REF_BIAS ? op1 >= i : op1 <= i))
68 err = "IRMref op1 out of range";
69 break;
70 case IRMlit: break;
71 case IRMcst:
72 if (i >= REF_BIAS) { err = "constant in IR range"; break; }
73 if (irt_is64(ir->t) && ir->o != IR_KNULL)
74 i++;
75 continue;
76 }
77 switch (irm_op2(mode)) {
78 case IRMnone:
79 if (op2) err = "IRMnone op2 used";
80 break;
81 case IRMref:
82 if (op2 < nk || (i >= REF_BIAS ? op2 >= i : op2 <= i))
83 err = "IRMref op2 out of range";
84 break;
85 case IRMlit: break;
86 case IRMcst: err = "IRMcst op2"; break;
87 }
88 if (!err && ir->prev) {
89 if (ir->prev < nk || (i >= REF_BIAS ? ir->prev >= i : ir->prev <= i))
90 err = "chain out of range";
91 else if (ir->o != IR_NOP && IR(ir->prev)->o != ir->o)
92 err = "chain to different op";
93 }
94 lj_assertJ(!err, "bad IR %04d op %d(%04d,%04d): %s",
95 i-REF_BIAS,
96 ir->o,
97 irm_op1(mode) == IRMref ? op1-REF_BIAS : op1,
98 irm_op2(mode) == IRMref ? op2-REF_BIAS : op2,
99 err);
100 }
101}
102
103/* Compare stack slots and frames of the recorder and the VM. */
104static void rec_check_slots(jit_State *J)
105{
106 BCReg s, nslots = J->baseslot + J->maxslot;
107 int32_t depth = 0;
108 cTValue *base = J->L->base - J->baseslot;
109 lj_assertJ(J->baseslot >= 1+LJ_FR2, "bad baseslot");
110 lj_assertJ(J->baseslot == 1+LJ_FR2 || (J->slot[J->baseslot-1] & TREF_FRAME),
111 "baseslot does not point to frame");
112 lj_assertJ(nslots <= LJ_MAX_JSLOTS, "slot overflow");
113 for (s = 0; s < nslots; s++) {
114 TRef tr = J->slot[s];
115 if (tr) {
116 cTValue *tv = &base[s];
117 IRRef ref = tref_ref(tr);
118 IRIns *ir = NULL; /* Silence compiler. */
119 if (!LJ_FR2 || ref || !(tr & (TREF_FRAME | TREF_CONT))) {
120 lj_assertJ(ref >= J->cur.nk && ref < J->cur.nins,
121 "slot %d ref %04d out of range", s, ref - REF_BIAS);
122 ir = IR(ref);
123 lj_assertJ(irt_t(ir->t) == tref_t(tr), "slot %d IR type mismatch", s);
124 }
125 if (s == 0) {
126 lj_assertJ(tref_isfunc(tr), "frame slot 0 is not a function");
127#if LJ_FR2
128 } else if (s == 1) {
129 lj_assertJ((tr & ~TREF_FRAME) == 0, "bad frame slot 1");
130#endif
131 } else if ((tr & TREF_FRAME)) {
132 GCfunc *fn = gco2func(frame_gc(tv));
133 BCReg delta = (BCReg)(tv - frame_prev(tv));
134#if LJ_FR2
135 lj_assertJ(!ref || ir_knum(ir)->u64 == tv->u64,
136 "frame slot %d PC mismatch", s);
137 tr = J->slot[s-1];
138 ir = IR(tref_ref(tr));
139#endif
140 lj_assertJ(tref_isfunc(tr),
141 "frame slot %d is not a function", s-LJ_FR2);
142 lj_assertJ(!tref_isk(tr) || fn == ir_kfunc(ir),
143 "frame slot %d function mismatch", s-LJ_FR2);
144 lj_assertJ(s > delta + LJ_FR2 ? (J->slot[s-delta] & TREF_FRAME)
145 : (s == delta + LJ_FR2),
146 "frame slot %d broken chain", s-LJ_FR2);
147 depth++;
148 } else if ((tr & TREF_CONT)) {
149#if LJ_FR2
150 lj_assertJ(!ref || ir_knum(ir)->u64 == tv->u64,
151 "cont slot %d continuation mismatch", s);
152#else
153 lj_assertJ(ir_kptr(ir) == gcrefp(tv->gcr, void),
154 "cont slot %d continuation mismatch", s);
155#endif
156 lj_assertJ((J->slot[s+1+LJ_FR2] & TREF_FRAME),
157 "cont slot %d not followed by frame", s);
158 depth++;
159 } else {
160 /* Number repr. may differ, but other types must be the same. */
161 lj_assertJ(tvisnumber(tv) ? tref_isnumber(tr) :
162 itype2irt(tv) == tref_type(tr),
163 "slot %d type mismatch: stack type %d vs IR type %d",
164 s, itypemap(tv), tref_type(tr));
165 if (tref_isk(tr)) { /* Compare constants. */
166 TValue tvk;
167 lj_ir_kvalue(J->L, &tvk, ir);
168 lj_assertJ((tvisnum(&tvk) && tvisnan(&tvk)) ?
169 (tvisnum(tv) && tvisnan(tv)) :
170 lj_obj_equal(tv, &tvk),
171 "slot %d const mismatch: stack %016llx vs IR %016llx",
172 s, tv->u64, tvk.u64);
173 }
174 }
175 }
176 }
177 lj_assertJ(J->framedepth == depth,
178 "frame depth mismatch %d vs %d", J->framedepth, depth);
179}
180#endif
181
182/* -- Type handling and specialization ------------------------------------ */
183
184/* Note: these functions return tagged references (TRef). */
185
186/* Specialize a slot to a specific type. Note: slot can be negative! */
187static TRef sloadt(jit_State *J, int32_t slot, IRType t, int mode)
188{
189 /* Caller may set IRT_GUARD in t. */
190 TRef ref = emitir_raw(IRT(IR_SLOAD, t), (int32_t)J->baseslot+slot, mode);
191 J->base[slot] = ref;
192 return ref;
193}
194
195/* Specialize a slot to the runtime type. Note: slot can be negative! */
196static TRef sload(jit_State *J, int32_t slot)
197{
198 IRType t = itype2irt(&J->L->base[slot]);
199 TRef ref = emitir_raw(IRTG(IR_SLOAD, t), (int32_t)J->baseslot+slot,
200 IRSLOAD_TYPECHECK);
201 if (irtype_ispri(t)) ref = TREF_PRI(t); /* Canonicalize primitive refs. */
202 J->base[slot] = ref;
203 return ref;
204}
205
206/* Get TRef from slot. Load slot and specialize if not done already. */
207#define getslot(J, s) (J->base[(s)] ? J->base[(s)] : sload(J, (int32_t)(s)))
208
209/* Get TRef for current function. */
210static TRef getcurrf(jit_State *J)
211{
212 if (J->base[-1-LJ_FR2])
213 return J->base[-1-LJ_FR2];
214 /* Non-base frame functions ought to be loaded already. */
215 lj_assertJ(J->baseslot == 1+LJ_FR2, "bad baseslot");
216 return sloadt(J, -1-LJ_FR2, IRT_FUNC, IRSLOAD_READONLY);
217}
218
219/* Compare for raw object equality.
220** Returns 0 if the objects are the same.
221** Returns 1 if they are different, but the same type.
222** Returns 2 for two different types.
223** Comparisons between primitives always return 1 -- no caller cares about it.
224*/
225int lj_record_objcmp(jit_State *J, TRef a, TRef b, cTValue *av, cTValue *bv)
226{
227 int diff = !lj_obj_equal(av, bv);
228 if (!tref_isk2(a, b)) { /* Shortcut, also handles primitives. */
229 IRType ta = tref_isinteger(a) ? IRT_INT : tref_type(a);
230 IRType tb = tref_isinteger(b) ? IRT_INT : tref_type(b);
231 if (ta != tb) {
232 /* Widen mixed number/int comparisons to number/number comparison. */
233 if (ta == IRT_INT && tb == IRT_NUM) {
234 a = emitir(IRTN(IR_CONV), a, IRCONV_NUM_INT);
235 ta = IRT_NUM;
236 } else if (ta == IRT_NUM && tb == IRT_INT) {
237 b = emitir(IRTN(IR_CONV), b, IRCONV_NUM_INT);
238 } else {
239 return 2; /* Two different types are never equal. */
240 }
241 }
242 emitir(IRTG(diff ? IR_NE : IR_EQ, ta), a, b);
243 }
244 return diff;
245}
246
247/* Constify a value. Returns 0 for non-representable object types. */
248TRef lj_record_constify(jit_State *J, cTValue *o)
249{
250 if (tvisgcv(o))
251 return lj_ir_kgc(J, gcV(o), itype2irt(o));
252 else if (tvisint(o))
253 return lj_ir_kint(J, intV(o));
254 else if (tvisnum(o))
255 return lj_ir_knumint(J, numV(o));
256 else if (tvisbool(o))
257 return TREF_PRI(itype2irt(o));
258 else
259 return 0; /* Can't represent lightuserdata (pointless). */
260}
261
262/* -- Record loop ops ----------------------------------------------------- */
263
264/* Loop event. */
265typedef enum {
266 LOOPEV_LEAVE, /* Loop is left or not entered. */
267 LOOPEV_ENTERLO, /* Loop is entered with a low iteration count left. */
268 LOOPEV_ENTER /* Loop is entered. */
269} LoopEvent;
270
271/* Canonicalize slots: convert integers to numbers. */
272static void canonicalize_slots(jit_State *J)
273{
274 BCReg s;
275 if (LJ_DUALNUM) return;
276 for (s = J->baseslot+J->maxslot-1; s >= 1; s--) {
277 TRef tr = J->slot[s];
278 if (tref_isinteger(tr)) {
279 IRIns *ir = IR(tref_ref(tr));
280 if (!(ir->o == IR_SLOAD && (ir->op2 & IRSLOAD_READONLY)))
281 J->slot[s] = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT);
282 }
283 }
284}
285
286/* Stop recording. */
287void lj_record_stop(jit_State *J, TraceLink linktype, TraceNo lnk)
288{
289#ifdef LUAJIT_ENABLE_TABLE_BUMP
290 if (J->retryrec)
291 lj_trace_err(J, LJ_TRERR_RETRY);
292#endif
293 lj_trace_end(J);
294 J->cur.linktype = (uint8_t)linktype;
295 J->cur.link = (uint16_t)lnk;
296 /* Looping back at the same stack level? */
297 if (lnk == J->cur.traceno && J->framedepth + J->retdepth == 0) {
298 if ((J->flags & JIT_F_OPT_LOOP)) /* Shall we try to create a loop? */
299 goto nocanon; /* Do not canonicalize or we lose the narrowing. */
300 if (J->cur.root) /* Otherwise ensure we always link to the root trace. */
301 J->cur.link = J->cur.root;
302 }
303 canonicalize_slots(J);
304nocanon:
305 /* Note: all loop ops must set J->pc to the following instruction! */
306 lj_snap_add(J); /* Add loop snapshot. */
307 J->needsnap = 0;
308 J->mergesnap = 1; /* In case recording continues. */
309}
310
311/* Search bytecode backwards for a int/num constant slot initializer. */
312static TRef find_kinit(jit_State *J, const BCIns *endpc, BCReg slot, IRType t)
313{
314 /* This algorithm is rather simplistic and assumes quite a bit about
315 ** how the bytecode is generated. It works fine for FORI initializers,
316 ** but it won't necessarily work in other cases (e.g. iterator arguments).
317 ** It doesn't do anything fancy, either (like backpropagating MOVs).
318 */
319 const BCIns *pc, *startpc = proto_bc(J->pt);
320 for (pc = endpc-1; pc > startpc; pc--) {
321 BCIns ins = *pc;
322 BCOp op = bc_op(ins);
323 /* First try to find the last instruction that stores to this slot. */
324 if (bcmode_a(op) == BCMbase && bc_a(ins) <= slot) {
325 return 0; /* Multiple results, e.g. from a CALL or KNIL. */
326 } else if (bcmode_a(op) == BCMdst && bc_a(ins) == slot) {
327 if (op == BC_KSHORT || op == BC_KNUM) { /* Found const. initializer. */
328 /* Now try to verify there's no forward jump across it. */
329 const BCIns *kpc = pc;
330 for (; pc > startpc; pc--)
331 if (bc_op(*pc) == BC_JMP) {
332 const BCIns *target = pc+bc_j(*pc)+1;
333 if (target > kpc && target <= endpc)
334 return 0; /* Conditional assignment. */
335 }
336 if (op == BC_KSHORT) {
337 int32_t k = (int32_t)(int16_t)bc_d(ins);
338 return t == IRT_INT ? lj_ir_kint(J, k) : lj_ir_knum(J, (lua_Number)k);
339 } else {
340 cTValue *tv = proto_knumtv(J->pt, bc_d(ins));
341 if (t == IRT_INT) {
342 int32_t k = numberVint(tv);
343 if (tvisint(tv) || numV(tv) == (lua_Number)k) /* -0 is ok here. */
344 return lj_ir_kint(J, k);
345 return 0; /* Type mismatch. */
346 } else {
347 return lj_ir_knum(J, numberVnum(tv));
348 }
349 }
350 }
351 return 0; /* Non-constant initializer. */
352 }
353 }
354 return 0; /* No assignment to this slot found? */
355}
356
357/* Load and optionally convert a FORI argument from a slot. */
358static TRef fori_load(jit_State *J, BCReg slot, IRType t, int mode)
359{
360 int conv = (tvisint(&J->L->base[slot]) != (t==IRT_INT)) ? IRSLOAD_CONVERT : 0;
361 return sloadt(J, (int32_t)slot,
362 t + (((mode & IRSLOAD_TYPECHECK) ||
363 (conv && t == IRT_INT && !(mode >> 16))) ?
364 IRT_GUARD : 0),
365 mode + conv);
366}
367
368/* Peek before FORI to find a const initializer. Otherwise load from slot. */
369static TRef fori_arg(jit_State *J, const BCIns *fori, BCReg slot,
370 IRType t, int mode)
371{
372 TRef tr = J->base[slot];
373 if (!tr) {
374 tr = find_kinit(J, fori, slot, t);
375 if (!tr)
376 tr = fori_load(J, slot, t, mode);
377 }
378 return tr;
379}
380
381/* Return the direction of the FOR loop iterator.
382** It's important to exactly reproduce the semantics of the interpreter.
383*/
384static int rec_for_direction(cTValue *o)
385{
386 return (tvisint(o) ? intV(o) : (int32_t)o->u32.hi) >= 0;
387}
388
389/* Simulate the runtime behavior of the FOR loop iterator. */
390static LoopEvent rec_for_iter(IROp *op, cTValue *o, int isforl)
391{
392 lua_Number stopv = numberVnum(&o[FORL_STOP]);
393 lua_Number idxv = numberVnum(&o[FORL_IDX]);
394 lua_Number stepv = numberVnum(&o[FORL_STEP]);
395 if (isforl)
396 idxv += stepv;
397 if (rec_for_direction(&o[FORL_STEP])) {
398 if (idxv <= stopv) {
399 *op = IR_LE;
400 return idxv + 2*stepv > stopv ? LOOPEV_ENTERLO : LOOPEV_ENTER;
401 }
402 *op = IR_GT; return LOOPEV_LEAVE;
403 } else {
404 if (stopv <= idxv) {
405 *op = IR_GE;
406 return idxv + 2*stepv < stopv ? LOOPEV_ENTERLO : LOOPEV_ENTER;
407 }
408 *op = IR_LT; return LOOPEV_LEAVE;
409 }
410}
411
412/* Record checks for FOR loop overflow and step direction. */
413static void rec_for_check(jit_State *J, IRType t, int dir,
414 TRef stop, TRef step, int init)
415{
416 if (!tref_isk(step)) {
417 /* Non-constant step: need a guard for the direction. */
418 TRef zero = (t == IRT_INT) ? lj_ir_kint(J, 0) : lj_ir_knum_zero(J);
419 emitir(IRTG(dir ? IR_GE : IR_LT, t), step, zero);
420 /* Add hoistable overflow checks for a narrowed FORL index. */
421 if (init && t == IRT_INT) {
422 if (tref_isk(stop)) {
423 /* Constant stop: optimize check away or to a range check for step. */
424 int32_t k = IR(tref_ref(stop))->i;
425 if (dir) {
426 if (k > 0)
427 emitir(IRTGI(IR_LE), step, lj_ir_kint(J, (int32_t)0x7fffffff-k));
428 } else {
429 if (k < 0)
430 emitir(IRTGI(IR_GE), step, lj_ir_kint(J, (int32_t)0x80000000-k));
431 }
432 } else {
433 /* Stop+step variable: need full overflow check. */
434 TRef tr = emitir(IRTGI(IR_ADDOV), step, stop);
435 emitir(IRTI(IR_USE), tr, 0); /* ADDOV is weak. Avoid dead result. */
436 }
437 }
438 } else if (init && t == IRT_INT && !tref_isk(stop)) {
439 /* Constant step: optimize overflow check to a range check for stop. */
440 int32_t k = IR(tref_ref(step))->i;
441 k = (int32_t)(dir ? 0x7fffffff : 0x80000000) - k;
442 emitir(IRTGI(dir ? IR_LE : IR_GE), stop, lj_ir_kint(J, k));
443 }
444}
445
446/* Record a FORL instruction. */
447static void rec_for_loop(jit_State *J, const BCIns *fori, ScEvEntry *scev,
448 int init)
449{
450 BCReg ra = bc_a(*fori);
451 cTValue *tv = &J->L->base[ra];
452 TRef idx = J->base[ra+FORL_IDX];
453 IRType t = idx ? tref_type(idx) :
454 (init || LJ_DUALNUM) ? lj_opt_narrow_forl(J, tv) : IRT_NUM;
455 int mode = IRSLOAD_INHERIT +
456 ((!LJ_DUALNUM || tvisint(tv) == (t == IRT_INT)) ? IRSLOAD_READONLY : 0);
457 TRef stop = fori_arg(J, fori, ra+FORL_STOP, t, mode);
458 TRef step = fori_arg(J, fori, ra+FORL_STEP, t, mode);
459 int tc, dir = rec_for_direction(&tv[FORL_STEP]);
460 lj_assertJ(bc_op(*fori) == BC_FORI || bc_op(*fori) == BC_JFORI,
461 "bad bytecode %d instead of FORI/JFORI", bc_op(*fori));
462 scev->t.irt = t;
463 scev->dir = dir;
464 scev->stop = tref_ref(stop);
465 scev->step = tref_ref(step);
466 rec_for_check(J, t, dir, stop, step, init);
467 scev->start = tref_ref(find_kinit(J, fori, ra+FORL_IDX, IRT_INT));
468 tc = (LJ_DUALNUM &&
469 !(scev->start && irref_isk(scev->stop) && irref_isk(scev->step) &&
470 tvisint(&tv[FORL_IDX]) == (t == IRT_INT))) ?
471 IRSLOAD_TYPECHECK : 0;
472 if (tc) {
473 J->base[ra+FORL_STOP] = stop;
474 J->base[ra+FORL_STEP] = step;
475 }
476 if (!idx)
477 idx = fori_load(J, ra+FORL_IDX, t,
478 IRSLOAD_INHERIT + tc + (J->scev.start << 16));
479 if (!init)
480 J->base[ra+FORL_IDX] = idx = emitir(IRT(IR_ADD, t), idx, step);
481 J->base[ra+FORL_EXT] = idx;
482 scev->idx = tref_ref(idx);
483 setmref(scev->pc, fori);
484 J->maxslot = ra+FORL_EXT+1;
485}
486
487/* Record FORL/JFORL or FORI/JFORI. */
488static LoopEvent rec_for(jit_State *J, const BCIns *fori, int isforl)
489{
490 BCReg ra = bc_a(*fori);
491 TValue *tv = &J->L->base[ra];
492 TRef *tr = &J->base[ra];
493 IROp op;
494 LoopEvent ev;
495 TRef stop;
496 IRType t;
497 if (isforl) { /* Handle FORL/JFORL opcodes. */
498 TRef idx = tr[FORL_IDX];
499 if (mref(J->scev.pc, const BCIns) == fori && tref_ref(idx) == J->scev.idx) {
500 t = J->scev.t.irt;
501 stop = J->scev.stop;
502 idx = emitir(IRT(IR_ADD, t), idx, J->scev.step);
503 tr[FORL_EXT] = tr[FORL_IDX] = idx;
504 } else {
505 ScEvEntry scev;
506 rec_for_loop(J, fori, &scev, 0);
507 t = scev.t.irt;
508 stop = scev.stop;
509 }
510 } else { /* Handle FORI/JFORI opcodes. */
511 BCReg i;
512 lj_meta_for(J->L, tv);
513 t = (LJ_DUALNUM || tref_isint(tr[FORL_IDX])) ? lj_opt_narrow_forl(J, tv) :
514 IRT_NUM;
515 for (i = FORL_IDX; i <= FORL_STEP; i++) {
516 if (!tr[i]) sload(J, ra+i);
517 lj_assertJ(tref_isnumber_str(tr[i]), "bad FORI argument type");
518 if (tref_isstr(tr[i]))
519 tr[i] = emitir(IRTG(IR_STRTO, IRT_NUM), tr[i], 0);
520 if (t == IRT_INT) {
521 if (!tref_isinteger(tr[i]))
522 tr[i] = emitir(IRTGI(IR_CONV), tr[i], IRCONV_INT_NUM|IRCONV_CHECK);
523 } else {
524 if (!tref_isnum(tr[i]))
525 tr[i] = emitir(IRTN(IR_CONV), tr[i], IRCONV_NUM_INT);
526 }
527 }
528 tr[FORL_EXT] = tr[FORL_IDX];
529 stop = tr[FORL_STOP];
530 rec_for_check(J, t, rec_for_direction(&tv[FORL_STEP]),
531 stop, tr[FORL_STEP], 1);
532 }
533
534 ev = rec_for_iter(&op, tv, isforl);
535 if (ev == LOOPEV_LEAVE) {
536 J->maxslot = ra+FORL_EXT+1;
537 J->pc = fori+1;
538 } else {
539 J->maxslot = ra;
540 J->pc = fori+bc_j(*fori)+1;
541 }
542 lj_snap_add(J);
543
544 emitir(IRTG(op, t), tr[FORL_IDX], stop);
545
546 if (ev == LOOPEV_LEAVE) {
547 J->maxslot = ra;
548 J->pc = fori+bc_j(*fori)+1;
549 } else {
550 J->maxslot = ra+FORL_EXT+1;
551 J->pc = fori+1;
552 }
553 J->needsnap = 1;
554 return ev;
555}
556
557/* Record ITERL/JITERL. */
558static LoopEvent rec_iterl(jit_State *J, const BCIns iterins)
559{
560 BCReg ra = bc_a(iterins);
561 if (!tref_isnil(getslot(J, ra))) { /* Looping back? */
562 J->base[ra-1] = J->base[ra]; /* Copy result of ITERC to control var. */
563 J->maxslot = ra-1+bc_b(J->pc[-1]);
564 J->pc += bc_j(iterins)+1;
565 return LOOPEV_ENTER;
566 } else {
567 J->maxslot = ra-3;
568 J->pc++;
569 return LOOPEV_LEAVE;
570 }
571}
572
573/* Record LOOP/JLOOP. Now, that was easy. */
574static LoopEvent rec_loop(jit_State *J, BCReg ra, int skip)
575{
576 if (ra < J->maxslot) J->maxslot = ra;
577 J->pc += skip;
578 return LOOPEV_ENTER;
579}
580
581/* Check if a loop repeatedly failed to trace because it didn't loop back. */
582static int innerloopleft(jit_State *J, const BCIns *pc)
583{
584 ptrdiff_t i;
585 for (i = 0; i < PENALTY_SLOTS; i++)
586 if (mref(J->penalty[i].pc, const BCIns) == pc) {
587 if ((J->penalty[i].reason == LJ_TRERR_LLEAVE ||
588 J->penalty[i].reason == LJ_TRERR_LINNER) &&
589 J->penalty[i].val >= 2*PENALTY_MIN)
590 return 1;
591 break;
592 }
593 return 0;
594}
595
596/* Handle the case when an interpreted loop op is hit. */
597static void rec_loop_interp(jit_State *J, const BCIns *pc, LoopEvent ev)
598{
599 if (J->parent == 0 && J->exitno == 0) {
600 if (pc == J->startpc && J->framedepth + J->retdepth == 0) {
601 /* Same loop? */
602 if (ev == LOOPEV_LEAVE) /* Must loop back to form a root trace. */
603 lj_trace_err(J, LJ_TRERR_LLEAVE);
604 lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno); /* Looping trace. */
605 } else if (ev != LOOPEV_LEAVE) { /* Entering inner loop? */
606 /* It's usually better to abort here and wait until the inner loop
607 ** is traced. But if the inner loop repeatedly didn't loop back,
608 ** this indicates a low trip count. In this case try unrolling
609 ** an inner loop even in a root trace. But it's better to be a bit
610 ** more conservative here and only do it for very short loops.
611 */
612 if (bc_j(*pc) != -1 && !innerloopleft(J, pc))
613 lj_trace_err(J, LJ_TRERR_LINNER); /* Root trace hit an inner loop. */
614 if ((ev != LOOPEV_ENTERLO &&
615 J->loopref && J->cur.nins - J->loopref > 24) || --J->loopunroll < 0)
616 lj_trace_err(J, LJ_TRERR_LUNROLL); /* Limit loop unrolling. */
617 J->loopref = J->cur.nins;
618 }
619 } else if (ev != LOOPEV_LEAVE) { /* Side trace enters an inner loop. */
620 J->loopref = J->cur.nins;
621 if (--J->loopunroll < 0)
622 lj_trace_err(J, LJ_TRERR_LUNROLL); /* Limit loop unrolling. */
623 } /* Side trace continues across a loop that's left or not entered. */
624}
625
626/* Handle the case when an already compiled loop op is hit. */
627static void rec_loop_jit(jit_State *J, TraceNo lnk, LoopEvent ev)
628{
629 if (J->parent == 0 && J->exitno == 0) { /* Root trace hit an inner loop. */
630 /* Better let the inner loop spawn a side trace back here. */
631 lj_trace_err(J, LJ_TRERR_LINNER);
632 } else if (ev != LOOPEV_LEAVE) { /* Side trace enters a compiled loop. */
633 J->instunroll = 0; /* Cannot continue across a compiled loop op. */
634 if (J->pc == J->startpc && J->framedepth + J->retdepth == 0)
635 lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno); /* Form extra loop. */
636 else
637 lj_record_stop(J, LJ_TRLINK_ROOT, lnk); /* Link to the loop. */
638 } /* Side trace continues across a loop that's left or not entered. */
639}
640
641/* -- Record profiler hook checks ----------------------------------------- */
642
643#if LJ_HASPROFILE
644
645/* Need to insert profiler hook check? */
646static int rec_profile_need(jit_State *J, GCproto *pt, const BCIns *pc)
647{
648 GCproto *ppt;
649 lj_assertJ(J->prof_mode == 'f' || J->prof_mode == 'l',
650 "bad profiler mode %c", J->prof_mode);
651 if (!pt)
652 return 0;
653 ppt = J->prev_pt;
654 J->prev_pt = pt;
655 if (pt != ppt && ppt) {
656 J->prev_line = -1;
657 return 1;
658 }
659 if (J->prof_mode == 'l') {
660 BCLine line = lj_debug_line(pt, proto_bcpos(pt, pc));
661 BCLine pline = J->prev_line;
662 J->prev_line = line;
663 if (pline != line)
664 return 1;
665 }
666 return 0;
667}
668
669static void rec_profile_ins(jit_State *J, const BCIns *pc)
670{
671 if (J->prof_mode && rec_profile_need(J, J->pt, pc)) {
672 emitir(IRTG(IR_PROF, IRT_NIL), 0, 0);
673 lj_snap_add(J);
674 }
675}
676
677static void rec_profile_ret(jit_State *J)
678{
679 if (J->prof_mode == 'f') {
680 emitir(IRTG(IR_PROF, IRT_NIL), 0, 0);
681 J->prev_pt = NULL;
682 lj_snap_add(J);
683 }
684}
685
686#endif
687
688/* -- Record calls and returns -------------------------------------------- */
689
690/* Specialize to the runtime value of the called function or its prototype. */
691static TRef rec_call_specialize(jit_State *J, GCfunc *fn, TRef tr)
692{
693 TRef kfunc;
694 if (isluafunc(fn)) {
695 GCproto *pt = funcproto(fn);
696 /* Too many closures created? Probably not a monomorphic function. */
697 if (pt->flags >= PROTO_CLC_POLY) { /* Specialize to prototype instead. */
698 TRef trpt = emitir(IRT(IR_FLOAD, IRT_PGC), tr, IRFL_FUNC_PC);
699 emitir(IRTG(IR_EQ, IRT_PGC), trpt, lj_ir_kptr(J, proto_bc(pt)));
700 (void)lj_ir_kgc(J, obj2gco(pt), IRT_PROTO); /* Prevent GC of proto. */
701 return tr;
702 }
703 } else {
704 /* Don't specialize to non-monomorphic builtins. */
705 switch (fn->c.ffid) {
706 case FF_coroutine_wrap_aux:
707 case FF_string_gmatch_aux:
708 /* NYI: io_file_iter doesn't have an ffid, yet. */
709 { /* Specialize to the ffid. */
710 TRef trid = emitir(IRT(IR_FLOAD, IRT_U8), tr, IRFL_FUNC_FFID);
711 emitir(IRTG(IR_EQ, IRT_INT), trid, lj_ir_kint(J, fn->c.ffid));
712 }
713 return tr;
714 default:
715 /* NYI: don't specialize to non-monomorphic C functions. */
716 break;
717 }
718 }
719 /* Otherwise specialize to the function (closure) value itself. */
720 kfunc = lj_ir_kfunc(J, fn);
721 emitir(IRTG(IR_EQ, IRT_FUNC), tr, kfunc);
722 return kfunc;
723}
724
725/* Record call setup. */
726static void rec_call_setup(jit_State *J, BCReg func, ptrdiff_t nargs)
727{
728 RecordIndex ix;
729 TValue *functv = &J->L->base[func];
730 TRef kfunc, *fbase = &J->base[func];
731 ptrdiff_t i;
732 (void)getslot(J, func); /* Ensure func has a reference. */
733 for (i = 1; i <= nargs; i++)
734 (void)getslot(J, func+LJ_FR2+i); /* Ensure all args have a reference. */
735 if (!tref_isfunc(fbase[0])) { /* Resolve __call metamethod. */
736 ix.tab = fbase[0];
737 copyTV(J->L, &ix.tabv, functv);
738 if (!lj_record_mm_lookup(J, &ix, MM_call) || !tref_isfunc(ix.mobj))
739 lj_trace_err(J, LJ_TRERR_NOMM);
740 for (i = ++nargs; i > LJ_FR2; i--) /* Shift arguments up. */
741 fbase[i+LJ_FR2] = fbase[i+LJ_FR2-1];
742#if LJ_FR2
743 fbase[2] = fbase[0];
744#endif
745 fbase[0] = ix.mobj; /* Replace function. */
746 functv = &ix.mobjv;
747 }
748 kfunc = rec_call_specialize(J, funcV(functv), fbase[0]);
749#if LJ_FR2
750 fbase[0] = kfunc;
751 fbase[1] = TREF_FRAME;
752#else
753 fbase[0] = kfunc | TREF_FRAME;
754#endif
755 J->maxslot = (BCReg)nargs;
756}
757
758/* Record call. */
759void lj_record_call(jit_State *J, BCReg func, ptrdiff_t nargs)
760{
761 rec_call_setup(J, func, nargs);
762 /* Bump frame. */
763 J->framedepth++;
764 J->base += func+1+LJ_FR2;
765 J->baseslot += func+1+LJ_FR2;
766 if (J->baseslot + J->maxslot >= LJ_MAX_JSLOTS)
767 lj_trace_err(J, LJ_TRERR_STACKOV);
768}
769
770/* Record tail call. */
771void lj_record_tailcall(jit_State *J, BCReg func, ptrdiff_t nargs)
772{
773 rec_call_setup(J, func, nargs);
774 if (frame_isvarg(J->L->base - 1)) {
775 BCReg cbase = (BCReg)frame_delta(J->L->base - 1);
776 if (--J->framedepth < 0)
777 lj_trace_err(J, LJ_TRERR_NYIRETL);
778 J->baseslot -= (BCReg)cbase;
779 J->base -= cbase;
780 func += cbase;
781 }
782 /* Move func + args down. */
783 if (LJ_FR2 && J->baseslot == 2)
784 J->base[func+1] = TREF_FRAME;
785 memmove(&J->base[-1-LJ_FR2], &J->base[func], sizeof(TRef)*(J->maxslot+1+LJ_FR2));
786 /* Note: the new TREF_FRAME is now at J->base[-1] (even for slot #0). */
787 /* Tailcalls can form a loop, so count towards the loop unroll limit. */
788 if (++J->tailcalled > J->loopunroll)
789 lj_trace_err(J, LJ_TRERR_LUNROLL);
790}
791
792/* Check unroll limits for down-recursion. */
793static int check_downrec_unroll(jit_State *J, GCproto *pt)
794{
795 IRRef ptref;
796 for (ptref = J->chain[IR_KGC]; ptref; ptref = IR(ptref)->prev)
797 if (ir_kgc(IR(ptref)) == obj2gco(pt)) {
798 int count = 0;
799 IRRef ref;
800 for (ref = J->chain[IR_RETF]; ref; ref = IR(ref)->prev)
801 if (IR(ref)->op1 == ptref)
802 count++;
803 if (count) {
804 if (J->pc == J->startpc) {
805 if (count + J->tailcalled > J->param[JIT_P_recunroll])
806 return 1;
807 } else {
808 lj_trace_err(J, LJ_TRERR_DOWNREC);
809 }
810 }
811 }
812 return 0;
813}
814
815static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot);
816
817/* Record return. */
818void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults)
819{
820 TValue *frame = J->L->base - 1;
821 ptrdiff_t i;
822 for (i = 0; i < gotresults; i++)
823 (void)getslot(J, rbase+i); /* Ensure all results have a reference. */
824 while (frame_ispcall(frame)) { /* Immediately resolve pcall() returns. */
825 BCReg cbase = (BCReg)frame_delta(frame);
826 if (--J->framedepth <= 0)
827 lj_trace_err(J, LJ_TRERR_NYIRETL);
828 lj_assertJ(J->baseslot > 1+LJ_FR2, "bad baseslot for return");
829 gotresults++;
830 rbase += cbase;
831 J->baseslot -= (BCReg)cbase;
832 J->base -= cbase;
833 J->base[--rbase] = TREF_TRUE; /* Prepend true to results. */
834 frame = frame_prevd(frame);
835 J->needsnap = 1; /* Stop catching on-trace errors. */
836 }
837 /* Return to lower frame via interpreter for unhandled cases. */
838 if (J->framedepth == 0 && J->pt && bc_isret(bc_op(*J->pc)) &&
839 (!frame_islua(frame) ||
840 (J->parent == 0 && J->exitno == 0 &&
841 !bc_isret(bc_op(J->cur.startins))))) {
842 /* NYI: specialize to frame type and return directly, not via RET*. */
843 for (i = 0; i < (ptrdiff_t)rbase; i++)
844 J->base[i] = 0; /* Purge dead slots. */
845 J->maxslot = rbase + (BCReg)gotresults;
846 lj_record_stop(J, LJ_TRLINK_RETURN, 0); /* Return to interpreter. */
847 return;
848 }
849 if (frame_isvarg(frame)) {
850 BCReg cbase = (BCReg)frame_delta(frame);
851 if (--J->framedepth < 0) /* NYI: return of vararg func to lower frame. */
852 lj_trace_err(J, LJ_TRERR_NYIRETL);
853 lj_assertJ(J->baseslot > 1+LJ_FR2, "bad baseslot for return");
854 rbase += cbase;
855 J->baseslot -= (BCReg)cbase;
856 J->base -= cbase;
857 frame = frame_prevd(frame);
858 }
859 if (frame_islua(frame)) { /* Return to Lua frame. */
860 BCIns callins = *(frame_pc(frame)-1);
861 ptrdiff_t nresults = bc_b(callins) ? (ptrdiff_t)bc_b(callins)-1 :gotresults;
862 BCReg cbase = bc_a(callins);
863 GCproto *pt = funcproto(frame_func(frame - (cbase+1+LJ_FR2)));
864 if ((pt->flags & PROTO_NOJIT))
865 lj_trace_err(J, LJ_TRERR_CJITOFF);
866 if (J->framedepth == 0 && J->pt && frame == J->L->base - 1) {
867 if (check_downrec_unroll(J, pt)) {
868 J->maxslot = (BCReg)(rbase + gotresults);
869 lj_snap_purge(J);
870 lj_record_stop(J, LJ_TRLINK_DOWNREC, J->cur.traceno); /* Down-rec. */
871 return;
872 }
873 lj_snap_add(J);
874 }
875 for (i = 0; i < nresults; i++) /* Adjust results. */
876 J->base[i-1-LJ_FR2] = i < gotresults ? J->base[rbase+i] : TREF_NIL;
877 J->maxslot = cbase+(BCReg)nresults;
878 if (J->framedepth > 0) { /* Return to a frame that is part of the trace. */
879 J->framedepth--;
880 lj_assertJ(J->baseslot > cbase+1+LJ_FR2, "bad baseslot for return");
881 J->baseslot -= cbase+1+LJ_FR2;
882 J->base -= cbase+1+LJ_FR2;
883 } else if (J->parent == 0 && J->exitno == 0 &&
884 !bc_isret(bc_op(J->cur.startins))) {
885 /* Return to lower frame would leave the loop in a root trace. */
886 lj_trace_err(J, LJ_TRERR_LLEAVE);
887 } else if (J->needsnap) { /* Tailcalled to ff with side-effects. */
888 lj_trace_err(J, LJ_TRERR_NYIRETL); /* No way to insert snapshot here. */
889 } else { /* Return to lower frame. Guard for the target we return to. */
890 TRef trpt = lj_ir_kgc(J, obj2gco(pt), IRT_PROTO);
891 TRef trpc = lj_ir_kptr(J, (void *)frame_pc(frame));
892 emitir(IRTG(IR_RETF, IRT_PGC), trpt, trpc);
893 J->retdepth++;
894 J->needsnap = 1;
895 lj_assertJ(J->baseslot == 1+LJ_FR2, "bad baseslot for return");
896 /* Shift result slots up and clear the slots of the new frame below. */
897 memmove(J->base + cbase, J->base-1-LJ_FR2, sizeof(TRef)*nresults);
898 memset(J->base-1-LJ_FR2, 0, sizeof(TRef)*(cbase+1+LJ_FR2));
899 }
900 } else if (frame_iscont(frame)) { /* Return to continuation frame. */
901 ASMFunction cont = frame_contf(frame);
902 BCReg cbase = (BCReg)frame_delta(frame);
903 if ((J->framedepth -= 2) < 0)
904 lj_trace_err(J, LJ_TRERR_NYIRETL);
905 J->baseslot -= (BCReg)cbase;
906 J->base -= cbase;
907 J->maxslot = cbase-(2<<LJ_FR2);
908 if (cont == lj_cont_ra) {
909 /* Copy result to destination slot. */
910 BCReg dst = bc_a(*(frame_contpc(frame)-1));
911 J->base[dst] = gotresults ? J->base[cbase+rbase] : TREF_NIL;
912 if (dst >= J->maxslot) {
913 J->maxslot = dst+1;
914 }
915 } else if (cont == lj_cont_nop) {
916 /* Nothing to do here. */
917 } else if (cont == lj_cont_cat) {
918 BCReg bslot = bc_b(*(frame_contpc(frame)-1));
919 TRef tr = gotresults ? J->base[cbase+rbase] : TREF_NIL;
920 if (bslot != J->maxslot) { /* Concatenate the remainder. */
921 TValue *b = J->L->base, save; /* Simulate lower frame and result. */
922 J->base[J->maxslot] = tr;
923 copyTV(J->L, &save, b-(2<<LJ_FR2));
924 if (gotresults)
925 copyTV(J->L, b-(2<<LJ_FR2), b+rbase);
926 else
927 setnilV(b-(2<<LJ_FR2));
928 J->L->base = b - cbase;
929 tr = rec_cat(J, bslot, cbase-(2<<LJ_FR2));
930 b = J->L->base + cbase; /* Undo. */
931 J->L->base = b;
932 copyTV(J->L, b-(2<<LJ_FR2), &save);
933 }
934 if (tr) { /* Store final result. */
935 BCReg dst = bc_a(*(frame_contpc(frame)-1));
936 J->base[dst] = tr;
937 if (dst >= J->maxslot) {
938 J->maxslot = dst+1;
939 }
940 } /* Otherwise continue with another __concat call. */
941 } else {
942 /* Result type already specialized. */
943 lj_assertJ(cont == lj_cont_condf || cont == lj_cont_condt,
944 "bad continuation type");
945 }
946 } else {
947 lj_trace_err(J, LJ_TRERR_NYIRETL); /* NYI: handle return to C frame. */
948 }
949 lj_assertJ(J->baseslot >= 1+LJ_FR2, "bad baseslot for return");
950}
951
952/* -- Metamethod handling ------------------------------------------------- */
953
954/* Prepare to record call to metamethod. */
955static BCReg rec_mm_prep(jit_State *J, ASMFunction cont)
956{
957 BCReg s, top = cont == lj_cont_cat ? J->maxslot : curr_proto(J->L)->framesize;
958#if LJ_FR2
959 J->base[top] = lj_ir_k64(J, IR_KNUM, u64ptr(contptr(cont)));
960 J->base[top+1] = TREF_CONT;
961#else
962 J->base[top] = lj_ir_kptr(J, contptr(cont)) | TREF_CONT;
963#endif
964 J->framedepth++;
965 for (s = J->maxslot; s < top; s++)
966 J->base[s] = 0; /* Clear frame gap to avoid resurrecting previous refs. */
967 return top+1+LJ_FR2;
968}
969
970/* Record metamethod lookup. */
971int lj_record_mm_lookup(jit_State *J, RecordIndex *ix, MMS mm)
972{
973 RecordIndex mix;
974 GCtab *mt;
975 if (tref_istab(ix->tab)) {
976 mt = tabref(tabV(&ix->tabv)->metatable);
977 mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META);
978 } else if (tref_isudata(ix->tab)) {
979 int udtype = udataV(&ix->tabv)->udtype;
980 mt = tabref(udataV(&ix->tabv)->metatable);
981 /* The metatables of special userdata objects are treated as immutable. */
982 if (udtype != UDTYPE_USERDATA) {
983 cTValue *mo;
984 if (LJ_HASFFI && udtype == UDTYPE_FFI_CLIB) {
985 /* Specialize to the C library namespace object. */
986 emitir(IRTG(IR_EQ, IRT_PGC), ix->tab, lj_ir_kptr(J, udataV(&ix->tabv)));
987 } else {
988 /* Specialize to the type of userdata. */
989 TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), ix->tab, IRFL_UDATA_UDTYPE);
990 emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, udtype));
991 }
992 immutable_mt:
993 mo = lj_tab_getstr(mt, mmname_str(J2G(J), mm));
994 if (!mo || tvisnil(mo))
995 return 0; /* No metamethod. */
996 /* Treat metamethod or index table as immutable, too. */
997 if (!(tvisfunc(mo) || tvistab(mo)))
998 lj_trace_err(J, LJ_TRERR_BADTYPE);
999 copyTV(J->L, &ix->mobjv, mo);
1000 ix->mobj = lj_ir_kgc(J, gcV(mo), tvisfunc(mo) ? IRT_FUNC : IRT_TAB);
1001 ix->mtv = mt;
1002 ix->mt = TREF_NIL; /* Dummy value for comparison semantics. */
1003 return 1; /* Got metamethod or index table. */
1004 }
1005 mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_UDATA_META);
1006 } else {
1007 /* Specialize to base metatable. Must flush mcode in lua_setmetatable(). */
1008 mt = tabref(basemt_obj(J2G(J), &ix->tabv));
1009 if (mt == NULL) {
1010 ix->mt = TREF_NIL;
1011 return 0; /* No metamethod. */
1012 }
1013 /* The cdata metatable is treated as immutable. */
1014 if (LJ_HASFFI && tref_iscdata(ix->tab)) goto immutable_mt;
1015 ix->mt = mix.tab = lj_ir_ggfload(J, IRT_TAB,
1016 GG_OFS(g.gcroot[GCROOT_BASEMT+itypemap(&ix->tabv)]));
1017 goto nocheck;
1018 }
1019 ix->mt = mt ? mix.tab : TREF_NIL;
1020 emitir(IRTG(mt ? IR_NE : IR_EQ, IRT_TAB), mix.tab, lj_ir_knull(J, IRT_TAB));
1021nocheck:
1022 if (mt) {
1023 GCstr *mmstr = mmname_str(J2G(J), mm);
1024 cTValue *mo = lj_tab_getstr(mt, mmstr);
1025 if (mo && !tvisnil(mo))
1026 copyTV(J->L, &ix->mobjv, mo);
1027 ix->mtv = mt;
1028 settabV(J->L, &mix.tabv, mt);
1029 setstrV(J->L, &mix.keyv, mmstr);
1030 mix.key = lj_ir_kstr(J, mmstr);
1031 mix.val = 0;
1032 mix.idxchain = 0;
1033 ix->mobj = lj_record_idx(J, &mix);
1034 return !tref_isnil(ix->mobj); /* 1 if metamethod found, 0 if not. */
1035 }
1036 return 0; /* No metamethod. */
1037}
1038
1039/* Record call to arithmetic metamethod. */
1040static TRef rec_mm_arith(jit_State *J, RecordIndex *ix, MMS mm)
1041{
1042 /* Set up metamethod call first to save ix->tab and ix->tabv. */
1043 BCReg func = rec_mm_prep(J, mm == MM_concat ? lj_cont_cat : lj_cont_ra);
1044 TRef *base = J->base + func;
1045 TValue *basev = J->L->base + func;
1046 base[1+LJ_FR2] = ix->tab; base[2+LJ_FR2] = ix->key;
1047 copyTV(J->L, basev+1+LJ_FR2, &ix->tabv);
1048 copyTV(J->L, basev+2+LJ_FR2, &ix->keyv);
1049 if (!lj_record_mm_lookup(J, ix, mm)) { /* Lookup mm on 1st operand. */
1050 if (mm != MM_unm) {
1051 ix->tab = ix->key;
1052 copyTV(J->L, &ix->tabv, &ix->keyv);
1053 if (lj_record_mm_lookup(J, ix, mm)) /* Lookup mm on 2nd operand. */
1054 goto ok;
1055 }
1056 lj_trace_err(J, LJ_TRERR_NOMM);
1057 }
1058ok:
1059 base[0] = ix->mobj;
1060#if LJ_FR2
1061 base[1] = 0;
1062#endif
1063 copyTV(J->L, basev+0, &ix->mobjv);
1064 lj_record_call(J, func, 2);
1065 return 0; /* No result yet. */
1066}
1067
1068/* Record call to __len metamethod. */
1069static TRef rec_mm_len(jit_State *J, TRef tr, TValue *tv)
1070{
1071 RecordIndex ix;
1072 ix.tab = tr;
1073 copyTV(J->L, &ix.tabv, tv);
1074 if (lj_record_mm_lookup(J, &ix, MM_len)) {
1075 BCReg func = rec_mm_prep(J, lj_cont_ra);
1076 TRef *base = J->base + func;
1077 TValue *basev = J->L->base + func;
1078 base[0] = ix.mobj; copyTV(J->L, basev+0, &ix.mobjv);
1079 base += LJ_FR2;
1080 basev += LJ_FR2;
1081 base[1] = tr; copyTV(J->L, basev+1, tv);
1082#if LJ_52
1083 base[2] = tr; copyTV(J->L, basev+2, tv);
1084#else
1085 base[2] = TREF_NIL; setnilV(basev+2);
1086#endif
1087 lj_record_call(J, func, 2);
1088 } else {
1089 if (LJ_52 && tref_istab(tr))
1090 return emitir(IRTI(IR_ALEN), tr, TREF_NIL);
1091 lj_trace_err(J, LJ_TRERR_NOMM);
1092 }
1093 return 0; /* No result yet. */
1094}
1095
1096/* Call a comparison metamethod. */
1097static void rec_mm_callcomp(jit_State *J, RecordIndex *ix, int op)
1098{
1099 BCReg func = rec_mm_prep(J, (op&1) ? lj_cont_condf : lj_cont_condt);
1100 TRef *base = J->base + func + LJ_FR2;
1101 TValue *tv = J->L->base + func + LJ_FR2;
1102 base[-LJ_FR2] = ix->mobj; base[1] = ix->val; base[2] = ix->key;
1103 copyTV(J->L, tv-LJ_FR2, &ix->mobjv);
1104 copyTV(J->L, tv+1, &ix->valv);
1105 copyTV(J->L, tv+2, &ix->keyv);
1106 lj_record_call(J, func, 2);
1107}
1108
1109/* Record call to equality comparison metamethod (for tab and udata only). */
1110static void rec_mm_equal(jit_State *J, RecordIndex *ix, int op)
1111{
1112 ix->tab = ix->val;
1113 copyTV(J->L, &ix->tabv, &ix->valv);
1114 if (lj_record_mm_lookup(J, ix, MM_eq)) { /* Lookup mm on 1st operand. */
1115 cTValue *bv;
1116 TRef mo1 = ix->mobj;
1117 TValue mo1v;
1118 copyTV(J->L, &mo1v, &ix->mobjv);
1119 /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */
1120 bv = &ix->keyv;
1121 if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) {
1122 TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META);
1123 emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
1124 } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) {
1125 TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META);
1126 emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
1127 } else { /* Lookup metamethod on 2nd operand and compare both. */
1128 ix->tab = ix->key;
1129 copyTV(J->L, &ix->tabv, bv);
1130 if (!lj_record_mm_lookup(J, ix, MM_eq) ||
1131 lj_record_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv))
1132 return;
1133 }
1134 rec_mm_callcomp(J, ix, op);
1135 }
1136}
1137
1138/* Record call to ordered comparison metamethods (for arbitrary objects). */
1139static void rec_mm_comp(jit_State *J, RecordIndex *ix, int op)
1140{
1141 ix->tab = ix->val;
1142 copyTV(J->L, &ix->tabv, &ix->valv);
1143 while (1) {
1144 MMS mm = (op & 2) ? MM_le : MM_lt; /* Try __le + __lt or only __lt. */
1145#if LJ_52
1146 if (!lj_record_mm_lookup(J, ix, mm)) { /* Lookup mm on 1st operand. */
1147 ix->tab = ix->key;
1148 copyTV(J->L, &ix->tabv, &ix->keyv);
1149 if (!lj_record_mm_lookup(J, ix, mm)) /* Lookup mm on 2nd operand. */
1150 goto nomatch;
1151 }
1152 rec_mm_callcomp(J, ix, op);
1153 return;
1154#else
1155 if (lj_record_mm_lookup(J, ix, mm)) { /* Lookup mm on 1st operand. */
1156 cTValue *bv;
1157 TRef mo1 = ix->mobj;
1158 TValue mo1v;
1159 copyTV(J->L, &mo1v, &ix->mobjv);
1160 /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */
1161 bv = &ix->keyv;
1162 if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) {
1163 TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META);
1164 emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
1165 } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) {
1166 TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META);
1167 emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt);
1168 } else { /* Lookup metamethod on 2nd operand and compare both. */
1169 ix->tab = ix->key;
1170 copyTV(J->L, &ix->tabv, bv);
1171 if (!lj_record_mm_lookup(J, ix, mm) ||
1172 lj_record_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv))
1173 goto nomatch;
1174 }
1175 rec_mm_callcomp(J, ix, op);
1176 return;
1177 }
1178#endif
1179 nomatch:
1180 /* Lookup failed. Retry with __lt and swapped operands. */
1181 if (!(op & 2)) break; /* Already at __lt. Interpreter will throw. */
1182 ix->tab = ix->key; ix->key = ix->val; ix->val = ix->tab;
1183 copyTV(J->L, &ix->tabv, &ix->keyv);
1184 copyTV(J->L, &ix->keyv, &ix->valv);
1185 copyTV(J->L, &ix->valv, &ix->tabv);
1186 op ^= 3;
1187 }
1188}
1189
1190#if LJ_HASFFI
1191/* Setup call to cdata comparison metamethod. */
1192static void rec_mm_comp_cdata(jit_State *J, RecordIndex *ix, int op, MMS mm)
1193{
1194 lj_snap_add(J);
1195 if (tref_iscdata(ix->val)) {
1196 ix->tab = ix->val;
1197 copyTV(J->L, &ix->tabv, &ix->valv);
1198 } else {
1199 lj_assertJ(tref_iscdata(ix->key), "cdata expected");
1200 ix->tab = ix->key;
1201 copyTV(J->L, &ix->tabv, &ix->keyv);
1202 }
1203 lj_record_mm_lookup(J, ix, mm);
1204 rec_mm_callcomp(J, ix, op);
1205}
1206#endif
1207
1208/* -- Indexed access ------------------------------------------------------ */
1209
1210#ifdef LUAJIT_ENABLE_TABLE_BUMP
1211/* Bump table allocations in bytecode when they grow during recording. */
1212static void rec_idx_bump(jit_State *J, RecordIndex *ix)
1213{
1214 RBCHashEntry *rbc = &J->rbchash[(ix->tab & (RBCHASH_SLOTS-1))];
1215 if (tref_ref(ix->tab) == rbc->ref) {
1216 const BCIns *pc = mref(rbc->pc, const BCIns);
1217 GCtab *tb = tabV(&ix->tabv);
1218 uint32_t nhbits;
1219 IRIns *ir;
1220 if (!tvisnil(&ix->keyv))
1221 (void)lj_tab_set(J->L, tb, &ix->keyv); /* Grow table right now. */
1222 nhbits = tb->hmask > 0 ? lj_fls(tb->hmask)+1 : 0;
1223 ir = IR(tref_ref(ix->tab));
1224 if (ir->o == IR_TNEW) {
1225 uint32_t ah = bc_d(*pc);
1226 uint32_t asize = ah & 0x7ff, hbits = ah >> 11;
1227 if (nhbits > hbits) hbits = nhbits;
1228 if (tb->asize > asize) {
1229 asize = tb->asize <= 0x7ff ? tb->asize : 0x7ff;
1230 }
1231 if ((asize | (hbits<<11)) != ah) { /* Has the size changed? */
1232 /* Patch bytecode, but continue recording (for more patching). */
1233 setbc_d(pc, (asize | (hbits<<11)));
1234 /* Patching TNEW operands is only safe if the trace is aborted. */
1235 ir->op1 = asize; ir->op2 = hbits;
1236 J->retryrec = 1; /* Abort the trace at the end of recording. */
1237 }
1238 } else if (ir->o == IR_TDUP) {
1239 GCtab *tpl = gco2tab(proto_kgc(&gcref(rbc->pt)->pt, ~(ptrdiff_t)bc_d(*pc)));
1240 /* Grow template table, but preserve keys with nil values. */
1241 if ((tb->asize > tpl->asize && (1u << nhbits)-1 == tpl->hmask) ||
1242 (tb->asize == tpl->asize && (1u << nhbits)-1 > tpl->hmask)) {
1243 Node *node = noderef(tpl->node);
1244 uint32_t i, hmask = tpl->hmask, asize;
1245 TValue *array;
1246 for (i = 0; i <= hmask; i++) {
1247 if (!tvisnil(&node[i].key) && tvisnil(&node[i].val))
1248 settabV(J->L, &node[i].val, tpl);
1249 }
1250 if (!tvisnil(&ix->keyv) && tref_isk(ix->key)) {
1251 TValue *o = lj_tab_set(J->L, tpl, &ix->keyv);
1252 if (tvisnil(o)) settabV(J->L, o, tpl);
1253 }
1254 lj_tab_resize(J->L, tpl, tb->asize, nhbits);
1255 node = noderef(tpl->node);
1256 hmask = tpl->hmask;
1257 for (i = 0; i <= hmask; i++) {
1258 /* This is safe, since template tables only hold immutable values. */
1259 if (tvistab(&node[i].val))
1260 setnilV(&node[i].val);
1261 }
1262 /* The shape of the table may have changed. Clean up array part, too. */
1263 asize = tpl->asize;
1264 array = tvref(tpl->array);
1265 for (i = 0; i < asize; i++) {
1266 if (tvistab(&array[i]))
1267 setnilV(&array[i]);
1268 }
1269 J->retryrec = 1; /* Abort the trace at the end of recording. */
1270 }
1271 }
1272 }
1273}
1274#endif
1275
1276/* Record bounds-check. */
1277static void rec_idx_abc(jit_State *J, TRef asizeref, TRef ikey, uint32_t asize)
1278{
1279 /* Try to emit invariant bounds checks. */
1280 if ((J->flags & (JIT_F_OPT_LOOP|JIT_F_OPT_ABC)) ==
1281 (JIT_F_OPT_LOOP|JIT_F_OPT_ABC)) {
1282 IRRef ref = tref_ref(ikey);
1283 IRIns *ir = IR(ref);
1284 int32_t ofs = 0;
1285 IRRef ofsref = 0;
1286 /* Handle constant offsets. */
1287 if (ir->o == IR_ADD && irref_isk(ir->op2)) {
1288 ofsref = ir->op2;
1289 ofs = IR(ofsref)->i;
1290 ref = ir->op1;
1291 ir = IR(ref);
1292 }
1293 /* Got scalar evolution analysis results for this reference? */
1294 if (ref == J->scev.idx) {
1295 int32_t stop;
1296 lj_assertJ(irt_isint(J->scev.t) && ir->o == IR_SLOAD,
1297 "only int SCEV supported");
1298 stop = numberVint(&(J->L->base - J->baseslot)[ir->op1 + FORL_STOP]);
1299 /* Runtime value for stop of loop is within bounds? */
1300 if ((uint64_t)stop + ofs < (uint64_t)asize) {
1301 /* Emit invariant bounds check for stop. */
1302 emitir(IRTG(IR_ABC, IRT_P32), asizeref, ofs == 0 ? J->scev.stop :
1303 emitir(IRTI(IR_ADD), J->scev.stop, ofsref));
1304 /* Emit invariant bounds check for start, if not const or negative. */
1305 if (!(J->scev.dir && J->scev.start &&
1306 (int64_t)IR(J->scev.start)->i + ofs >= 0))
1307 emitir(IRTG(IR_ABC, IRT_P32), asizeref, ikey);
1308 return;
1309 }
1310 }
1311 }
1312 emitir(IRTGI(IR_ABC), asizeref, ikey); /* Emit regular bounds check. */
1313}
1314
1315/* Record indexed key lookup. */
1316static TRef rec_idx_key(jit_State *J, RecordIndex *ix, IRRef *rbref,
1317 IRType1 *rbguard)
1318{
1319 TRef key;
1320 GCtab *t = tabV(&ix->tabv);
1321 ix->oldv = lj_tab_get(J->L, t, &ix->keyv); /* Lookup previous value. */
1322 *rbref = 0;
1323 rbguard->irt = 0;
1324
1325 /* Integer keys are looked up in the array part first. */
1326 key = ix->key;
1327 if (tref_isnumber(key)) {
1328 int32_t k = numberVint(&ix->keyv);
1329 if (!tvisint(&ix->keyv) && numV(&ix->keyv) != (lua_Number)k)
1330 k = LJ_MAX_ASIZE;
1331 if ((MSize)k < LJ_MAX_ASIZE) { /* Potential array key? */
1332 TRef ikey = lj_opt_narrow_index(J, key);
1333 TRef asizeref = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE);
1334 if ((MSize)k < t->asize) { /* Currently an array key? */
1335 TRef arrayref;
1336 rec_idx_abc(J, asizeref, ikey, t->asize);
1337 arrayref = emitir(IRT(IR_FLOAD, IRT_PGC), ix->tab, IRFL_TAB_ARRAY);
1338 return emitir(IRT(IR_AREF, IRT_PGC), arrayref, ikey);
1339 } else { /* Currently not in array (may be an array extension)? */
1340 emitir(IRTGI(IR_ULE), asizeref, ikey); /* Inv. bounds check. */
1341 if (k == 0 && tref_isk(key))
1342 key = lj_ir_knum_zero(J); /* Canonicalize 0 or +-0.0 to +0.0. */
1343 /* And continue with the hash lookup. */
1344 }
1345 } else if (!tref_isk(key)) {
1346 /* We can rule out const numbers which failed the integerness test
1347 ** above. But all other numbers are potential array keys.
1348 */
1349 if (t->asize == 0) { /* True sparse tables have an empty array part. */
1350 /* Guard that the array part stays empty. */
1351 TRef tmp = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE);
1352 emitir(IRTGI(IR_EQ), tmp, lj_ir_kint(J, 0));
1353 } else {
1354 lj_trace_err(J, LJ_TRERR_NYITMIX);
1355 }
1356 }
1357 }
1358
1359 /* Otherwise the key is located in the hash part. */
1360 if (t->hmask == 0) { /* Shortcut for empty hash part. */
1361 /* Guard that the hash part stays empty. */
1362 TRef tmp = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK);
1363 emitir(IRTGI(IR_EQ), tmp, lj_ir_kint(J, 0));
1364 return lj_ir_kkptr(J, niltvg(J2G(J)));
1365 }
1366 if (tref_isinteger(key)) /* Hash keys are based on numbers, not ints. */
1367 key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT);
1368 if (tref_isk(key)) {
1369 /* Optimize lookup of constant hash keys. */
1370 MSize hslot = (MSize)((char *)ix->oldv - (char *)&noderef(t->node)[0].val);
1371 if (t->hmask > 0 && hslot <= t->hmask*(MSize)sizeof(Node) &&
1372 hslot <= 65535*(MSize)sizeof(Node)) {
1373 TRef node, kslot, hm;
1374 *rbref = J->cur.nins; /* Mark possible rollback point. */
1375 *rbguard = J->guardemit;
1376 hm = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK);
1377 emitir(IRTGI(IR_EQ), hm, lj_ir_kint(J, (int32_t)t->hmask));
1378 node = emitir(IRT(IR_FLOAD, IRT_PGC), ix->tab, IRFL_TAB_NODE);
1379 kslot = lj_ir_kslot(J, key, hslot / sizeof(Node));
1380 return emitir(IRTG(IR_HREFK, IRT_PGC), node, kslot);
1381 }
1382 }
1383 /* Fall back to a regular hash lookup. */
1384 return emitir(IRT(IR_HREF, IRT_PGC), ix->tab, key);
1385}
1386
1387/* Determine whether a key is NOT one of the fast metamethod names. */
1388static int nommstr(jit_State *J, TRef key)
1389{
1390 if (tref_isstr(key)) {
1391 if (tref_isk(key)) {
1392 GCstr *str = ir_kstr(IR(tref_ref(key)));
1393 uint32_t mm;
1394 for (mm = 0; mm <= MM_FAST; mm++)
1395 if (mmname_str(J2G(J), mm) == str)
1396 return 0; /* MUST be one the fast metamethod names. */
1397 } else {
1398 return 0; /* Variable string key MAY be a metamethod name. */
1399 }
1400 }
1401 return 1; /* CANNOT be a metamethod name. */
1402}
1403
1404/* Record indexed load/store. */
1405TRef lj_record_idx(jit_State *J, RecordIndex *ix)
1406{
1407 TRef xref;
1408 IROp xrefop, loadop;
1409 IRRef rbref;
1410 IRType1 rbguard;
1411 cTValue *oldv;
1412
1413 while (!tref_istab(ix->tab)) { /* Handle non-table lookup. */
1414 /* Never call raw lj_record_idx() on non-table. */
1415 lj_assertJ(ix->idxchain != 0, "bad usage");
1416 if (!lj_record_mm_lookup(J, ix, ix->val ? MM_newindex : MM_index))
1417 lj_trace_err(J, LJ_TRERR_NOMM);
1418 handlemm:
1419 if (tref_isfunc(ix->mobj)) { /* Handle metamethod call. */
1420 BCReg func = rec_mm_prep(J, ix->val ? lj_cont_nop : lj_cont_ra);
1421 TRef *base = J->base + func + LJ_FR2;
1422 TValue *tv = J->L->base + func + LJ_FR2;
1423 base[-LJ_FR2] = ix->mobj; base[1] = ix->tab; base[2] = ix->key;
1424 setfuncV(J->L, tv-LJ_FR2, funcV(&ix->mobjv));
1425 copyTV(J->L, tv+1, &ix->tabv);
1426 copyTV(J->L, tv+2, &ix->keyv);
1427 if (ix->val) {
1428 base[3] = ix->val;
1429 copyTV(J->L, tv+3, &ix->valv);
1430 lj_record_call(J, func, 3); /* mobj(tab, key, val) */
1431 return 0;
1432 } else {
1433 lj_record_call(J, func, 2); /* res = mobj(tab, key) */
1434 return 0; /* No result yet. */
1435 }
1436 }
1437 /* Otherwise retry lookup with metaobject. */
1438 ix->tab = ix->mobj;
1439 copyTV(J->L, &ix->tabv, &ix->mobjv);
1440 if (--ix->idxchain == 0)
1441 lj_trace_err(J, LJ_TRERR_IDXLOOP);
1442 }
1443
1444 /* First catch nil and NaN keys for tables. */
1445 if (tvisnil(&ix->keyv) || (tvisnum(&ix->keyv) && tvisnan(&ix->keyv))) {
1446 if (ix->val) /* Better fail early. */
1447 lj_trace_err(J, LJ_TRERR_STORENN);
1448 if (tref_isk(ix->key)) {
1449 if (ix->idxchain && lj_record_mm_lookup(J, ix, MM_index))
1450 goto handlemm;
1451 return TREF_NIL;
1452 }
1453 }
1454
1455 /* Record the key lookup. */
1456 xref = rec_idx_key(J, ix, &rbref, &rbguard);
1457 xrefop = IR(tref_ref(xref))->o;
1458 loadop = xrefop == IR_AREF ? IR_ALOAD : IR_HLOAD;
1459 /* The lj_meta_tset() inconsistency is gone, but better play safe. */
1460 oldv = xrefop == IR_KKPTR ? (cTValue *)ir_kptr(IR(tref_ref(xref))) : ix->oldv;
1461
1462 if (ix->val == 0) { /* Indexed load */
1463 IRType t = itype2irt(oldv);
1464 TRef res;
1465 if (oldv == niltvg(J2G(J))) {
1466 emitir(IRTG(IR_EQ, IRT_PGC), xref, lj_ir_kkptr(J, niltvg(J2G(J))));
1467 res = TREF_NIL;
1468 } else {
1469 res = emitir(IRTG(loadop, t), xref, 0);
1470 }
1471 if (tref_ref(res) < rbref) { /* HREFK + load forwarded? */
1472 lj_ir_rollback(J, rbref); /* Rollback to eliminate hmask guard. */
1473 J->guardemit = rbguard;
1474 }
1475 if (t == IRT_NIL && ix->idxchain && lj_record_mm_lookup(J, ix, MM_index))
1476 goto handlemm;
1477 if (irtype_ispri(t)) res = TREF_PRI(t); /* Canonicalize primitives. */
1478 return res;
1479 } else { /* Indexed store. */
1480 GCtab *mt = tabref(tabV(&ix->tabv)->metatable);
1481 int keybarrier = tref_isgcv(ix->key) && !tref_isnil(ix->val);
1482 if (tref_ref(xref) < rbref) { /* HREFK forwarded? */
1483 lj_ir_rollback(J, rbref); /* Rollback to eliminate hmask guard. */
1484 J->guardemit = rbguard;
1485 }
1486 if (tvisnil(oldv)) { /* Previous value was nil? */
1487 /* Need to duplicate the hasmm check for the early guards. */
1488 int hasmm = 0;
1489 if (ix->idxchain && mt) {
1490 cTValue *mo = lj_tab_getstr(mt, mmname_str(J2G(J), MM_newindex));
1491 hasmm = mo && !tvisnil(mo);
1492 }
1493 if (hasmm)
1494 emitir(IRTG(loadop, IRT_NIL), xref, 0); /* Guard for nil value. */
1495 else if (xrefop == IR_HREF)
1496 emitir(IRTG(oldv == niltvg(J2G(J)) ? IR_EQ : IR_NE, IRT_PGC),
1497 xref, lj_ir_kkptr(J, niltvg(J2G(J))));
1498 if (ix->idxchain && lj_record_mm_lookup(J, ix, MM_newindex)) {
1499 lj_assertJ(hasmm, "inconsistent metamethod handling");
1500 goto handlemm;
1501 }
1502 lj_assertJ(!hasmm, "inconsistent metamethod handling");
1503 if (oldv == niltvg(J2G(J))) { /* Need to insert a new key. */
1504 TRef key = ix->key;
1505 if (tref_isinteger(key)) /* NEWREF needs a TValue as a key. */
1506 key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT);
1507 xref = emitir(IRT(IR_NEWREF, IRT_PGC), ix->tab, key);
1508 keybarrier = 0; /* NEWREF already takes care of the key barrier. */
1509#ifdef LUAJIT_ENABLE_TABLE_BUMP
1510 if ((J->flags & JIT_F_OPT_SINK)) /* Avoid a separate flag. */
1511 rec_idx_bump(J, ix);
1512#endif
1513 }
1514 } else if (!lj_opt_fwd_wasnonnil(J, loadop, tref_ref(xref))) {
1515 /* Cannot derive that the previous value was non-nil, must do checks. */
1516 if (xrefop == IR_HREF) /* Guard against store to niltv. */
1517 emitir(IRTG(IR_NE, IRT_PGC), xref, lj_ir_kkptr(J, niltvg(J2G(J))));
1518 if (ix->idxchain) { /* Metamethod lookup required? */
1519 /* A check for NULL metatable is cheaper (hoistable) than a load. */
1520 if (!mt) {
1521 TRef mtref = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META);
1522 emitir(IRTG(IR_EQ, IRT_TAB), mtref, lj_ir_knull(J, IRT_TAB));
1523 } else {
1524 IRType t = itype2irt(oldv);
1525 emitir(IRTG(loadop, t), xref, 0); /* Guard for non-nil value. */
1526 }
1527 }
1528 } else {
1529 keybarrier = 0; /* Previous non-nil value kept the key alive. */
1530 }
1531 /* Convert int to number before storing. */
1532 if (!LJ_DUALNUM && tref_isinteger(ix->val))
1533 ix->val = emitir(IRTN(IR_CONV), ix->val, IRCONV_NUM_INT);
1534 emitir(IRT(loadop+IRDELTA_L2S, tref_type(ix->val)), xref, ix->val);
1535 if (keybarrier || tref_isgcv(ix->val))
1536 emitir(IRT(IR_TBAR, IRT_NIL), ix->tab, 0);
1537 /* Invalidate neg. metamethod cache for stores with certain string keys. */
1538 if (!nommstr(J, ix->key)) {
1539 TRef fref = emitir(IRT(IR_FREF, IRT_PGC), ix->tab, IRFL_TAB_NOMM);
1540 emitir(IRT(IR_FSTORE, IRT_U8), fref, lj_ir_kint(J, 0));
1541 }
1542 J->needsnap = 1;
1543 return 0;
1544 }
1545}
1546
1547static void rec_tsetm(jit_State *J, BCReg ra, BCReg rn, int32_t i)
1548{
1549 RecordIndex ix;
1550 cTValue *basev = J->L->base;
1551 GCtab *t = tabV(&basev[ra-1]);
1552 settabV(J->L, &ix.tabv, t);
1553 ix.tab = getslot(J, ra-1);
1554 ix.idxchain = 0;
1555#ifdef LUAJIT_ENABLE_TABLE_BUMP
1556 if ((J->flags & JIT_F_OPT_SINK)) {
1557 if (t->asize < i+rn-ra)
1558 lj_tab_reasize(J->L, t, i+rn-ra);
1559 setnilV(&ix.keyv);
1560 rec_idx_bump(J, &ix);
1561 }
1562#endif
1563 for (; ra < rn; i++, ra++) {
1564 setintV(&ix.keyv, i);
1565 ix.key = lj_ir_kint(J, i);
1566 copyTV(J->L, &ix.valv, &basev[ra]);
1567 ix.val = getslot(J, ra);
1568 lj_record_idx(J, &ix);
1569 }
1570}
1571
1572/* -- Upvalue access ------------------------------------------------------ */
1573
1574/* Check whether upvalue is immutable and ok to constify. */
1575static int rec_upvalue_constify(jit_State *J, GCupval *uvp)
1576{
1577 if (uvp->immutable) {
1578 cTValue *o = uvval(uvp);
1579 /* Don't constify objects that may retain large amounts of memory. */
1580#if LJ_HASFFI
1581 if (tviscdata(o)) {
1582 GCcdata *cd = cdataV(o);
1583 if (!cdataisv(cd) && !(cd->marked & LJ_GC_CDATA_FIN)) {
1584 CType *ct = ctype_raw(ctype_ctsG(J2G(J)), cd->ctypeid);
1585 if (!ctype_hassize(ct->info) || ct->size <= 16)
1586 return 1;
1587 }
1588 return 0;
1589 }
1590#else
1591 UNUSED(J);
1592#endif
1593 if (!(tvistab(o) || tvisudata(o) || tvisthread(o)))
1594 return 1;
1595 }
1596 return 0;
1597}
1598
1599/* Record upvalue load/store. */
1600static TRef rec_upvalue(jit_State *J, uint32_t uv, TRef val)
1601{
1602 GCupval *uvp = &gcref(J->fn->l.uvptr[uv])->uv;
1603 TRef fn = getcurrf(J);
1604 IRRef uref;
1605 int needbarrier = 0;
1606 if (rec_upvalue_constify(J, uvp)) { /* Try to constify immutable upvalue. */
1607 TRef tr, kfunc;
1608 lj_assertJ(val == 0, "bad usage");
1609 if (!tref_isk(fn)) { /* Late specialization of current function. */
1610 if (J->pt->flags >= PROTO_CLC_POLY)
1611 goto noconstify;
1612 kfunc = lj_ir_kfunc(J, J->fn);
1613 emitir(IRTG(IR_EQ, IRT_FUNC), fn, kfunc);
1614#if LJ_FR2
1615 J->base[-2] = kfunc;
1616#else
1617 J->base[-1] = kfunc | TREF_FRAME;
1618#endif
1619 fn = kfunc;
1620 }
1621 tr = lj_record_constify(J, uvval(uvp));
1622 if (tr)
1623 return tr;
1624 }
1625noconstify:
1626 /* Note: this effectively limits LJ_MAX_UPVAL to 127. */
1627 uv = (uv << 8) | (hashrot(uvp->dhash, uvp->dhash + HASH_BIAS) & 0xff);
1628 if (!uvp->closed) {
1629 uref = tref_ref(emitir(IRTG(IR_UREFO, IRT_PGC), fn, uv));
1630 /* In current stack? */
1631 if (uvval(uvp) >= tvref(J->L->stack) &&
1632 uvval(uvp) < tvref(J->L->maxstack)) {
1633 int32_t slot = (int32_t)(uvval(uvp) - (J->L->base - J->baseslot));
1634 if (slot >= 0) { /* Aliases an SSA slot? */
1635 emitir(IRTG(IR_EQ, IRT_PGC),
1636 REF_BASE,
1637 emitir(IRT(IR_ADD, IRT_PGC), uref,
1638 lj_ir_kint(J, (slot - 1 - LJ_FR2) * -8)));
1639 slot -= (int32_t)J->baseslot; /* Note: slot number may be negative! */
1640 if (val == 0) {
1641 return getslot(J, slot);
1642 } else {
1643 J->base[slot] = val;
1644 if (slot >= (int32_t)J->maxslot) J->maxslot = (BCReg)(slot+1);
1645 return 0;
1646 }
1647 }
1648 }
1649 emitir(IRTG(IR_UGT, IRT_PGC),
1650 emitir(IRT(IR_SUB, IRT_PGC), uref, REF_BASE),
1651 lj_ir_kint(J, (J->baseslot + J->maxslot) * 8));
1652 } else {
1653 needbarrier = 1;
1654 uref = tref_ref(emitir(IRTG(IR_UREFC, IRT_PGC), fn, uv));
1655 }
1656 if (val == 0) { /* Upvalue load */
1657 IRType t = itype2irt(uvval(uvp));
1658 TRef res = emitir(IRTG(IR_ULOAD, t), uref, 0);
1659 if (irtype_ispri(t)) res = TREF_PRI(t); /* Canonicalize primitive refs. */
1660 return res;
1661 } else { /* Upvalue store. */
1662 /* Convert int to number before storing. */
1663 if (!LJ_DUALNUM && tref_isinteger(val))
1664 val = emitir(IRTN(IR_CONV), val, IRCONV_NUM_INT);
1665 emitir(IRT(IR_USTORE, tref_type(val)), uref, val);
1666 if (needbarrier && tref_isgcv(val))
1667 emitir(IRT(IR_OBAR, IRT_NIL), uref, val);
1668 J->needsnap = 1;
1669 return 0;
1670 }
1671}
1672
1673/* -- Record calls to Lua functions --------------------------------------- */
1674
1675/* Check unroll limits for calls. */
1676static void check_call_unroll(jit_State *J, TraceNo lnk)
1677{
1678 cTValue *frame = J->L->base - 1;
1679 void *pc = mref(frame_func(frame)->l.pc, void);
1680 int32_t depth = J->framedepth;
1681 int32_t count = 0;
1682 if ((J->pt->flags & PROTO_VARARG)) depth--; /* Vararg frame still missing. */
1683 for (; depth > 0; depth--) { /* Count frames with same prototype. */
1684 if (frame_iscont(frame)) depth--;
1685 frame = frame_prev(frame);
1686 if (mref(frame_func(frame)->l.pc, void) == pc)
1687 count++;
1688 }
1689 if (J->pc == J->startpc) {
1690 if (count + J->tailcalled > J->param[JIT_P_recunroll]) {
1691 J->pc++;
1692 if (J->framedepth + J->retdepth == 0)
1693 lj_record_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno); /* Tail-rec. */
1694 else
1695 lj_record_stop(J, LJ_TRLINK_UPREC, J->cur.traceno); /* Up-recursion. */
1696 }
1697 } else {
1698 if (count > J->param[JIT_P_callunroll]) {
1699 if (lnk) { /* Possible tail- or up-recursion. */
1700 lj_trace_flush(J, lnk); /* Flush trace that only returns. */
1701 /* Set a small, pseudo-random hotcount for a quick retry of JFUNC*. */
1702 hotcount_set(J2GG(J), J->pc+1, lj_prng_u64(&J2G(J)->prng) & 15u);
1703 }
1704 lj_trace_err(J, LJ_TRERR_CUNROLL);
1705 }
1706 }
1707}
1708
1709/* Record Lua function setup. */
1710static void rec_func_setup(jit_State *J)
1711{
1712 GCproto *pt = J->pt;
1713 BCReg s, numparams = pt->numparams;
1714 if ((pt->flags & PROTO_NOJIT))
1715 lj_trace_err(J, LJ_TRERR_CJITOFF);
1716 if (J->baseslot + pt->framesize >= LJ_MAX_JSLOTS)
1717 lj_trace_err(J, LJ_TRERR_STACKOV);
1718 /* Fill up missing parameters with nil. */
1719 for (s = J->maxslot; s < numparams; s++)
1720 J->base[s] = TREF_NIL;
1721 /* The remaining slots should never be read before they are written. */
1722 J->maxslot = numparams;
1723}
1724
1725/* Record Lua vararg function setup. */
1726static void rec_func_vararg(jit_State *J)
1727{
1728 GCproto *pt = J->pt;
1729 BCReg s, fixargs, vframe = J->maxslot+1+LJ_FR2;
1730 lj_assertJ((pt->flags & PROTO_VARARG), "FUNCV in non-vararg function");
1731 if (J->baseslot + vframe + pt->framesize >= LJ_MAX_JSLOTS)
1732 lj_trace_err(J, LJ_TRERR_STACKOV);
1733 J->base[vframe-1-LJ_FR2] = J->base[-1-LJ_FR2]; /* Copy function up. */
1734#if LJ_FR2
1735 J->base[vframe-1] = TREF_FRAME;
1736#endif
1737 /* Copy fixarg slots up and set their original slots to nil. */
1738 fixargs = pt->numparams < J->maxslot ? pt->numparams : J->maxslot;
1739 for (s = 0; s < fixargs; s++) {
1740 J->base[vframe+s] = J->base[s];
1741 J->base[s] = TREF_NIL;
1742 }
1743 J->maxslot = fixargs;
1744 J->framedepth++;
1745 J->base += vframe;
1746 J->baseslot += vframe;
1747}
1748
1749/* Record entry to a Lua function. */
1750static void rec_func_lua(jit_State *J)
1751{
1752 rec_func_setup(J);
1753 check_call_unroll(J, 0);
1754}
1755
1756/* Record entry to an already compiled function. */
1757static void rec_func_jit(jit_State *J, TraceNo lnk)
1758{
1759 GCtrace *T;
1760 rec_func_setup(J);
1761 T = traceref(J, lnk);
1762 if (T->linktype == LJ_TRLINK_RETURN) { /* Trace returns to interpreter? */
1763 check_call_unroll(J, lnk);
1764 /* Temporarily unpatch JFUNC* to continue recording across function. */
1765 J->patchins = *J->pc;
1766 J->patchpc = (BCIns *)J->pc;
1767 *J->patchpc = T->startins;
1768 return;
1769 }
1770 J->instunroll = 0; /* Cannot continue across a compiled function. */
1771 if (J->pc == J->startpc && J->framedepth + J->retdepth == 0)
1772 lj_record_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno); /* Extra tail-rec. */
1773 else
1774 lj_record_stop(J, LJ_TRLINK_ROOT, lnk); /* Link to the function. */
1775}
1776
1777/* -- Vararg handling ----------------------------------------------------- */
1778
1779/* Detect y = select(x, ...) idiom. */
1780static int select_detect(jit_State *J)
1781{
1782 BCIns ins = J->pc[1];
1783 if (bc_op(ins) == BC_CALLM && bc_b(ins) == 2 && bc_c(ins) == 1) {
1784 cTValue *func = &J->L->base[bc_a(ins)];
1785 if (tvisfunc(func) && funcV(func)->c.ffid == FF_select) {
1786 TRef kfunc = lj_ir_kfunc(J, funcV(func));
1787 emitir(IRTG(IR_EQ, IRT_FUNC), getslot(J, bc_a(ins)), kfunc);
1788 return 1;
1789 }
1790 }
1791 return 0;
1792}
1793
1794/* Record vararg instruction. */
1795static void rec_varg(jit_State *J, BCReg dst, ptrdiff_t nresults)
1796{
1797 int32_t numparams = J->pt->numparams;
1798 ptrdiff_t nvararg = frame_delta(J->L->base-1) - numparams - 1 - LJ_FR2;
1799 lj_assertJ(frame_isvarg(J->L->base-1), "VARG in non-vararg frame");
1800 if (LJ_FR2 && dst > J->maxslot)
1801 J->base[dst-1] = 0; /* Prevent resurrection of unrelated slot. */
1802 if (J->framedepth > 0) { /* Simple case: varargs defined on-trace. */
1803 ptrdiff_t i;
1804 if (nvararg < 0) nvararg = 0;
1805 if (nresults == -1) {
1806 nresults = nvararg;
1807 J->maxslot = dst + (BCReg)nvararg;
1808 } else if (dst + nresults > J->maxslot) {
1809 J->maxslot = dst + (BCReg)nresults;
1810 }
1811 for (i = 0; i < nresults; i++)
1812 J->base[dst+i] = i < nvararg ? getslot(J, i - nvararg - 1 - LJ_FR2) : TREF_NIL;
1813 } else { /* Unknown number of varargs passed to trace. */
1814 TRef fr = emitir(IRTI(IR_SLOAD), LJ_FR2, IRSLOAD_READONLY|IRSLOAD_FRAME);
1815 int32_t frofs = 8*(1+LJ_FR2+numparams)+FRAME_VARG;
1816 if (nresults >= 0) { /* Known fixed number of results. */
1817 ptrdiff_t i;
1818 if (nvararg > 0) {
1819 ptrdiff_t nload = nvararg >= nresults ? nresults : nvararg;
1820 TRef vbase;
1821 if (nvararg >= nresults)
1822 emitir(IRTGI(IR_GE), fr, lj_ir_kint(J, frofs+8*(int32_t)nresults));
1823 else
1824 emitir(IRTGI(IR_EQ), fr,
1825 lj_ir_kint(J, (int32_t)frame_ftsz(J->L->base-1)));
1826 vbase = emitir(IRT(IR_SUB, IRT_IGC), REF_BASE, fr);
1827 vbase = emitir(IRT(IR_ADD, IRT_PGC), vbase, lj_ir_kint(J, frofs-8));
1828 for (i = 0; i < nload; i++) {
1829 IRType t = itype2irt(&J->L->base[i-1-LJ_FR2-nvararg]);
1830 TRef aref = emitir(IRT(IR_AREF, IRT_PGC),
1831 vbase, lj_ir_kint(J, (int32_t)i));
1832 TRef tr = emitir(IRTG(IR_VLOAD, t), aref, 0);
1833 if (irtype_ispri(t)) tr = TREF_PRI(t); /* Canonicalize primitives. */
1834 J->base[dst+i] = tr;
1835 }
1836 } else {
1837 emitir(IRTGI(IR_LE), fr, lj_ir_kint(J, frofs));
1838 nvararg = 0;
1839 }
1840 for (i = nvararg; i < nresults; i++)
1841 J->base[dst+i] = TREF_NIL;
1842 if (dst + (BCReg)nresults > J->maxslot)
1843 J->maxslot = dst + (BCReg)nresults;
1844 } else if (select_detect(J)) { /* y = select(x, ...) */
1845 TRef tridx = J->base[dst-1];
1846 TRef tr = TREF_NIL;
1847 ptrdiff_t idx = lj_ffrecord_select_mode(J, tridx, &J->L->base[dst-1]);
1848 if (idx < 0) goto nyivarg;
1849 if (idx != 0 && !tref_isinteger(tridx))
1850 tridx = emitir(IRTGI(IR_CONV), tridx, IRCONV_INT_NUM|IRCONV_INDEX);
1851 if (idx != 0 && tref_isk(tridx)) {
1852 emitir(IRTGI(idx <= nvararg ? IR_GE : IR_LT),
1853 fr, lj_ir_kint(J, frofs+8*(int32_t)idx));
1854 frofs -= 8; /* Bias for 1-based index. */
1855 } else if (idx <= nvararg) { /* Compute size. */
1856 TRef tmp = emitir(IRTI(IR_ADD), fr, lj_ir_kint(J, -frofs));
1857 if (numparams)
1858 emitir(IRTGI(IR_GE), tmp, lj_ir_kint(J, 0));
1859 tr = emitir(IRTI(IR_BSHR), tmp, lj_ir_kint(J, 3));
1860 if (idx != 0) {
1861 tridx = emitir(IRTI(IR_ADD), tridx, lj_ir_kint(J, -1));
1862 rec_idx_abc(J, tr, tridx, (uint32_t)nvararg);
1863 }
1864 } else {
1865 TRef tmp = lj_ir_kint(J, frofs);
1866 if (idx != 0) {
1867 TRef tmp2 = emitir(IRTI(IR_BSHL), tridx, lj_ir_kint(J, 3));
1868 tmp = emitir(IRTI(IR_ADD), tmp2, tmp);
1869 } else {
1870 tr = lj_ir_kint(J, 0);
1871 }
1872 emitir(IRTGI(IR_LT), fr, tmp);
1873 }
1874 if (idx != 0 && idx <= nvararg) {
1875 IRType t;
1876 TRef aref, vbase = emitir(IRT(IR_SUB, IRT_IGC), REF_BASE, fr);
1877 vbase = emitir(IRT(IR_ADD, IRT_PGC), vbase,
1878 lj_ir_kint(J, frofs-(8<<LJ_FR2)));
1879 t = itype2irt(&J->L->base[idx-2-LJ_FR2-nvararg]);
1880 aref = emitir(IRT(IR_AREF, IRT_PGC), vbase, tridx);
1881 tr = emitir(IRTG(IR_VLOAD, t), aref, 0);
1882 if (irtype_ispri(t)) tr = TREF_PRI(t); /* Canonicalize primitives. */
1883 }
1884 J->base[dst-2-LJ_FR2] = tr;
1885 J->maxslot = dst-1-LJ_FR2;
1886 J->bcskip = 2; /* Skip CALLM + select. */
1887 } else {
1888 nyivarg:
1889 setintV(&J->errinfo, BC_VARG);
1890 lj_trace_err_info(J, LJ_TRERR_NYIBC);
1891 }
1892 }
1893 if (J->baseslot + J->maxslot >= LJ_MAX_JSLOTS)
1894 lj_trace_err(J, LJ_TRERR_STACKOV);
1895}
1896
1897/* -- Record allocations -------------------------------------------------- */
1898
1899static TRef rec_tnew(jit_State *J, uint32_t ah)
1900{
1901 uint32_t asize = ah & 0x7ff;
1902 uint32_t hbits = ah >> 11;
1903 TRef tr;
1904 if (asize == 0x7ff) asize = 0x801;
1905 tr = emitir(IRTG(IR_TNEW, IRT_TAB), asize, hbits);
1906#ifdef LUAJIT_ENABLE_TABLE_BUMP
1907 J->rbchash[(tr & (RBCHASH_SLOTS-1))].ref = tref_ref(tr);
1908 setmref(J->rbchash[(tr & (RBCHASH_SLOTS-1))].pc, J->pc);
1909 setgcref(J->rbchash[(tr & (RBCHASH_SLOTS-1))].pt, obj2gco(J->pt));
1910#endif
1911 return tr;
1912}
1913
1914/* -- Concatenation ------------------------------------------------------- */
1915
1916static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot)
1917{
1918 TRef *top = &J->base[topslot];
1919 TValue savetv[5];
1920 BCReg s;
1921 RecordIndex ix;
1922 lj_assertJ(baseslot < topslot, "bad CAT arg");
1923 for (s = baseslot; s <= topslot; s++)
1924 (void)getslot(J, s); /* Ensure all arguments have a reference. */
1925 if (tref_isnumber_str(top[0]) && tref_isnumber_str(top[-1])) {
1926 TRef tr, hdr, *trp, *xbase, *base = &J->base[baseslot];
1927 /* First convert numbers to strings. */
1928 for (trp = top; trp >= base; trp--) {
1929 if (tref_isnumber(*trp))
1930 *trp = emitir(IRT(IR_TOSTR, IRT_STR), *trp,
1931 tref_isnum(*trp) ? IRTOSTR_NUM : IRTOSTR_INT);
1932 else if (!tref_isstr(*trp))
1933 break;
1934 }
1935 xbase = ++trp;
1936 tr = hdr = emitir(IRT(IR_BUFHDR, IRT_PGC),
1937 lj_ir_kptr(J, &J2G(J)->tmpbuf), IRBUFHDR_RESET);
1938 do {
1939 tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr, *trp++);
1940 } while (trp <= top);
1941 tr = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr);
1942 J->maxslot = (BCReg)(xbase - J->base);
1943 if (xbase == base) return tr; /* Return simple concatenation result. */
1944 /* Pass partial result. */
1945 topslot = J->maxslot--;
1946 *xbase = tr;
1947 top = xbase;
1948 setstrV(J->L, &ix.keyv, &J2G(J)->strempty); /* Simulate string result. */
1949 } else {
1950 J->maxslot = topslot-1;
1951 copyTV(J->L, &ix.keyv, &J->L->base[topslot]);
1952 }
1953 copyTV(J->L, &ix.tabv, &J->L->base[topslot-1]);
1954 ix.tab = top[-1];
1955 ix.key = top[0];
1956 memcpy(savetv, &J->L->base[topslot-1], sizeof(savetv)); /* Save slots. */
1957 rec_mm_arith(J, &ix, MM_concat); /* Call __concat metamethod. */
1958 memcpy(&J->L->base[topslot-1], savetv, sizeof(savetv)); /* Restore slots. */
1959 return 0; /* No result yet. */
1960}
1961
1962/* -- Record bytecode ops ------------------------------------------------- */
1963
1964/* Prepare for comparison. */
1965static void rec_comp_prep(jit_State *J)
1966{
1967 /* Prevent merging with snapshot #0 (GC exit) since we fixup the PC. */
1968 if (J->cur.nsnap == 1 && J->cur.snap[0].ref == J->cur.nins)
1969 emitir_raw(IRT(IR_NOP, IRT_NIL), 0, 0);
1970 lj_snap_add(J);
1971}
1972
1973/* Fixup comparison. */
1974static void rec_comp_fixup(jit_State *J, const BCIns *pc, int cond)
1975{
1976 BCIns jmpins = pc[1];
1977 const BCIns *npc = pc + 2 + (cond ? bc_j(jmpins) : 0);
1978 SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
1979 /* Set PC to opposite target to avoid re-recording the comp. in side trace. */
1980#if LJ_FR2
1981 SnapEntry *flink = &J->cur.snapmap[snap->mapofs + snap->nent];
1982 uint64_t pcbase;
1983 memcpy(&pcbase, flink, sizeof(uint64_t));
1984 pcbase = (pcbase & 0xff) | (u64ptr(npc) << 8);
1985 memcpy(flink, &pcbase, sizeof(uint64_t));
1986#else
1987 J->cur.snapmap[snap->mapofs + snap->nent] = SNAP_MKPC(npc);
1988#endif
1989 J->needsnap = 1;
1990 if (bc_a(jmpins) < J->maxslot) J->maxslot = bc_a(jmpins);
1991 lj_snap_shrink(J); /* Shrink last snapshot if possible. */
1992}
1993
1994/* Record the next bytecode instruction (_before_ it's executed). */
1995void lj_record_ins(jit_State *J)
1996{
1997 cTValue *lbase;
1998 RecordIndex ix;
1999 const BCIns *pc;
2000 BCIns ins;
2001 BCOp op;
2002 TRef ra, rb, rc;
2003
2004 /* Perform post-processing action before recording the next instruction. */
2005 if (LJ_UNLIKELY(J->postproc != LJ_POST_NONE)) {
2006 switch (J->postproc) {
2007 case LJ_POST_FIXCOMP: /* Fixup comparison. */
2008 pc = (const BCIns *)(uintptr_t)J2G(J)->tmptv.u64;
2009 rec_comp_fixup(J, pc, (!tvistruecond(&J2G(J)->tmptv2) ^ (bc_op(*pc)&1)));
2010 /* fallthrough */
2011 case LJ_POST_FIXGUARD: /* Fixup and emit pending guard. */
2012 case LJ_POST_FIXGUARDSNAP: /* Fixup and emit pending guard and snapshot. */
2013 if (!tvistruecond(&J2G(J)->tmptv2)) {
2014 J->fold.ins.o ^= 1; /* Flip guard to opposite. */
2015 if (J->postproc == LJ_POST_FIXGUARDSNAP) {
2016 SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
2017 J->cur.snapmap[snap->mapofs+snap->nent-1]--; /* False -> true. */
2018 }
2019 }
2020 lj_opt_fold(J); /* Emit pending guard. */
2021 /* fallthrough */
2022 case LJ_POST_FIXBOOL:
2023 if (!tvistruecond(&J2G(J)->tmptv2)) {
2024 BCReg s;
2025 TValue *tv = J->L->base;
2026 for (s = 0; s < J->maxslot; s++) /* Fixup stack slot (if any). */
2027 if (J->base[s] == TREF_TRUE && tvisfalse(&tv[s])) {
2028 J->base[s] = TREF_FALSE;
2029 break;
2030 }
2031 }
2032 break;
2033 case LJ_POST_FIXCONST:
2034 {
2035 BCReg s;
2036 TValue *tv = J->L->base;
2037 for (s = 0; s < J->maxslot; s++) /* Constify stack slots (if any). */
2038 if (J->base[s] == TREF_NIL && !tvisnil(&tv[s]))
2039 J->base[s] = lj_record_constify(J, &tv[s]);
2040 }
2041 break;
2042 case LJ_POST_FFRETRY: /* Suppress recording of retried fast function. */
2043 if (bc_op(*J->pc) >= BC__MAX)
2044 return;
2045 break;
2046 default: lj_assertJ(0, "bad post-processing mode"); break;
2047 }
2048 J->postproc = LJ_POST_NONE;
2049 }
2050
2051 /* Need snapshot before recording next bytecode (e.g. after a store). */
2052 if (J->needsnap) {
2053 J->needsnap = 0;
2054 if (J->pt) lj_snap_purge(J);
2055 lj_snap_add(J);
2056 J->mergesnap = 1;
2057 }
2058
2059 /* Skip some bytecodes. */
2060 if (LJ_UNLIKELY(J->bcskip > 0)) {
2061 J->bcskip--;
2062 return;
2063 }
2064
2065 /* Record only closed loops for root traces. */
2066 pc = J->pc;
2067 if (J->framedepth == 0 &&
2068 (MSize)((char *)pc - (char *)J->bc_min) >= J->bc_extent)
2069 lj_trace_err(J, LJ_TRERR_LLEAVE);
2070
2071#ifdef LUA_USE_ASSERT
2072 rec_check_slots(J);
2073 rec_check_ir(J);
2074#endif
2075
2076#if LJ_HASPROFILE
2077 rec_profile_ins(J, pc);
2078#endif
2079
2080 /* Keep a copy of the runtime values of var/num/str operands. */
2081#define rav (&ix.valv)
2082#define rbv (&ix.tabv)
2083#define rcv (&ix.keyv)
2084
2085 lbase = J->L->base;
2086 ins = *pc;
2087 op = bc_op(ins);
2088 ra = bc_a(ins);
2089 ix.val = 0;
2090 switch (bcmode_a(op)) {
2091 case BCMvar:
2092 copyTV(J->L, rav, &lbase[ra]); ix.val = ra = getslot(J, ra); break;
2093 default: break; /* Handled later. */
2094 }
2095 rb = bc_b(ins);
2096 rc = bc_c(ins);
2097 switch (bcmode_b(op)) {
2098 case BCMnone: rb = 0; rc = bc_d(ins); break; /* Upgrade rc to 'rd'. */
2099 case BCMvar:
2100 copyTV(J->L, rbv, &lbase[rb]); ix.tab = rb = getslot(J, rb); break;
2101 default: break; /* Handled later. */
2102 }
2103 switch (bcmode_c(op)) {
2104 case BCMvar:
2105 copyTV(J->L, rcv, &lbase[rc]); ix.key = rc = getslot(J, rc); break;
2106 case BCMpri: setpriV(rcv, ~rc); ix.key = rc = TREF_PRI(IRT_NIL+rc); break;
2107 case BCMnum: { cTValue *tv = proto_knumtv(J->pt, rc);
2108 copyTV(J->L, rcv, tv); ix.key = rc = tvisint(tv) ? lj_ir_kint(J, intV(tv)) :
2109 lj_ir_knumint(J, numV(tv)); } break;
2110 case BCMstr: { GCstr *s = gco2str(proto_kgc(J->pt, ~(ptrdiff_t)rc));
2111 setstrV(J->L, rcv, s); ix.key = rc = lj_ir_kstr(J, s); } break;
2112 default: break; /* Handled later. */
2113 }
2114
2115 switch (op) {
2116
2117 /* -- Comparison ops ---------------------------------------------------- */
2118
2119 case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
2120#if LJ_HASFFI
2121 if (tref_iscdata(ra) || tref_iscdata(rc)) {
2122 rec_mm_comp_cdata(J, &ix, op, ((int)op & 2) ? MM_le : MM_lt);
2123 break;
2124 }
2125#endif
2126 /* Emit nothing for two numeric or string consts. */
2127 if (!(tref_isk2(ra,rc) && tref_isnumber_str(ra) && tref_isnumber_str(rc))) {
2128 IRType ta = tref_isinteger(ra) ? IRT_INT : tref_type(ra);
2129 IRType tc = tref_isinteger(rc) ? IRT_INT : tref_type(rc);
2130 int irop;
2131 if (ta != tc) {
2132 /* Widen mixed number/int comparisons to number/number comparison. */
2133 if (ta == IRT_INT && tc == IRT_NUM) {
2134 ra = emitir(IRTN(IR_CONV), ra, IRCONV_NUM_INT);
2135 ta = IRT_NUM;
2136 } else if (ta == IRT_NUM && tc == IRT_INT) {
2137 rc = emitir(IRTN(IR_CONV), rc, IRCONV_NUM_INT);
2138 } else if (LJ_52) {
2139 ta = IRT_NIL; /* Force metamethod for different types. */
2140 } else if (!((ta == IRT_FALSE || ta == IRT_TRUE) &&
2141 (tc == IRT_FALSE || tc == IRT_TRUE))) {
2142 break; /* Interpreter will throw for two different types. */
2143 }
2144 }
2145 rec_comp_prep(J);
2146 irop = (int)op - (int)BC_ISLT + (int)IR_LT;
2147 if (ta == IRT_NUM) {
2148 if ((irop & 1)) irop ^= 4; /* ISGE/ISGT are unordered. */
2149 if (!lj_ir_numcmp(numberVnum(rav), numberVnum(rcv), (IROp)irop))
2150 irop ^= 5;
2151 } else if (ta == IRT_INT) {
2152 if (!lj_ir_numcmp(numberVnum(rav), numberVnum(rcv), (IROp)irop))
2153 irop ^= 1;
2154 } else if (ta == IRT_STR) {
2155 if (!lj_ir_strcmp(strV(rav), strV(rcv), (IROp)irop)) irop ^= 1;
2156 ra = lj_ir_call(J, IRCALL_lj_str_cmp, ra, rc);
2157 rc = lj_ir_kint(J, 0);
2158 ta = IRT_INT;
2159 } else {
2160 rec_mm_comp(J, &ix, (int)op);
2161 break;
2162 }
2163 emitir(IRTG(irop, ta), ra, rc);
2164 rec_comp_fixup(J, J->pc, ((int)op ^ irop) & 1);
2165 }
2166 break;
2167
2168 case BC_ISEQV: case BC_ISNEV:
2169 case BC_ISEQS: case BC_ISNES:
2170 case BC_ISEQN: case BC_ISNEN:
2171 case BC_ISEQP: case BC_ISNEP:
2172#if LJ_HASFFI
2173 if (tref_iscdata(ra) || tref_iscdata(rc)) {
2174 rec_mm_comp_cdata(J, &ix, op, MM_eq);
2175 break;
2176 }
2177#endif
2178 /* Emit nothing for two non-table, non-udata consts. */
2179 if (!(tref_isk2(ra, rc) && !(tref_istab(ra) || tref_isudata(ra)))) {
2180 int diff;
2181 rec_comp_prep(J);
2182 diff = lj_record_objcmp(J, ra, rc, rav, rcv);
2183 if (diff == 2 || !(tref_istab(ra) || tref_isudata(ra)))
2184 rec_comp_fixup(J, J->pc, ((int)op & 1) == !diff);
2185 else if (diff == 1) /* Only check __eq if different, but same type. */
2186 rec_mm_equal(J, &ix, (int)op);
2187 }
2188 break;
2189
2190 /* -- Unary test and copy ops ------------------------------------------- */
2191
2192 case BC_ISTC: case BC_ISFC:
2193 if ((op & 1) == tref_istruecond(rc))
2194 rc = 0; /* Don't store if condition is not true. */
2195 /* fallthrough */
2196 case BC_IST: case BC_ISF: /* Type specialization suffices. */
2197 if (bc_a(pc[1]) < J->maxslot)
2198 J->maxslot = bc_a(pc[1]); /* Shrink used slots. */
2199 break;
2200
2201 case BC_ISTYPE: case BC_ISNUM:
2202 /* These coercions need to correspond with lj_meta_istype(). */
2203 if (LJ_DUALNUM && rc == ~LJ_TNUMX+1)
2204 ra = lj_opt_narrow_toint(J, ra);
2205 else if (rc == ~LJ_TNUMX+2)
2206 ra = lj_ir_tonum(J, ra);
2207 else if (rc == ~LJ_TSTR+1)
2208 ra = lj_ir_tostr(J, ra);
2209 /* else: type specialization suffices. */
2210 J->base[bc_a(ins)] = ra;
2211 break;
2212
2213 /* -- Unary ops --------------------------------------------------------- */
2214
2215 case BC_NOT:
2216 /* Type specialization already forces const result. */
2217 rc = tref_istruecond(rc) ? TREF_FALSE : TREF_TRUE;
2218 break;
2219
2220 case BC_LEN:
2221 if (tref_isstr(rc))
2222 rc = emitir(IRTI(IR_FLOAD), rc, IRFL_STR_LEN);
2223 else if (!LJ_52 && tref_istab(rc))
2224 rc = emitir(IRTI(IR_ALEN), rc, TREF_NIL);
2225 else
2226 rc = rec_mm_len(J, rc, rcv);
2227 break;
2228
2229 /* -- Arithmetic ops ---------------------------------------------------- */
2230
2231 case BC_UNM:
2232 if (tref_isnumber_str(rc)) {
2233 rc = lj_opt_narrow_unm(J, rc, rcv);
2234 } else {
2235 ix.tab = rc;
2236 copyTV(J->L, &ix.tabv, rcv);
2237 rc = rec_mm_arith(J, &ix, MM_unm);
2238 }
2239 break;
2240
2241 case BC_ADDNV: case BC_SUBNV: case BC_MULNV: case BC_DIVNV: case BC_MODNV:
2242 /* Swap rb/rc and rbv/rcv. rav is temp. */
2243 ix.tab = rc; ix.key = rc = rb; rb = ix.tab;
2244 copyTV(J->L, rav, rbv);
2245 copyTV(J->L, rbv, rcv);
2246 copyTV(J->L, rcv, rav);
2247 if (op == BC_MODNV)
2248 goto recmod;
2249 /* fallthrough */
2250 case BC_ADDVN: case BC_SUBVN: case BC_MULVN: case BC_DIVVN:
2251 case BC_ADDVV: case BC_SUBVV: case BC_MULVV: case BC_DIVVV: {
2252 MMS mm = bcmode_mm(op);
2253 if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
2254 rc = lj_opt_narrow_arith(J, rb, rc, rbv, rcv,
2255 (int)mm - (int)MM_add + (int)IR_ADD);
2256 else
2257 rc = rec_mm_arith(J, &ix, mm);
2258 break;
2259 }
2260
2261 case BC_MODVN: case BC_MODVV:
2262 recmod:
2263 if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
2264 rc = lj_opt_narrow_mod(J, rb, rc, rbv, rcv);
2265 else
2266 rc = rec_mm_arith(J, &ix, MM_mod);
2267 break;
2268
2269 case BC_POW:
2270 if (tref_isnumber_str(rb) && tref_isnumber_str(rc))
2271 rc = lj_opt_narrow_pow(J, rb, rc, rbv, rcv);
2272 else
2273 rc = rec_mm_arith(J, &ix, MM_pow);
2274 break;
2275
2276 /* -- Miscellaneous ops ------------------------------------------------- */
2277
2278 case BC_CAT:
2279 rc = rec_cat(J, rb, rc);
2280 break;
2281
2282 /* -- Constant and move ops --------------------------------------------- */
2283
2284 case BC_MOV:
2285 /* Clear gap of method call to avoid resurrecting previous refs. */
2286 if (ra > J->maxslot) {
2287#if LJ_FR2
2288 memset(J->base + J->maxslot, 0, (ra - J->maxslot) * sizeof(TRef));
2289#else
2290 J->base[ra-1] = 0;
2291#endif
2292 }
2293 break;
2294 case BC_KSTR: case BC_KNUM: case BC_KPRI:
2295 break;
2296 case BC_KSHORT:
2297 rc = lj_ir_kint(J, (int32_t)(int16_t)rc);
2298 break;
2299 case BC_KNIL:
2300 if (LJ_FR2 && ra > J->maxslot)
2301 J->base[ra-1] = 0;
2302 while (ra <= rc)
2303 J->base[ra++] = TREF_NIL;
2304 if (rc >= J->maxslot) J->maxslot = rc+1;
2305 break;
2306#if LJ_HASFFI
2307 case BC_KCDATA:
2308 rc = lj_ir_kgc(J, proto_kgc(J->pt, ~(ptrdiff_t)rc), IRT_CDATA);
2309 break;
2310#endif
2311
2312 /* -- Upvalue and function ops ------------------------------------------ */
2313
2314 case BC_UGET:
2315 rc = rec_upvalue(J, rc, 0);
2316 break;
2317 case BC_USETV: case BC_USETS: case BC_USETN: case BC_USETP:
2318 rec_upvalue(J, ra, rc);
2319 break;
2320
2321 /* -- Table ops --------------------------------------------------------- */
2322
2323 case BC_GGET: case BC_GSET:
2324 settabV(J->L, &ix.tabv, tabref(J->fn->l.env));
2325 ix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), getcurrf(J), IRFL_FUNC_ENV);
2326 ix.idxchain = LJ_MAX_IDXCHAIN;
2327 rc = lj_record_idx(J, &ix);
2328 break;
2329
2330 case BC_TGETB: case BC_TSETB:
2331 setintV(&ix.keyv, (int32_t)rc);
2332 ix.key = lj_ir_kint(J, (int32_t)rc);
2333 /* fallthrough */
2334 case BC_TGETV: case BC_TGETS: case BC_TSETV: case BC_TSETS:
2335 ix.idxchain = LJ_MAX_IDXCHAIN;
2336 rc = lj_record_idx(J, &ix);
2337 break;
2338 case BC_TGETR: case BC_TSETR:
2339 ix.idxchain = 0;
2340 rc = lj_record_idx(J, &ix);
2341 break;
2342
2343 case BC_TSETM:
2344 rec_tsetm(J, ra, (BCReg)(J->L->top - J->L->base), (int32_t)rcv->u32.lo);
2345 break;
2346
2347 case BC_TNEW:
2348 rc = rec_tnew(J, rc);
2349 break;
2350 case BC_TDUP:
2351 rc = emitir(IRTG(IR_TDUP, IRT_TAB),
2352 lj_ir_ktab(J, gco2tab(proto_kgc(J->pt, ~(ptrdiff_t)rc))), 0);
2353#ifdef LUAJIT_ENABLE_TABLE_BUMP
2354 J->rbchash[(rc & (RBCHASH_SLOTS-1))].ref = tref_ref(rc);
2355 setmref(J->rbchash[(rc & (RBCHASH_SLOTS-1))].pc, pc);
2356 setgcref(J->rbchash[(rc & (RBCHASH_SLOTS-1))].pt, obj2gco(J->pt));
2357#endif
2358 break;
2359
2360 /* -- Calls and vararg handling ----------------------------------------- */
2361
2362 case BC_ITERC:
2363 J->base[ra] = getslot(J, ra-3);
2364 J->base[ra+1+LJ_FR2] = getslot(J, ra-2);
2365 J->base[ra+2+LJ_FR2] = getslot(J, ra-1);
2366 { /* Do the actual copy now because lj_record_call needs the values. */
2367 TValue *b = &J->L->base[ra];
2368 copyTV(J->L, b, b-3);
2369 copyTV(J->L, b+1+LJ_FR2, b-2);
2370 copyTV(J->L, b+2+LJ_FR2, b-1);
2371 }
2372 lj_record_call(J, ra, (ptrdiff_t)rc-1);
2373 break;
2374
2375 /* L->top is set to L->base+ra+rc+NARGS-1+1. See lj_dispatch_ins(). */
2376 case BC_CALLM:
2377 rc = (BCReg)(J->L->top - J->L->base) - ra - LJ_FR2;
2378 /* fallthrough */
2379 case BC_CALL:
2380 lj_record_call(J, ra, (ptrdiff_t)rc-1);
2381 break;
2382
2383 case BC_CALLMT:
2384 rc = (BCReg)(J->L->top - J->L->base) - ra - LJ_FR2;
2385 /* fallthrough */
2386 case BC_CALLT:
2387 lj_record_tailcall(J, ra, (ptrdiff_t)rc-1);
2388 break;
2389
2390 case BC_VARG:
2391 rec_varg(J, ra, (ptrdiff_t)rb-1);
2392 break;
2393
2394 /* -- Returns ----------------------------------------------------------- */
2395
2396 case BC_RETM:
2397 /* L->top is set to L->base+ra+rc+NRESULTS-1, see lj_dispatch_ins(). */
2398 rc = (BCReg)(J->L->top - J->L->base) - ra + 1;
2399 /* fallthrough */
2400 case BC_RET: case BC_RET0: case BC_RET1:
2401#if LJ_HASPROFILE
2402 rec_profile_ret(J);
2403#endif
2404 lj_record_ret(J, ra, (ptrdiff_t)rc-1);
2405 break;
2406
2407 /* -- Loops and branches ------------------------------------------------ */
2408
2409 case BC_FORI:
2410 if (rec_for(J, pc, 0) != LOOPEV_LEAVE)
2411 J->loopref = J->cur.nins;
2412 break;
2413 case BC_JFORI:
2414 lj_assertJ(bc_op(pc[(ptrdiff_t)rc-BCBIAS_J]) == BC_JFORL,
2415 "JFORI does not point to JFORL");
2416 if (rec_for(J, pc, 0) != LOOPEV_LEAVE) /* Link to existing loop. */
2417 lj_record_stop(J, LJ_TRLINK_ROOT, bc_d(pc[(ptrdiff_t)rc-BCBIAS_J]));
2418 /* Continue tracing if the loop is not entered. */
2419 break;
2420
2421 case BC_FORL:
2422 rec_loop_interp(J, pc, rec_for(J, pc+((ptrdiff_t)rc-BCBIAS_J), 1));
2423 break;
2424 case BC_ITERL:
2425 rec_loop_interp(J, pc, rec_iterl(J, *pc));
2426 break;
2427 case BC_LOOP:
2428 rec_loop_interp(J, pc, rec_loop(J, ra, 1));
2429 break;
2430
2431 case BC_JFORL:
2432 rec_loop_jit(J, rc, rec_for(J, pc+bc_j(traceref(J, rc)->startins), 1));
2433 break;
2434 case BC_JITERL:
2435 rec_loop_jit(J, rc, rec_iterl(J, traceref(J, rc)->startins));
2436 break;
2437 case BC_JLOOP:
2438 rec_loop_jit(J, rc, rec_loop(J, ra,
2439 !bc_isret(bc_op(traceref(J, rc)->startins))));
2440 break;
2441
2442 case BC_IFORL:
2443 case BC_IITERL:
2444 case BC_ILOOP:
2445 case BC_IFUNCF:
2446 case BC_IFUNCV:
2447 lj_trace_err(J, LJ_TRERR_BLACKL);
2448 break;
2449
2450 case BC_JMP:
2451 if (ra < J->maxslot)
2452 J->maxslot = ra; /* Shrink used slots. */
2453 break;
2454
2455 /* -- Function headers -------------------------------------------------- */
2456
2457 case BC_FUNCF:
2458 rec_func_lua(J);
2459 break;
2460 case BC_JFUNCF:
2461 rec_func_jit(J, rc);
2462 break;
2463
2464 case BC_FUNCV:
2465 rec_func_vararg(J);
2466 rec_func_lua(J);
2467 break;
2468 case BC_JFUNCV:
2469 /* Cannot happen. No hotcall counting for varag funcs. */
2470 lj_assertJ(0, "unsupported vararg hotcall");
2471 break;
2472
2473 case BC_FUNCC:
2474 case BC_FUNCCW:
2475 lj_ffrecord_func(J);
2476 break;
2477
2478 default:
2479 if (op >= BC__MAX) {
2480 lj_ffrecord_func(J);
2481 break;
2482 }
2483 /* fallthrough */
2484 case BC_ITERN:
2485 case BC_ISNEXT:
2486 case BC_UCLO:
2487 case BC_FNEW:
2488 setintV(&J->errinfo, (int32_t)op);
2489 lj_trace_err_info(J, LJ_TRERR_NYIBC);
2490 break;
2491 }
2492
2493 /* rc == 0 if we have no result yet, e.g. pending __index metamethod call. */
2494 if (bcmode_a(op) == BCMdst && rc) {
2495 J->base[ra] = rc;
2496 if (ra >= J->maxslot) {
2497#if LJ_FR2
2498 if (ra > J->maxslot) J->base[ra-1] = 0;
2499#endif
2500 J->maxslot = ra+1;
2501 }
2502 }
2503
2504#undef rav
2505#undef rbv
2506#undef rcv
2507
2508 /* Limit the number of recorded IR instructions and constants. */
2509 if (J->cur.nins > REF_FIRST+(IRRef)J->param[JIT_P_maxrecord] ||
2510 J->cur.nk < REF_BIAS-(IRRef)J->param[JIT_P_maxirconst])
2511 lj_trace_err(J, LJ_TRERR_TRACEOV);
2512}
2513
2514/* -- Recording setup ----------------------------------------------------- */
2515
2516/* Setup recording for a root trace started by a hot loop. */
2517static const BCIns *rec_setup_root(jit_State *J)
2518{
2519 /* Determine the next PC and the bytecode range for the loop. */
2520 const BCIns *pcj, *pc = J->pc;
2521 BCIns ins = *pc;
2522 BCReg ra = bc_a(ins);
2523 switch (bc_op(ins)) {
2524 case BC_FORL:
2525 J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
2526 pc += 1+bc_j(ins);
2527 J->bc_min = pc;
2528 break;
2529 case BC_ITERL:
2530 lj_assertJ(bc_op(pc[-1]) == BC_ITERC, "no ITERC before ITERL");
2531 J->maxslot = ra + bc_b(pc[-1]) - 1;
2532 J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
2533 pc += 1+bc_j(ins);
2534 lj_assertJ(bc_op(pc[-1]) == BC_JMP, "ITERL does not point to JMP+1");
2535 J->bc_min = pc;
2536 break;
2537 case BC_LOOP:
2538 /* Only check BC range for real loops, but not for "repeat until true". */
2539 pcj = pc + bc_j(ins);
2540 ins = *pcj;
2541 if (bc_op(ins) == BC_JMP && bc_j(ins) < 0) {
2542 J->bc_min = pcj+1 + bc_j(ins);
2543 J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns);
2544 }
2545 J->maxslot = ra;
2546 pc++;
2547 break;
2548 case BC_RET:
2549 case BC_RET0:
2550 case BC_RET1:
2551 /* No bytecode range check for down-recursive root traces. */
2552 J->maxslot = ra + bc_d(ins) - 1;
2553 break;
2554 case BC_FUNCF:
2555 /* No bytecode range check for root traces started by a hot call. */
2556 J->maxslot = J->pt->numparams;
2557 pc++;
2558 break;
2559 case BC_CALLM:
2560 case BC_CALL:
2561 case BC_ITERC:
2562 /* No bytecode range check for stitched traces. */
2563 pc++;
2564 break;
2565 default:
2566 lj_assertJ(0, "bad root trace start bytecode %d", bc_op(ins));
2567 break;
2568 }
2569 return pc;
2570}
2571
2572/* Setup for recording a new trace. */
2573void lj_record_setup(jit_State *J)
2574{
2575 uint32_t i;
2576
2577 /* Initialize state related to current trace. */
2578 memset(J->slot, 0, sizeof(J->slot));
2579 memset(J->chain, 0, sizeof(J->chain));
2580#ifdef LUAJIT_ENABLE_TABLE_BUMP
2581 memset(J->rbchash, 0, sizeof(J->rbchash));
2582#endif
2583 memset(J->bpropcache, 0, sizeof(J->bpropcache));
2584 J->scev.idx = REF_NIL;
2585 setmref(J->scev.pc, NULL);
2586
2587 J->baseslot = 1+LJ_FR2; /* Invoking function is at base[-1-LJ_FR2]. */
2588 J->base = J->slot + J->baseslot;
2589 J->maxslot = 0;
2590 J->framedepth = 0;
2591 J->retdepth = 0;
2592
2593 J->instunroll = J->param[JIT_P_instunroll];
2594 J->loopunroll = J->param[JIT_P_loopunroll];
2595 J->tailcalled = 0;
2596 J->loopref = 0;
2597
2598 J->bc_min = NULL; /* Means no limit. */
2599 J->bc_extent = ~(MSize)0;
2600
2601 /* Emit instructions for fixed references. Also triggers initial IR alloc. */
2602 emitir_raw(IRT(IR_BASE, IRT_PGC), J->parent, J->exitno);
2603 for (i = 0; i <= 2; i++) {
2604 IRIns *ir = IR(REF_NIL-i);
2605 ir->i = 0;
2606 ir->t.irt = (uint8_t)(IRT_NIL+i);
2607 ir->o = IR_KPRI;
2608 ir->prev = 0;
2609 }
2610 J->cur.nk = REF_TRUE;
2611
2612 J->startpc = J->pc;
2613 setmref(J->cur.startpc, J->pc);
2614 if (J->parent) { /* Side trace. */
2615 GCtrace *T = traceref(J, J->parent);
2616 TraceNo root = T->root ? T->root : J->parent;
2617 J->cur.root = (uint16_t)root;
2618 J->cur.startins = BCINS_AD(BC_JMP, 0, 0);
2619 /* Check whether we could at least potentially form an extra loop. */
2620 if (J->exitno == 0 && T->snap[0].nent == 0) {
2621 /* We can narrow a FORL for some side traces, too. */
2622 if (J->pc > proto_bc(J->pt) && bc_op(J->pc[-1]) == BC_JFORI &&
2623 bc_d(J->pc[bc_j(J->pc[-1])-1]) == root) {
2624 lj_snap_add(J);
2625 rec_for_loop(J, J->pc-1, &J->scev, 1);
2626 goto sidecheck;
2627 }
2628 } else {
2629 J->startpc = NULL; /* Prevent forming an extra loop. */
2630 }
2631 lj_snap_replay(J, T);
2632 sidecheck:
2633 if (traceref(J, J->cur.root)->nchild >= J->param[JIT_P_maxside] ||
2634 T->snap[J->exitno].count >= J->param[JIT_P_hotexit] +
2635 J->param[JIT_P_tryside]) {
2636 lj_record_stop(J, LJ_TRLINK_INTERP, 0);
2637 }
2638 } else { /* Root trace. */
2639 J->cur.root = 0;
2640 J->cur.startins = *J->pc;
2641 J->pc = rec_setup_root(J);
2642 /* Note: the loop instruction itself is recorded at the end and not
2643 ** at the start! So snapshot #0 needs to point to the *next* instruction.
2644 */
2645 lj_snap_add(J);
2646 if (bc_op(J->cur.startins) == BC_FORL)
2647 rec_for_loop(J, J->pc-1, &J->scev, 1);
2648 else if (bc_op(J->cur.startins) == BC_ITERC)
2649 J->startpc = NULL;
2650 if (1 + J->pt->framesize >= LJ_MAX_JSLOTS)
2651 lj_trace_err(J, LJ_TRERR_STACKOV);
2652 }
2653#if LJ_HASPROFILE
2654 J->prev_pt = NULL;
2655 J->prev_line = -1;
2656#endif
2657#ifdef LUAJIT_ENABLE_CHECKHOOK
2658 /* Regularly check for instruction/line hooks from compiled code and
2659 ** exit to the interpreter if the hooks are set.
2660 **
2661 ** This is a compile-time option and disabled by default, since the
2662 ** hook checks may be quite expensive in tight loops.
2663 **
2664 ** Note this is only useful if hooks are *not* set most of the time.
2665 ** Use this only if you want to *asynchronously* interrupt the execution.
2666 **
2667 ** You can set the instruction hook via lua_sethook() with a count of 1
2668 ** from a signal handler or another native thread. Please have a look
2669 ** at the first few functions in luajit.c for an example (Ctrl-C handler).
2670 */
2671 {
2672 TRef tr = emitir(IRT(IR_XLOAD, IRT_U8),
2673 lj_ir_kptr(J, &J2G(J)->hookmask), IRXLOAD_VOLATILE);
2674 tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (LUA_MASKLINE|LUA_MASKCOUNT)));
2675 emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, 0));
2676 }
2677#endif
2678}
2679
2680#undef IR
2681#undef emitir_raw
2682#undef emitir
2683
2684#endif
2685