1/*
2** Trace management.
3** Copyright (C) 2005-2021 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#define lj_trace_c
7#define LUA_CORE
8
9#include "lj_obj.h"
10
11#if LJ_HASJIT
12
13#include "lj_gc.h"
14#include "lj_err.h"
15#include "lj_debug.h"
16#include "lj_str.h"
17#include "lj_frame.h"
18#include "lj_state.h"
19#include "lj_bc.h"
20#include "lj_ir.h"
21#include "lj_jit.h"
22#include "lj_iropt.h"
23#include "lj_mcode.h"
24#include "lj_trace.h"
25#include "lj_snap.h"
26#include "lj_gdbjit.h"
27#include "lj_record.h"
28#include "lj_asm.h"
29#include "lj_dispatch.h"
30#include "lj_vm.h"
31#include "lj_vmevent.h"
32#include "lj_target.h"
33#include "lj_prng.h"
34
35/* -- Error handling ------------------------------------------------------ */
36
37/* Synchronous abort with error message. */
38void lj_trace_err(jit_State *J, TraceError e)
39{
40 setnilV(&J->errinfo); /* No error info. */
41 setintV(J->L->top++, (int32_t)e);
42 lj_err_throw(J->L, LUA_ERRRUN);
43}
44
45/* Synchronous abort with error message and error info. */
46void lj_trace_err_info(jit_State *J, TraceError e)
47{
48 setintV(J->L->top++, (int32_t)e);
49 lj_err_throw(J->L, LUA_ERRRUN);
50}
51
52/* -- Trace management ---------------------------------------------------- */
53
54/* The current trace is first assembled in J->cur. The variable length
55** arrays point to shared, growable buffers (J->irbuf etc.). When trace
56** recording ends successfully, the current trace and its data structures
57** are copied to a new (compact) GCtrace object.
58*/
59
60/* Find a free trace number. */
61static TraceNo trace_findfree(jit_State *J)
62{
63 MSize osz, lim;
64 if (J->freetrace == 0)
65 J->freetrace = 1;
66 for (; J->freetrace < J->sizetrace; J->freetrace++)
67 if (traceref(J, J->freetrace) == NULL)
68 return J->freetrace++;
69 /* Need to grow trace array. */
70 lim = (MSize)J->param[JIT_P_maxtrace] + 1;
71 if (lim < 2) lim = 2; else if (lim > 65535) lim = 65535;
72 osz = J->sizetrace;
73 if (osz >= lim)
74 return 0; /* Too many traces. */
75 lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, GCRef);
76 for (; osz < J->sizetrace; osz++)
77 setgcrefnull(J->trace[osz]);
78 return J->freetrace;
79}
80
81#define TRACE_APPENDVEC(field, szfield, tp) \
82 T->field = (tp *)p; \
83 memcpy(p, J->cur.field, J->cur.szfield*sizeof(tp)); \
84 p += J->cur.szfield*sizeof(tp);
85
86#ifdef LUAJIT_USE_PERFTOOLS
87/*
88** Create symbol table of JIT-compiled code. For use with Linux perf tools.
89** Example usage:
90** perf record -f -e cycles luajit test.lua
91** perf report -s symbol
92** rm perf.data /tmp/perf-*.map
93*/
94#include <stdio.h>
95#include <unistd.h>
96
97static void perftools_addtrace(GCtrace *T)
98{
99 static FILE *fp;
100 GCproto *pt = &gcref(T->startpt)->pt;
101 const BCIns *startpc = mref(T->startpc, const BCIns);
102 const char *name = proto_chunknamestr(pt);
103 BCLine lineno;
104 if (name[0] == '@' || name[0] == '=')
105 name++;
106 else
107 name = "(string)";
108 lj_assertX(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc,
109 "trace PC out of range");
110 lineno = lj_debug_line(pt, proto_bcpos(pt, startpc));
111 if (!fp) {
112 char fname[40];
113 sprintf(fname, "/tmp/perf-%d.map", getpid());
114 if (!(fp = fopen(fname, "w"))) return;
115 setlinebuf(fp);
116 }
117 fprintf(fp, "%lx %x TRACE_%d::%s:%u\n",
118 (long)T->mcode, T->szmcode, T->traceno, name, lineno);
119}
120#endif
121
122/* Allocate space for copy of T. */
123GCtrace * LJ_FASTCALL lj_trace_alloc(lua_State *L, GCtrace *T)
124{
125 size_t sztr = ((sizeof(GCtrace)+7)&~7);
126 size_t szins = (T->nins-T->nk)*sizeof(IRIns);
127 size_t sz = sztr + szins +
128 T->nsnap*sizeof(SnapShot) +
129 T->nsnapmap*sizeof(SnapEntry);
130 GCtrace *T2 = lj_mem_newt(L, (MSize)sz, GCtrace);
131 char *p = (char *)T2 + sztr;
132 T2->gct = ~LJ_TTRACE;
133 T2->marked = 0;
134 T2->traceno = 0;
135 T2->ir = (IRIns *)p - T->nk;
136 T2->nins = T->nins;
137 T2->nk = T->nk;
138 T2->nsnap = T->nsnap;
139 T2->nsnapmap = T->nsnapmap;
140 memcpy(p, T->ir + T->nk, szins);
141 return T2;
142}
143
144/* Save current trace by copying and compacting it. */
145static void trace_save(jit_State *J, GCtrace *T)
146{
147 size_t sztr = ((sizeof(GCtrace)+7)&~7);
148 size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns);
149 char *p = (char *)T + sztr;
150 memcpy(T, &J->cur, sizeof(GCtrace));
151 setgcrefr(T->nextgc, J2G(J)->gc.root);
152 setgcrefp(J2G(J)->gc.root, T);
153 newwhite(J2G(J), T);
154 T->gct = ~LJ_TTRACE;
155 T->ir = (IRIns *)p - J->cur.nk; /* The IR has already been copied above. */
156 p += szins;
157 TRACE_APPENDVEC(snap, nsnap, SnapShot)
158 TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry)
159 J->cur.traceno = 0;
160 J->curfinal = NULL;
161 setgcrefp(J->trace[T->traceno], T);
162 lj_gc_barriertrace(J2G(J), T->traceno);
163 lj_gdbjit_addtrace(J, T);
164#ifdef LUAJIT_USE_PERFTOOLS
165 perftools_addtrace(T);
166#endif
167}
168
169void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T)
170{
171 jit_State *J = G2J(g);
172 if (T->traceno) {
173 lj_gdbjit_deltrace(J, T);
174 if (T->traceno < J->freetrace)
175 J->freetrace = T->traceno;
176 setgcrefnull(J->trace[T->traceno]);
177 }
178 lj_mem_free(g, T,
179 ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
180 T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry));
181}
182
183/* Re-enable compiling a prototype by unpatching any modified bytecode. */
184void lj_trace_reenableproto(GCproto *pt)
185{
186 if ((pt->flags & PROTO_ILOOP)) {
187 BCIns *bc = proto_bc(pt);
188 BCPos i, sizebc = pt->sizebc;
189 pt->flags &= ~PROTO_ILOOP;
190 if (bc_op(bc[0]) == BC_IFUNCF)
191 setbc_op(&bc[0], BC_FUNCF);
192 for (i = 1; i < sizebc; i++) {
193 BCOp op = bc_op(bc[i]);
194 if (op == BC_IFORL || op == BC_IITERL || op == BC_ILOOP)
195 setbc_op(&bc[i], (int)op+(int)BC_LOOP-(int)BC_ILOOP);
196 }
197 }
198}
199
200/* Unpatch the bytecode modified by a root trace. */
201static void trace_unpatch(jit_State *J, GCtrace *T)
202{
203 BCOp op = bc_op(T->startins);
204 BCIns *pc = mref(T->startpc, BCIns);
205 UNUSED(J);
206 if (op == BC_JMP)
207 return; /* No need to unpatch branches in parent traces (yet). */
208 switch (bc_op(*pc)) {
209 case BC_JFORL:
210 lj_assertJ(traceref(J, bc_d(*pc)) == T, "JFORL references other trace");
211 *pc = T->startins;
212 pc += bc_j(T->startins);
213 lj_assertJ(bc_op(*pc) == BC_JFORI, "FORL does not point to JFORI");
214 setbc_op(pc, BC_FORI);
215 break;
216 case BC_JITERL:
217 case BC_JLOOP:
218 lj_assertJ(op == BC_ITERL || op == BC_LOOP || bc_isret(op),
219 "bad original bytecode %d", op);
220 *pc = T->startins;
221 break;
222 case BC_JMP:
223 lj_assertJ(op == BC_ITERL, "bad original bytecode %d", op);
224 pc += bc_j(*pc)+2;
225 if (bc_op(*pc) == BC_JITERL) {
226 lj_assertJ(traceref(J, bc_d(*pc)) == T, "JITERL references other trace");
227 *pc = T->startins;
228 }
229 break;
230 case BC_JFUNCF:
231 lj_assertJ(op == BC_FUNCF, "bad original bytecode %d", op);
232 *pc = T->startins;
233 break;
234 default: /* Already unpatched. */
235 break;
236 }
237}
238
239/* Flush a root trace. */
240static void trace_flushroot(jit_State *J, GCtrace *T)
241{
242 GCproto *pt = &gcref(T->startpt)->pt;
243 lj_assertJ(T->root == 0, "not a root trace");
244 lj_assertJ(pt != NULL, "trace has no prototype");
245 /* First unpatch any modified bytecode. */
246 trace_unpatch(J, T);
247 /* Unlink root trace from chain anchored in prototype. */
248 if (pt->trace == T->traceno) { /* Trace is first in chain. Easy. */
249 pt->trace = T->nextroot;
250 } else if (pt->trace) { /* Otherwise search in chain of root traces. */
251 GCtrace *T2 = traceref(J, pt->trace);
252 if (T2) {
253 for (; T2->nextroot; T2 = traceref(J, T2->nextroot))
254 if (T2->nextroot == T->traceno) {
255 T2->nextroot = T->nextroot; /* Unlink from chain. */
256 break;
257 }
258 }
259 }
260}
261
262/* Flush a trace. Only root traces are considered. */
263void lj_trace_flush(jit_State *J, TraceNo traceno)
264{
265 if (traceno > 0 && traceno < J->sizetrace) {
266 GCtrace *T = traceref(J, traceno);
267 if (T && T->root == 0)
268 trace_flushroot(J, T);
269 }
270}
271
272/* Flush all traces associated with a prototype. */
273void lj_trace_flushproto(global_State *g, GCproto *pt)
274{
275 while (pt->trace != 0)
276 trace_flushroot(G2J(g), traceref(G2J(g), pt->trace));
277}
278
279/* Flush all traces. */
280int lj_trace_flushall(lua_State *L)
281{
282 jit_State *J = L2J(L);
283 ptrdiff_t i;
284 if ((J2G(J)->hookmask & HOOK_GC))
285 return 1;
286 for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--) {
287 GCtrace *T = traceref(J, i);
288 if (T) {
289 if (T->root == 0)
290 trace_flushroot(J, T);
291 lj_gdbjit_deltrace(J, T);
292 T->traceno = T->link = 0; /* Blacklist the link for cont_stitch. */
293 setgcrefnull(J->trace[i]);
294 }
295 }
296 J->cur.traceno = 0;
297 J->freetrace = 0;
298 /* Clear penalty cache. */
299 memset(J->penalty, 0, sizeof(J->penalty));
300 /* Free the whole machine code and invalidate all exit stub groups. */
301 lj_mcode_free(J);
302 memset(J->exitstubgroup, 0, sizeof(J->exitstubgroup));
303 lj_vmevent_send(L, TRACE,
304 setstrV(L, L->top++, lj_str_newlit(L, "flush"));
305 );
306 return 0;
307}
308
309/* Initialize JIT compiler state. */
310void lj_trace_initstate(global_State *g)
311{
312 jit_State *J = G2J(g);
313 TValue *tv;
314
315 /* Initialize aligned SIMD constants. */
316 tv = LJ_KSIMD(J, LJ_KSIMD_ABS);
317 tv[0].u64 = U64x(7fffffff,ffffffff);
318 tv[1].u64 = U64x(7fffffff,ffffffff);
319 tv = LJ_KSIMD(J, LJ_KSIMD_NEG);
320 tv[0].u64 = U64x(80000000,00000000);
321 tv[1].u64 = U64x(80000000,00000000);
322
323 /* Initialize 32/64 bit constants. */
324#if LJ_TARGET_X86ORX64
325 J->k64[LJ_K64_TOBIT].u64 = U64x(43380000,00000000);
326#if LJ_32
327 J->k64[LJ_K64_M2P64_31].u64 = U64x(c1e00000,00000000);
328#endif
329 J->k64[LJ_K64_2P64].u64 = U64x(43f00000,00000000);
330 J->k32[LJ_K32_M2P64_31] = LJ_64 ? 0xdf800000 : 0xcf000000;
331#endif
332#if LJ_TARGET_X86ORX64 || LJ_TARGET_MIPS64
333 J->k64[LJ_K64_M2P64].u64 = U64x(c3f00000,00000000);
334#endif
335#if LJ_TARGET_PPC
336 J->k32[LJ_K32_2P52_2P31] = 0x59800004;
337 J->k32[LJ_K32_2P52] = 0x59800000;
338#endif
339#if LJ_TARGET_PPC || LJ_TARGET_MIPS
340 J->k32[LJ_K32_2P31] = 0x4f000000;
341#endif
342#if LJ_TARGET_MIPS
343 J->k64[LJ_K64_2P31].u64 = U64x(41e00000,00000000);
344#if LJ_64
345 J->k64[LJ_K64_2P63].u64 = U64x(43e00000,00000000);
346 J->k32[LJ_K32_2P63] = 0x5f000000;
347 J->k32[LJ_K32_M2P64] = 0xdf800000;
348#endif
349#endif
350}
351
352/* Free everything associated with the JIT compiler state. */
353void lj_trace_freestate(global_State *g)
354{
355 jit_State *J = G2J(g);
356#ifdef LUA_USE_ASSERT
357 { /* This assumes all traces have already been freed. */
358 ptrdiff_t i;
359 for (i = 1; i < (ptrdiff_t)J->sizetrace; i++)
360 lj_assertG(i == (ptrdiff_t)J->cur.traceno || traceref(J, i) == NULL,
361 "trace still allocated");
362 }
363#endif
364 lj_mcode_free(J);
365 lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry);
366 lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot);
367 lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns);
368 lj_mem_freevec(g, J->trace, J->sizetrace, GCRef);
369}
370
371/* -- Penalties and blacklisting ------------------------------------------ */
372
373/* Blacklist a bytecode instruction. */
374static void blacklist_pc(GCproto *pt, BCIns *pc)
375{
376 setbc_op(pc, (int)bc_op(*pc)+(int)BC_ILOOP-(int)BC_LOOP);
377 pt->flags |= PROTO_ILOOP;
378}
379
380/* Penalize a bytecode instruction. */
381static void penalty_pc(jit_State *J, GCproto *pt, BCIns *pc, TraceError e)
382{
383 uint32_t i, val = PENALTY_MIN;
384 for (i = 0; i < PENALTY_SLOTS; i++)
385 if (mref(J->penalty[i].pc, const BCIns) == pc) { /* Cache slot found? */
386 /* First try to bump its hotcount several times. */
387 val = ((uint32_t)J->penalty[i].val << 1) +
388 (lj_prng_u64(&J2G(J)->prng) & ((1u<<PENALTY_RNDBITS)-1));
389 if (val > PENALTY_MAX) {
390 blacklist_pc(pt, pc); /* Blacklist it, if that didn't help. */
391 return;
392 }
393 goto setpenalty;
394 }
395 /* Assign a new penalty cache slot. */
396 i = J->penaltyslot;
397 J->penaltyslot = (J->penaltyslot + 1) & (PENALTY_SLOTS-1);
398 setmref(J->penalty[i].pc, pc);
399setpenalty:
400 J->penalty[i].val = (uint16_t)val;
401 J->penalty[i].reason = e;
402 hotcount_set(J2GG(J), pc+1, val);
403}
404
405/* -- Trace compiler state machine ---------------------------------------- */
406
407/* Start tracing. */
408static void trace_start(jit_State *J)
409{
410 lua_State *L;
411 TraceNo traceno;
412
413 if ((J->pt->flags & PROTO_NOJIT)) { /* JIT disabled for this proto? */
414 if (J->parent == 0 && J->exitno == 0) {
415 /* Lazy bytecode patching to disable hotcount events. */
416 lj_assertJ(bc_op(*J->pc) == BC_FORL || bc_op(*J->pc) == BC_ITERL ||
417 bc_op(*J->pc) == BC_LOOP || bc_op(*J->pc) == BC_FUNCF,
418 "bad hot bytecode %d", bc_op(*J->pc));
419 setbc_op(J->pc, (int)bc_op(*J->pc)+(int)BC_ILOOP-(int)BC_LOOP);
420 J->pt->flags |= PROTO_ILOOP;
421 }
422 J->state = LJ_TRACE_IDLE; /* Silently ignored. */
423 return;
424 }
425
426 /* Get a new trace number. */
427 traceno = trace_findfree(J);
428 if (LJ_UNLIKELY(traceno == 0)) { /* No free trace? */
429 lj_assertJ((J2G(J)->hookmask & HOOK_GC) == 0,
430 "recorder called from GC hook");
431 lj_trace_flushall(J->L);
432 J->state = LJ_TRACE_IDLE; /* Silently ignored. */
433 return;
434 }
435 setgcrefp(J->trace[traceno], &J->cur);
436
437 /* Setup enough of the current trace to be able to send the vmevent. */
438 memset(&J->cur, 0, sizeof(GCtrace));
439 J->cur.traceno = traceno;
440 J->cur.nins = J->cur.nk = REF_BASE;
441 J->cur.ir = J->irbuf;
442 J->cur.snap = J->snapbuf;
443 J->cur.snapmap = J->snapmapbuf;
444 J->mergesnap = 0;
445 J->needsnap = 0;
446 J->bcskip = 0;
447 J->guardemit.irt = 0;
448 J->postproc = LJ_POST_NONE;
449 lj_resetsplit(J);
450 J->retryrec = 0;
451 J->ktrace = 0;
452 setgcref(J->cur.startpt, obj2gco(J->pt));
453
454 L = J->L;
455 lj_vmevent_send(L, TRACE,
456 setstrV(L, L->top++, lj_str_newlit(L, "start"));
457 setintV(L->top++, traceno);
458 setfuncV(L, L->top++, J->fn);
459 setintV(L->top++, proto_bcpos(J->pt, J->pc));
460 if (J->parent) {
461 setintV(L->top++, J->parent);
462 setintV(L->top++, J->exitno);
463 } else {
464 BCOp op = bc_op(*J->pc);
465 if (op == BC_CALLM || op == BC_CALL || op == BC_ITERC) {
466 setintV(L->top++, J->exitno); /* Parent of stitched trace. */
467 setintV(L->top++, -1);
468 }
469 }
470 );
471 lj_record_setup(J);
472}
473
474/* Stop tracing. */
475static void trace_stop(jit_State *J)
476{
477 BCIns *pc = mref(J->cur.startpc, BCIns);
478 BCOp op = bc_op(J->cur.startins);
479 GCproto *pt = &gcref(J->cur.startpt)->pt;
480 TraceNo traceno = J->cur.traceno;
481 GCtrace *T = J->curfinal;
482 lua_State *L;
483
484 switch (op) {
485 case BC_FORL:
486 setbc_op(pc+bc_j(J->cur.startins), BC_JFORI); /* Patch FORI, too. */
487 /* fallthrough */
488 case BC_LOOP:
489 case BC_ITERL:
490 case BC_FUNCF:
491 /* Patch bytecode of starting instruction in root trace. */
492 setbc_op(pc, (int)op+(int)BC_JLOOP-(int)BC_LOOP);
493 setbc_d(pc, traceno);
494 addroot:
495 /* Add to root trace chain in prototype. */
496 J->cur.nextroot = pt->trace;
497 pt->trace = (TraceNo1)traceno;
498 break;
499 case BC_RET:
500 case BC_RET0:
501 case BC_RET1:
502 *pc = BCINS_AD(BC_JLOOP, J->cur.snap[0].nslots, traceno);
503 goto addroot;
504 case BC_JMP:
505 /* Patch exit branch in parent to side trace entry. */
506 lj_assertJ(J->parent != 0 && J->cur.root != 0, "not a side trace");
507 lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode);
508 /* Avoid compiling a side trace twice (stack resizing uses parent exit). */
509 traceref(J, J->parent)->snap[J->exitno].count = SNAPCOUNT_DONE;
510 /* Add to side trace chain in root trace. */
511 {
512 GCtrace *root = traceref(J, J->cur.root);
513 root->nchild++;
514 J->cur.nextside = root->nextside;
515 root->nextside = (TraceNo1)traceno;
516 }
517 break;
518 case BC_CALLM:
519 case BC_CALL:
520 case BC_ITERC:
521 /* Trace stitching: patch link of previous trace. */
522 traceref(J, J->exitno)->link = traceno;
523 break;
524 default:
525 lj_assertJ(0, "bad stop bytecode %d", op);
526 break;
527 }
528
529 /* Commit new mcode only after all patching is done. */
530 lj_mcode_commit(J, J->cur.mcode);
531 J->postproc = LJ_POST_NONE;
532 trace_save(J, T);
533
534 L = J->L;
535 lj_vmevent_send(L, TRACE,
536 setstrV(L, L->top++, lj_str_newlit(L, "stop"));
537 setintV(L->top++, traceno);
538 setfuncV(L, L->top++, J->fn);
539 );
540}
541
542/* Start a new root trace for down-recursion. */
543static int trace_downrec(jit_State *J)
544{
545 /* Restart recording at the return instruction. */
546 lj_assertJ(J->pt != NULL, "no active prototype");
547 lj_assertJ(bc_isret(bc_op(*J->pc)), "not at a return bytecode");
548 if (bc_op(*J->pc) == BC_RETM)
549 return 0; /* NYI: down-recursion with RETM. */
550 J->parent = 0;
551 J->exitno = 0;
552 J->state = LJ_TRACE_RECORD;
553 trace_start(J);
554 return 1;
555}
556
557/* Abort tracing. */
558static int trace_abort(jit_State *J)
559{
560 lua_State *L = J->L;
561 TraceError e = LJ_TRERR_RECERR;
562 TraceNo traceno;
563
564 J->postproc = LJ_POST_NONE;
565 lj_mcode_abort(J);
566 if (J->curfinal) {
567 lj_trace_free(J2G(J), J->curfinal);
568 J->curfinal = NULL;
569 }
570 if (tvisnumber(L->top-1))
571 e = (TraceError)numberVint(L->top-1);
572 if (e == LJ_TRERR_MCODELM) {
573 L->top--; /* Remove error object */
574 J->state = LJ_TRACE_ASM;
575 return 1; /* Retry ASM with new MCode area. */
576 }
577 /* Penalize or blacklist starting bytecode instruction. */
578 if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) {
579 if (J->exitno == 0) {
580 BCIns *startpc = mref(J->cur.startpc, BCIns);
581 if (e == LJ_TRERR_RETRY)
582 hotcount_set(J2GG(J), startpc+1, 1); /* Immediate retry. */
583 else
584 penalty_pc(J, &gcref(J->cur.startpt)->pt, startpc, e);
585 } else {
586 traceref(J, J->exitno)->link = J->exitno; /* Self-link is blacklisted. */
587 }
588 }
589
590 /* Is there anything to abort? */
591 traceno = J->cur.traceno;
592 if (traceno) {
593 ptrdiff_t errobj = savestack(L, L->top-1); /* Stack may be resized. */
594 J->cur.link = 0;
595 J->cur.linktype = LJ_TRLINK_NONE;
596 lj_vmevent_send(L, TRACE,
597 TValue *frame;
598 const BCIns *pc;
599 GCfunc *fn;
600 setstrV(L, L->top++, lj_str_newlit(L, "abort"));
601 setintV(L->top++, traceno);
602 /* Find original Lua function call to generate a better error message. */
603 frame = J->L->base-1;
604 pc = J->pc;
605 while (!isluafunc(frame_func(frame))) {
606 pc = (frame_iscont(frame) ? frame_contpc(frame) : frame_pc(frame)) - 1;
607 frame = frame_prev(frame);
608 }
609 fn = frame_func(frame);
610 setfuncV(L, L->top++, fn);
611 setintV(L->top++, proto_bcpos(funcproto(fn), pc));
612 copyTV(L, L->top++, restorestack(L, errobj));
613 copyTV(L, L->top++, &J->errinfo);
614 );
615 /* Drop aborted trace after the vmevent (which may still access it). */
616 setgcrefnull(J->trace[traceno]);
617 if (traceno < J->freetrace)
618 J->freetrace = traceno;
619 J->cur.traceno = 0;
620 }
621 L->top--; /* Remove error object */
622 if (e == LJ_TRERR_DOWNREC)
623 return trace_downrec(J);
624 else if (e == LJ_TRERR_MCODEAL)
625 lj_trace_flushall(L);
626 return 0;
627}
628
629/* Perform pending re-patch of a bytecode instruction. */
630static LJ_AINLINE void trace_pendpatch(jit_State *J, int force)
631{
632 if (LJ_UNLIKELY(J->patchpc)) {
633 if (force || J->bcskip == 0) {
634 *J->patchpc = J->patchins;
635 J->patchpc = NULL;
636 } else {
637 J->bcskip = 0;
638 }
639 }
640}
641
642/* State machine for the trace compiler. Protected callback. */
643static TValue *trace_state(lua_State *L, lua_CFunction dummy, void *ud)
644{
645 jit_State *J = (jit_State *)ud;
646 UNUSED(dummy);
647 do {
648 retry:
649 switch (J->state) {
650 case LJ_TRACE_START:
651 J->state = LJ_TRACE_RECORD; /* trace_start() may change state. */
652 trace_start(J);
653 lj_dispatch_update(J2G(J));
654 break;
655
656 case LJ_TRACE_RECORD:
657 trace_pendpatch(J, 0);
658 setvmstate(J2G(J), RECORD);
659 lj_vmevent_send_(L, RECORD,
660 /* Save/restore tmptv state for trace recorder. */
661 TValue savetv = J2G(J)->tmptv;
662 TValue savetv2 = J2G(J)->tmptv2;
663 setintV(L->top++, J->cur.traceno);
664 setfuncV(L, L->top++, J->fn);
665 setintV(L->top++, J->pt ? (int32_t)proto_bcpos(J->pt, J->pc) : -1);
666 setintV(L->top++, J->framedepth);
667 ,
668 J2G(J)->tmptv = savetv;
669 J2G(J)->tmptv2 = savetv2;
670 );
671 lj_record_ins(J);
672 break;
673
674 case LJ_TRACE_END:
675 trace_pendpatch(J, 1);
676 J->loopref = 0;
677 if ((J->flags & JIT_F_OPT_LOOP) &&
678 J->cur.link == J->cur.traceno && J->framedepth + J->retdepth == 0) {
679 setvmstate(J2G(J), OPT);
680 lj_opt_dce(J);
681 if (lj_opt_loop(J)) { /* Loop optimization failed? */
682 J->cur.link = 0;
683 J->cur.linktype = LJ_TRLINK_NONE;
684 J->loopref = J->cur.nins;
685 J->state = LJ_TRACE_RECORD; /* Try to continue recording. */
686 break;
687 }
688 J->loopref = J->chain[IR_LOOP]; /* Needed by assembler. */
689 }
690 lj_opt_split(J);
691 lj_opt_sink(J);
692 if (!J->loopref) J->cur.snap[J->cur.nsnap-1].count = SNAPCOUNT_DONE;
693 J->state = LJ_TRACE_ASM;
694 break;
695
696 case LJ_TRACE_ASM:
697 setvmstate(J2G(J), ASM);
698 lj_asm_trace(J, &J->cur);
699 trace_stop(J);
700 setvmstate(J2G(J), INTERP);
701 J->state = LJ_TRACE_IDLE;
702 lj_dispatch_update(J2G(J));
703 return NULL;
704
705 default: /* Trace aborted asynchronously. */
706 setintV(L->top++, (int32_t)LJ_TRERR_RECERR);
707 /* fallthrough */
708 case LJ_TRACE_ERR:
709 trace_pendpatch(J, 1);
710 if (trace_abort(J))
711 goto retry;
712 setvmstate(J2G(J), INTERP);
713 J->state = LJ_TRACE_IDLE;
714 lj_dispatch_update(J2G(J));
715 return NULL;
716 }
717 } while (J->state > LJ_TRACE_RECORD);
718 return NULL;
719}
720
721/* -- Event handling ------------------------------------------------------ */
722
723/* A bytecode instruction is about to be executed. Record it. */
724void lj_trace_ins(jit_State *J, const BCIns *pc)
725{
726 /* Note: J->L must already be set. pc is the true bytecode PC here. */
727 J->pc = pc;
728 J->fn = curr_func(J->L);
729 J->pt = isluafunc(J->fn) ? funcproto(J->fn) : NULL;
730 while (lj_vm_cpcall(J->L, NULL, (void *)J, trace_state) != 0)
731 J->state = LJ_TRACE_ERR;
732}
733
734/* A hotcount triggered. Start recording a root trace. */
735void LJ_FASTCALL lj_trace_hot(jit_State *J, const BCIns *pc)
736{
737 /* Note: pc is the interpreter bytecode PC here. It's offset by 1. */
738 ERRNO_SAVE
739 /* Reset hotcount. */
740 hotcount_set(J2GG(J), pc, J->param[JIT_P_hotloop]*HOTCOUNT_LOOP);
741 /* Only start a new trace if not recording or inside __gc call or vmevent. */
742 if (J->state == LJ_TRACE_IDLE &&
743 !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
744 J->parent = 0; /* Root trace. */
745 J->exitno = 0;
746 J->state = LJ_TRACE_START;
747 lj_trace_ins(J, pc-1);
748 }
749 ERRNO_RESTORE
750}
751
752/* Check for a hot side exit. If yes, start recording a side trace. */
753static void trace_hotside(jit_State *J, const BCIns *pc)
754{
755 SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno];
756 if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) &&
757 isluafunc(curr_func(J->L)) &&
758 snap->count != SNAPCOUNT_DONE &&
759 ++snap->count >= J->param[JIT_P_hotexit]) {
760 lj_assertJ(J->state == LJ_TRACE_IDLE, "hot side exit while recording");
761 /* J->parent is non-zero for a side trace. */
762 J->state = LJ_TRACE_START;
763 lj_trace_ins(J, pc);
764 }
765}
766
767/* Stitch a new trace to the previous trace. */
768void LJ_FASTCALL lj_trace_stitch(jit_State *J, const BCIns *pc)
769{
770 /* Only start a new trace if not recording or inside __gc call or vmevent. */
771 if (J->state == LJ_TRACE_IDLE &&
772 !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
773 J->parent = 0; /* Have to treat it like a root trace. */
774 /* J->exitno is set to the invoking trace. */
775 J->state = LJ_TRACE_START;
776 lj_trace_ins(J, pc);
777 }
778}
779
780
781/* Tiny struct to pass data to protected call. */
782typedef struct ExitDataCP {
783 jit_State *J;
784 void *exptr; /* Pointer to exit state. */
785 const BCIns *pc; /* Restart interpreter at this PC. */
786} ExitDataCP;
787
788/* Need to protect lj_snap_restore because it may throw. */
789static TValue *trace_exit_cp(lua_State *L, lua_CFunction dummy, void *ud)
790{
791 ExitDataCP *exd = (ExitDataCP *)ud;
792 /* Always catch error here and don't call error function. */
793 cframe_errfunc(L->cframe) = 0;
794 cframe_nres(L->cframe) = -2*LUAI_MAXSTACK*(int)sizeof(TValue);
795 exd->pc = lj_snap_restore(exd->J, exd->exptr);
796 UNUSED(dummy);
797 return NULL;
798}
799
800#ifndef LUAJIT_DISABLE_VMEVENT
801/* Push all registers from exit state. */
802static void trace_exit_regs(lua_State *L, ExitState *ex)
803{
804 int32_t i;
805 setintV(L->top++, RID_NUM_GPR);
806 setintV(L->top++, RID_NUM_FPR);
807 for (i = 0; i < RID_NUM_GPR; i++) {
808 if (sizeof(ex->gpr[i]) == sizeof(int32_t))
809 setintV(L->top++, (int32_t)ex->gpr[i]);
810 else
811 setnumV(L->top++, (lua_Number)ex->gpr[i]);
812 }
813#if !LJ_SOFTFP
814 for (i = 0; i < RID_NUM_FPR; i++) {
815 setnumV(L->top, ex->fpr[i]);
816 if (LJ_UNLIKELY(tvisnan(L->top)))
817 setnanV(L->top);
818 L->top++;
819 }
820#endif
821}
822#endif
823
824#if defined(EXITSTATE_PCREG) || (LJ_UNWIND_JIT && !EXITTRACE_VMSTATE)
825/* Determine trace number from pc of exit instruction. */
826static TraceNo trace_exit_find(jit_State *J, MCode *pc)
827{
828 TraceNo traceno;
829 for (traceno = 1; traceno < J->sizetrace; traceno++) {
830 GCtrace *T = traceref(J, traceno);
831 if (T && pc >= T->mcode && pc < (MCode *)((char *)T->mcode + T->szmcode))
832 return traceno;
833 }
834 lj_assertJ(0, "bad exit pc");
835 return 0;
836}
837#endif
838
839/* A trace exited. Restore interpreter state. */
840int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
841{
842 ERRNO_SAVE
843 lua_State *L = J->L;
844 ExitState *ex = (ExitState *)exptr;
845 ExitDataCP exd;
846 int errcode, exitcode = J->exitcode;
847 TValue exiterr;
848 const BCIns *pc;
849 void *cf;
850 GCtrace *T;
851
852 setnilV(&exiterr);
853 if (exitcode) { /* Trace unwound with error code. */
854 J->exitcode = 0;
855 copyTV(L, &exiterr, L->top-1);
856 }
857
858#ifdef EXITSTATE_PCREG
859 J->parent = trace_exit_find(J, (MCode *)(intptr_t)ex->gpr[EXITSTATE_PCREG]);
860#endif
861 T = traceref(J, J->parent); UNUSED(T);
862#ifdef EXITSTATE_CHECKEXIT
863 if (J->exitno == T->nsnap) { /* Treat stack check like a parent exit. */
864 lj_assertJ(T->root != 0, "stack check in root trace");
865 J->exitno = T->ir[REF_BASE].op2;
866 J->parent = T->ir[REF_BASE].op1;
867 T = traceref(J, J->parent);
868 }
869#endif
870 lj_assertJ(T != NULL && J->exitno < T->nsnap, "bad trace or exit number");
871 exd.J = J;
872 exd.exptr = exptr;
873 errcode = lj_vm_cpcall(L, NULL, &exd, trace_exit_cp);
874 if (errcode)
875 return -errcode; /* Return negated error code. */
876
877 if (exitcode) copyTV(L, L->top++, &exiterr); /* Anchor the error object. */
878
879 if (!(LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)))
880 lj_vmevent_send(L, TEXIT,
881 lj_state_checkstack(L, 4+RID_NUM_GPR+RID_NUM_FPR+LUA_MINSTACK);
882 setintV(L->top++, J->parent);
883 setintV(L->top++, J->exitno);
884 trace_exit_regs(L, ex);
885 );
886
887 pc = exd.pc;
888 cf = cframe_raw(L->cframe);
889 setcframe_pc(cf, pc);
890 if (exitcode) {
891 return -exitcode;
892 } else if (LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)) {
893 /* Just exit to interpreter. */
894 } else if (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize) {
895 if (!(G(L)->hookmask & HOOK_GC))
896 lj_gc_step(L); /* Exited because of GC: drive GC forward. */
897 } else {
898 trace_hotside(J, pc);
899 }
900 if (bc_op(*pc) == BC_JLOOP) {
901 BCIns *retpc = &traceref(J, bc_d(*pc))->startins;
902 if (bc_isret(bc_op(*retpc))) {
903 if (J->state == LJ_TRACE_RECORD) {
904 J->patchins = *pc;
905 J->patchpc = (BCIns *)pc;
906 *J->patchpc = *retpc;
907 J->bcskip = 1;
908 } else {
909 pc = retpc;
910 setcframe_pc(cf, pc);
911 }
912 }
913 }
914 /* Return MULTRES or 0. */
915 ERRNO_RESTORE
916 switch (bc_op(*pc)) {
917 case BC_CALLM: case BC_CALLMT:
918 return (int)((BCReg)(L->top - L->base) - bc_a(*pc) - bc_c(*pc) - LJ_FR2);
919 case BC_RETM:
920 return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc) - bc_d(*pc));
921 case BC_TSETM:
922 return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc));
923 default:
924 if (bc_op(*pc) >= BC_FUNCF)
925 return (int)((BCReg)(L->top - L->base) + 1);
926 return 0;
927 }
928}
929
930#if LJ_UNWIND_JIT
931/* Given an mcode address determine trace exit address for unwinding. */
932uintptr_t LJ_FASTCALL lj_trace_unwind(jit_State *J, uintptr_t addr, ExitNo *ep)
933{
934#if EXITTRACE_VMSTATE
935 TraceNo traceno = J2G(J)->vmstate;
936#else
937 TraceNo traceno = trace_exit_find(J, (MCode *)addr);
938#endif
939 GCtrace *T = traceref(J, traceno);
940 if (T
941#if EXITTRACE_VMSTATE
942 && addr >= (uintptr_t)T->mcode && addr < (uintptr_t)T->mcode + T->szmcode
943#endif
944 ) {
945 SnapShot *snap = T->snap;
946 SnapNo lo = 0, exitno = T->nsnap;
947 uintptr_t ofs = (uintptr_t)((MCode *)addr - T->mcode); /* MCode units! */
948 /* Rightmost binary search for mcode offset to determine exit number. */
949 do {
950 SnapNo mid = (lo+exitno) >> 1;
951 if (ofs < snap[mid].mcofs) exitno = mid; else lo = mid + 1;
952 } while (lo < exitno);
953 exitno--;
954 *ep = exitno;
955#ifdef EXITSTUBS_PER_GROUP
956 return (uintptr_t)exitstub_addr(J, exitno);
957#else
958 return (uintptr_t)exitstub_trace_addr(T, exitno);
959#endif
960 }
961 /* Cannot correlate addr with trace/exit. This will be fatal. */
962 lj_assertJ(0, "bad exit pc");
963 return 0;
964}
965#endif
966
967#endif
968