1/*
2** Trace management.
3** Copyright (C) 2005-2014 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#define lj_trace_c
7#define LUA_CORE
8
9#include "lj_obj.h"
10
11#if LJ_HASJIT
12
13#include "lj_gc.h"
14#include "lj_err.h"
15#include "lj_debug.h"
16#include "lj_str.h"
17#include "lj_frame.h"
18#include "lj_state.h"
19#include "lj_bc.h"
20#include "lj_ir.h"
21#include "lj_jit.h"
22#include "lj_iropt.h"
23#include "lj_mcode.h"
24#include "lj_trace.h"
25#include "lj_snap.h"
26#include "lj_gdbjit.h"
27#include "lj_record.h"
28#include "lj_asm.h"
29#include "lj_dispatch.h"
30#include "lj_vm.h"
31#include "lj_vmevent.h"
32#include "lj_target.h"
33
34/* -- Error handling ------------------------------------------------------ */
35
36/* Synchronous abort with error message. */
37void lj_trace_err(jit_State *J, TraceError e)
38{
39 setnilV(&J->errinfo); /* No error info. */
40 setintV(J->L->top++, (int32_t)e);
41 lj_err_throw(J->L, LUA_ERRRUN);
42}
43
44/* Synchronous abort with error message and error info. */
45void lj_trace_err_info(jit_State *J, TraceError e)
46{
47 setintV(J->L->top++, (int32_t)e);
48 lj_err_throw(J->L, LUA_ERRRUN);
49}
50
51/* -- Trace management ---------------------------------------------------- */
52
53/* The current trace is first assembled in J->cur. The variable length
54** arrays point to shared, growable buffers (J->irbuf etc.). When trace
55** recording ends successfully, the current trace and its data structures
56** are copied to a new (compact) GCtrace object.
57*/
58
59/* Find a free trace number. */
60static TraceNo trace_findfree(jit_State *J)
61{
62 MSize osz, lim;
63 if (J->freetrace == 0)
64 J->freetrace = 1;
65 for (; J->freetrace < J->sizetrace; J->freetrace++)
66 if (traceref(J, J->freetrace) == NULL)
67 return J->freetrace++;
68 /* Need to grow trace array. */
69 lim = (MSize)J->param[JIT_P_maxtrace] + 1;
70 if (lim < 2) lim = 2; else if (lim > 65535) lim = 65535;
71 osz = J->sizetrace;
72 if (osz >= lim)
73 return 0; /* Too many traces. */
74 lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, GCRef);
75 for (; osz < J->sizetrace; osz++)
76 setgcrefnull(J->trace[osz]);
77 return J->freetrace;
78}
79
80#define TRACE_APPENDVEC(field, szfield, tp) \
81 T->field = (tp *)p; \
82 memcpy(p, J->cur.field, J->cur.szfield*sizeof(tp)); \
83 p += J->cur.szfield*sizeof(tp);
84
85#ifdef LUAJIT_USE_PERFTOOLS
86/*
87** Create symbol table of JIT-compiled code. For use with Linux perf tools.
88** Example usage:
89** perf record -f -e cycles luajit test.lua
90** perf report -s symbol
91** rm perf.data /tmp/perf-*.map
92*/
93#include <stdio.h>
94#include <unistd.h>
95
96static void perftools_addtrace(GCtrace *T)
97{
98 static FILE *fp;
99 GCproto *pt = &gcref(T->startpt)->pt;
100 const BCIns *startpc = mref(T->startpc, const BCIns);
101 const char *name = proto_chunknamestr(pt);
102 BCLine lineno;
103 if (name[0] == '@' || name[0] == '=')
104 name++;
105 else
106 name = "(string)";
107 lua_assert(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc);
108 lineno = lj_debug_line(pt, proto_bcpos(pt, startpc));
109 if (!fp) {
110 char fname[40];
111 sprintf(fname, "/tmp/perf-%d.map", getpid());
112 if (!(fp = fopen(fname, "w"))) return;
113 setlinebuf(fp);
114 }
115 fprintf(fp, "%lx %x TRACE_%d::%s:%u\n",
116 (long)T->mcode, T->szmcode, T->traceno, name, lineno);
117}
118#endif
119
120/* Save current trace by copying and compacting it. */
121static void trace_save(jit_State *J)
122{
123 size_t sztr = ((sizeof(GCtrace)+7)&~7);
124 size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns);
125 size_t sz = sztr + szins +
126 J->cur.nsnap*sizeof(SnapShot) +
127 J->cur.nsnapmap*sizeof(SnapEntry);
128 GCtrace *T = lj_mem_newt(J->L, (MSize)sz, GCtrace);
129 char *p = (char *)T + sztr;
130 memcpy(T, &J->cur, sizeof(GCtrace));
131 setgcrefr(T->nextgc, J2G(J)->gc.root);
132 setgcrefp(J2G(J)->gc.root, T);
133 newwhite(J2G(J), T);
134 T->gct = ~LJ_TTRACE;
135 T->ir = (IRIns *)p - J->cur.nk;
136 memcpy(p, J->cur.ir+J->cur.nk, szins);
137 p += szins;
138 TRACE_APPENDVEC(snap, nsnap, SnapShot)
139 TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry)
140 J->cur.traceno = 0;
141 setgcrefp(J->trace[T->traceno], T);
142 lj_gc_barriertrace(J2G(J), T->traceno);
143 lj_gdbjit_addtrace(J, T);
144#ifdef LUAJIT_USE_PERFTOOLS
145 perftools_addtrace(T);
146#endif
147}
148
149void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T)
150{
151 jit_State *J = G2J(g);
152 if (T->traceno) {
153 lj_gdbjit_deltrace(J, T);
154 if (T->traceno < J->freetrace)
155 J->freetrace = T->traceno;
156 setgcrefnull(J->trace[T->traceno]);
157 }
158 lj_mem_free(g, T,
159 ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
160 T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry));
161}
162
163/* Re-enable compiling a prototype by unpatching any modified bytecode. */
164void lj_trace_reenableproto(GCproto *pt)
165{
166 if ((pt->flags & PROTO_ILOOP)) {
167 BCIns *bc = proto_bc(pt);
168 BCPos i, sizebc = pt->sizebc;;
169 pt->flags &= ~PROTO_ILOOP;
170 if (bc_op(bc[0]) == BC_IFUNCF)
171 setbc_op(&bc[0], BC_FUNCF);
172 for (i = 1; i < sizebc; i++) {
173 BCOp op = bc_op(bc[i]);
174 if (op == BC_IFORL || op == BC_IITERL || op == BC_ILOOP)
175 setbc_op(&bc[i], (int)op+(int)BC_LOOP-(int)BC_ILOOP);
176 }
177 }
178}
179
180/* Unpatch the bytecode modified by a root trace. */
181static void trace_unpatch(jit_State *J, GCtrace *T)
182{
183 BCOp op = bc_op(T->startins);
184 BCIns *pc = mref(T->startpc, BCIns);
185 UNUSED(J);
186 if (op == BC_JMP)
187 return; /* No need to unpatch branches in parent traces (yet). */
188 switch (bc_op(*pc)) {
189 case BC_JFORL:
190 lua_assert(traceref(J, bc_d(*pc)) == T);
191 *pc = T->startins;
192 pc += bc_j(T->startins);
193 lua_assert(bc_op(*pc) == BC_JFORI);
194 setbc_op(pc, BC_FORI);
195 break;
196 case BC_JITERL:
197 case BC_JLOOP:
198 lua_assert(op == BC_ITERL || op == BC_LOOP || bc_isret(op));
199 *pc = T->startins;
200 break;
201 case BC_JMP:
202 lua_assert(op == BC_ITERL);
203 pc += bc_j(*pc)+2;
204 if (bc_op(*pc) == BC_JITERL) {
205 lua_assert(traceref(J, bc_d(*pc)) == T);
206 *pc = T->startins;
207 }
208 break;
209 case BC_JFUNCF:
210 lua_assert(op == BC_FUNCF);
211 *pc = T->startins;
212 break;
213 default: /* Already unpatched. */
214 break;
215 }
216}
217
218/* Flush a root trace. */
219static void trace_flushroot(jit_State *J, GCtrace *T)
220{
221 GCproto *pt = &gcref(T->startpt)->pt;
222 lua_assert(T->root == 0 && pt != NULL);
223 /* First unpatch any modified bytecode. */
224 trace_unpatch(J, T);
225 /* Unlink root trace from chain anchored in prototype. */
226 if (pt->trace == T->traceno) { /* Trace is first in chain. Easy. */
227 pt->trace = T->nextroot;
228 } else if (pt->trace) { /* Otherwise search in chain of root traces. */
229 GCtrace *T2 = traceref(J, pt->trace);
230 if (T2) {
231 for (; T2->nextroot; T2 = traceref(J, T2->nextroot))
232 if (T2->nextroot == T->traceno) {
233 T2->nextroot = T->nextroot; /* Unlink from chain. */
234 break;
235 }
236 }
237 }
238}
239
240/* Flush a trace. Only root traces are considered. */
241void lj_trace_flush(jit_State *J, TraceNo traceno)
242{
243 if (traceno > 0 && traceno < J->sizetrace) {
244 GCtrace *T = traceref(J, traceno);
245 if (T && T->root == 0)
246 trace_flushroot(J, T);
247 }
248}
249
250/* Flush all traces associated with a prototype. */
251void lj_trace_flushproto(global_State *g, GCproto *pt)
252{
253 while (pt->trace != 0)
254 trace_flushroot(G2J(g), traceref(G2J(g), pt->trace));
255}
256
257/* Flush all traces. */
258int lj_trace_flushall(lua_State *L)
259{
260 jit_State *J = L2J(L);
261 ptrdiff_t i;
262 if ((J2G(J)->hookmask & HOOK_GC))
263 return 1;
264 for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--) {
265 GCtrace *T = traceref(J, i);
266 if (T) {
267 if (T->root == 0)
268 trace_flushroot(J, T);
269 lj_gdbjit_deltrace(J, T);
270 T->traceno = 0;
271 setgcrefnull(J->trace[i]);
272 }
273 }
274 J->cur.traceno = 0;
275 J->freetrace = 0;
276 /* Clear penalty cache. */
277 memset(J->penalty, 0, sizeof(J->penalty));
278 /* Free the whole machine code and invalidate all exit stub groups. */
279 lj_mcode_free(J);
280 memset(J->exitstubgroup, 0, sizeof(J->exitstubgroup));
281 lj_vmevent_send(L, TRACE,
282 setstrV(L, L->top++, lj_str_newlit(L, "flush"));
283 );
284 return 0;
285}
286
287/* Initialize JIT compiler state. */
288void lj_trace_initstate(global_State *g)
289{
290 jit_State *J = G2J(g);
291 TValue *tv;
292 /* Initialize SIMD constants. */
293 tv = LJ_KSIMD(J, LJ_KSIMD_ABS);
294 tv[0].u64 = U64x(7fffffff,ffffffff);
295 tv[1].u64 = U64x(7fffffff,ffffffff);
296 tv = LJ_KSIMD(J, LJ_KSIMD_NEG);
297 tv[0].u64 = U64x(80000000,00000000);
298 tv[1].u64 = U64x(80000000,00000000);
299}
300
301/* Free everything associated with the JIT compiler state. */
302void lj_trace_freestate(global_State *g)
303{
304 jit_State *J = G2J(g);
305#ifdef LUA_USE_ASSERT
306 { /* This assumes all traces have already been freed. */
307 ptrdiff_t i;
308 for (i = 1; i < (ptrdiff_t)J->sizetrace; i++)
309 lua_assert(i == (ptrdiff_t)J->cur.traceno || traceref(J, i) == NULL);
310 }
311#endif
312 lj_mcode_free(J);
313 lj_ir_k64_freeall(J);
314 lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry);
315 lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot);
316 lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns);
317 lj_mem_freevec(g, J->trace, J->sizetrace, GCRef);
318}
319
320/* -- Penalties and blacklisting ------------------------------------------ */
321
322/* Blacklist a bytecode instruction. */
323static void blacklist_pc(GCproto *pt, BCIns *pc)
324{
325 setbc_op(pc, (int)bc_op(*pc)+(int)BC_ILOOP-(int)BC_LOOP);
326 pt->flags |= PROTO_ILOOP;
327}
328
329/* Penalize a bytecode instruction. */
330static void penalty_pc(jit_State *J, GCproto *pt, BCIns *pc, TraceError e)
331{
332 uint32_t i, val = PENALTY_MIN;
333 for (i = 0; i < PENALTY_SLOTS; i++)
334 if (mref(J->penalty[i].pc, const BCIns) == pc) { /* Cache slot found? */
335 /* First try to bump its hotcount several times. */
336 val = ((uint32_t)J->penalty[i].val << 1) +
337 LJ_PRNG_BITS(J, PENALTY_RNDBITS);
338 if (val > PENALTY_MAX) {
339 blacklist_pc(pt, pc); /* Blacklist it, if that didn't help. */
340 return;
341 }
342 goto setpenalty;
343 }
344 /* Assign a new penalty cache slot. */
345 i = J->penaltyslot;
346 J->penaltyslot = (J->penaltyslot + 1) & (PENALTY_SLOTS-1);
347 setmref(J->penalty[i].pc, pc);
348setpenalty:
349 J->penalty[i].val = (uint16_t)val;
350 J->penalty[i].reason = e;
351 hotcount_set(J2GG(J), pc+1, val);
352}
353
354/* -- Trace compiler state machine ---------------------------------------- */
355
356/* Start tracing. */
357static void trace_start(jit_State *J)
358{
359 lua_State *L;
360 TraceNo traceno;
361
362 if ((J->pt->flags & PROTO_NOJIT)) { /* JIT disabled for this proto? */
363 if (J->parent == 0) {
364 /* Lazy bytecode patching to disable hotcount events. */
365 lua_assert(bc_op(*J->pc) == BC_FORL || bc_op(*J->pc) == BC_ITERL ||
366 bc_op(*J->pc) == BC_LOOP || bc_op(*J->pc) == BC_FUNCF);
367 setbc_op(J->pc, (int)bc_op(*J->pc)+(int)BC_ILOOP-(int)BC_LOOP);
368 J->pt->flags |= PROTO_ILOOP;
369 }
370 J->state = LJ_TRACE_IDLE; /* Silently ignored. */
371 return;
372 }
373
374 /* Get a new trace number. */
375 traceno = trace_findfree(J);
376 if (LJ_UNLIKELY(traceno == 0)) { /* No free trace? */
377 lua_assert((J2G(J)->hookmask & HOOK_GC) == 0);
378 lj_trace_flushall(J->L);
379 J->state = LJ_TRACE_IDLE; /* Silently ignored. */
380 return;
381 }
382 setgcrefp(J->trace[traceno], &J->cur);
383
384 /* Setup enough of the current trace to be able to send the vmevent. */
385 memset(&J->cur, 0, sizeof(GCtrace));
386 J->cur.traceno = traceno;
387 J->cur.nins = J->cur.nk = REF_BASE;
388 J->cur.ir = J->irbuf;
389 J->cur.snap = J->snapbuf;
390 J->cur.snapmap = J->snapmapbuf;
391 J->mergesnap = 0;
392 J->needsnap = 0;
393 J->bcskip = 0;
394 J->guardemit.irt = 0;
395 J->postproc = LJ_POST_NONE;
396 lj_resetsplit(J);
397 setgcref(J->cur.startpt, obj2gco(J->pt));
398
399 L = J->L;
400 lj_vmevent_send(L, TRACE,
401 setstrV(L, L->top++, lj_str_newlit(L, "start"));
402 setintV(L->top++, traceno);
403 setfuncV(L, L->top++, J->fn);
404 setintV(L->top++, proto_bcpos(J->pt, J->pc));
405 if (J->parent) {
406 setintV(L->top++, J->parent);
407 setintV(L->top++, J->exitno);
408 }
409 );
410 lj_record_setup(J);
411}
412
413/* Stop tracing. */
414static void trace_stop(jit_State *J)
415{
416 BCIns *pc = mref(J->cur.startpc, BCIns);
417 BCOp op = bc_op(J->cur.startins);
418 GCproto *pt = &gcref(J->cur.startpt)->pt;
419 TraceNo traceno = J->cur.traceno;
420 lua_State *L;
421
422 switch (op) {
423 case BC_FORL:
424 setbc_op(pc+bc_j(J->cur.startins), BC_JFORI); /* Patch FORI, too. */
425 /* fallthrough */
426 case BC_LOOP:
427 case BC_ITERL:
428 case BC_FUNCF:
429 /* Patch bytecode of starting instruction in root trace. */
430 setbc_op(pc, (int)op+(int)BC_JLOOP-(int)BC_LOOP);
431 setbc_d(pc, traceno);
432 addroot:
433 /* Add to root trace chain in prototype. */
434 J->cur.nextroot = pt->trace;
435 pt->trace = (TraceNo1)traceno;
436 break;
437 case BC_RET:
438 case BC_RET0:
439 case BC_RET1:
440 *pc = BCINS_AD(BC_JLOOP, J->cur.snap[0].nslots, traceno);
441 goto addroot;
442 case BC_JMP:
443 /* Patch exit branch in parent to side trace entry. */
444 lua_assert(J->parent != 0 && J->cur.root != 0);
445 lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode);
446 /* Avoid compiling a side trace twice (stack resizing uses parent exit). */
447 traceref(J, J->parent)->snap[J->exitno].count = SNAPCOUNT_DONE;
448 /* Add to side trace chain in root trace. */
449 {
450 GCtrace *root = traceref(J, J->cur.root);
451 root->nchild++;
452 J->cur.nextside = root->nextside;
453 root->nextside = (TraceNo1)traceno;
454 }
455 break;
456 default:
457 lua_assert(0);
458 break;
459 }
460
461 /* Commit new mcode only after all patching is done. */
462 lj_mcode_commit(J, J->cur.mcode);
463 J->postproc = LJ_POST_NONE;
464 trace_save(J);
465
466 L = J->L;
467 lj_vmevent_send(L, TRACE,
468 setstrV(L, L->top++, lj_str_newlit(L, "stop"));
469 setintV(L->top++, traceno);
470 );
471}
472
473/* Start a new root trace for down-recursion. */
474static int trace_downrec(jit_State *J)
475{
476 /* Restart recording at the return instruction. */
477 lua_assert(J->pt != NULL);
478 lua_assert(bc_isret(bc_op(*J->pc)));
479 if (bc_op(*J->pc) == BC_RETM)
480 return 0; /* NYI: down-recursion with RETM. */
481 J->parent = 0;
482 J->exitno = 0;
483 J->state = LJ_TRACE_RECORD;
484 trace_start(J);
485 return 1;
486}
487
488/* Abort tracing. */
489static int trace_abort(jit_State *J)
490{
491 lua_State *L = J->L;
492 TraceError e = LJ_TRERR_RECERR;
493 TraceNo traceno;
494
495 J->postproc = LJ_POST_NONE;
496 lj_mcode_abort(J);
497 if (tvisnumber(L->top-1))
498 e = (TraceError)numberVint(L->top-1);
499 if (e == LJ_TRERR_MCODELM) {
500 L->top--; /* Remove error object */
501 J->state = LJ_TRACE_ASM;
502 return 1; /* Retry ASM with new MCode area. */
503 }
504 /* Penalize or blacklist starting bytecode instruction. */
505 if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins)))
506 penalty_pc(J, &gcref(J->cur.startpt)->pt, mref(J->cur.startpc, BCIns), e);
507
508 /* Is there anything to abort? */
509 traceno = J->cur.traceno;
510 if (traceno) {
511 ptrdiff_t errobj = savestack(L, L->top-1); /* Stack may be resized. */
512 J->cur.link = 0;
513 J->cur.linktype = LJ_TRLINK_NONE;
514 lj_vmevent_send(L, TRACE,
515 TValue *frame;
516 const BCIns *pc;
517 GCfunc *fn;
518 setstrV(L, L->top++, lj_str_newlit(L, "abort"));
519 setintV(L->top++, traceno);
520 /* Find original Lua function call to generate a better error message. */
521 frame = J->L->base-1;
522 pc = J->pc;
523 while (!isluafunc(frame_func(frame))) {
524 pc = (frame_iscont(frame) ? frame_contpc(frame) : frame_pc(frame)) - 1;
525 frame = frame_prev(frame);
526 }
527 fn = frame_func(frame);
528 setfuncV(L, L->top++, fn);
529 setintV(L->top++, proto_bcpos(funcproto(fn), pc));
530 copyTV(L, L->top++, restorestack(L, errobj));
531 copyTV(L, L->top++, &J->errinfo);
532 );
533 /* Drop aborted trace after the vmevent (which may still access it). */
534 setgcrefnull(J->trace[traceno]);
535 if (traceno < J->freetrace)
536 J->freetrace = traceno;
537 J->cur.traceno = 0;
538 }
539 L->top--; /* Remove error object */
540 if (e == LJ_TRERR_DOWNREC)
541 return trace_downrec(J);
542 else if (e == LJ_TRERR_MCODEAL)
543 lj_trace_flushall(L);
544 return 0;
545}
546
547/* Perform pending re-patch of a bytecode instruction. */
548static LJ_AINLINE void trace_pendpatch(jit_State *J, int force)
549{
550 if (LJ_UNLIKELY(J->patchpc)) {
551 if (force || J->bcskip == 0) {
552 *J->patchpc = J->patchins;
553 J->patchpc = NULL;
554 } else {
555 J->bcskip = 0;
556 }
557 }
558}
559
560/* State machine for the trace compiler. Protected callback. */
561static TValue *trace_state(lua_State *L, lua_CFunction dummy, void *ud)
562{
563 jit_State *J = (jit_State *)ud;
564 UNUSED(dummy);
565 do {
566 retry:
567 switch (J->state) {
568 case LJ_TRACE_START:
569 J->state = LJ_TRACE_RECORD; /* trace_start() may change state. */
570 trace_start(J);
571 lj_dispatch_update(J2G(J));
572 break;
573
574 case LJ_TRACE_RECORD:
575 trace_pendpatch(J, 0);
576 setvmstate(J2G(J), RECORD);
577 lj_vmevent_send_(L, RECORD,
578 /* Save/restore tmptv state for trace recorder. */
579 TValue savetv = J2G(J)->tmptv;
580 TValue savetv2 = J2G(J)->tmptv2;
581 setintV(L->top++, J->cur.traceno);
582 setfuncV(L, L->top++, J->fn);
583 setintV(L->top++, J->pt ? (int32_t)proto_bcpos(J->pt, J->pc) : -1);
584 setintV(L->top++, J->framedepth);
585 ,
586 J2G(J)->tmptv = savetv;
587 J2G(J)->tmptv2 = savetv2;
588 );
589 lj_record_ins(J);
590 break;
591
592 case LJ_TRACE_END:
593 trace_pendpatch(J, 1);
594 J->loopref = 0;
595 if ((J->flags & JIT_F_OPT_LOOP) &&
596 J->cur.link == J->cur.traceno && J->framedepth + J->retdepth == 0) {
597 setvmstate(J2G(J), OPT);
598 lj_opt_dce(J);
599 if (lj_opt_loop(J)) { /* Loop optimization failed? */
600 J->cur.link = 0;
601 J->cur.linktype = LJ_TRLINK_NONE;
602 J->loopref = J->cur.nins;
603 J->state = LJ_TRACE_RECORD; /* Try to continue recording. */
604 break;
605 }
606 J->loopref = J->chain[IR_LOOP]; /* Needed by assembler. */
607 }
608 lj_opt_split(J);
609 lj_opt_sink(J);
610 J->state = LJ_TRACE_ASM;
611 break;
612
613 case LJ_TRACE_ASM:
614 setvmstate(J2G(J), ASM);
615 lj_asm_trace(J, &J->cur);
616 trace_stop(J);
617 setvmstate(J2G(J), INTERP);
618 J->state = LJ_TRACE_IDLE;
619 lj_dispatch_update(J2G(J));
620 return NULL;
621
622 default: /* Trace aborted asynchronously. */
623 setintV(L->top++, (int32_t)LJ_TRERR_RECERR);
624 /* fallthrough */
625 case LJ_TRACE_ERR:
626 trace_pendpatch(J, 1);
627 if (trace_abort(J))
628 goto retry;
629 setvmstate(J2G(J), INTERP);
630 J->state = LJ_TRACE_IDLE;
631 lj_dispatch_update(J2G(J));
632 return NULL;
633 }
634 } while (J->state > LJ_TRACE_RECORD);
635 return NULL;
636}
637
638/* -- Event handling ------------------------------------------------------ */
639
640/* A bytecode instruction is about to be executed. Record it. */
641void lj_trace_ins(jit_State *J, const BCIns *pc)
642{
643 /* Note: J->L must already be set. pc is the true bytecode PC here. */
644 J->pc = pc;
645 J->fn = curr_func(J->L);
646 J->pt = isluafunc(J->fn) ? funcproto(J->fn) : NULL;
647 while (lj_vm_cpcall(J->L, NULL, (void *)J, trace_state) != 0)
648 J->state = LJ_TRACE_ERR;
649}
650
651/* A hotcount triggered. Start recording a root trace. */
652void LJ_FASTCALL lj_trace_hot(jit_State *J, const BCIns *pc)
653{
654 /* Note: pc is the interpreter bytecode PC here. It's offset by 1. */
655 ERRNO_SAVE
656 /* Reset hotcount. */
657 hotcount_set(J2GG(J), pc, J->param[JIT_P_hotloop]*HOTCOUNT_LOOP);
658 /* Only start a new trace if not recording or inside __gc call or vmevent. */
659 if (J->state == LJ_TRACE_IDLE &&
660 !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
661 J->parent = 0; /* Root trace. */
662 J->exitno = 0;
663 J->state = LJ_TRACE_START;
664 lj_trace_ins(J, pc-1);
665 }
666 ERRNO_RESTORE
667}
668
669/* Check for a hot side exit. If yes, start recording a side trace. */
670static void trace_hotside(jit_State *J, const BCIns *pc)
671{
672 SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno];
673 if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) &&
674 snap->count != SNAPCOUNT_DONE &&
675 ++snap->count >= J->param[JIT_P_hotexit]) {
676 lua_assert(J->state == LJ_TRACE_IDLE);
677 /* J->parent is non-zero for a side trace. */
678 J->state = LJ_TRACE_START;
679 lj_trace_ins(J, pc);
680 }
681}
682
683/* Tiny struct to pass data to protected call. */
684typedef struct ExitDataCP {
685 jit_State *J;
686 void *exptr; /* Pointer to exit state. */
687 const BCIns *pc; /* Restart interpreter at this PC. */
688} ExitDataCP;
689
690/* Need to protect lj_snap_restore because it may throw. */
691static TValue *trace_exit_cp(lua_State *L, lua_CFunction dummy, void *ud)
692{
693 ExitDataCP *exd = (ExitDataCP *)ud;
694 cframe_errfunc(L->cframe) = -1; /* Inherit error function. */
695 exd->pc = lj_snap_restore(exd->J, exd->exptr);
696 UNUSED(dummy);
697 return NULL;
698}
699
700#ifndef LUAJIT_DISABLE_VMEVENT
701/* Push all registers from exit state. */
702static void trace_exit_regs(lua_State *L, ExitState *ex)
703{
704 int32_t i;
705 setintV(L->top++, RID_NUM_GPR);
706 setintV(L->top++, RID_NUM_FPR);
707 for (i = 0; i < RID_NUM_GPR; i++) {
708 if (sizeof(ex->gpr[i]) == sizeof(int32_t))
709 setintV(L->top++, (int32_t)ex->gpr[i]);
710 else
711 setnumV(L->top++, (lua_Number)ex->gpr[i]);
712 }
713#if !LJ_SOFTFP
714 for (i = 0; i < RID_NUM_FPR; i++) {
715 setnumV(L->top, ex->fpr[i]);
716 if (LJ_UNLIKELY(tvisnan(L->top)))
717 setnanV(L->top);
718 L->top++;
719 }
720#endif
721}
722#endif
723
724#ifdef EXITSTATE_PCREG
725/* Determine trace number from pc of exit instruction. */
726static TraceNo trace_exit_find(jit_State *J, MCode *pc)
727{
728 TraceNo traceno;
729 for (traceno = 1; traceno < J->sizetrace; traceno++) {
730 GCtrace *T = traceref(J, traceno);
731 if (T && pc >= T->mcode && pc < (MCode *)((char *)T->mcode + T->szmcode))
732 return traceno;
733 }
734 lua_assert(0);
735 return 0;
736}
737#endif
738
739/* A trace exited. Restore interpreter state. */
740int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr)
741{
742 ERRNO_SAVE
743 lua_State *L = J->L;
744 ExitState *ex = (ExitState *)exptr;
745 ExitDataCP exd;
746 int errcode;
747 const BCIns *pc;
748 void *cf;
749 GCtrace *T;
750#ifdef EXITSTATE_PCREG
751 J->parent = trace_exit_find(J, (MCode *)(intptr_t)ex->gpr[EXITSTATE_PCREG]);
752#endif
753 T = traceref(J, J->parent); UNUSED(T);
754#ifdef EXITSTATE_CHECKEXIT
755 if (J->exitno == T->nsnap) { /* Treat stack check like a parent exit. */
756 lua_assert(T->root != 0);
757 J->exitno = T->ir[REF_BASE].op2;
758 J->parent = T->ir[REF_BASE].op1;
759 T = traceref(J, J->parent);
760 }
761#endif
762 lua_assert(T != NULL && J->exitno < T->nsnap);
763 exd.J = J;
764 exd.exptr = exptr;
765 errcode = lj_vm_cpcall(L, NULL, &exd, trace_exit_cp);
766 if (errcode)
767 return -errcode; /* Return negated error code. */
768
769 lj_vmevent_send(L, TEXIT,
770 lj_state_checkstack(L, 4+RID_NUM_GPR+RID_NUM_FPR+LUA_MINSTACK);
771 setintV(L->top++, J->parent);
772 setintV(L->top++, J->exitno);
773 trace_exit_regs(L, ex);
774 );
775
776 pc = exd.pc;
777 cf = cframe_raw(L->cframe);
778 setcframe_pc(cf, pc);
779 if (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize) {
780 if (!(G(L)->hookmask & HOOK_GC))
781 lj_gc_step(L); /* Exited because of GC: drive GC forward. */
782 } else {
783 trace_hotside(J, pc);
784 }
785 if (bc_op(*pc) == BC_JLOOP) {
786 BCIns *retpc = &traceref(J, bc_d(*pc))->startins;
787 if (bc_isret(bc_op(*retpc))) {
788 if (J->state == LJ_TRACE_RECORD) {
789 J->patchins = *pc;
790 J->patchpc = (BCIns *)pc;
791 *J->patchpc = *retpc;
792 J->bcskip = 1;
793 } else {
794 pc = retpc;
795 setcframe_pc(cf, pc);
796 }
797 }
798 }
799 /* Return MULTRES or 0. */
800 ERRNO_RESTORE
801 switch (bc_op(*pc)) {
802 case BC_CALLM: case BC_CALLMT:
803 return (int)((BCReg)(L->top - L->base) - bc_a(*pc) - bc_c(*pc));
804 case BC_RETM:
805 return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc) - bc_d(*pc));
806 case BC_TSETM:
807 return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc));
808 default:
809 if (bc_op(*pc) >= BC_FUNCF)
810 return (int)((BCReg)(L->top - L->base) + 1);
811 return 0;
812 }
813}
814
815#endif
816