1/*
2** Instruction dispatch handling.
3** Copyright (C) 2005-2021 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#define lj_dispatch_c
7#define LUA_CORE
8
9#include "lj_obj.h"
10#include "lj_err.h"
11#include "lj_buf.h"
12#include "lj_func.h"
13#include "lj_str.h"
14#include "lj_tab.h"
15#include "lj_meta.h"
16#include "lj_debug.h"
17#include "lj_state.h"
18#include "lj_frame.h"
19#include "lj_bc.h"
20#include "lj_ff.h"
21#include "lj_strfmt.h"
22#if LJ_HASJIT
23#include "lj_jit.h"
24#endif
25#if LJ_HASFFI
26#include "lj_ccallback.h"
27#endif
28#include "lj_trace.h"
29#include "lj_dispatch.h"
30#if LJ_HASPROFILE
31#include "lj_profile.h"
32#endif
33#include "lj_vm.h"
34#include "luajit.h"
35
36/* Bump GG_NUM_ASMFF in lj_dispatch.h as needed. Ugly. */
37LJ_STATIC_ASSERT(GG_NUM_ASMFF == FF_NUM_ASMFUNC);
38
39/* -- Dispatch table management ------------------------------------------- */
40
41#if LJ_TARGET_MIPS
42#include <math.h>
43LJ_FUNCA_NORET void LJ_FASTCALL lj_ffh_coroutine_wrap_err(lua_State *L,
44 lua_State *co);
45#if !LJ_HASJIT
46#define lj_dispatch_stitch lj_dispatch_ins
47#endif
48#if !LJ_HASPROFILE
49#define lj_dispatch_profile lj_dispatch_ins
50#endif
51
52#define GOTFUNC(name) (ASMFunction)name,
53static const ASMFunction dispatch_got[] = {
54 GOTDEF(GOTFUNC)
55};
56#undef GOTFUNC
57#endif
58
59/* Initialize instruction dispatch table and hot counters. */
60void lj_dispatch_init(GG_State *GG)
61{
62 uint32_t i;
63 ASMFunction *disp = GG->dispatch;
64 for (i = 0; i < GG_LEN_SDISP; i++)
65 disp[GG_LEN_DDISP+i] = disp[i] = makeasmfunc(lj_bc_ofs[i]);
66 for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
67 disp[i] = makeasmfunc(lj_bc_ofs[i]);
68 /* The JIT engine is off by default. luaopen_jit() turns it on. */
69 disp[BC_FORL] = disp[BC_IFORL];
70 disp[BC_ITERL] = disp[BC_IITERL];
71 disp[BC_LOOP] = disp[BC_ILOOP];
72 disp[BC_FUNCF] = disp[BC_IFUNCF];
73 disp[BC_FUNCV] = disp[BC_IFUNCV];
74 GG->g.bc_cfunc_ext = GG->g.bc_cfunc_int = BCINS_AD(BC_FUNCC, LUA_MINSTACK, 0);
75 for (i = 0; i < GG_NUM_ASMFF; i++)
76 GG->bcff[i] = BCINS_AD(BC__MAX+i, 0, 0);
77#if LJ_TARGET_MIPS
78 memcpy(GG->got, dispatch_got, LJ_GOT__MAX*sizeof(ASMFunction *));
79#endif
80}
81
82#if LJ_HASJIT
83/* Initialize hotcount table. */
84void lj_dispatch_init_hotcount(global_State *g)
85{
86 int32_t hotloop = G2J(g)->param[JIT_P_hotloop];
87 HotCount start = (HotCount)(hotloop*HOTCOUNT_LOOP - 1);
88 HotCount *hotcount = G2GG(g)->hotcount;
89 uint32_t i;
90 for (i = 0; i < HOTCOUNT_SIZE; i++)
91 hotcount[i] = start;
92}
93#endif
94
95/* Internal dispatch mode bits. */
96#define DISPMODE_CALL 0x01 /* Override call dispatch. */
97#define DISPMODE_RET 0x02 /* Override return dispatch. */
98#define DISPMODE_INS 0x04 /* Override instruction dispatch. */
99#define DISPMODE_JIT 0x10 /* JIT compiler on. */
100#define DISPMODE_REC 0x20 /* Recording active. */
101#define DISPMODE_PROF 0x40 /* Profiling active. */
102
103/* Update dispatch table depending on various flags. */
104void lj_dispatch_update(global_State *g)
105{
106 uint8_t oldmode = g->dispatchmode;
107 uint8_t mode = 0;
108#if LJ_HASJIT
109 mode |= (G2J(g)->flags & JIT_F_ON) ? DISPMODE_JIT : 0;
110 mode |= G2J(g)->state != LJ_TRACE_IDLE ?
111 (DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0;
112#endif
113#if LJ_HASPROFILE
114 mode |= (g->hookmask & HOOK_PROFILE) ? (DISPMODE_PROF|DISPMODE_INS) : 0;
115#endif
116 mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0;
117 mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0;
118 mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0;
119 if (oldmode != mode) { /* Mode changed? */
120 ASMFunction *disp = G2GG(g)->dispatch;
121 ASMFunction f_forl, f_iterl, f_loop, f_funcf, f_funcv;
122 g->dispatchmode = mode;
123
124 /* Hotcount if JIT is on, but not while recording. */
125 if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) {
126 f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]);
127 f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]);
128 f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]);
129 f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]);
130 f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]);
131 } else { /* Otherwise use the non-hotcounting instructions. */
132 f_forl = disp[GG_LEN_DDISP+BC_IFORL];
133 f_iterl = disp[GG_LEN_DDISP+BC_IITERL];
134 f_loop = disp[GG_LEN_DDISP+BC_ILOOP];
135 f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]);
136 f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]);
137 }
138 /* Init static counting instruction dispatch first (may be copied below). */
139 disp[GG_LEN_DDISP+BC_FORL] = f_forl;
140 disp[GG_LEN_DDISP+BC_ITERL] = f_iterl;
141 disp[GG_LEN_DDISP+BC_LOOP] = f_loop;
142
143 /* Set dynamic instruction dispatch. */
144 if ((oldmode ^ mode) & (DISPMODE_PROF|DISPMODE_REC|DISPMODE_INS)) {
145 /* Need to update the whole table. */
146 if (!(mode & DISPMODE_INS)) { /* No ins dispatch? */
147 /* Copy static dispatch table to dynamic dispatch table. */
148 memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction));
149 /* Overwrite with dynamic return dispatch. */
150 if ((mode & DISPMODE_RET)) {
151 disp[BC_RETM] = lj_vm_rethook;
152 disp[BC_RET] = lj_vm_rethook;
153 disp[BC_RET0] = lj_vm_rethook;
154 disp[BC_RET1] = lj_vm_rethook;
155 }
156 } else {
157 /* The recording dispatch also checks for hooks. */
158 ASMFunction f = (mode & DISPMODE_PROF) ? lj_vm_profhook :
159 (mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook;
160 uint32_t i;
161 for (i = 0; i < GG_LEN_SDISP; i++)
162 disp[i] = f;
163 }
164 } else if (!(mode & DISPMODE_INS)) {
165 /* Otherwise set dynamic counting ins. */
166 disp[BC_FORL] = f_forl;
167 disp[BC_ITERL] = f_iterl;
168 disp[BC_LOOP] = f_loop;
169 /* Set dynamic return dispatch. */
170 if ((mode & DISPMODE_RET)) {
171 disp[BC_RETM] = lj_vm_rethook;
172 disp[BC_RET] = lj_vm_rethook;
173 disp[BC_RET0] = lj_vm_rethook;
174 disp[BC_RET1] = lj_vm_rethook;
175 } else {
176 disp[BC_RETM] = disp[GG_LEN_DDISP+BC_RETM];
177 disp[BC_RET] = disp[GG_LEN_DDISP+BC_RET];
178 disp[BC_RET0] = disp[GG_LEN_DDISP+BC_RET0];
179 disp[BC_RET1] = disp[GG_LEN_DDISP+BC_RET1];
180 }
181 }
182
183 /* Set dynamic call dispatch. */
184 if ((oldmode ^ mode) & DISPMODE_CALL) { /* Update the whole table? */
185 uint32_t i;
186 if ((mode & DISPMODE_CALL) == 0) { /* No call hooks? */
187 for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
188 disp[i] = makeasmfunc(lj_bc_ofs[i]);
189 } else {
190 for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
191 disp[i] = lj_vm_callhook;
192 }
193 }
194 if (!(mode & DISPMODE_CALL)) { /* Overwrite dynamic counting ins. */
195 disp[BC_FUNCF] = f_funcf;
196 disp[BC_FUNCV] = f_funcv;
197 }
198
199#if LJ_HASJIT
200 /* Reset hotcounts for JIT off to on transition. */
201 if ((mode & DISPMODE_JIT) && !(oldmode & DISPMODE_JIT))
202 lj_dispatch_init_hotcount(g);
203#endif
204 }
205}
206
207/* -- JIT mode setting ---------------------------------------------------- */
208
209#if LJ_HASJIT
210/* Set JIT mode for a single prototype. */
211static void setptmode(global_State *g, GCproto *pt, int mode)
212{
213 if ((mode & LUAJIT_MODE_ON)) { /* (Re-)enable JIT compilation. */
214 pt->flags &= ~PROTO_NOJIT;
215 lj_trace_reenableproto(pt); /* Unpatch all ILOOP etc. bytecodes. */
216 } else { /* Flush and/or disable JIT compilation. */
217 if (!(mode & LUAJIT_MODE_FLUSH))
218 pt->flags |= PROTO_NOJIT;
219 lj_trace_flushproto(g, pt); /* Flush all traces of prototype. */
220 }
221}
222
223/* Recursively set the JIT mode for all children of a prototype. */
224static void setptmode_all(global_State *g, GCproto *pt, int mode)
225{
226 ptrdiff_t i;
227 if (!(pt->flags & PROTO_CHILD)) return;
228 for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) {
229 GCobj *o = proto_kgc(pt, i);
230 if (o->gch.gct == ~LJ_TPROTO) {
231 setptmode(g, gco2pt(o), mode);
232 setptmode_all(g, gco2pt(o), mode);
233 }
234 }
235}
236#endif
237
238/* Public API function: control the JIT engine. */
239int luaJIT_setmode(lua_State *L, int idx, int mode)
240{
241 global_State *g = G(L);
242 int mm = mode & LUAJIT_MODE_MASK;
243 lj_trace_abort(g); /* Abort recording on any state change. */
244 /* Avoid pulling the rug from under our own feet. */
245 if ((g->hookmask & HOOK_GC))
246 lj_err_caller(L, LJ_ERR_NOGCMM);
247 switch (mm) {
248#if LJ_HASJIT
249 case LUAJIT_MODE_ENGINE:
250 if ((mode & LUAJIT_MODE_FLUSH)) {
251 lj_trace_flushall(L);
252 } else {
253 if (!(mode & LUAJIT_MODE_ON))
254 G2J(g)->flags &= ~(uint32_t)JIT_F_ON;
255 else
256 G2J(g)->flags |= (uint32_t)JIT_F_ON;
257 lj_dispatch_update(g);
258 }
259 break;
260 case LUAJIT_MODE_FUNC:
261 case LUAJIT_MODE_ALLFUNC:
262 case LUAJIT_MODE_ALLSUBFUNC: {
263 cTValue *tv = idx == 0 ? frame_prev(L->base-1)-LJ_FR2 :
264 idx > 0 ? L->base + (idx-1) : L->top + idx;
265 GCproto *pt;
266 if ((idx == 0 || tvisfunc(tv)) && isluafunc(&gcval(tv)->fn))
267 pt = funcproto(&gcval(tv)->fn); /* Cannot use funcV() for frame slot. */
268 else if (tvisproto(tv))
269 pt = protoV(tv);
270 else
271 return 0; /* Failed. */
272 if (mm != LUAJIT_MODE_ALLSUBFUNC)
273 setptmode(g, pt, mode);
274 if (mm != LUAJIT_MODE_FUNC)
275 setptmode_all(g, pt, mode);
276 break;
277 }
278 case LUAJIT_MODE_TRACE:
279 if (!(mode & LUAJIT_MODE_FLUSH))
280 return 0; /* Failed. */
281 lj_trace_flush(G2J(g), idx);
282 break;
283#else
284 case LUAJIT_MODE_ENGINE:
285 case LUAJIT_MODE_FUNC:
286 case LUAJIT_MODE_ALLFUNC:
287 case LUAJIT_MODE_ALLSUBFUNC:
288 UNUSED(idx);
289 if ((mode & LUAJIT_MODE_ON))
290 return 0; /* Failed. */
291 break;
292#endif
293 case LUAJIT_MODE_WRAPCFUNC:
294 if ((mode & LUAJIT_MODE_ON)) {
295 if (idx != 0) {
296 cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx;
297 if (tvislightud(tv))
298 g->wrapf = (lua_CFunction)lightudV(g, tv);
299 else
300 return 0; /* Failed. */
301 } else {
302 return 0; /* Failed. */
303 }
304 g->bc_cfunc_ext = BCINS_AD(BC_FUNCCW, 0, 0);
305 } else {
306 g->bc_cfunc_ext = BCINS_AD(BC_FUNCC, 0, 0);
307 }
308 break;
309 default:
310 return 0; /* Failed. */
311 }
312 return 1; /* OK. */
313}
314
315/* Enforce (dynamic) linker error for version mismatches. See luajit.c. */
316LUA_API void LUAJIT_VERSION_SYM(void)
317{
318}
319
320/* -- Hooks --------------------------------------------------------------- */
321
322/* This function can be called asynchronously (e.g. during a signal). */
323LUA_API int lua_sethook(lua_State *L, lua_Hook func, int mask, int count)
324{
325 global_State *g = G(L);
326 mask &= HOOK_EVENTMASK;
327 if (func == NULL || mask == 0) { mask = 0; func = NULL; } /* Consistency. */
328 g->hookf = func;
329 g->hookcount = g->hookcstart = (int32_t)count;
330 g->hookmask = (uint8_t)((g->hookmask & ~HOOK_EVENTMASK) | mask);
331 lj_trace_abort(g); /* Abort recording on any hook change. */
332 lj_dispatch_update(g);
333 return 1;
334}
335
336LUA_API lua_Hook lua_gethook(lua_State *L)
337{
338 return G(L)->hookf;
339}
340
341LUA_API int lua_gethookmask(lua_State *L)
342{
343 return G(L)->hookmask & HOOK_EVENTMASK;
344}
345
346LUA_API int lua_gethookcount(lua_State *L)
347{
348 return (int)G(L)->hookcstart;
349}
350
351/* Call a hook. */
352static void callhook(lua_State *L, int event, BCLine line)
353{
354 global_State *g = G(L);
355 lua_Hook hookf = g->hookf;
356 if (hookf && !hook_active(g)) {
357 lua_Debug ar;
358 lj_trace_abort(g); /* Abort recording on any hook call. */
359 ar.event = event;
360 ar.currentline = line;
361 /* Top frame, nextframe = NULL. */
362 ar.i_ci = (int)((L->base-1) - tvref(L->stack));
363 lj_state_checkstack(L, 1+LUA_MINSTACK);
364#if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
365 lj_profile_hook_enter(g);
366#else
367 hook_enter(g);
368#endif
369 hookf(L, &ar);
370 lj_assertG(hook_active(g), "active hook flag removed");
371 setgcref(g->cur_L, obj2gco(L));
372#if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
373 lj_profile_hook_leave(g);
374#else
375 hook_leave(g);
376#endif
377 }
378}
379
380/* -- Dispatch callbacks -------------------------------------------------- */
381
382/* Calculate number of used stack slots in the current frame. */
383static BCReg cur_topslot(GCproto *pt, const BCIns *pc, uint32_t nres)
384{
385 BCIns ins = pc[-1];
386 if (bc_op(ins) == BC_UCLO)
387 ins = pc[bc_j(ins)];
388 switch (bc_op(ins)) {
389 case BC_CALLM: case BC_CALLMT: return bc_a(ins) + bc_c(ins) + nres-1+1+LJ_FR2;
390 case BC_RETM: return bc_a(ins) + bc_d(ins) + nres-1;
391 case BC_TSETM: return bc_a(ins) + nres-1;
392 default: return pt->framesize;
393 }
394}
395
396/* Instruction dispatch. Used by instr/line/return hooks or when recording. */
397void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc)
398{
399 ERRNO_SAVE
400 GCfunc *fn = curr_func(L);
401 GCproto *pt = funcproto(fn);
402 void *cf = cframe_raw(L->cframe);
403 const BCIns *oldpc = cframe_pc(cf);
404 global_State *g = G(L);
405 BCReg slots;
406 setcframe_pc(cf, pc);
407 slots = cur_topslot(pt, pc, cframe_multres_n(cf));
408 L->top = L->base + slots; /* Fix top. */
409#if LJ_HASJIT
410 {
411 jit_State *J = G2J(g);
412 if (J->state != LJ_TRACE_IDLE) {
413#ifdef LUA_USE_ASSERT
414 ptrdiff_t delta = L->top - L->base;
415#endif
416 J->L = L;
417 lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */
418 lj_assertG(L->top - L->base == delta,
419 "unbalanced stack after tracing of instruction");
420 }
421 }
422#endif
423 if ((g->hookmask & LUA_MASKCOUNT) && g->hookcount == 0) {
424 g->hookcount = g->hookcstart;
425 callhook(L, LUA_HOOKCOUNT, -1);
426 L->top = L->base + slots; /* Fix top again. */
427 }
428 if ((g->hookmask & LUA_MASKLINE)) {
429 BCPos npc = proto_bcpos(pt, pc) - 1;
430 BCPos opc = proto_bcpos(pt, oldpc) - 1;
431 BCLine line = lj_debug_line(pt, npc);
432 if (pc <= oldpc || opc >= pt->sizebc || line != lj_debug_line(pt, opc)) {
433 callhook(L, LUA_HOOKLINE, line);
434 L->top = L->base + slots; /* Fix top again. */
435 }
436 }
437 if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1])))
438 callhook(L, LUA_HOOKRET, -1);
439 ERRNO_RESTORE
440}
441
442/* Initialize call. Ensure stack space and return # of missing parameters. */
443static int call_init(lua_State *L, GCfunc *fn)
444{
445 if (isluafunc(fn)) {
446 GCproto *pt = funcproto(fn);
447 int numparams = pt->numparams;
448 int gotparams = (int)(L->top - L->base);
449 int need = pt->framesize;
450 if ((pt->flags & PROTO_VARARG)) need += 1+gotparams;
451 lj_state_checkstack(L, (MSize)need);
452 numparams -= gotparams;
453 return numparams >= 0 ? numparams : 0;
454 } else {
455 lj_state_checkstack(L, LUA_MINSTACK);
456 return 0;
457 }
458}
459
460/* Call dispatch. Used by call hooks, hot calls or when recording. */
461ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
462{
463 ERRNO_SAVE
464 GCfunc *fn = curr_func(L);
465 BCOp op;
466 global_State *g = G(L);
467#if LJ_HASJIT
468 jit_State *J = G2J(g);
469#endif
470 int missing = call_init(L, fn);
471#if LJ_HASJIT
472 J->L = L;
473 if ((uintptr_t)pc & 1) { /* Marker for hot call. */
474#ifdef LUA_USE_ASSERT
475 ptrdiff_t delta = L->top - L->base;
476#endif
477 pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1);
478 lj_trace_hot(J, pc);
479 lj_assertG(L->top - L->base == delta,
480 "unbalanced stack after hot call");
481 goto out;
482 } else if (J->state != LJ_TRACE_IDLE &&
483 !(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
484#ifdef LUA_USE_ASSERT
485 ptrdiff_t delta = L->top - L->base;
486#endif
487 /* Record the FUNC* bytecodes, too. */
488 lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */
489 lj_assertG(L->top - L->base == delta,
490 "unbalanced stack after hot instruction");
491 }
492#endif
493 if ((g->hookmask & LUA_MASKCALL)) {
494 int i;
495 for (i = 0; i < missing; i++) /* Add missing parameters. */
496 setnilV(L->top++);
497 callhook(L, LUA_HOOKCALL, -1);
498 /* Preserve modifications of missing parameters by lua_setlocal(). */
499 while (missing-- > 0 && tvisnil(L->top - 1))
500 L->top--;
501 }
502#if LJ_HASJIT
503out:
504#endif
505 op = bc_op(pc[-1]); /* Get FUNC* op. */
506#if LJ_HASJIT
507 /* Use the non-hotcounting variants if JIT is off or while recording. */
508 if ((!(J->flags & JIT_F_ON) || J->state != LJ_TRACE_IDLE) &&
509 (op == BC_FUNCF || op == BC_FUNCV))
510 op = (BCOp)((int)op+(int)BC_IFUNCF-(int)BC_FUNCF);
511#endif
512 ERRNO_RESTORE
513 return makeasmfunc(lj_bc_ofs[op]); /* Return static dispatch target. */
514}
515
516#if LJ_HASJIT
517/* Stitch a new trace. */
518void LJ_FASTCALL lj_dispatch_stitch(jit_State *J, const BCIns *pc)
519{
520 ERRNO_SAVE
521 lua_State *L = J->L;
522 void *cf = cframe_raw(L->cframe);
523 const BCIns *oldpc = cframe_pc(cf);
524 setcframe_pc(cf, pc);
525 /* Before dispatch, have to bias PC by 1. */
526 L->top = L->base + cur_topslot(curr_proto(L), pc+1, cframe_multres_n(cf));
527 lj_trace_stitch(J, pc-1); /* Point to the CALL instruction. */
528 setcframe_pc(cf, oldpc);
529 ERRNO_RESTORE
530}
531#endif
532
533#if LJ_HASPROFILE
534/* Profile dispatch. */
535void LJ_FASTCALL lj_dispatch_profile(lua_State *L, const BCIns *pc)
536{
537 ERRNO_SAVE
538 GCfunc *fn = curr_func(L);
539 GCproto *pt = funcproto(fn);
540 void *cf = cframe_raw(L->cframe);
541 const BCIns *oldpc = cframe_pc(cf);
542 global_State *g;
543 setcframe_pc(cf, pc);
544 L->top = L->base + cur_topslot(pt, pc, cframe_multres_n(cf));
545 lj_profile_interpreter(L);
546 setcframe_pc(cf, oldpc);
547 g = G(L);
548 setgcref(g->cur_L, obj2gco(L));
549 setvmstate(g, INTERP);
550 ERRNO_RESTORE
551}
552#endif
553
554