1/*
2** Instruction dispatch handling.
3** Copyright (C) 2005-2014 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#define lj_dispatch_c
7#define LUA_CORE
8
9#include "lj_obj.h"
10#include "lj_err.h"
11#include "lj_func.h"
12#include "lj_str.h"
13#include "lj_tab.h"
14#include "lj_meta.h"
15#include "lj_debug.h"
16#include "lj_state.h"
17#include "lj_frame.h"
18#include "lj_bc.h"
19#include "lj_ff.h"
20#if LJ_HASJIT
21#include "lj_jit.h"
22#endif
23#if LJ_HASFFI
24#include "lj_ccallback.h"
25#endif
26#include "lj_trace.h"
27#include "lj_dispatch.h"
28#include "lj_vm.h"
29#include "luajit.h"
30
31/* Bump GG_NUM_ASMFF in lj_dispatch.h as needed. Ugly. */
32LJ_STATIC_ASSERT(GG_NUM_ASMFF == FF_NUM_ASMFUNC);
33
34/* -- Dispatch table management ------------------------------------------- */
35
36#if LJ_TARGET_MIPS
37#include <math.h>
38LJ_FUNCA_NORET void LJ_FASTCALL lj_ffh_coroutine_wrap_err(lua_State *L,
39 lua_State *co);
40
41#define GOTFUNC(name) (ASMFunction)name,
42static const ASMFunction dispatch_got[] = {
43 GOTDEF(GOTFUNC)
44};
45#undef GOTFUNC
46#endif
47
48/* Initialize instruction dispatch table and hot counters. */
49void lj_dispatch_init(GG_State *GG)
50{
51 uint32_t i;
52 ASMFunction *disp = GG->dispatch;
53 for (i = 0; i < GG_LEN_SDISP; i++)
54 disp[GG_LEN_DDISP+i] = disp[i] = makeasmfunc(lj_bc_ofs[i]);
55 for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
56 disp[i] = makeasmfunc(lj_bc_ofs[i]);
57 /* The JIT engine is off by default. luaopen_jit() turns it on. */
58 disp[BC_FORL] = disp[BC_IFORL];
59 disp[BC_ITERL] = disp[BC_IITERL];
60 disp[BC_LOOP] = disp[BC_ILOOP];
61 disp[BC_FUNCF] = disp[BC_IFUNCF];
62 disp[BC_FUNCV] = disp[BC_IFUNCV];
63 GG->g.bc_cfunc_ext = GG->g.bc_cfunc_int = BCINS_AD(BC_FUNCC, LUA_MINSTACK, 0);
64 for (i = 0; i < GG_NUM_ASMFF; i++)
65 GG->bcff[i] = BCINS_AD(BC__MAX+i, 0, 0);
66#if LJ_TARGET_MIPS
67 memcpy(GG->got, dispatch_got, LJ_GOT__MAX*4);
68#endif
69}
70
71#if LJ_HASJIT
72/* Initialize hotcount table. */
73void lj_dispatch_init_hotcount(global_State *g)
74{
75 int32_t hotloop = G2J(g)->param[JIT_P_hotloop];
76 HotCount start = (HotCount)(hotloop*HOTCOUNT_LOOP - 1);
77 HotCount *hotcount = G2GG(g)->hotcount;
78 uint32_t i;
79 for (i = 0; i < HOTCOUNT_SIZE; i++)
80 hotcount[i] = start;
81}
82#endif
83
84/* Internal dispatch mode bits. */
85#define DISPMODE_JIT 0x01 /* JIT compiler on. */
86#define DISPMODE_REC 0x02 /* Recording active. */
87#define DISPMODE_INS 0x04 /* Override instruction dispatch. */
88#define DISPMODE_CALL 0x08 /* Override call dispatch. */
89#define DISPMODE_RET 0x10 /* Override return dispatch. */
90
91/* Update dispatch table depending on various flags. */
92void lj_dispatch_update(global_State *g)
93{
94 uint8_t oldmode = g->dispatchmode;
95 uint8_t mode = 0;
96#if LJ_HASJIT
97 mode |= (G2J(g)->flags & JIT_F_ON) ? DISPMODE_JIT : 0;
98 mode |= G2J(g)->state != LJ_TRACE_IDLE ?
99 (DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0;
100#endif
101 mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0;
102 mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0;
103 mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0;
104 if (oldmode != mode) { /* Mode changed? */
105 ASMFunction *disp = G2GG(g)->dispatch;
106 ASMFunction f_forl, f_iterl, f_loop, f_funcf, f_funcv;
107 g->dispatchmode = mode;
108
109 /* Hotcount if JIT is on, but not while recording. */
110 if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) {
111 f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]);
112 f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]);
113 f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]);
114 f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]);
115 f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]);
116 } else { /* Otherwise use the non-hotcounting instructions. */
117 f_forl = disp[GG_LEN_DDISP+BC_IFORL];
118 f_iterl = disp[GG_LEN_DDISP+BC_IITERL];
119 f_loop = disp[GG_LEN_DDISP+BC_ILOOP];
120 f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]);
121 f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]);
122 }
123 /* Init static counting instruction dispatch first (may be copied below). */
124 disp[GG_LEN_DDISP+BC_FORL] = f_forl;
125 disp[GG_LEN_DDISP+BC_ITERL] = f_iterl;
126 disp[GG_LEN_DDISP+BC_LOOP] = f_loop;
127
128 /* Set dynamic instruction dispatch. */
129 if ((oldmode ^ mode) & (DISPMODE_REC|DISPMODE_INS)) {
130 /* Need to update the whole table. */
131 if (!(mode & (DISPMODE_REC|DISPMODE_INS))) { /* No ins dispatch? */
132 /* Copy static dispatch table to dynamic dispatch table. */
133 memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction));
134 /* Overwrite with dynamic return dispatch. */
135 if ((mode & DISPMODE_RET)) {
136 disp[BC_RETM] = lj_vm_rethook;
137 disp[BC_RET] = lj_vm_rethook;
138 disp[BC_RET0] = lj_vm_rethook;
139 disp[BC_RET1] = lj_vm_rethook;
140 }
141 } else {
142 /* The recording dispatch also checks for hooks. */
143 ASMFunction f = (mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook;
144 uint32_t i;
145 for (i = 0; i < GG_LEN_SDISP; i++)
146 disp[i] = f;
147 }
148 } else if (!(mode & (DISPMODE_REC|DISPMODE_INS))) {
149 /* Otherwise set dynamic counting ins. */
150 disp[BC_FORL] = f_forl;
151 disp[BC_ITERL] = f_iterl;
152 disp[BC_LOOP] = f_loop;
153 /* Set dynamic return dispatch. */
154 if ((mode & DISPMODE_RET)) {
155 disp[BC_RETM] = lj_vm_rethook;
156 disp[BC_RET] = lj_vm_rethook;
157 disp[BC_RET0] = lj_vm_rethook;
158 disp[BC_RET1] = lj_vm_rethook;
159 } else {
160 disp[BC_RETM] = disp[GG_LEN_DDISP+BC_RETM];
161 disp[BC_RET] = disp[GG_LEN_DDISP+BC_RET];
162 disp[BC_RET0] = disp[GG_LEN_DDISP+BC_RET0];
163 disp[BC_RET1] = disp[GG_LEN_DDISP+BC_RET1];
164 }
165 }
166
167 /* Set dynamic call dispatch. */
168 if ((oldmode ^ mode) & DISPMODE_CALL) { /* Update the whole table? */
169 uint32_t i;
170 if ((mode & DISPMODE_CALL) == 0) { /* No call hooks? */
171 for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
172 disp[i] = makeasmfunc(lj_bc_ofs[i]);
173 } else {
174 for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
175 disp[i] = lj_vm_callhook;
176 }
177 }
178 if (!(mode & DISPMODE_CALL)) { /* Overwrite dynamic counting ins. */
179 disp[BC_FUNCF] = f_funcf;
180 disp[BC_FUNCV] = f_funcv;
181 }
182
183#if LJ_HASJIT
184 /* Reset hotcounts for JIT off to on transition. */
185 if ((mode & DISPMODE_JIT) && !(oldmode & DISPMODE_JIT))
186 lj_dispatch_init_hotcount(g);
187#endif
188 }
189}
190
191/* -- JIT mode setting ---------------------------------------------------- */
192
193#if LJ_HASJIT
194/* Set JIT mode for a single prototype. */
195static void setptmode(global_State *g, GCproto *pt, int mode)
196{
197 if ((mode & LUAJIT_MODE_ON)) { /* (Re-)enable JIT compilation. */
198 pt->flags &= ~PROTO_NOJIT;
199 lj_trace_reenableproto(pt); /* Unpatch all ILOOP etc. bytecodes. */
200 } else { /* Flush and/or disable JIT compilation. */
201 if (!(mode & LUAJIT_MODE_FLUSH))
202 pt->flags |= PROTO_NOJIT;
203 lj_trace_flushproto(g, pt); /* Flush all traces of prototype. */
204 }
205}
206
207/* Recursively set the JIT mode for all children of a prototype. */
208static void setptmode_all(global_State *g, GCproto *pt, int mode)
209{
210 ptrdiff_t i;
211 if (!(pt->flags & PROTO_CHILD)) return;
212 for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) {
213 GCobj *o = proto_kgc(pt, i);
214 if (o->gch.gct == ~LJ_TPROTO) {
215 setptmode(g, gco2pt(o), mode);
216 setptmode_all(g, gco2pt(o), mode);
217 }
218 }
219}
220#endif
221
222/* Public API function: control the JIT engine. */
223int luaJIT_setmode(lua_State *L, int idx, int mode)
224{
225 global_State *g = G(L);
226 int mm = mode & LUAJIT_MODE_MASK;
227 lj_trace_abort(g); /* Abort recording on any state change. */
228 /* Avoid pulling the rug from under our own feet. */
229 if ((g->hookmask & HOOK_GC))
230 lj_err_caller(L, LJ_ERR_NOGCMM);
231 switch (mm) {
232#if LJ_HASJIT
233 case LUAJIT_MODE_ENGINE:
234 if ((mode & LUAJIT_MODE_FLUSH)) {
235 lj_trace_flushall(L);
236 } else {
237 if (!(mode & LUAJIT_MODE_ON))
238 G2J(g)->flags &= ~(uint32_t)JIT_F_ON;
239#if LJ_TARGET_X86ORX64
240 else if ((G2J(g)->flags & JIT_F_SSE2))
241 G2J(g)->flags |= (uint32_t)JIT_F_ON;
242 else
243 return 0; /* Don't turn on JIT compiler without SSE2 support. */
244#else
245 else
246 G2J(g)->flags |= (uint32_t)JIT_F_ON;
247#endif
248 lj_dispatch_update(g);
249 }
250 break;
251 case LUAJIT_MODE_FUNC:
252 case LUAJIT_MODE_ALLFUNC:
253 case LUAJIT_MODE_ALLSUBFUNC: {
254 cTValue *tv = idx == 0 ? frame_prev(L->base-1) :
255 idx > 0 ? L->base + (idx-1) : L->top + idx;
256 GCproto *pt;
257 if ((idx == 0 || tvisfunc(tv)) && isluafunc(&gcval(tv)->fn))
258 pt = funcproto(&gcval(tv)->fn); /* Cannot use funcV() for frame slot. */
259 else if (tvisproto(tv))
260 pt = protoV(tv);
261 else
262 return 0; /* Failed. */
263 if (mm != LUAJIT_MODE_ALLSUBFUNC)
264 setptmode(g, pt, mode);
265 if (mm != LUAJIT_MODE_FUNC)
266 setptmode_all(g, pt, mode);
267 break;
268 }
269 case LUAJIT_MODE_TRACE:
270 if (!(mode & LUAJIT_MODE_FLUSH))
271 return 0; /* Failed. */
272 lj_trace_flush(G2J(g), idx);
273 break;
274#else
275 case LUAJIT_MODE_ENGINE:
276 case LUAJIT_MODE_FUNC:
277 case LUAJIT_MODE_ALLFUNC:
278 case LUAJIT_MODE_ALLSUBFUNC:
279 UNUSED(idx);
280 if ((mode & LUAJIT_MODE_ON))
281 return 0; /* Failed. */
282 break;
283#endif
284 case LUAJIT_MODE_WRAPCFUNC:
285 if ((mode & LUAJIT_MODE_ON)) {
286 if (idx != 0) {
287 cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx;
288 if (tvislightud(tv))
289 g->wrapf = (lua_CFunction)lightudV(tv);
290 else
291 return 0; /* Failed. */
292 } else {
293 return 0; /* Failed. */
294 }
295 g->bc_cfunc_ext = BCINS_AD(BC_FUNCCW, 0, 0);
296 } else {
297 g->bc_cfunc_ext = BCINS_AD(BC_FUNCC, 0, 0);
298 }
299 break;
300 default:
301 return 0; /* Failed. */
302 }
303 return 1; /* OK. */
304}
305
306/* Enforce (dynamic) linker error for version mismatches. See luajit.c. */
307LUA_API void LUAJIT_VERSION_SYM(void)
308{
309}
310
311/* -- Hooks --------------------------------------------------------------- */
312
313/* This function can be called asynchronously (e.g. during a signal). */
314LUA_API int lua_sethook(lua_State *L, lua_Hook func, int mask, int count)
315{
316 global_State *g = G(L);
317 mask &= HOOK_EVENTMASK;
318 if (func == NULL || mask == 0) { mask = 0; func = NULL; } /* Consistency. */
319 g->hookf = func;
320 g->hookcount = g->hookcstart = (int32_t)count;
321 g->hookmask = (uint8_t)((g->hookmask & ~HOOK_EVENTMASK) | mask);
322 lj_trace_abort(g); /* Abort recording on any hook change. */
323 lj_dispatch_update(g);
324 return 1;
325}
326
327LUA_API lua_Hook lua_gethook(lua_State *L)
328{
329 return G(L)->hookf;
330}
331
332LUA_API int lua_gethookmask(lua_State *L)
333{
334 return G(L)->hookmask & HOOK_EVENTMASK;
335}
336
337LUA_API int lua_gethookcount(lua_State *L)
338{
339 return (int)G(L)->hookcstart;
340}
341
342/* Call a hook. */
343static void callhook(lua_State *L, int event, BCLine line)
344{
345 global_State *g = G(L);
346 lua_Hook hookf = g->hookf;
347 if (hookf && !hook_active(g)) {
348 lua_Debug ar;
349 lj_trace_abort(g); /* Abort recording on any hook call. */
350 ar.event = event;
351 ar.currentline = line;
352 /* Top frame, nextframe = NULL. */
353 ar.i_ci = (int)((L->base-1) - tvref(L->stack));
354 lj_state_checkstack(L, 1+LUA_MINSTACK);
355 hook_enter(g);
356 hookf(L, &ar);
357 lua_assert(hook_active(g));
358 hook_leave(g);
359 }
360}
361
362/* -- Dispatch callbacks -------------------------------------------------- */
363
364/* Calculate number of used stack slots in the current frame. */
365static BCReg cur_topslot(GCproto *pt, const BCIns *pc, uint32_t nres)
366{
367 BCIns ins = pc[-1];
368 if (bc_op(ins) == BC_UCLO)
369 ins = pc[bc_j(ins)];
370 switch (bc_op(ins)) {
371 case BC_CALLM: case BC_CALLMT: return bc_a(ins) + bc_c(ins) + nres-1+1;
372 case BC_RETM: return bc_a(ins) + bc_d(ins) + nres-1;
373 case BC_TSETM: return bc_a(ins) + nres-1;
374 default: return pt->framesize;
375 }
376}
377
378/* Instruction dispatch. Used by instr/line/return hooks or when recording. */
379void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc)
380{
381 ERRNO_SAVE
382 GCfunc *fn = curr_func(L);
383 GCproto *pt = funcproto(fn);
384 void *cf = cframe_raw(L->cframe);
385 const BCIns *oldpc = cframe_pc(cf);
386 global_State *g = G(L);
387 BCReg slots;
388 setcframe_pc(cf, pc);
389 slots = cur_topslot(pt, pc, cframe_multres_n(cf));
390 L->top = L->base + slots; /* Fix top. */
391#if LJ_HASJIT
392 {
393 jit_State *J = G2J(g);
394 if (J->state != LJ_TRACE_IDLE) {
395#ifdef LUA_USE_ASSERT
396 ptrdiff_t delta = L->top - L->base;
397#endif
398 J->L = L;
399 lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */
400 lua_assert(L->top - L->base == delta);
401 }
402 }
403#endif
404 if ((g->hookmask & LUA_MASKCOUNT) && g->hookcount == 0) {
405 g->hookcount = g->hookcstart;
406 callhook(L, LUA_HOOKCOUNT, -1);
407 L->top = L->base + slots; /* Fix top again. */
408 }
409 if ((g->hookmask & LUA_MASKLINE)) {
410 BCPos npc = proto_bcpos(pt, pc) - 1;
411 BCPos opc = proto_bcpos(pt, oldpc) - 1;
412 BCLine line = lj_debug_line(pt, npc);
413 if (pc <= oldpc || opc >= pt->sizebc || line != lj_debug_line(pt, opc)) {
414 callhook(L, LUA_HOOKLINE, line);
415 L->top = L->base + slots; /* Fix top again. */
416 }
417 }
418 if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1])))
419 callhook(L, LUA_HOOKRET, -1);
420 ERRNO_RESTORE
421}
422
423/* Initialize call. Ensure stack space and return # of missing parameters. */
424static int call_init(lua_State *L, GCfunc *fn)
425{
426 if (isluafunc(fn)) {
427 GCproto *pt = funcproto(fn);
428 int numparams = pt->numparams;
429 int gotparams = (int)(L->top - L->base);
430 int need = pt->framesize;
431 if ((pt->flags & PROTO_VARARG)) need += 1+gotparams;
432 lj_state_checkstack(L, (MSize)need);
433 numparams -= gotparams;
434 return numparams >= 0 ? numparams : 0;
435 } else {
436 lj_state_checkstack(L, LUA_MINSTACK);
437 return 0;
438 }
439}
440
441/* Call dispatch. Used by call hooks, hot calls or when recording. */
442ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
443{
444 ERRNO_SAVE
445 GCfunc *fn = curr_func(L);
446 BCOp op;
447 global_State *g = G(L);
448#if LJ_HASJIT
449 jit_State *J = G2J(g);
450#endif
451 int missing = call_init(L, fn);
452#if LJ_HASJIT
453 J->L = L;
454 if ((uintptr_t)pc & 1) { /* Marker for hot call. */
455#ifdef LUA_USE_ASSERT
456 ptrdiff_t delta = L->top - L->base;
457#endif
458 pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1);
459 lj_trace_hot(J, pc);
460 lua_assert(L->top - L->base == delta);
461 goto out;
462 } else if (J->state != LJ_TRACE_IDLE &&
463 !(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
464#ifdef LUA_USE_ASSERT
465 ptrdiff_t delta = L->top - L->base;
466#endif
467 /* Record the FUNC* bytecodes, too. */
468 lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */
469 lua_assert(L->top - L->base == delta);
470 }
471#endif
472 if ((g->hookmask & LUA_MASKCALL)) {
473 int i;
474 for (i = 0; i < missing; i++) /* Add missing parameters. */
475 setnilV(L->top++);
476 callhook(L, LUA_HOOKCALL, -1);
477 /* Preserve modifications of missing parameters by lua_setlocal(). */
478 while (missing-- > 0 && tvisnil(L->top - 1))
479 L->top--;
480 }
481#if LJ_HASJIT
482out:
483#endif
484 op = bc_op(pc[-1]); /* Get FUNC* op. */
485#if LJ_HASJIT
486 /* Use the non-hotcounting variants if JIT is off or while recording. */
487 if ((!(J->flags & JIT_F_ON) || J->state != LJ_TRACE_IDLE) &&
488 (op == BC_FUNCF || op == BC_FUNCV))
489 op = (BCOp)((int)op+(int)BC_IFUNCF-(int)BC_FUNCF);
490#endif
491 ERRNO_RESTORE
492 return makeasmfunc(lj_bc_ofs[op]); /* Return static dispatch target. */
493}
494
495