1/*
2** Machine code management.
3** Copyright (C) 2005-2021 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#define lj_mcode_c
7#define LUA_CORE
8
9#include "lj_obj.h"
10#if LJ_HASJIT
11#include "lj_gc.h"
12#include "lj_err.h"
13#include "lj_jit.h"
14#include "lj_mcode.h"
15#include "lj_trace.h"
16#include "lj_dispatch.h"
17#include "lj_prng.h"
18#endif
19#if LJ_HASJIT || LJ_HASFFI
20#include "lj_vm.h"
21#endif
22
23/* -- OS-specific functions ----------------------------------------------- */
24
25#if LJ_HASJIT || LJ_HASFFI
26
27/* Define this if you want to run LuaJIT with Valgrind. */
28#ifdef LUAJIT_USE_VALGRIND
29#include <valgrind/valgrind.h>
30#endif
31
32#if LJ_TARGET_IOS
33void sys_icache_invalidate(void *start, size_t len);
34#endif
35
36/* Synchronize data/instruction cache. */
37void lj_mcode_sync(void *start, void *end)
38{
39#ifdef LUAJIT_USE_VALGRIND
40 VALGRIND_DISCARD_TRANSLATIONS(start, (char *)end-(char *)start);
41#endif
42#if LJ_TARGET_X86ORX64
43 UNUSED(start); UNUSED(end);
44#elif LJ_TARGET_IOS
45 sys_icache_invalidate(start, (char *)end-(char *)start);
46#elif LJ_TARGET_PPC
47 lj_vm_cachesync(start, end);
48#elif defined(__GNUC__) || defined(__clang__)
49 __clear_cache(start, end);
50#else
51#error "Missing builtin to flush instruction cache"
52#endif
53}
54
55#endif
56
57#if LJ_HASJIT
58
59#if LJ_TARGET_WINDOWS
60
61#define WIN32_LEAN_AND_MEAN
62#include <windows.h>
63
64#define MCPROT_RW PAGE_READWRITE
65#define MCPROT_RX PAGE_EXECUTE_READ
66#define MCPROT_RWX PAGE_EXECUTE_READWRITE
67
68static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, DWORD prot)
69{
70 void *p = LJ_WIN_VALLOC((void *)hint, sz,
71 MEM_RESERVE|MEM_COMMIT|MEM_TOP_DOWN, prot);
72 if (!p && !hint)
73 lj_trace_err(J, LJ_TRERR_MCODEAL);
74 return p;
75}
76
77static void mcode_free(jit_State *J, void *p, size_t sz)
78{
79 UNUSED(J); UNUSED(sz);
80 VirtualFree(p, 0, MEM_RELEASE);
81}
82
83static int mcode_setprot(void *p, size_t sz, DWORD prot)
84{
85 DWORD oprot;
86 return !LJ_WIN_VPROTECT(p, sz, prot, &oprot);
87}
88
89#elif LJ_TARGET_POSIX
90
91#include <sys/mman.h>
92
93#ifndef MAP_ANONYMOUS
94#define MAP_ANONYMOUS MAP_ANON
95#endif
96
97#define MCPROT_RW (PROT_READ|PROT_WRITE)
98#define MCPROT_RX (PROT_READ|PROT_EXEC)
99#define MCPROT_RWX (PROT_READ|PROT_WRITE|PROT_EXEC)
100
101static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot)
102{
103 void *p = mmap((void *)hint, sz, prot, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
104 if (p == MAP_FAILED) {
105 if (!hint) lj_trace_err(J, LJ_TRERR_MCODEAL);
106 p = NULL;
107 }
108 return p;
109}
110
111static void mcode_free(jit_State *J, void *p, size_t sz)
112{
113 UNUSED(J);
114 munmap(p, sz);
115}
116
117static int mcode_setprot(void *p, size_t sz, int prot)
118{
119 return mprotect(p, sz, prot);
120}
121
122#else
123
124#error "Missing OS support for explicit placement of executable memory"
125
126#endif
127
128/* -- MCode area protection ----------------------------------------------- */
129
130#if LUAJIT_SECURITY_MCODE == 0
131
132/* Define this ONLY if page protection twiddling becomes a bottleneck.
133**
134** It's generally considered to be a potential security risk to have
135** pages with simultaneous write *and* execute access in a process.
136**
137** Do not even think about using this mode for server processes or
138** apps handling untrusted external data.
139**
140** The security risk is not in LuaJIT itself -- but if an adversary finds
141** any *other* flaw in your C application logic, then any RWX memory pages
142** simplify writing an exploit considerably.
143*/
144#define MCPROT_GEN MCPROT_RWX
145#define MCPROT_RUN MCPROT_RWX
146
147static void mcode_protect(jit_State *J, int prot)
148{
149 UNUSED(J); UNUSED(prot); UNUSED(mcode_setprot);
150}
151
152#else
153
154/* This is the default behaviour and much safer:
155**
156** Most of the time the memory pages holding machine code are executable,
157** but NONE of them is writable.
158**
159** The current memory area is marked read-write (but NOT executable) only
160** during the short time window while the assembler generates machine code.
161*/
162#define MCPROT_GEN MCPROT_RW
163#define MCPROT_RUN MCPROT_RX
164
165/* Protection twiddling failed. Probably due to kernel security. */
166static LJ_NOINLINE void mcode_protfail(jit_State *J)
167{
168 lua_CFunction panic = J2G(J)->panic;
169 if (panic) {
170 lua_State *L = J->L;
171 setstrV(L, L->top++, lj_err_str(L, LJ_ERR_JITPROT));
172 panic(L);
173 }
174}
175
176/* Change protection of MCode area. */
177static void mcode_protect(jit_State *J, int prot)
178{
179 if (J->mcprot != prot) {
180 if (LJ_UNLIKELY(mcode_setprot(J->mcarea, J->szmcarea, prot)))
181 mcode_protfail(J);
182 J->mcprot = prot;
183 }
184}
185
186#endif
187
188/* -- MCode area allocation ----------------------------------------------- */
189
190#if LJ_64
191#define mcode_validptr(p) (p)
192#else
193#define mcode_validptr(p) ((p) && (uintptr_t)(p) < 0xffff0000)
194#endif
195
196#ifdef LJ_TARGET_JUMPRANGE
197
198/* Get memory within relative jump distance of our code in 64 bit mode. */
199static void *mcode_alloc(jit_State *J, size_t sz)
200{
201 /* Target an address in the static assembler code (64K aligned).
202 ** Try addresses within a distance of target-range/2+1MB..target+range/2-1MB.
203 ** Use half the jump range so every address in the range can reach any other.
204 */
205#if LJ_TARGET_MIPS
206 /* Use the middle of the 256MB-aligned region. */
207 uintptr_t target = ((uintptr_t)(void *)lj_vm_exit_handler &
208 ~(uintptr_t)0x0fffffffu) + 0x08000000u;
209#else
210 uintptr_t target = (uintptr_t)(void *)lj_vm_exit_handler & ~(uintptr_t)0xffff;
211#endif
212 const uintptr_t range = (1u << (LJ_TARGET_JUMPRANGE-1)) - (1u << 21);
213 /* First try a contiguous area below the last one. */
214 uintptr_t hint = J->mcarea ? (uintptr_t)J->mcarea - sz : 0;
215 int i;
216 /* Limit probing iterations, depending on the available pool size. */
217 for (i = 0; i < LJ_TARGET_JUMPRANGE; i++) {
218 if (mcode_validptr(hint)) {
219 void *p = mcode_alloc_at(J, hint, sz, MCPROT_GEN);
220
221 if (mcode_validptr(p) &&
222 ((uintptr_t)p + sz - target < range || target - (uintptr_t)p < range))
223 return p;
224 if (p) mcode_free(J, p, sz); /* Free badly placed area. */
225 }
226 /* Next try probing 64K-aligned pseudo-random addresses. */
227 do {
228 hint = lj_prng_u64(&J2G(J)->prng) & ((1u<<LJ_TARGET_JUMPRANGE)-0x10000);
229 } while (!(hint + sz < range+range));
230 hint = target + hint - range;
231 }
232 lj_trace_err(J, LJ_TRERR_MCODEAL); /* Give up. OS probably ignores hints? */
233 return NULL;
234}
235
236#else
237
238/* All memory addresses are reachable by relative jumps. */
239static void *mcode_alloc(jit_State *J, size_t sz)
240{
241#if defined(__OpenBSD__) || LJ_TARGET_UWP
242 /* Allow better executable memory allocation for OpenBSD W^X mode. */
243 void *p = mcode_alloc_at(J, 0, sz, MCPROT_RUN);
244 if (p && mcode_setprot(p, sz, MCPROT_GEN)) {
245 mcode_free(J, p, sz);
246 return NULL;
247 }
248 return p;
249#else
250 return mcode_alloc_at(J, 0, sz, MCPROT_GEN);
251#endif
252}
253
254#endif
255
256/* -- MCode area management ----------------------------------------------- */
257
258/* Allocate a new MCode area. */
259static void mcode_allocarea(jit_State *J)
260{
261 MCode *oldarea = J->mcarea;
262 size_t sz = (size_t)J->param[JIT_P_sizemcode] << 10;
263 sz = (sz + LJ_PAGESIZE-1) & ~(size_t)(LJ_PAGESIZE - 1);
264 J->mcarea = (MCode *)mcode_alloc(J, sz);
265 J->szmcarea = sz;
266 J->mcprot = MCPROT_GEN;
267 J->mctop = (MCode *)((char *)J->mcarea + J->szmcarea);
268 J->mcbot = (MCode *)((char *)J->mcarea + sizeof(MCLink));
269 ((MCLink *)J->mcarea)->next = oldarea;
270 ((MCLink *)J->mcarea)->size = sz;
271 J->szallmcarea += sz;
272 J->mcbot = (MCode *)lj_err_register_mcode(J->mcarea, sz, (uint8_t *)J->mcbot);
273}
274
275/* Free all MCode areas. */
276void lj_mcode_free(jit_State *J)
277{
278 MCode *mc = J->mcarea;
279 J->mcarea = NULL;
280 J->szallmcarea = 0;
281 while (mc) {
282 MCode *next = ((MCLink *)mc)->next;
283 size_t sz = ((MCLink *)mc)->size;
284 lj_err_deregister_mcode(mc, sz, (uint8_t *)mc + sizeof(MCLink));
285 mcode_free(J, mc, sz);
286 mc = next;
287 }
288}
289
290/* -- MCode transactions -------------------------------------------------- */
291
292/* Reserve the remainder of the current MCode area. */
293MCode *lj_mcode_reserve(jit_State *J, MCode **lim)
294{
295 if (!J->mcarea)
296 mcode_allocarea(J);
297 else
298 mcode_protect(J, MCPROT_GEN);
299 *lim = J->mcbot;
300 return J->mctop;
301}
302
303/* Commit the top part of the current MCode area. */
304void lj_mcode_commit(jit_State *J, MCode *top)
305{
306 J->mctop = top;
307 mcode_protect(J, MCPROT_RUN);
308}
309
310/* Abort the reservation. */
311void lj_mcode_abort(jit_State *J)
312{
313 if (J->mcarea)
314 mcode_protect(J, MCPROT_RUN);
315}
316
317/* Set/reset protection to allow patching of MCode areas. */
318MCode *lj_mcode_patch(jit_State *J, MCode *ptr, int finish)
319{
320 if (finish) {
321#if LUAJIT_SECURITY_MCODE
322 if (J->mcarea == ptr)
323 mcode_protect(J, MCPROT_RUN);
324 else if (LJ_UNLIKELY(mcode_setprot(ptr, ((MCLink *)ptr)->size, MCPROT_RUN)))
325 mcode_protfail(J);
326#endif
327 return NULL;
328 } else {
329 MCode *mc = J->mcarea;
330 /* Try current area first to use the protection cache. */
331 if (ptr >= mc && ptr < (MCode *)((char *)mc + J->szmcarea)) {
332#if LUAJIT_SECURITY_MCODE
333 mcode_protect(J, MCPROT_GEN);
334#endif
335 return mc;
336 }
337 /* Otherwise search through the list of MCode areas. */
338 for (;;) {
339 mc = ((MCLink *)mc)->next;
340 lj_assertJ(mc != NULL, "broken MCode area chain");
341 if (ptr >= mc && ptr < (MCode *)((char *)mc + ((MCLink *)mc)->size)) {
342#if LUAJIT_SECURITY_MCODE
343 if (LJ_UNLIKELY(mcode_setprot(mc, ((MCLink *)mc)->size, MCPROT_GEN)))
344 mcode_protfail(J);
345#endif
346 return mc;
347 }
348 }
349 }
350}
351
352/* Limit of MCode reservation reached. */
353void lj_mcode_limiterr(jit_State *J, size_t need)
354{
355 size_t sizemcode, maxmcode;
356 lj_mcode_abort(J);
357 sizemcode = (size_t)J->param[JIT_P_sizemcode] << 10;
358 sizemcode = (sizemcode + LJ_PAGESIZE-1) & ~(size_t)(LJ_PAGESIZE - 1);
359 maxmcode = (size_t)J->param[JIT_P_maxmcode] << 10;
360 if ((size_t)need > sizemcode)
361 lj_trace_err(J, LJ_TRERR_MCODEOV); /* Too long for any area. */
362 if (J->szallmcarea + sizemcode > maxmcode)
363 lj_trace_err(J, LJ_TRERR_MCODEAL);
364 mcode_allocarea(J);
365 lj_trace_err(J, LJ_TRERR_MCODELM); /* Retry with new area. */
366}
367
368#endif
369