1/*
2** LuaJIT VM tags, values and objects.
3** Copyright (C) 2005-2021 Mike Pall. See Copyright Notice in luajit.h
4**
5** Portions taken verbatim or adapted from the Lua interpreter.
6** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
7*/
8
9#ifndef _LJ_OBJ_H
10#define _LJ_OBJ_H
11
12#include "lua.h"
13#include "lj_def.h"
14#include "lj_arch.h"
15
16/* -- Memory references --------------------------------------------------- */
17
18/* Memory and GC object sizes. */
19typedef uint32_t MSize;
20#if LJ_GC64
21typedef uint64_t GCSize;
22#else
23typedef uint32_t GCSize;
24#endif
25
26/* Memory reference */
27typedef struct MRef {
28#if LJ_GC64
29 uint64_t ptr64; /* True 64 bit pointer. */
30#else
31 uint32_t ptr32; /* Pseudo 32 bit pointer. */
32#endif
33} MRef;
34
35#if LJ_GC64
36#define mref(r, t) ((t *)(void *)(r).ptr64)
37
38#define setmref(r, p) ((r).ptr64 = (uint64_t)(void *)(p))
39#define setmrefr(r, v) ((r).ptr64 = (v).ptr64)
40#else
41#define mref(r, t) ((t *)(void *)(uintptr_t)(r).ptr32)
42
43#define setmref(r, p) ((r).ptr32 = (uint32_t)(uintptr_t)(void *)(p))
44#define setmrefr(r, v) ((r).ptr32 = (v).ptr32)
45#endif
46
47/* -- GC object references ------------------------------------------------ */
48
49/* GCobj reference */
50typedef struct GCRef {
51#if LJ_GC64
52 uint64_t gcptr64; /* True 64 bit pointer. */
53#else
54 uint32_t gcptr32; /* Pseudo 32 bit pointer. */
55#endif
56} GCRef;
57
58/* Common GC header for all collectable objects. */
59#define GCHeader GCRef nextgc; uint8_t marked; uint8_t gct
60/* This occupies 6 bytes, so use the next 2 bytes for non-32 bit fields. */
61
62#if LJ_GC64
63#define gcref(r) ((GCobj *)(r).gcptr64)
64#define gcrefp(r, t) ((t *)(void *)(r).gcptr64)
65#define gcrefu(r) ((r).gcptr64)
66#define gcrefeq(r1, r2) ((r1).gcptr64 == (r2).gcptr64)
67
68#define setgcref(r, gc) ((r).gcptr64 = (uint64_t)&(gc)->gch)
69#define setgcreft(r, gc, it) \
70 (r).gcptr64 = (uint64_t)&(gc)->gch | (((uint64_t)(it)) << 47)
71#define setgcrefp(r, p) ((r).gcptr64 = (uint64_t)(p))
72#define setgcrefnull(r) ((r).gcptr64 = 0)
73#define setgcrefr(r, v) ((r).gcptr64 = (v).gcptr64)
74#else
75#define gcref(r) ((GCobj *)(uintptr_t)(r).gcptr32)
76#define gcrefp(r, t) ((t *)(void *)(uintptr_t)(r).gcptr32)
77#define gcrefu(r) ((r).gcptr32)
78#define gcrefeq(r1, r2) ((r1).gcptr32 == (r2).gcptr32)
79
80#define setgcref(r, gc) ((r).gcptr32 = (uint32_t)(uintptr_t)&(gc)->gch)
81#define setgcrefp(r, p) ((r).gcptr32 = (uint32_t)(uintptr_t)(p))
82#define setgcrefnull(r) ((r).gcptr32 = 0)
83#define setgcrefr(r, v) ((r).gcptr32 = (v).gcptr32)
84#endif
85
86#define gcnext(gc) (gcref((gc)->gch.nextgc))
87
88/* IMPORTANT NOTE:
89**
90** All uses of the setgcref* macros MUST be accompanied with a write barrier.
91**
92** This is to ensure the integrity of the incremental GC. The invariant
93** to preserve is that a black object never points to a white object.
94** I.e. never store a white object into a field of a black object.
95**
96** It's ok to LEAVE OUT the write barrier ONLY in the following cases:
97** - The source is not a GC object (NULL).
98** - The target is a GC root. I.e. everything in global_State.
99** - The target is a lua_State field (threads are never black).
100** - The target is a stack slot, see setgcV et al.
101** - The target is an open upvalue, i.e. pointing to a stack slot.
102** - The target is a newly created object (i.e. marked white). But make
103** sure nothing invokes the GC inbetween.
104** - The target and the source are the same object (self-reference).
105** - The target already contains the object (e.g. moving elements around).
106**
107** The most common case is a store to a stack slot. All other cases where
108** a barrier has been omitted are annotated with a NOBARRIER comment.
109**
110** The same logic applies for stores to table slots (array part or hash
111** part). ALL uses of lj_tab_set* require a barrier for the stored value
112** *and* the stored key, based on the above rules. In practice this means
113** a barrier is needed if *either* of the key or value are a GC object.
114**
115** It's ok to LEAVE OUT the write barrier in the following special cases:
116** - The stored value is nil. The key doesn't matter because it's either
117** not resurrected or lj_tab_newkey() will take care of the key barrier.
118** - The key doesn't matter if the *previously* stored value is guaranteed
119** to be non-nil (because the key is kept alive in the table).
120** - The key doesn't matter if it's guaranteed not to be part of the table,
121** since lj_tab_newkey() takes care of the key barrier. This applies
122** trivially to new tables, but watch out for resurrected keys. Storing
123** a nil value leaves the key in the table!
124**
125** In case of doubt use lj_gc_anybarriert() as it's rather cheap. It's used
126** by the interpreter for all table stores.
127**
128** Note: In contrast to Lua's GC, LuaJIT's GC does *not* specially mark
129** dead keys in tables. The reference is left in, but it's guaranteed to
130** be never dereferenced as long as the value is nil. It's ok if the key is
131** freed or if any object subsequently gets the same address.
132**
133** Not destroying dead keys helps to keep key hash slots stable. This avoids
134** specialization back-off for HREFK when a value flips between nil and
135** non-nil and the GC gets in the way. It also allows safely hoisting
136** HREF/HREFK across GC steps. Dead keys are only removed if a table is
137** resized (i.e. by NEWREF) and xREF must not be CSEd across a resize.
138**
139** The trade-off is that a write barrier for tables must take the key into
140** account, too. Implicitly resurrecting the key by storing a non-nil value
141** may invalidate the incremental GC invariant.
142*/
143
144/* -- Common type definitions --------------------------------------------- */
145
146/* Types for handling bytecodes. Need this here, details in lj_bc.h. */
147typedef uint32_t BCIns; /* Bytecode instruction. */
148typedef uint32_t BCPos; /* Bytecode position. */
149typedef uint32_t BCReg; /* Bytecode register. */
150typedef int32_t BCLine; /* Bytecode line number. */
151
152/* Internal assembler functions. Never call these directly from C. */
153typedef void (*ASMFunction)(void);
154
155/* Resizable string buffer. Need this here, details in lj_buf.h. */
156typedef struct SBuf {
157 MRef p; /* String buffer pointer. */
158 MRef e; /* String buffer end pointer. */
159 MRef b; /* String buffer base. */
160 MRef L; /* lua_State, used for buffer resizing. */
161} SBuf;
162
163/* -- Tags and values ----------------------------------------------------- */
164
165/* Frame link. */
166typedef union {
167 int32_t ftsz; /* Frame type and size of previous frame. */
168 MRef pcr; /* Or PC for Lua frames. */
169} FrameLink;
170
171/* Tagged value. */
172typedef LJ_ALIGN(8) union TValue {
173 uint64_t u64; /* 64 bit pattern overlaps number. */
174 lua_Number n; /* Number object overlaps split tag/value object. */
175#if LJ_GC64
176 GCRef gcr; /* GCobj reference with tag. */
177 int64_t it64;
178 struct {
179 LJ_ENDIAN_LOHI(
180 int32_t i; /* Integer value. */
181 , uint32_t it; /* Internal object tag. Must overlap MSW of number. */
182 )
183 };
184#else
185 struct {
186 LJ_ENDIAN_LOHI(
187 union {
188 GCRef gcr; /* GCobj reference (if any). */
189 int32_t i; /* Integer value. */
190 };
191 , uint32_t it; /* Internal object tag. Must overlap MSW of number. */
192 )
193 };
194#endif
195#if LJ_FR2
196 int64_t ftsz; /* Frame type and size of previous frame, or PC. */
197#else
198 struct {
199 LJ_ENDIAN_LOHI(
200 GCRef func; /* Function for next frame (or dummy L). */
201 , FrameLink tp; /* Link to previous frame. */
202 )
203 } fr;
204#endif
205 struct {
206 LJ_ENDIAN_LOHI(
207 uint32_t lo; /* Lower 32 bits of number. */
208 , uint32_t hi; /* Upper 32 bits of number. */
209 )
210 } u32;
211} TValue;
212
213typedef const TValue cTValue;
214
215#define tvref(r) (mref(r, TValue))
216
217/* More external and GCobj tags for internal objects. */
218#define LAST_TT LUA_TTHREAD
219#define LUA_TPROTO (LAST_TT+1)
220#define LUA_TCDATA (LAST_TT+2)
221
222/* Internal object tags.
223**
224** Format for 32 bit GC references (!LJ_GC64):
225**
226** Internal tags overlap the MSW of a number object (must be a double).
227** Interpreted as a double these are special NaNs. The FPU only generates
228** one type of NaN (0xfff8_0000_0000_0000). So MSWs > 0xfff80000 are available
229** for use as internal tags. Small negative numbers are used to shorten the
230** encoding of type comparisons (reg/mem against sign-ext. 8 bit immediate).
231**
232** ---MSW---.---LSW---
233** primitive types | itype | |
234** lightuserdata | itype | void * | (32 bit platforms)
235** lightuserdata |ffff|seg| ofs | (64 bit platforms)
236** GC objects | itype | GCRef |
237** int (LJ_DUALNUM)| itype | int |
238** number -------double------
239**
240** Format for 64 bit GC references (LJ_GC64):
241**
242** The upper 13 bits must be 1 (0xfff8...) for a special NaN. The next
243** 4 bits hold the internal tag. The lowest 47 bits either hold a pointer,
244** a zero-extended 32 bit integer or all bits set to 1 for primitive types.
245**
246** ------MSW------.------LSW------
247** primitive types |1..1|itype|1..................1|
248** GC objects |1..1|itype|-------GCRef--------|
249** lightuserdata |1..1|itype|seg|------ofs-------|
250** int (LJ_DUALNUM) |1..1|itype|0..0|-----int-------|
251** number ------------double-------------
252**
253** ORDER LJ_T
254** Primitive types nil/false/true must be first, lightuserdata next.
255** GC objects are at the end, table/userdata must be lowest.
256** Also check lj_ir.h for similar ordering constraints.
257*/
258#define LJ_TNIL (~0u)
259#define LJ_TFALSE (~1u)
260#define LJ_TTRUE (~2u)
261#define LJ_TLIGHTUD (~3u)
262#define LJ_TSTR (~4u)
263#define LJ_TUPVAL (~5u)
264#define LJ_TTHREAD (~6u)
265#define LJ_TPROTO (~7u)
266#define LJ_TFUNC (~8u)
267#define LJ_TTRACE (~9u)
268#define LJ_TCDATA (~10u)
269#define LJ_TTAB (~11u)
270#define LJ_TUDATA (~12u)
271/* This is just the canonical number type used in some places. */
272#define LJ_TNUMX (~13u)
273
274/* Integers have itype == LJ_TISNUM doubles have itype < LJ_TISNUM */
275#if LJ_64 && !LJ_GC64
276#define LJ_TISNUM 0xfffeffffu
277#else
278#define LJ_TISNUM LJ_TNUMX
279#endif
280#define LJ_TISTRUECOND LJ_TFALSE
281#define LJ_TISPRI LJ_TTRUE
282#define LJ_TISGCV (LJ_TSTR+1)
283#define LJ_TISTABUD LJ_TTAB
284
285#if LJ_GC64
286#define LJ_GCVMASK (((uint64_t)1 << 47) - 1)
287#endif
288
289#if LJ_64
290/* To stay within 47 bits, lightuserdata is segmented. */
291#define LJ_LIGHTUD_BITS_SEG 8
292#define LJ_LIGHTUD_BITS_LO (47 - LJ_LIGHTUD_BITS_SEG)
293#endif
294
295/* -- String object ------------------------------------------------------- */
296
297typedef uint32_t StrHash; /* String hash value. */
298typedef uint32_t StrID; /* String ID. */
299
300/* String object header. String payload follows. */
301typedef struct GCstr {
302 GCHeader;
303 uint8_t reserved; /* Used by lexer for fast lookup of reserved words. */
304 uint8_t hashalg; /* Hash algorithm. */
305 StrID sid; /* Interned string ID. */
306 StrHash hash; /* Hash of string. */
307 MSize len; /* Size of string. */
308} GCstr;
309
310#define strref(r) (&gcref((r))->str)
311#define strdata(s) ((const char *)((s)+1))
312#define strdatawr(s) ((char *)((s)+1))
313#define strVdata(o) strdata(strV(o))
314
315/* -- Userdata object ----------------------------------------------------- */
316
317/* Userdata object. Payload follows. */
318typedef struct GCudata {
319 GCHeader;
320 uint8_t udtype; /* Userdata type. */
321 uint8_t unused2;
322 GCRef env; /* Should be at same offset in GCfunc. */
323 MSize len; /* Size of payload. */
324 GCRef metatable; /* Must be at same offset in GCtab. */
325 uint32_t align1; /* To force 8 byte alignment of the payload. */
326} GCudata;
327
328/* Userdata types. */
329enum {
330 UDTYPE_USERDATA, /* Regular userdata. */
331 UDTYPE_IO_FILE, /* I/O library FILE. */
332 UDTYPE_FFI_CLIB, /* FFI C library namespace. */
333 UDTYPE__MAX
334};
335
336#define uddata(u) ((void *)((u)+1))
337#define sizeudata(u) (sizeof(struct GCudata)+(u)->len)
338
339/* -- C data object ------------------------------------------------------- */
340
341/* C data object. Payload follows. */
342typedef struct GCcdata {
343 GCHeader;
344 uint16_t ctypeid; /* C type ID. */
345} GCcdata;
346
347/* Prepended to variable-sized or realigned C data objects. */
348typedef struct GCcdataVar {
349 uint16_t offset; /* Offset to allocated memory (relative to GCcdata). */
350 uint16_t extra; /* Extra space allocated (incl. GCcdata + GCcdatav). */
351 MSize len; /* Size of payload. */
352} GCcdataVar;
353
354#define cdataptr(cd) ((void *)((cd)+1))
355#define cdataisv(cd) ((cd)->marked & 0x80)
356#define cdatav(cd) ((GCcdataVar *)((char *)(cd) - sizeof(GCcdataVar)))
357#define cdatavlen(cd) check_exp(cdataisv(cd), cdatav(cd)->len)
358#define sizecdatav(cd) (cdatavlen(cd) + cdatav(cd)->extra)
359#define memcdatav(cd) ((void *)((char *)(cd) - cdatav(cd)->offset))
360
361/* -- Prototype object ---------------------------------------------------- */
362
363#define SCALE_NUM_GCO ((int32_t)sizeof(lua_Number)/sizeof(GCRef))
364#define round_nkgc(n) (((n) + SCALE_NUM_GCO-1) & ~(SCALE_NUM_GCO-1))
365
366typedef struct GCproto {
367 GCHeader;
368 uint8_t numparams; /* Number of parameters. */
369 uint8_t framesize; /* Fixed frame size. */
370 MSize sizebc; /* Number of bytecode instructions. */
371#if LJ_GC64
372 uint32_t unused_gc64;
373#endif
374 GCRef gclist;
375 MRef k; /* Split constant array (points to the middle). */
376 MRef uv; /* Upvalue list. local slot|0x8000 or parent uv idx. */
377 MSize sizekgc; /* Number of collectable constants. */
378 MSize sizekn; /* Number of lua_Number constants. */
379 MSize sizept; /* Total size including colocated arrays. */
380 uint8_t sizeuv; /* Number of upvalues. */
381 uint8_t flags; /* Miscellaneous flags (see below). */
382 uint16_t trace; /* Anchor for chain of root traces. */
383 /* ------ The following fields are for debugging/tracebacks only ------ */
384 GCRef chunkname; /* Name of the chunk this function was defined in. */
385 BCLine firstline; /* First line of the function definition. */
386 BCLine numline; /* Number of lines for the function definition. */
387 MRef lineinfo; /* Compressed map from bytecode ins. to source line. */
388 MRef uvinfo; /* Upvalue names. */
389 MRef varinfo; /* Names and compressed extents of local variables. */
390} GCproto;
391
392/* Flags for prototype. */
393#define PROTO_CHILD 0x01 /* Has child prototypes. */
394#define PROTO_VARARG 0x02 /* Vararg function. */
395#define PROTO_FFI 0x04 /* Uses BC_KCDATA for FFI datatypes. */
396#define PROTO_NOJIT 0x08 /* JIT disabled for this function. */
397#define PROTO_ILOOP 0x10 /* Patched bytecode with ILOOP etc. */
398/* Only used during parsing. */
399#define PROTO_HAS_RETURN 0x20 /* Already emitted a return. */
400#define PROTO_FIXUP_RETURN 0x40 /* Need to fixup emitted returns. */
401/* Top bits used for counting created closures. */
402#define PROTO_CLCOUNT 0x20 /* Base of saturating 3 bit counter. */
403#define PROTO_CLC_BITS 3
404#define PROTO_CLC_POLY (3*PROTO_CLCOUNT) /* Polymorphic threshold. */
405
406#define PROTO_UV_LOCAL 0x8000 /* Upvalue for local slot. */
407#define PROTO_UV_IMMUTABLE 0x4000 /* Immutable upvalue. */
408
409#define proto_kgc(pt, idx) \
410 check_exp((uintptr_t)(intptr_t)(idx) >= (uintptr_t)-(intptr_t)(pt)->sizekgc, \
411 gcref(mref((pt)->k, GCRef)[(idx)]))
412#define proto_knumtv(pt, idx) \
413 check_exp((uintptr_t)(idx) < (pt)->sizekn, &mref((pt)->k, TValue)[(idx)])
414#define proto_bc(pt) ((BCIns *)((char *)(pt) + sizeof(GCproto)))
415#define proto_bcpos(pt, pc) ((BCPos)((pc) - proto_bc(pt)))
416#define proto_uv(pt) (mref((pt)->uv, uint16_t))
417
418#define proto_chunkname(pt) (strref((pt)->chunkname))
419#define proto_chunknamestr(pt) (strdata(proto_chunkname((pt))))
420#define proto_lineinfo(pt) (mref((pt)->lineinfo, const void))
421#define proto_uvinfo(pt) (mref((pt)->uvinfo, const uint8_t))
422#define proto_varinfo(pt) (mref((pt)->varinfo, const uint8_t))
423
424/* -- Upvalue object ------------------------------------------------------ */
425
426typedef struct GCupval {
427 GCHeader;
428 uint8_t closed; /* Set if closed (i.e. uv->v == &uv->u.value). */
429 uint8_t immutable; /* Immutable value. */
430 union {
431 TValue tv; /* If closed: the value itself. */
432 struct { /* If open: double linked list, anchored at thread. */
433 GCRef prev;
434 GCRef next;
435 };
436 };
437 MRef v; /* Points to stack slot (open) or above (closed). */
438 uint32_t dhash; /* Disambiguation hash: dh1 != dh2 => cannot alias. */
439} GCupval;
440
441#define uvprev(uv_) (&gcref((uv_)->prev)->uv)
442#define uvnext(uv_) (&gcref((uv_)->next)->uv)
443#define uvval(uv_) (mref((uv_)->v, TValue))
444
445/* -- Function object (closures) ------------------------------------------ */
446
447/* Common header for functions. env should be at same offset in GCudata. */
448#define GCfuncHeader \
449 GCHeader; uint8_t ffid; uint8_t nupvalues; \
450 GCRef env; GCRef gclist; MRef pc
451
452typedef struct GCfuncC {
453 GCfuncHeader;
454 lua_CFunction f; /* C function to be called. */
455 TValue upvalue[1]; /* Array of upvalues (TValue). */
456} GCfuncC;
457
458typedef struct GCfuncL {
459 GCfuncHeader;
460 GCRef uvptr[1]; /* Array of _pointers_ to upvalue objects (GCupval). */
461} GCfuncL;
462
463typedef union GCfunc {
464 GCfuncC c;
465 GCfuncL l;
466} GCfunc;
467
468#define FF_LUA 0
469#define FF_C 1
470#define isluafunc(fn) ((fn)->c.ffid == FF_LUA)
471#define iscfunc(fn) ((fn)->c.ffid == FF_C)
472#define isffunc(fn) ((fn)->c.ffid > FF_C)
473#define funcproto(fn) \
474 check_exp(isluafunc(fn), (GCproto *)(mref((fn)->l.pc, char)-sizeof(GCproto)))
475#define sizeCfunc(n) (sizeof(GCfuncC)-sizeof(TValue)+sizeof(TValue)*(n))
476#define sizeLfunc(n) (sizeof(GCfuncL)-sizeof(GCRef)+sizeof(GCRef)*(n))
477
478/* -- Table object -------------------------------------------------------- */
479
480/* Hash node. */
481typedef struct Node {
482 TValue val; /* Value object. Must be first field. */
483 TValue key; /* Key object. */
484 MRef next; /* Hash chain. */
485#if !LJ_GC64
486 MRef freetop; /* Top of free elements (stored in t->node[0]). */
487#endif
488} Node;
489
490LJ_STATIC_ASSERT(offsetof(Node, val) == 0);
491
492typedef struct GCtab {
493 GCHeader;
494 uint8_t nomm; /* Negative cache for fast metamethods. */
495 int8_t colo; /* Array colocation. */
496 MRef array; /* Array part. */
497 GCRef gclist;
498 GCRef metatable; /* Must be at same offset in GCudata. */
499 MRef node; /* Hash part. */
500 uint32_t asize; /* Size of array part (keys [0, asize-1]). */
501 uint32_t hmask; /* Hash part mask (size of hash part - 1). */
502#if LJ_GC64
503 MRef freetop; /* Top of free elements. */
504#endif
505} GCtab;
506
507#define sizetabcolo(n) ((n)*sizeof(TValue) + sizeof(GCtab))
508#define tabref(r) (&gcref((r))->tab)
509#define noderef(r) (mref((r), Node))
510#define nextnode(n) (mref((n)->next, Node))
511#if LJ_GC64
512#define getfreetop(t, n) (noderef((t)->freetop))
513#define setfreetop(t, n, v) (setmref((t)->freetop, (v)))
514#else
515#define getfreetop(t, n) (noderef((n)->freetop))
516#define setfreetop(t, n, v) (setmref((n)->freetop, (v)))
517#endif
518
519/* -- State objects ------------------------------------------------------- */
520
521/* VM states. */
522enum {
523 LJ_VMST_INTERP, /* Interpreter. */
524 LJ_VMST_C, /* C function. */
525 LJ_VMST_GC, /* Garbage collector. */
526 LJ_VMST_EXIT, /* Trace exit handler. */
527 LJ_VMST_RECORD, /* Trace recorder. */
528 LJ_VMST_OPT, /* Optimizer. */
529 LJ_VMST_ASM, /* Assembler. */
530 LJ_VMST__MAX
531};
532
533#define setvmstate(g, st) ((g)->vmstate = ~LJ_VMST_##st)
534
535/* Metamethods. ORDER MM */
536#ifdef LJ_HASFFI
537#define MMDEF_FFI(_) _(new)
538#else
539#define MMDEF_FFI(_)
540#endif
541
542#if LJ_52 || LJ_HASFFI
543#define MMDEF_PAIRS(_) _(pairs) _(ipairs)
544#else
545#define MMDEF_PAIRS(_)
546#define MM_pairs 255
547#define MM_ipairs 255
548#endif
549
550#define MMDEF(_) \
551 _(index) _(newindex) _(gc) _(mode) _(eq) _(len) \
552 /* Only the above (fast) metamethods are negative cached (max. 8). */ \
553 _(lt) _(le) _(concat) _(call) \
554 /* The following must be in ORDER ARITH. */ \
555 _(add) _(sub) _(mul) _(div) _(mod) _(pow) _(unm) \
556 /* The following are used in the standard libraries. */ \
557 _(metatable) _(tostring) MMDEF_FFI(_) MMDEF_PAIRS(_)
558
559typedef enum {
560#define MMENUM(name) MM_##name,
561MMDEF(MMENUM)
562#undef MMENUM
563 MM__MAX,
564 MM____ = MM__MAX,
565 MM_FAST = MM_len
566} MMS;
567
568/* GC root IDs. */
569typedef enum {
570 GCROOT_MMNAME, /* Metamethod names. */
571 GCROOT_MMNAME_LAST = GCROOT_MMNAME + MM__MAX-1,
572 GCROOT_BASEMT, /* Metatables for base types. */
573 GCROOT_BASEMT_NUM = GCROOT_BASEMT + ~LJ_TNUMX,
574 GCROOT_IO_INPUT, /* Userdata for default I/O input file. */
575 GCROOT_IO_OUTPUT, /* Userdata for default I/O output file. */
576 GCROOT_MAX
577} GCRootID;
578
579#define basemt_it(g, it) ((g)->gcroot[GCROOT_BASEMT+~(it)])
580#define basemt_obj(g, o) ((g)->gcroot[GCROOT_BASEMT+itypemap(o)])
581#define mmname_str(g, mm) (strref((g)->gcroot[GCROOT_MMNAME+(mm)]))
582
583/* Garbage collector state. */
584typedef struct GCState {
585 GCSize total; /* Memory currently allocated. */
586 GCSize threshold; /* Memory threshold. */
587 uint8_t currentwhite; /* Current white color. */
588 uint8_t state; /* GC state. */
589 uint8_t nocdatafin; /* No cdata finalizer called. */
590#if LJ_64
591 uint8_t lightudnum; /* Number of lightuserdata segments - 1. */
592#else
593 uint8_t unused1;
594#endif
595 MSize sweepstr; /* Sweep position in string table. */
596 GCRef root; /* List of all collectable objects. */
597 MRef sweep; /* Sweep position in root list. */
598 GCRef gray; /* List of gray objects. */
599 GCRef grayagain; /* List of objects for atomic traversal. */
600 GCRef weak; /* List of weak tables (to be cleared). */
601 GCRef mmudata; /* List of userdata (to be finalized). */
602 GCSize debt; /* Debt (how much GC is behind schedule). */
603 GCSize estimate; /* Estimate of memory actually in use. */
604 MSize stepmul; /* Incremental GC step granularity. */
605 MSize pause; /* Pause between successive GC cycles. */
606#if LJ_64
607 MRef lightudseg; /* Upper bits of lightuserdata segments. */
608#endif
609} GCState;
610
611/* String interning state. */
612typedef struct StrInternState {
613 GCRef *tab; /* String hash table anchors. */
614 MSize mask; /* String hash mask (size of hash table - 1). */
615 MSize num; /* Number of strings in hash table. */
616 StrID id; /* Next string ID. */
617 uint8_t idreseed; /* String ID reseed counter. */
618 uint8_t second; /* String interning table uses secondary hashing. */
619 uint8_t unused1;
620 uint8_t unused2;
621 LJ_ALIGN(8) uint64_t seed; /* Random string seed. */
622} StrInternState;
623
624/* Global state, shared by all threads of a Lua universe. */
625typedef struct global_State {
626 lua_Alloc allocf; /* Memory allocator. */
627 void *allocd; /* Memory allocator data. */
628 GCState gc; /* Garbage collector. */
629 GCstr strempty; /* Empty string. */
630 uint8_t stremptyz; /* Zero terminator of empty string. */
631 uint8_t hookmask; /* Hook mask. */
632 uint8_t dispatchmode; /* Dispatch mode. */
633 uint8_t vmevmask; /* VM event mask. */
634 StrInternState str; /* String interning. */
635 volatile int32_t vmstate; /* VM state or current JIT code trace number. */
636 GCRef mainthref; /* Link to main thread. */
637 SBuf tmpbuf; /* Temporary string buffer. */
638 TValue tmptv, tmptv2; /* Temporary TValues. */
639 Node nilnode; /* Fallback 1-element hash part (nil key and value). */
640 TValue registrytv; /* Anchor for registry. */
641 GCupval uvhead; /* Head of double-linked list of all open upvalues. */
642 int32_t hookcount; /* Instruction hook countdown. */
643 int32_t hookcstart; /* Start count for instruction hook counter. */
644 lua_Hook hookf; /* Hook function. */
645 lua_CFunction wrapf; /* Wrapper for C function calls. */
646 lua_CFunction panic; /* Called as a last resort for errors. */
647 BCIns bc_cfunc_int; /* Bytecode for internal C function calls. */
648 BCIns bc_cfunc_ext; /* Bytecode for external C function calls. */
649 GCRef cur_L; /* Currently executing lua_State. */
650 MRef jit_base; /* Current JIT code L->base or NULL. */
651 MRef ctype_state; /* Pointer to C type state. */
652 PRNGState prng; /* Global PRNG state. */
653 GCRef gcroot[GCROOT_MAX]; /* GC roots. */
654} global_State;
655
656#define mainthread(g) (&gcref(g->mainthref)->th)
657#define niltv(L) \
658 check_exp(tvisnil(&G(L)->nilnode.val), &G(L)->nilnode.val)
659#define niltvg(g) \
660 check_exp(tvisnil(&(g)->nilnode.val), &(g)->nilnode.val)
661
662/* Hook management. Hook event masks are defined in lua.h. */
663#define HOOK_EVENTMASK 0x0f
664#define HOOK_ACTIVE 0x10
665#define HOOK_ACTIVE_SHIFT 4
666#define HOOK_VMEVENT 0x20
667#define HOOK_GC 0x40
668#define HOOK_PROFILE 0x80
669#define hook_active(g) ((g)->hookmask & HOOK_ACTIVE)
670#define hook_enter(g) ((g)->hookmask |= HOOK_ACTIVE)
671#define hook_entergc(g) \
672 ((g)->hookmask = ((g)->hookmask | (HOOK_ACTIVE|HOOK_GC)) & ~HOOK_PROFILE)
673#define hook_vmevent(g) ((g)->hookmask |= (HOOK_ACTIVE|HOOK_VMEVENT))
674#define hook_leave(g) ((g)->hookmask &= ~HOOK_ACTIVE)
675#define hook_save(g) ((g)->hookmask & ~HOOK_EVENTMASK)
676#define hook_restore(g, h) \
677 ((g)->hookmask = ((g)->hookmask & HOOK_EVENTMASK) | (h))
678
679/* Per-thread state object. */
680struct lua_State {
681 GCHeader;
682 uint8_t dummy_ffid; /* Fake FF_C for curr_funcisL() on dummy frames. */
683 uint8_t status; /* Thread status. */
684 MRef glref; /* Link to global state. */
685 GCRef gclist; /* GC chain. */
686 TValue *base; /* Base of currently executing function. */
687 TValue *top; /* First free slot in the stack. */
688 MRef maxstack; /* Last free slot in the stack. */
689 MRef stack; /* Stack base. */
690 GCRef openupval; /* List of open upvalues in the stack. */
691 GCRef env; /* Thread environment (table of globals). */
692 void *cframe; /* End of C stack frame chain. */
693 MSize stacksize; /* True stack size (incl. LJ_STACK_EXTRA). */
694};
695
696#define G(L) (mref(L->glref, global_State))
697#define registry(L) (&G(L)->registrytv)
698
699/* Macros to access the currently executing (Lua) function. */
700#if LJ_GC64
701#define curr_func(L) (&gcval(L->base-2)->fn)
702#elif LJ_FR2
703#define curr_func(L) (&gcref((L->base-2)->gcr)->fn)
704#else
705#define curr_func(L) (&gcref((L->base-1)->fr.func)->fn)
706#endif
707#define curr_funcisL(L) (isluafunc(curr_func(L)))
708#define curr_proto(L) (funcproto(curr_func(L)))
709#define curr_topL(L) (L->base + curr_proto(L)->framesize)
710#define curr_top(L) (curr_funcisL(L) ? curr_topL(L) : L->top)
711
712#if defined(LUA_USE_ASSERT) || defined(LUA_USE_APICHECK)
713LJ_FUNC_NORET void lj_assert_fail(global_State *g, const char *file, int line,
714 const char *func, const char *fmt, ...);
715#endif
716
717/* -- GC object definition and conversions -------------------------------- */
718
719/* GC header for generic access to common fields of GC objects. */
720typedef struct GChead {
721 GCHeader;
722 uint8_t unused1;
723 uint8_t unused2;
724 GCRef env;
725 GCRef gclist;
726 GCRef metatable;
727} GChead;
728
729/* The env field SHOULD be at the same offset for all GC objects. */
730LJ_STATIC_ASSERT(offsetof(GChead, env) == offsetof(GCfuncL, env));
731LJ_STATIC_ASSERT(offsetof(GChead, env) == offsetof(GCudata, env));
732
733/* The metatable field MUST be at the same offset for all GC objects. */
734LJ_STATIC_ASSERT(offsetof(GChead, metatable) == offsetof(GCtab, metatable));
735LJ_STATIC_ASSERT(offsetof(GChead, metatable) == offsetof(GCudata, metatable));
736
737/* The gclist field MUST be at the same offset for all GC objects. */
738LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(lua_State, gclist));
739LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCproto, gclist));
740LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCfuncL, gclist));
741LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCtab, gclist));
742
743typedef union GCobj {
744 GChead gch;
745 GCstr str;
746 GCupval uv;
747 lua_State th;
748 GCproto pt;
749 GCfunc fn;
750 GCcdata cd;
751 GCtab tab;
752 GCudata ud;
753} GCobj;
754
755/* Macros to convert a GCobj pointer into a specific value. */
756#define gco2str(o) check_exp((o)->gch.gct == ~LJ_TSTR, &(o)->str)
757#define gco2uv(o) check_exp((o)->gch.gct == ~LJ_TUPVAL, &(o)->uv)
758#define gco2th(o) check_exp((o)->gch.gct == ~LJ_TTHREAD, &(o)->th)
759#define gco2pt(o) check_exp((o)->gch.gct == ~LJ_TPROTO, &(o)->pt)
760#define gco2func(o) check_exp((o)->gch.gct == ~LJ_TFUNC, &(o)->fn)
761#define gco2cd(o) check_exp((o)->gch.gct == ~LJ_TCDATA, &(o)->cd)
762#define gco2tab(o) check_exp((o)->gch.gct == ~LJ_TTAB, &(o)->tab)
763#define gco2ud(o) check_exp((o)->gch.gct == ~LJ_TUDATA, &(o)->ud)
764
765/* Macro to convert any collectable object into a GCobj pointer. */
766#define obj2gco(v) ((GCobj *)(v))
767
768/* -- TValue getters/setters ---------------------------------------------- */
769
770/* Macros to test types. */
771#if LJ_GC64
772#define itype(o) ((uint32_t)((o)->it64 >> 47))
773#define tvisnil(o) ((o)->it64 == -1)
774#else
775#define itype(o) ((o)->it)
776#define tvisnil(o) (itype(o) == LJ_TNIL)
777#endif
778#define tvisfalse(o) (itype(o) == LJ_TFALSE)
779#define tvistrue(o) (itype(o) == LJ_TTRUE)
780#define tvisbool(o) (tvisfalse(o) || tvistrue(o))
781#if LJ_64 && !LJ_GC64
782#define tvislightud(o) (((int32_t)itype(o) >> 15) == -2)
783#else
784#define tvislightud(o) (itype(o) == LJ_TLIGHTUD)
785#endif
786#define tvisstr(o) (itype(o) == LJ_TSTR)
787#define tvisfunc(o) (itype(o) == LJ_TFUNC)
788#define tvisthread(o) (itype(o) == LJ_TTHREAD)
789#define tvisproto(o) (itype(o) == LJ_TPROTO)
790#define tviscdata(o) (itype(o) == LJ_TCDATA)
791#define tvistab(o) (itype(o) == LJ_TTAB)
792#define tvisudata(o) (itype(o) == LJ_TUDATA)
793#define tvisnumber(o) (itype(o) <= LJ_TISNUM)
794#define tvisint(o) (LJ_DUALNUM && itype(o) == LJ_TISNUM)
795#define tvisnum(o) (itype(o) < LJ_TISNUM)
796
797#define tvistruecond(o) (itype(o) < LJ_TISTRUECOND)
798#define tvispri(o) (itype(o) >= LJ_TISPRI)
799#define tvistabud(o) (itype(o) <= LJ_TISTABUD) /* && !tvisnum() */
800#define tvisgcv(o) ((itype(o) - LJ_TISGCV) > (LJ_TNUMX - LJ_TISGCV))
801
802/* Special macros to test numbers for NaN, +0, -0, +1 and raw equality. */
803#define tvisnan(o) ((o)->n != (o)->n)
804#if LJ_64
805#define tviszero(o) (((o)->u64 << 1) == 0)
806#else
807#define tviszero(o) (((o)->u32.lo | ((o)->u32.hi << 1)) == 0)
808#endif
809#define tvispzero(o) ((o)->u64 == 0)
810#define tvismzero(o) ((o)->u64 == U64x(80000000,00000000))
811#define tvispone(o) ((o)->u64 == U64x(3ff00000,00000000))
812#define rawnumequal(o1, o2) ((o1)->u64 == (o2)->u64)
813
814/* Macros to convert type ids. */
815#if LJ_64 && !LJ_GC64
816#define itypemap(o) \
817 (tvisnumber(o) ? ~LJ_TNUMX : tvislightud(o) ? ~LJ_TLIGHTUD : ~itype(o))
818#else
819#define itypemap(o) (tvisnumber(o) ? ~LJ_TNUMX : ~itype(o))
820#endif
821
822/* Macros to get tagged values. */
823#if LJ_GC64
824#define gcval(o) ((GCobj *)(gcrefu((o)->gcr) & LJ_GCVMASK))
825#else
826#define gcval(o) (gcref((o)->gcr))
827#endif
828#define boolV(o) check_exp(tvisbool(o), (LJ_TFALSE - itype(o)))
829#if LJ_64
830#define lightudseg(u) \
831 (((u) >> LJ_LIGHTUD_BITS_LO) & ((1 << LJ_LIGHTUD_BITS_SEG)-1))
832#define lightudlo(u) \
833 ((u) & (((uint64_t)1 << LJ_LIGHTUD_BITS_LO) - 1))
834#define lightudup(p) \
835 ((uint32_t)(((p) >> LJ_LIGHTUD_BITS_LO) << (LJ_LIGHTUD_BITS_LO-32)))
836static LJ_AINLINE void *lightudV(global_State *g, cTValue *o)
837{
838 uint64_t u = o->u64;
839 uint64_t seg = lightudseg(u);
840 uint32_t *segmap = mref(g->gc.lightudseg, uint32_t);
841 lj_assertG(tvislightud(o), "lightuserdata expected");
842 lj_assertG(seg <= g->gc.lightudnum, "bad lightuserdata segment %d", seg);
843 return (void *)(((uint64_t)segmap[seg] << 32) | lightudlo(u));
844}
845#else
846#define lightudV(g, o) check_exp(tvislightud(o), gcrefp((o)->gcr, void))
847#endif
848#define gcV(o) check_exp(tvisgcv(o), gcval(o))
849#define strV(o) check_exp(tvisstr(o), &gcval(o)->str)
850#define funcV(o) check_exp(tvisfunc(o), &gcval(o)->fn)
851#define threadV(o) check_exp(tvisthread(o), &gcval(o)->th)
852#define protoV(o) check_exp(tvisproto(o), &gcval(o)->pt)
853#define cdataV(o) check_exp(tviscdata(o), &gcval(o)->cd)
854#define tabV(o) check_exp(tvistab(o), &gcval(o)->tab)
855#define udataV(o) check_exp(tvisudata(o), &gcval(o)->ud)
856#define numV(o) check_exp(tvisnum(o), (o)->n)
857#define intV(o) check_exp(tvisint(o), (int32_t)(o)->i)
858
859/* Macros to set tagged values. */
860#if LJ_GC64
861#define setitype(o, i) ((o)->it = ((i) << 15))
862#define setnilV(o) ((o)->it64 = -1)
863#define setpriV(o, x) ((o)->it64 = (int64_t)~((uint64_t)~(x)<<47))
864#define setboolV(o, x) ((o)->it64 = (int64_t)~((uint64_t)((x)+1)<<47))
865#else
866#define setitype(o, i) ((o)->it = (i))
867#define setnilV(o) ((o)->it = LJ_TNIL)
868#define setboolV(o, x) ((o)->it = LJ_TFALSE-(uint32_t)(x))
869#define setpriV(o, i) (setitype((o), (i)))
870#endif
871
872static LJ_AINLINE void setrawlightudV(TValue *o, void *p)
873{
874#if LJ_GC64
875 o->u64 = (uint64_t)p | (((uint64_t)LJ_TLIGHTUD) << 47);
876#elif LJ_64
877 o->u64 = (uint64_t)p | (((uint64_t)0xffff) << 48);
878#else
879 setgcrefp(o->gcr, p); setitype(o, LJ_TLIGHTUD);
880#endif
881}
882
883#if LJ_FR2 || LJ_32
884#define contptr(f) ((void *)(f))
885#define setcont(o, f) ((o)->u64 = (uint64_t)(uintptr_t)contptr(f))
886#else
887#define contptr(f) \
888 ((void *)(uintptr_t)(uint32_t)((intptr_t)(f) - (intptr_t)lj_vm_asm_begin))
889#define setcont(o, f) \
890 ((o)->u64 = (uint64_t)(void *)(f) - (uint64_t)lj_vm_asm_begin)
891#endif
892
893static LJ_AINLINE void checklivetv(lua_State *L, TValue *o, const char *msg)
894{
895 UNUSED(L); UNUSED(o); UNUSED(msg);
896#if LUA_USE_ASSERT
897 if (tvisgcv(o)) {
898 lj_assertL(~itype(o) == gcval(o)->gch.gct,
899 "mismatch of TValue type %d vs GC type %d",
900 ~itype(o), gcval(o)->gch.gct);
901 /* Copy of isdead check from lj_gc.h to avoid circular include. */
902 lj_assertL(!(gcval(o)->gch.marked & (G(L)->gc.currentwhite ^ 3) & 3), msg);
903 }
904#endif
905}
906
907static LJ_AINLINE void setgcVraw(TValue *o, GCobj *v, uint32_t itype)
908{
909#if LJ_GC64
910 setgcreft(o->gcr, v, itype);
911#else
912 setgcref(o->gcr, v); setitype(o, itype);
913#endif
914}
915
916static LJ_AINLINE void setgcV(lua_State *L, TValue *o, GCobj *v, uint32_t it)
917{
918 setgcVraw(o, v, it);
919 checklivetv(L, o, "store to dead GC object");
920}
921
922#define define_setV(name, type, tag) \
923static LJ_AINLINE void name(lua_State *L, TValue *o, type *v) \
924{ \
925 setgcV(L, o, obj2gco(v), tag); \
926}
927define_setV(setstrV, GCstr, LJ_TSTR)
928define_setV(setthreadV, lua_State, LJ_TTHREAD)
929define_setV(setprotoV, GCproto, LJ_TPROTO)
930define_setV(setfuncV, GCfunc, LJ_TFUNC)
931define_setV(setcdataV, GCcdata, LJ_TCDATA)
932define_setV(settabV, GCtab, LJ_TTAB)
933define_setV(setudataV, GCudata, LJ_TUDATA)
934
935#define setnumV(o, x) ((o)->n = (x))
936#define setnanV(o) ((o)->u64 = U64x(fff80000,00000000))
937#define setpinfV(o) ((o)->u64 = U64x(7ff00000,00000000))
938#define setminfV(o) ((o)->u64 = U64x(fff00000,00000000))
939
940static LJ_AINLINE void setintV(TValue *o, int32_t i)
941{
942#if LJ_DUALNUM
943 o->i = (uint32_t)i; setitype(o, LJ_TISNUM);
944#else
945 o->n = (lua_Number)i;
946#endif
947}
948
949static LJ_AINLINE void setint64V(TValue *o, int64_t i)
950{
951 if (LJ_DUALNUM && LJ_LIKELY(i == (int64_t)(int32_t)i))
952 setintV(o, (int32_t)i);
953 else
954 setnumV(o, (lua_Number)i);
955}
956
957#if LJ_64
958#define setintptrV(o, i) setint64V((o), (i))
959#else
960#define setintptrV(o, i) setintV((o), (i))
961#endif
962
963/* Copy tagged values. */
964static LJ_AINLINE void copyTV(lua_State *L, TValue *o1, const TValue *o2)
965{
966 *o1 = *o2;
967 checklivetv(L, o1, "copy of dead GC object");
968}
969
970/* -- Number to integer conversion ---------------------------------------- */
971
972#if LJ_SOFTFP
973LJ_ASMF int32_t lj_vm_tobit(double x);
974#if LJ_TARGET_MIPS64
975LJ_ASMF int32_t lj_vm_tointg(double x);
976#endif
977#endif
978
979static LJ_AINLINE int32_t lj_num2bit(lua_Number n)
980{
981#if LJ_SOFTFP
982 return lj_vm_tobit(n);
983#else
984 TValue o;
985 o.n = n + 6755399441055744.0; /* 2^52 + 2^51 */
986 return (int32_t)o.u32.lo;
987#endif
988}
989
990#define lj_num2int(n) ((int32_t)(n))
991
992/*
993** This must match the JIT backend behavior. In particular for archs
994** that don't have a common hardware instruction for this conversion.
995** Note that signed FP to unsigned int conversions have an undefined
996** result and should never be relied upon in portable FFI code.
997** See also: C99 or C11 standard, 6.3.1.4, footnote of (1).
998*/
999static LJ_AINLINE uint64_t lj_num2u64(lua_Number n)
1000{
1001#if LJ_TARGET_X86ORX64 || LJ_TARGET_MIPS
1002 int64_t i = (int64_t)n;
1003 if (i < 0) i = (int64_t)(n - 18446744073709551616.0);
1004 return (uint64_t)i;
1005#else
1006 return (uint64_t)n;
1007#endif
1008}
1009
1010static LJ_AINLINE int32_t numberVint(cTValue *o)
1011{
1012 if (LJ_LIKELY(tvisint(o)))
1013 return intV(o);
1014 else
1015 return lj_num2int(numV(o));
1016}
1017
1018static LJ_AINLINE lua_Number numberVnum(cTValue *o)
1019{
1020 if (LJ_UNLIKELY(tvisint(o)))
1021 return (lua_Number)intV(o);
1022 else
1023 return numV(o);
1024}
1025
1026/* -- Miscellaneous object handling --------------------------------------- */
1027
1028/* Names and maps for internal and external object tags. */
1029LJ_DATA const char *const lj_obj_typename[1+LUA_TCDATA+1];
1030LJ_DATA const char *const lj_obj_itypename[~LJ_TNUMX+1];
1031
1032#define lj_typename(o) (lj_obj_itypename[itypemap(o)])
1033
1034/* Compare two objects without calling metamethods. */
1035LJ_FUNC int LJ_FASTCALL lj_obj_equal(cTValue *o1, cTValue *o2);
1036LJ_FUNC const void * LJ_FASTCALL lj_obj_ptr(global_State *g, cTValue *o);
1037
1038#endif
1039