1//
2// m3_exec.h
3//
4// Created by Steven Massey on 4/17/19.
5// Copyright © 2019 Steven Massey. All rights reserved.
6
7
8#ifndef m3_exec_h
9#define m3_exec_h
10
11// TODO: all these functions could move over to the .c at some point. normally, I'd say screw it,
12// but it might prove useful to be able to compile m3_exec alone w/ optimizations while the remaining
13// code is at debug O0
14
15
16// About the naming convention of these operations/macros (_rs, _sr_, _ss, _srs, etc.)
17//------------------------------------------------------------------------------------------------------
18// - 'r' means register and 's' means slot
19// - the first letter is the top of the stack
20//
21// so, for example, _rs means the first operand (the first thing pushed to the stack) is in a slot
22// and the second operand (the top of the stack) is in a register
23//------------------------------------------------------------------------------------------------------
24
25#ifndef M3_COMPILE_OPCODES
26# error "Opcodes should only be included in one compilation unit"
27#endif
28
29#include "m3_math_utils.h"
30#include "m3_compile.h"
31#include "m3_env.h"
32#include "m3_info.h"
33#include "m3_exec_defs.h"
34
35#include <limits.h>
36
37d_m3BeginExternC
38
39# define rewrite_op(OP) * ((void **) (_pc-1)) = (void*)(OP)
40
41# define immediate(TYPE) * ((TYPE *) _pc++)
42# define skip_immediate(TYPE) (_pc++)
43
44# define slot(TYPE) * (TYPE *) (_sp + immediate (i32))
45# define slot_ptr(TYPE) (TYPE *) (_sp + immediate (i32))
46
47
48# if d_m3EnableOpProfiling
49 d_m3RetSig profileOp (d_m3OpSig, cstr_t i_operationName);
50# define nextOp() return profileOp (d_m3OpAllArgs, __FUNCTION__)
51# elif d_m3EnableOpTracing
52 d_m3RetSig debugOp (d_m3OpSig, cstr_t i_operationName);
53# define nextOp() return debugOp (d_m3OpAllArgs, __FUNCTION__)
54# else
55# define nextOp() nextOpDirect()
56# endif
57
58#define jumpOp(PC) jumpOpDirect(PC)
59
60#if d_m3RecordBacktraces
61 #define pushBacktraceFrame() (PushBacktraceFrame (_mem->runtime, _pc - 1))
62 #define fillBacktraceFrame(FUNCTION) (FillBacktraceFunctionInfo (_mem->runtime, function))
63
64 #define newTrap(err) return (pushBacktraceFrame (), err)
65 #define forwardTrap(err) return err
66#else
67 #define pushBacktraceFrame() do {} while (0)
68 #define fillBacktraceFrame(FUNCTION) do {} while (0)
69
70 #define newTrap(err) return err
71 #define forwardTrap(err) return err
72#endif
73
74
75#if d_m3EnableStrace == 1
76 // Flat trace
77 #define d_m3TracePrepare
78 #define d_m3TracePrint(fmt, ...) fprintf(stderr, fmt "\n", ##__VA_ARGS__)
79#elif d_m3EnableStrace >= 2
80 // Structured trace
81 #define d_m3TracePrepare const IM3Runtime trace_rt = m3MemRuntime(_mem);
82 #define d_m3TracePrint(fmt, ...) fprintf(stderr, "%*s" fmt "\n", (trace_rt->callDepth)*2, "", ##__VA_ARGS__)
83#else
84 #define d_m3TracePrepare
85 #define d_m3TracePrint(fmt, ...)
86#endif
87
88#if d_m3EnableStrace >= 3
89 #define d_m3TraceLoad(TYPE,offset,val) d_m3TracePrint("load." #TYPE " 0x%x = %" PRI##TYPE, offset, val)
90 #define d_m3TraceStore(TYPE,offset,val) d_m3TracePrint("store." #TYPE " 0x%x , %" PRI##TYPE, offset, val)
91#else
92 #define d_m3TraceLoad(TYPE,offset,val)
93 #define d_m3TraceStore(TYPE,offset,val)
94#endif
95
96#ifdef DEBUG
97 #define d_outOfBounds newTrap (ErrorRuntime (m3Err_trapOutOfBoundsMemoryAccess, \
98 _mem->runtime, "memory size: %zu; access offset: %zu", \
99 _mem->length, operand))
100
101# define d_outOfBoundsMemOp(OFFSET, SIZE) newTrap (ErrorRuntime (m3Err_trapOutOfBoundsMemoryAccess, \
102 _mem->runtime, "memory size: %zu; access offset: %zu; size: %u", \
103 _mem->length, OFFSET, SIZE))
104#else
105 #define d_outOfBounds newTrap (m3Err_trapOutOfBoundsMemoryAccess)
106
107# define d_outOfBoundsMemOp(OFFSET, SIZE) newTrap (m3Err_trapOutOfBoundsMemoryAccess)
108
109#endif
110
111
112d_m3RetSig Call (d_m3OpSig)
113{
114 m3ret_t possible_trap = m3_Yield ();
115 if (M3_UNLIKELY(possible_trap)) return possible_trap;
116
117 nextOpDirect();
118}
119
120// TODO: OK, this needs some explanation here ;0
121
122#define d_m3CommutativeOpMacro(RES, REG, TYPE, NAME, OP, ...) \
123d_m3Op(TYPE##_##NAME##_rs) \
124{ \
125 TYPE operand = slot (TYPE); \
126 OP((RES), operand, ((TYPE) REG), ##__VA_ARGS__); \
127 nextOp (); \
128} \
129d_m3Op(TYPE##_##NAME##_ss) \
130{ \
131 TYPE operand2 = slot (TYPE); \
132 TYPE operand1 = slot (TYPE); \
133 OP((RES), operand1, operand2, ##__VA_ARGS__); \
134 nextOp (); \
135}
136
137#define d_m3OpMacro(RES, REG, TYPE, NAME, OP, ...) \
138d_m3Op(TYPE##_##NAME##_sr) \
139{ \
140 TYPE operand = slot (TYPE); \
141 OP((RES), ((TYPE) REG), operand, ##__VA_ARGS__); \
142 nextOp (); \
143} \
144d_m3CommutativeOpMacro(RES, REG, TYPE,NAME, OP, ##__VA_ARGS__)
145
146// Accept macros
147#define d_m3CommutativeOpMacro_i(TYPE, NAME, MACRO, ...) d_m3CommutativeOpMacro ( _r0, _r0, TYPE, NAME, MACRO, ##__VA_ARGS__)
148#define d_m3OpMacro_i(TYPE, NAME, MACRO, ...) d_m3OpMacro ( _r0, _r0, TYPE, NAME, MACRO, ##__VA_ARGS__)
149#define d_m3CommutativeOpMacro_f(TYPE, NAME, MACRO, ...) d_m3CommutativeOpMacro (_fp0, _fp0, TYPE, NAME, MACRO, ##__VA_ARGS__)
150#define d_m3OpMacro_f(TYPE, NAME, MACRO, ...) d_m3OpMacro (_fp0, _fp0, TYPE, NAME, MACRO, ##__VA_ARGS__)
151
152#define M3_FUNC(RES, A, B, OP) (RES) = OP((A), (B)) // Accept functions: res = OP(a,b)
153#define M3_OPER(RES, A, B, OP) (RES) = ((A) OP (B)) // Accept operators: res = a OP b
154
155#define d_m3CommutativeOpFunc_i(TYPE, NAME, OP) d_m3CommutativeOpMacro_i (TYPE, NAME, M3_FUNC, OP)
156#define d_m3OpFunc_i(TYPE, NAME, OP) d_m3OpMacro_i (TYPE, NAME, M3_FUNC, OP)
157#define d_m3CommutativeOpFunc_f(TYPE, NAME, OP) d_m3CommutativeOpMacro_f (TYPE, NAME, M3_FUNC, OP)
158#define d_m3OpFunc_f(TYPE, NAME, OP) d_m3OpMacro_f (TYPE, NAME, M3_FUNC, OP)
159
160#define d_m3CommutativeOp_i(TYPE, NAME, OP) d_m3CommutativeOpMacro_i (TYPE, NAME, M3_OPER, OP)
161#define d_m3Op_i(TYPE, NAME, OP) d_m3OpMacro_i (TYPE, NAME, M3_OPER, OP)
162#define d_m3CommutativeOp_f(TYPE, NAME, OP) d_m3CommutativeOpMacro_f (TYPE, NAME, M3_OPER, OP)
163#define d_m3Op_f(TYPE, NAME, OP) d_m3OpMacro_f (TYPE, NAME, M3_OPER, OP)
164
165// compare needs to be distinct for fp 'cause the result must be _r0
166#define d_m3CompareOp_f(TYPE, NAME, OP) d_m3OpMacro (_r0, _fp0, TYPE, NAME, M3_OPER, OP)
167#define d_m3CommutativeCmpOp_f(TYPE, NAME, OP) d_m3CommutativeOpMacro (_r0, _fp0, TYPE, NAME, M3_OPER, OP)
168
169
170//-----------------------
171
172// signed
173d_m3CommutativeOp_i (i32, Equal, ==) d_m3CommutativeOp_i (i64, Equal, ==)
174d_m3CommutativeOp_i (i32, NotEqual, !=) d_m3CommutativeOp_i (i64, NotEqual, !=)
175
176d_m3Op_i (i32, LessThan, < ) d_m3Op_i (i64, LessThan, < )
177d_m3Op_i (i32, GreaterThan, > ) d_m3Op_i (i64, GreaterThan, > )
178d_m3Op_i (i32, LessThanOrEqual, <=) d_m3Op_i (i64, LessThanOrEqual, <=)
179d_m3Op_i (i32, GreaterThanOrEqual, >=) d_m3Op_i (i64, GreaterThanOrEqual, >=)
180
181// unsigned
182d_m3Op_i (u32, LessThan, < ) d_m3Op_i (u64, LessThan, < )
183d_m3Op_i (u32, GreaterThan, > ) d_m3Op_i (u64, GreaterThan, > )
184d_m3Op_i (u32, LessThanOrEqual, <=) d_m3Op_i (u64, LessThanOrEqual, <=)
185d_m3Op_i (u32, GreaterThanOrEqual, >=) d_m3Op_i (u64, GreaterThanOrEqual, >=)
186
187#if d_m3HasFloat
188d_m3CommutativeCmpOp_f (f32, Equal, ==) d_m3CommutativeCmpOp_f (f64, Equal, ==)
189d_m3CommutativeCmpOp_f (f32, NotEqual, !=) d_m3CommutativeCmpOp_f (f64, NotEqual, !=)
190d_m3CompareOp_f (f32, LessThan, < ) d_m3CompareOp_f (f64, LessThan, < )
191d_m3CompareOp_f (f32, GreaterThan, > ) d_m3CompareOp_f (f64, GreaterThan, > )
192d_m3CompareOp_f (f32, LessThanOrEqual, <=) d_m3CompareOp_f (f64, LessThanOrEqual, <=)
193d_m3CompareOp_f (f32, GreaterThanOrEqual, >=) d_m3CompareOp_f (f64, GreaterThanOrEqual, >=)
194#endif
195
196d_m3CommutativeOp_i (i32, Add, +) d_m3CommutativeOp_i (i64, Add, +)
197d_m3CommutativeOp_i (i32, Multiply, *) d_m3CommutativeOp_i (i64, Multiply, *)
198
199d_m3Op_i (i32, Subtract, -) d_m3Op_i (i64, Subtract, -)
200
201#define OP_SHL_32(X,N) ((X) << ((u32)(N) % 32))
202#define OP_SHL_64(X,N) ((X) << ((u64)(N) % 64))
203#define OP_SHR_32(X,N) ((X) >> ((u32)(N) % 32))
204#define OP_SHR_64(X,N) ((X) >> ((u64)(N) % 64))
205
206d_m3OpFunc_i (u32, ShiftLeft, OP_SHL_32) d_m3OpFunc_i (u64, ShiftLeft, OP_SHL_64)
207d_m3OpFunc_i (i32, ShiftRight, OP_SHR_32) d_m3OpFunc_i (i64, ShiftRight, OP_SHR_64)
208d_m3OpFunc_i (u32, ShiftRight, OP_SHR_32) d_m3OpFunc_i (u64, ShiftRight, OP_SHR_64)
209
210d_m3CommutativeOp_i (u32, And, &)
211d_m3CommutativeOp_i (u32, Or, |)
212d_m3CommutativeOp_i (u32, Xor, ^)
213
214d_m3CommutativeOp_i (u64, And, &)
215d_m3CommutativeOp_i (u64, Or, |)
216d_m3CommutativeOp_i (u64, Xor, ^)
217
218#if d_m3HasFloat
219d_m3CommutativeOp_f (f32, Add, +) d_m3CommutativeOp_f (f64, Add, +)
220d_m3CommutativeOp_f (f32, Multiply, *) d_m3CommutativeOp_f (f64, Multiply, *)
221d_m3Op_f (f32, Subtract, -) d_m3Op_f (f64, Subtract, -)
222d_m3Op_f (f32, Divide, /) d_m3Op_f (f64, Divide, /)
223#endif
224
225d_m3OpFunc_i(u32, Rotl, rotl32)
226d_m3OpFunc_i(u32, Rotr, rotr32)
227d_m3OpFunc_i(u64, Rotl, rotl64)
228d_m3OpFunc_i(u64, Rotr, rotr64)
229
230d_m3OpMacro_i(u32, Divide, OP_DIV_U);
231d_m3OpMacro_i(i32, Divide, OP_DIV_S, INT32_MIN);
232d_m3OpMacro_i(u64, Divide, OP_DIV_U);
233d_m3OpMacro_i(i64, Divide, OP_DIV_S, INT64_MIN);
234
235d_m3OpMacro_i(u32, Remainder, OP_REM_U);
236d_m3OpMacro_i(i32, Remainder, OP_REM_S, INT32_MIN);
237d_m3OpMacro_i(u64, Remainder, OP_REM_U);
238d_m3OpMacro_i(i64, Remainder, OP_REM_S, INT64_MIN);
239
240#if d_m3HasFloat
241d_m3OpFunc_f(f32, Min, min_f32);
242d_m3OpFunc_f(f32, Max, max_f32);
243d_m3OpFunc_f(f64, Min, min_f64);
244d_m3OpFunc_f(f64, Max, max_f64);
245
246d_m3OpFunc_f(f32, CopySign, copysignf);
247d_m3OpFunc_f(f64, CopySign, copysign);
248#endif
249
250// Unary operations
251// Note: This macro follows the principle of d_m3OpMacro
252
253#define d_m3UnaryMacro(RES, REG, TYPE, NAME, OP, ...) \
254d_m3Op(TYPE##_##NAME##_r) \
255{ \
256 OP((RES), (TYPE) REG, ##__VA_ARGS__); \
257 nextOp (); \
258} \
259d_m3Op(TYPE##_##NAME##_s) \
260{ \
261 TYPE operand = slot (TYPE); \
262 OP((RES), operand, ##__VA_ARGS__); \
263 nextOp (); \
264}
265
266#define M3_UNARY(RES, X, OP) (RES) = OP(X)
267#define d_m3UnaryOp_i(TYPE, NAME, OPERATION) d_m3UnaryMacro( _r0, _r0, TYPE, NAME, M3_UNARY, OPERATION)
268#define d_m3UnaryOp_f(TYPE, NAME, OPERATION) d_m3UnaryMacro(_fp0, _fp0, TYPE, NAME, M3_UNARY, OPERATION)
269
270#if d_m3HasFloat
271d_m3UnaryOp_f (f32, Abs, fabsf); d_m3UnaryOp_f (f64, Abs, fabs);
272d_m3UnaryOp_f (f32, Ceil, ceilf); d_m3UnaryOp_f (f64, Ceil, ceil);
273d_m3UnaryOp_f (f32, Floor, floorf); d_m3UnaryOp_f (f64, Floor, floor);
274d_m3UnaryOp_f (f32, Trunc, truncf); d_m3UnaryOp_f (f64, Trunc, trunc);
275d_m3UnaryOp_f (f32, Sqrt, sqrtf); d_m3UnaryOp_f (f64, Sqrt, sqrt);
276d_m3UnaryOp_f (f32, Nearest, rintf); d_m3UnaryOp_f (f64, Nearest, rint);
277d_m3UnaryOp_f (f32, Negate, -); d_m3UnaryOp_f (f64, Negate, -);
278#endif
279
280#define OP_EQZ(x) ((x) == 0)
281
282d_m3UnaryOp_i (i32, EqualToZero, OP_EQZ)
283d_m3UnaryOp_i (i64, EqualToZero, OP_EQZ)
284
285// clz(0), ctz(0) results are undefined for rest platforms, fix it
286#if (defined(__i386__) || defined(__x86_64__)) && !(defined(__AVX2__) || (defined(__ABM__) && defined(__BMI__)))
287 #define OP_CLZ_32(x) (M3_UNLIKELY((x) == 0) ? 32 : __builtin_clz(x))
288 #define OP_CTZ_32(x) (M3_UNLIKELY((x) == 0) ? 32 : __builtin_ctz(x))
289 // for 64-bit instructions branchless approach more preferable
290 #define OP_CLZ_64(x) (__builtin_clzll((x) | (1LL << 0)) + OP_EQZ(x))
291 #define OP_CTZ_64(x) (__builtin_ctzll((x) | (1LL << 63)) + OP_EQZ(x))
292#elif defined(__ppc__) || defined(__ppc64__)
293// PowerPC is defined for __builtin_clz(0) and __builtin_ctz(0).
294// See (https://github.com/aquynh/capstone/blob/master/MathExtras.h#L99)
295 #define OP_CLZ_32(x) __builtin_clz(x)
296 #define OP_CTZ_32(x) __builtin_ctz(x)
297 #define OP_CLZ_64(x) __builtin_clzll(x)
298 #define OP_CTZ_64(x) __builtin_ctzll(x)
299#else
300 #define OP_CLZ_32(x) (M3_UNLIKELY((x) == 0) ? 32 : __builtin_clz(x))
301 #define OP_CTZ_32(x) (M3_UNLIKELY((x) == 0) ? 32 : __builtin_ctz(x))
302 #define OP_CLZ_64(x) (M3_UNLIKELY((x) == 0) ? 64 : __builtin_clzll(x))
303 #define OP_CTZ_64(x) (M3_UNLIKELY((x) == 0) ? 64 : __builtin_ctzll(x))
304#endif
305
306d_m3UnaryOp_i (u32, Clz, OP_CLZ_32)
307d_m3UnaryOp_i (u64, Clz, OP_CLZ_64)
308
309d_m3UnaryOp_i (u32, Ctz, OP_CTZ_32)
310d_m3UnaryOp_i (u64, Ctz, OP_CTZ_64)
311
312d_m3UnaryOp_i (u32, Popcnt, __builtin_popcount)
313d_m3UnaryOp_i (u64, Popcnt, __builtin_popcountll)
314
315#define OP_WRAP_I64(X) ((X) & 0x00000000ffffffff)
316
317d_m3Op(i32_Wrap_i64_r)
318{
319 _r0 = OP_WRAP_I64((i64) _r0);
320 nextOp ();
321}
322
323d_m3Op(i32_Wrap_i64_s)
324{
325 i64 operand = slot (i64);
326 _r0 = OP_WRAP_I64(operand);
327 nextOp ();
328}
329
330// Integer sign extension operations
331#define OP_EXTEND8_S_I32(X) ((int32_t)(int8_t)(X))
332#define OP_EXTEND16_S_I32(X) ((int32_t)(int16_t)(X))
333#define OP_EXTEND8_S_I64(X) ((int64_t)(int8_t)(X))
334#define OP_EXTEND16_S_I64(X) ((int64_t)(int16_t)(X))
335#define OP_EXTEND32_S_I64(X) ((int64_t)(int32_t)(X))
336
337d_m3UnaryOp_i (i32, Extend8_s, OP_EXTEND8_S_I32)
338d_m3UnaryOp_i (i32, Extend16_s, OP_EXTEND16_S_I32)
339d_m3UnaryOp_i (i64, Extend8_s, OP_EXTEND8_S_I64)
340d_m3UnaryOp_i (i64, Extend16_s, OP_EXTEND16_S_I64)
341d_m3UnaryOp_i (i64, Extend32_s, OP_EXTEND32_S_I64)
342
343#define d_m3TruncMacro(DEST, SRC, TYPE, NAME, FROM, OP, ...) \
344d_m3Op(TYPE##_##NAME##_##FROM##_r_r) \
345{ \
346 OP((DEST), (FROM) SRC, ##__VA_ARGS__); \
347 nextOp (); \
348} \
349d_m3Op(TYPE##_##NAME##_##FROM##_r_s) \
350{ \
351 FROM * stack = slot_ptr (FROM); \
352 OP((DEST), (* stack), ##__VA_ARGS__); \
353 nextOp (); \
354} \
355d_m3Op(TYPE##_##NAME##_##FROM##_s_r) \
356{ \
357 TYPE * dest = slot_ptr (TYPE); \
358 OP((* dest), (FROM) SRC, ##__VA_ARGS__); \
359 nextOp (); \
360} \
361d_m3Op(TYPE##_##NAME##_##FROM##_s_s) \
362{ \
363 FROM * stack = slot_ptr (FROM); \
364 TYPE * dest = slot_ptr (TYPE); \
365 OP((* dest), (* stack), ##__VA_ARGS__); \
366 nextOp (); \
367}
368
369#if d_m3HasFloat
370d_m3TruncMacro(_r0, _fp0, i32, Trunc, f32, OP_I32_TRUNC_F32)
371d_m3TruncMacro(_r0, _fp0, u32, Trunc, f32, OP_U32_TRUNC_F32)
372d_m3TruncMacro(_r0, _fp0, i32, Trunc, f64, OP_I32_TRUNC_F64)
373d_m3TruncMacro(_r0, _fp0, u32, Trunc, f64, OP_U32_TRUNC_F64)
374
375d_m3TruncMacro(_r0, _fp0, i64, Trunc, f32, OP_I64_TRUNC_F32)
376d_m3TruncMacro(_r0, _fp0, u64, Trunc, f32, OP_U64_TRUNC_F32)
377d_m3TruncMacro(_r0, _fp0, i64, Trunc, f64, OP_I64_TRUNC_F64)
378d_m3TruncMacro(_r0, _fp0, u64, Trunc, f64, OP_U64_TRUNC_F64)
379
380d_m3TruncMacro(_r0, _fp0, i32, TruncSat, f32, OP_I32_TRUNC_SAT_F32)
381d_m3TruncMacro(_r0, _fp0, u32, TruncSat, f32, OP_U32_TRUNC_SAT_F32)
382d_m3TruncMacro(_r0, _fp0, i32, TruncSat, f64, OP_I32_TRUNC_SAT_F64)
383d_m3TruncMacro(_r0, _fp0, u32, TruncSat, f64, OP_U32_TRUNC_SAT_F64)
384
385d_m3TruncMacro(_r0, _fp0, i64, TruncSat, f32, OP_I64_TRUNC_SAT_F32)
386d_m3TruncMacro(_r0, _fp0, u64, TruncSat, f32, OP_U64_TRUNC_SAT_F32)
387d_m3TruncMacro(_r0, _fp0, i64, TruncSat, f64, OP_I64_TRUNC_SAT_F64)
388d_m3TruncMacro(_r0, _fp0, u64, TruncSat, f64, OP_U64_TRUNC_SAT_F64)
389#endif
390
391#define d_m3TypeModifyOp(REG_TO, REG_FROM, TO, NAME, FROM) \
392d_m3Op(TO##_##NAME##_##FROM##_r) \
393{ \
394 REG_TO = (TO) ((FROM) REG_FROM); \
395 nextOp (); \
396} \
397 \
398d_m3Op(TO##_##NAME##_##FROM##_s) \
399{ \
400 FROM from = slot (FROM); \
401 REG_TO = (TO) (from); \
402 nextOp (); \
403}
404
405// Int to int
406d_m3TypeModifyOp (_r0, _r0, i64, Extend, i32);
407d_m3TypeModifyOp (_r0, _r0, i64, Extend, u32);
408
409// Float to float
410#if d_m3HasFloat
411d_m3TypeModifyOp (_fp0, _fp0, f32, Demote, f64);
412d_m3TypeModifyOp (_fp0, _fp0, f64, Promote, f32);
413#endif
414
415#define d_m3TypeConvertOp(REG_TO, REG_FROM, TO, NAME, FROM) \
416d_m3Op(TO##_##NAME##_##FROM##_r_r) \
417{ \
418 REG_TO = (TO) ((FROM) REG_FROM); \
419 nextOp (); \
420} \
421 \
422d_m3Op(TO##_##NAME##_##FROM##_s_r) \
423{ \
424 slot (TO) = (TO) ((FROM) REG_FROM); \
425 nextOp (); \
426} \
427 \
428d_m3Op(TO##_##NAME##_##FROM##_r_s) \
429{ \
430 FROM from = slot (FROM); \
431 REG_TO = (TO) (from); \
432 nextOp (); \
433} \
434 \
435d_m3Op(TO##_##NAME##_##FROM##_s_s) \
436{ \
437 FROM from = slot (FROM); \
438 slot (TO) = (TO) (from); \
439 nextOp (); \
440}
441
442// Int to float
443#if d_m3HasFloat
444d_m3TypeConvertOp (_fp0, _r0, f64, Convert, i32);
445d_m3TypeConvertOp (_fp0, _r0, f64, Convert, u32);
446d_m3TypeConvertOp (_fp0, _r0, f64, Convert, i64);
447d_m3TypeConvertOp (_fp0, _r0, f64, Convert, u64);
448
449d_m3TypeConvertOp (_fp0, _r0, f32, Convert, i32);
450d_m3TypeConvertOp (_fp0, _r0, f32, Convert, u32);
451d_m3TypeConvertOp (_fp0, _r0, f32, Convert, i64);
452d_m3TypeConvertOp (_fp0, _r0, f32, Convert, u64);
453#endif
454
455#define d_m3ReinterpretOp(REG, TO, SRC, FROM) \
456d_m3Op(TO##_Reinterpret_##FROM##_r_r) \
457{ \
458 union { FROM c; TO t; } u; \
459 u.c = (FROM) SRC; \
460 REG = u.t; \
461 nextOp (); \
462} \
463 \
464d_m3Op(TO##_Reinterpret_##FROM##_r_s) \
465{ \
466 union { FROM c; TO t; } u; \
467 u.c = slot (FROM); \
468 REG = u.t; \
469 nextOp (); \
470} \
471 \
472d_m3Op(TO##_Reinterpret_##FROM##_s_r) \
473{ \
474 union { FROM c; TO t; } u; \
475 u.c = (FROM) SRC; \
476 slot (TO) = u.t; \
477 nextOp (); \
478} \
479 \
480d_m3Op(TO##_Reinterpret_##FROM##_s_s) \
481{ \
482 union { FROM c; TO t; } u; \
483 u.c = slot (FROM); \
484 slot (TO) = u.t; \
485 nextOp (); \
486}
487
488#if d_m3HasFloat
489d_m3ReinterpretOp (_r0, i32, _fp0, f32)
490d_m3ReinterpretOp (_r0, i64, _fp0, f64)
491d_m3ReinterpretOp (_fp0, f32, _r0, i32)
492d_m3ReinterpretOp (_fp0, f64, _r0, i64)
493#endif
494
495
496d_m3Op (GetGlobal_s32)
497{
498 u32 * global = immediate (u32 *);
499 slot (u32) = * global; // printf ("get global: %p %" PRIi64 "\n", global, *global);
500
501 nextOp ();
502}
503
504
505d_m3Op (GetGlobal_s64)
506{
507 u64 * global = immediate (u64 *);
508 slot (u64) = * global; // printf ("get global: %p %" PRIi64 "\n", global, *global);
509
510 nextOp ();
511}
512
513
514d_m3Op (SetGlobal_i32)
515{
516 u32 * global = immediate (u32 *);
517 * global = (u32) _r0; // printf ("set global: %p %" PRIi64 "\n", global, _r0);
518
519 nextOp ();
520}
521
522
523d_m3Op (SetGlobal_i64)
524{
525 u64 * global = immediate (u64 *);
526 * global = (u64) _r0; // printf ("set global: %p %" PRIi64 "\n", global, _r0);
527
528 nextOp ();
529}
530
531
532d_m3Op (Call)
533{
534 pc_t callPC = immediate (pc_t);
535 i32 stackOffset = immediate (i32);
536 IM3Memory memory = m3MemInfo (_mem);
537
538 m3stack_t sp = _sp + stackOffset;
539
540 m3ret_t r = Call (callPC, sp, _mem, d_m3OpDefaultArgs);
541 _mem = memory->mallocated;
542
543 if (M3_LIKELY(not r))
544 nextOp ();
545 else
546 {
547 pushBacktraceFrame ();
548 forwardTrap (r);
549 }
550}
551
552
553d_m3Op (CallIndirect)
554{
555 u32 tableIndex = slot (u32);
556 IM3Module module = immediate (IM3Module);
557 IM3FuncType type = immediate (IM3FuncType);
558 i32 stackOffset = immediate (i32);
559 IM3Memory memory = m3MemInfo (_mem);
560
561 m3stack_t sp = _sp + stackOffset;
562
563 m3ret_t r = m3Err_none;
564
565 if (M3_LIKELY(tableIndex < module->table0Size))
566 {
567 IM3Function function = module->table0 [tableIndex];
568
569 if (M3_LIKELY(function))
570 {
571 if (M3_LIKELY(type == function->funcType))
572 {
573 if (M3_UNLIKELY(not function->compiled))
574 r = CompileFunction (function);
575
576 if (M3_LIKELY(not r))
577 {
578 r = Call (function->compiled, sp, _mem, d_m3OpDefaultArgs);
579 _mem = memory->mallocated;
580
581 if (M3_LIKELY(not r))
582 nextOpDirect ();
583 else
584 {
585 pushBacktraceFrame ();
586 forwardTrap (r);
587 }
588 }
589 }
590 else r = m3Err_trapIndirectCallTypeMismatch;
591 }
592 else r = m3Err_trapTableElementIsNull;
593 }
594 else r = m3Err_trapTableIndexOutOfRange;
595
596 if (M3_UNLIKELY(r))
597 newTrap (r);
598 else forwardTrap (r);
599}
600
601
602d_m3Op (CallRawFunction)
603{
604 d_m3TracePrepare
605
606 M3ImportContext ctx;
607
608 M3RawCall call = (M3RawCall) (* _pc++);
609 ctx.function = immediate (IM3Function);
610 ctx.userdata = immediate (void *);
611 u64* const sp = ((u64*)_sp);
612 IM3Memory memory = m3MemInfo (_mem);
613
614 IM3Runtime runtime = m3MemRuntime(_mem);
615
616#if d_m3EnableStrace
617 IM3FuncType ftype = ctx.function->funcType;
618
619 FILE* out = stderr;
620 char outbuff[1024];
621 char* outp = outbuff;
622 char* oute = outbuff+1024;
623
624 outp += snprintf(outp, oute-outp, "%s!%s(", ctx.function->import.moduleUtf8, ctx.function->import.fieldUtf8);
625
626 const int nArgs = ftype->numArgs;
627 const int nRets = ftype->numRets;
628 u64 * args = sp + nRets;
629 for (int i=0; i<nArgs; i++) {
630 const int type = ftype->types[nRets + i];
631 switch (type) {
632 case c_m3Type_i32: outp += snprintf(outp, oute-outp, "%" PRIi32, *(i32*)(args+i)); break;
633 case c_m3Type_i64: outp += snprintf(outp, oute-outp, "%" PRIi64, *(i64*)(args+i)); break;
634 case c_m3Type_f32: outp += snprintf(outp, oute-outp, "%" PRIf32, *(f32*)(args+i)); break;
635 case c_m3Type_f64: outp += snprintf(outp, oute-outp, "%" PRIf64, *(f64*)(args+i)); break;
636 default: outp += snprintf(outp, oute-outp, "<type %d>", type); break;
637 }
638 outp += snprintf(outp, oute-outp, (i < nArgs-1) ? ", " : ")");
639 }
640# if d_m3EnableStrace >= 2
641 outp += snprintf(outp, oute-outp, " { <native> }");
642# endif
643#endif
644
645 // m3_Call uses runtime->stack to set-up initial exported function stack.
646 // Reconfigure the stack to enable recursive invocations of m3_Call.
647 // I.e. exported/table function can be called from an impoted function.
648 void* stack_backup = runtime->stack;
649 runtime->stack = sp;
650 m3ret_t possible_trap = call (runtime, &ctx, sp, m3MemData(_mem));
651 runtime->stack = stack_backup;
652
653#if d_m3EnableStrace
654 if (M3_UNLIKELY(possible_trap)) {
655 d_m3TracePrint("%s -> %s", outbuff, (char*)possible_trap);
656 } else {
657 switch (GetSingleRetType(ftype)) {
658 case c_m3Type_none: d_m3TracePrint("%s", outbuff); break;
659 case c_m3Type_i32: d_m3TracePrint("%s = %" PRIi32, outbuff, *(i32*)sp); break;
660 case c_m3Type_i64: d_m3TracePrint("%s = %" PRIi64, outbuff, *(i64*)sp); break;
661 case c_m3Type_f32: d_m3TracePrint("%s = %" PRIf32, outbuff, *(f32*)sp); break;
662 case c_m3Type_f64: d_m3TracePrint("%s = %" PRIf64, outbuff, *(f64*)sp); break;
663 }
664 }
665#endif
666
667 if (M3_UNLIKELY(possible_trap)) {
668 _mem = memory->mallocated;
669 pushBacktraceFrame ();
670 }
671 forwardTrap (possible_trap);
672}
673
674
675d_m3Op (MemSize)
676{
677 IM3Memory memory = m3MemInfo (_mem);
678
679 _r0 = memory->numPages;
680
681 nextOp ();
682}
683
684
685d_m3Op (MemGrow)
686{
687 IM3Runtime runtime = m3MemRuntime(_mem);
688 IM3Memory memory = & runtime->memory;
689
690 u32 numPagesToGrow = (u32) _r0;
691 _r0 = memory->numPages;
692
693 if (M3_LIKELY(numPagesToGrow))
694 {
695 u32 requiredPages = memory->numPages + numPagesToGrow;
696
697 M3Result r = ResizeMemory (runtime, requiredPages);
698 if (r)
699 _r0 = -1;
700
701 _mem = memory->mallocated;
702 }
703
704 nextOp ();
705}
706
707
708d_m3Op (MemCopy)
709{
710 u32 size = (u32) _r0;
711 u64 source = slot (u32);
712 u64 destination = slot (u32);
713
714 if (M3_LIKELY(destination + size <= _mem->length))
715 {
716 if (M3_LIKELY(source + size <= _mem->length))
717 {
718 u8 * dst = m3MemData (_mem) + destination;
719 u8 * src = m3MemData (_mem) + source;
720 memmove (dst, src, size);
721
722 nextOp ();
723 }
724 else d_outOfBoundsMemOp (source, size);
725 }
726 else d_outOfBoundsMemOp (destination, size);
727}
728
729
730d_m3Op (MemFill)
731{
732 u32 size = (u32) _r0;
733 u32 byte = slot (u32);
734 u64 destination = slot (u32);
735
736 if (M3_LIKELY(destination + size <= _mem->length))
737 {
738 u8 * mem8 = m3MemData (_mem) + destination;
739 memset (mem8, (u8) byte, size);
740 nextOp ();
741 }
742 else d_outOfBoundsMemOp (destination, size);
743}
744
745
746// it's a debate: should the compilation be trigger be the caller or callee page.
747// it's a much easier to put it in the caller pager. if it's in the callee, either the entire page
748// has be left dangling or it's just a stub that jumps to a newly acquired page. In Gestalt, I opted
749// for the stub approach. Stubbing makes it easier to dynamically free the compilation. You can also
750// do both.
751d_m3Op (Compile)
752{
753 rewrite_op (op_Call);
754
755 IM3Function function = immediate (IM3Function);
756
757 m3ret_t result = m3Err_none;
758
759 if (M3_UNLIKELY(not function->compiled)) // check to see if function was compiled since this operation was emitted.
760 result = CompileFunction (function);
761
762 if (not result)
763 {
764 // patch up compiled pc and call rewritten op_Call
765 * ((void**) --_pc) = (void*) (function->compiled);
766 --_pc;
767 nextOpDirect ();
768 }
769
770 newTrap (result);
771}
772
773
774
775d_m3Op (Entry)
776{
777 d_m3ClearRegisters
778
779 d_m3TracePrepare
780
781 IM3Function function = immediate (IM3Function);
782 IM3Memory memory = m3MemInfo (_mem);
783
784#if d_m3SkipStackCheck
785 if (true)
786#else
787 if (M3_LIKELY ((void *) (_sp + function->maxStackSlots) < _mem->maxStack))
788#endif
789 {
790#if defined(DEBUG)
791 function->hits++;
792#endif
793 u8 * stack = (u8 *) ((m3slot_t *) _sp + function->numRetAndArgSlots);
794
795 memset (stack, 0x0, function->numLocalBytes);
796 stack += function->numLocalBytes;
797
798 if (function->constants)
799 {
800 memcpy (stack, function->constants, function->numConstantBytes);
801 }
802
803#if d_m3EnableStrace >= 2
804 d_m3TracePrint("%s %s {", m3_GetFunctionName(function), SPrintFunctionArgList (function, _sp + function->numRetSlots));
805 trace_rt->callDepth++;
806#endif
807
808 m3ret_t r = nextOpImpl ();
809
810#if d_m3EnableStrace >= 2
811 trace_rt->callDepth--;
812
813 if (r) {
814 d_m3TracePrint("} !trap = %s", (char*)r);
815 } else {
816 int rettype = GetSingleRetType(function->funcType);
817 if (rettype != c_m3Type_none) {
818 char str [128] = { 0 };
819 SPrintArg (str, 127, _sp, rettype);
820 d_m3TracePrint("} = %s", str);
821 } else {
822 d_m3TracePrint("}");
823 }
824 }
825#endif
826
827 if (M3_UNLIKELY(r)) {
828 _mem = memory->mallocated;
829 fillBacktraceFrame ();
830 }
831 forwardTrap (r);
832 }
833 else newTrap (m3Err_trapStackOverflow);
834}
835
836
837d_m3Op (Loop)
838{
839 d_m3TracePrepare
840
841 // regs are unused coming into a loop anyway
842 // this reduces code size & stack usage
843 d_m3ClearRegisters
844
845 m3ret_t r;
846
847 IM3Memory memory = m3MemInfo (_mem);
848
849 do
850 {
851#if d_m3EnableStrace >= 3
852 d_m3TracePrint("iter {");
853 trace_rt->callDepth++;
854#endif
855 r = nextOpImpl ();
856
857#if d_m3EnableStrace >= 3
858 trace_rt->callDepth--;
859 d_m3TracePrint("}");
860#endif
861 // linear memory pointer needs refreshed here because the block it's looping over
862 // can potentially invoke the grow operation.
863 _mem = memory->mallocated;
864 }
865 while (r == _pc);
866
867 forwardTrap (r);
868}
869
870
871d_m3Op (Branch)
872{
873 jumpOp (* _pc);
874}
875
876
877d_m3Op (If_r)
878{
879 i32 condition = (i32) _r0;
880
881 pc_t elsePC = immediate (pc_t);
882
883 if (condition)
884 nextOp ();
885 else
886 jumpOp (elsePC);
887}
888
889
890d_m3Op (If_s)
891{
892 i32 condition = slot (i32);
893
894 pc_t elsePC = immediate (pc_t);
895
896 if (condition)
897 nextOp ();
898 else
899 jumpOp (elsePC);
900}
901
902
903d_m3Op (BranchTable)
904{
905 u32 branchIndex = slot (u32); // branch index is always in a slot
906 u32 numTargets = immediate (u32);
907
908 pc_t * branches = (pc_t *) _pc;
909
910 if (branchIndex > numTargets)
911 branchIndex = numTargets; // the default index
912
913 jumpOp (branches [branchIndex]);
914}
915
916
917#define d_m3SetRegisterSetSlot(TYPE, REG) \
918d_m3Op (SetRegister_##TYPE) \
919{ \
920 REG = slot (TYPE); \
921 nextOp (); \
922} \
923 \
924d_m3Op (SetSlot_##TYPE) \
925{ \
926 slot (TYPE) = (TYPE) REG; \
927 nextOp (); \
928} \
929 \
930d_m3Op (PreserveSetSlot_##TYPE) \
931{ \
932 TYPE * stack = slot_ptr (TYPE); \
933 TYPE * preserve = slot_ptr (TYPE); \
934 \
935 * preserve = * stack; \
936 * stack = (TYPE) REG; \
937 \
938 nextOp (); \
939}
940
941d_m3SetRegisterSetSlot (i32, _r0)
942d_m3SetRegisterSetSlot (i64, _r0)
943#if d_m3HasFloat
944d_m3SetRegisterSetSlot (f32, _fp0)
945d_m3SetRegisterSetSlot (f64, _fp0)
946#endif
947
948d_m3Op (CopySlot_32)
949{
950 u32 * dst = slot_ptr (u32);
951 u32 * src = slot_ptr (u32);
952
953 * dst = * src;
954
955 nextOp ();
956}
957
958
959d_m3Op (PreserveCopySlot_32)
960{
961 u32 * dest = slot_ptr (u32);
962 u32 * src = slot_ptr (u32);
963 u32 * preserve = slot_ptr (u32);
964
965 * preserve = * dest;
966 * dest = * src;
967
968 nextOp ();
969}
970
971
972d_m3Op (CopySlot_64)
973{
974 u64 * dst = slot_ptr (u64);
975 u64 * src = slot_ptr (u64);
976
977 * dst = * src; // printf ("copy: %p <- %" PRIi64 " <- %p\n", dst, * dst, src);
978
979 nextOp ();
980}
981
982
983d_m3Op (PreserveCopySlot_64)
984{
985 u64 * dest = slot_ptr (u64);
986 u64 * src = slot_ptr (u64);
987 u64 * preserve = slot_ptr (u64);
988
989 * preserve = * dest;
990 * dest = * src;
991
992 nextOp ();
993}
994
995
996#if d_m3EnableOpTracing
997//--------------------------------------------------------------------------------------------------------
998d_m3Op (DumpStack)
999{
1000 u32 opcodeIndex = immediate (u32);
1001 u32 stackHeight = immediate (u32);
1002 IM3Function function = immediate (IM3Function);
1003
1004 cstr_t funcName = (function) ? m3_GetFunctionName(function) : "";
1005
1006 printf (" %4d ", opcodeIndex);
1007 printf (" %-25s r0: 0x%016" PRIx64 " i:%" PRIi64 " u:%" PRIu64 "\n", funcName, _r0, _r0, _r0);
1008#if d_m3HasFloat
1009 printf (" fp0: %" PRIf64 "\n", _fp0);
1010#endif
1011 m3stack_t sp = _sp;
1012
1013 for (u32 i = 0; i < stackHeight; ++i)
1014 {
1015 cstr_t kind = "";
1016
1017 printf ("%p %5s %2d: 0x%" PRIx64 " i:%" PRIi64 "\n", sp, kind, i, (u64) *(sp), (i64) *(sp));
1018
1019 ++sp;
1020 }
1021 printf ("---------------------------------------------------------------------------------------------------------\n");
1022
1023 nextOpDirect();
1024}
1025#endif
1026
1027
1028#define d_m3Select_i(TYPE, REG) \
1029d_m3Op (Select_##TYPE##_rss) \
1030{ \
1031 i32 condition = (i32) _r0; \
1032 \
1033 TYPE operand2 = slot (TYPE); \
1034 TYPE operand1 = slot (TYPE); \
1035 \
1036 REG = (condition) ? operand1 : operand2; \
1037 \
1038 nextOp (); \
1039} \
1040 \
1041d_m3Op (Select_##TYPE##_srs) \
1042{ \
1043 i32 condition = slot (i32); \
1044 \
1045 TYPE operand2 = (TYPE) REG; \
1046 TYPE operand1 = slot (TYPE); \
1047 \
1048 REG = (condition) ? operand1 : operand2; \
1049 \
1050 nextOp (); \
1051} \
1052 \
1053d_m3Op (Select_##TYPE##_ssr) \
1054{ \
1055 i32 condition = slot (i32); \
1056 \
1057 TYPE operand2 = slot (TYPE); \
1058 TYPE operand1 = (TYPE) REG; \
1059 \
1060 REG = (condition) ? operand1 : operand2; \
1061 \
1062 nextOp (); \
1063} \
1064 \
1065d_m3Op (Select_##TYPE##_sss) \
1066{ \
1067 i32 condition = slot (i32); \
1068 \
1069 TYPE operand2 = slot (TYPE); \
1070 TYPE operand1 = slot (TYPE); \
1071 \
1072 REG = (condition) ? operand1 : operand2; \
1073 \
1074 nextOp (); \
1075}
1076
1077
1078d_m3Select_i (i32, _r0)
1079d_m3Select_i (i64, _r0)
1080
1081
1082#define d_m3Select_f(TYPE, REG, LABEL, SELECTOR) \
1083d_m3Op (Select_##TYPE##_##LABEL##ss) \
1084{ \
1085 i32 condition = (i32) SELECTOR; \
1086 \
1087 TYPE operand2 = slot (TYPE); \
1088 TYPE operand1 = slot (TYPE); \
1089 \
1090 REG = (condition) ? operand1 : operand2; \
1091 \
1092 nextOp (); \
1093} \
1094 \
1095d_m3Op (Select_##TYPE##_##LABEL##rs) \
1096{ \
1097 i32 condition = (i32) SELECTOR; \
1098 \
1099 TYPE operand2 = (TYPE) REG; \
1100 TYPE operand1 = slot (TYPE); \
1101 \
1102 REG = (condition) ? operand1 : operand2; \
1103 \
1104 nextOp (); \
1105} \
1106 \
1107d_m3Op (Select_##TYPE##_##LABEL##sr) \
1108{ \
1109 i32 condition = (i32) SELECTOR; \
1110 \
1111 TYPE operand2 = slot (TYPE); \
1112 TYPE operand1 = (TYPE) REG; \
1113 \
1114 REG = (condition) ? operand1 : operand2; \
1115 \
1116 nextOp (); \
1117}
1118
1119#if d_m3HasFloat
1120d_m3Select_f (f32, _fp0, r, _r0)
1121d_m3Select_f (f32, _fp0, s, slot (i32))
1122
1123d_m3Select_f (f64, _fp0, r, _r0)
1124d_m3Select_f (f64, _fp0, s, slot (i32))
1125#endif
1126
1127d_m3Op (Return)
1128{
1129 m3StackCheck();
1130 return m3Err_none;
1131}
1132
1133
1134d_m3Op (BranchIf_r)
1135{
1136 i32 condition = (i32) _r0;
1137 pc_t branch = immediate (pc_t);
1138
1139 if (condition)
1140 {
1141 jumpOp (branch);
1142 }
1143 else nextOp ();
1144}
1145
1146
1147d_m3Op (BranchIf_s)
1148{
1149 i32 condition = slot (i32);
1150 pc_t branch = immediate (pc_t);
1151
1152 if (condition)
1153 {
1154 jumpOp (branch);
1155 }
1156 else nextOp ();
1157}
1158
1159
1160d_m3Op (BranchIfPrologue_r)
1161{
1162 i32 condition = (i32) _r0;
1163 pc_t branch = immediate (pc_t);
1164
1165 if (condition)
1166 {
1167 // this is the "prologue" that ends with
1168 // a plain branch to the actual target
1169 nextOp ();
1170 }
1171 else jumpOp (branch); // jump over the prologue
1172}
1173
1174
1175d_m3Op (BranchIfPrologue_s)
1176{
1177 i32 condition = slot (i32);
1178 pc_t branch = immediate (pc_t);
1179
1180 if (condition)
1181 {
1182 nextOp ();
1183 }
1184 else jumpOp (branch);
1185}
1186
1187
1188d_m3Op (ContinueLoop)
1189{
1190 m3StackCheck();
1191
1192 // TODO: this is where execution can "escape" the M3 code and callback to the client / fiber switch
1193 // OR it can go in the Loop operation. I think it's best to do here. adding code to the loop operation
1194 // has the potential to increase its native-stack usage. (don't forget ContinueLoopIf too.)
1195
1196 void * loopId = immediate (void *);
1197 return loopId;
1198}
1199
1200
1201d_m3Op (ContinueLoopIf)
1202{
1203 i32 condition = (i32) _r0;
1204 void * loopId = immediate (void *);
1205
1206 if (condition)
1207 {
1208 return loopId;
1209 }
1210 else nextOp ();
1211}
1212
1213
1214d_m3Op (Const32)
1215{
1216 u32 value = * (u32 *)_pc++;
1217 slot (u32) = value;
1218 nextOp ();
1219}
1220
1221
1222d_m3Op (Const64)
1223{
1224 u64 value = * (u64 *)_pc;
1225 _pc += (M3_SIZEOF_PTR == 4) ? 2 : 1;
1226 slot (u64) = value;
1227 nextOp ();
1228}
1229
1230d_m3Op (Unsupported)
1231{
1232 newTrap ("unsupported instruction executed");
1233}
1234
1235d_m3Op (Unreachable)
1236{
1237 m3StackCheck();
1238 newTrap (m3Err_trapUnreachable);
1239}
1240
1241
1242d_m3Op (End)
1243{
1244 m3StackCheck();
1245 return m3Err_none;
1246}
1247
1248
1249d_m3Op (SetGlobal_s32)
1250{
1251 u32 * global = immediate (u32 *);
1252 * global = slot (u32);
1253
1254 nextOp ();
1255}
1256
1257
1258d_m3Op (SetGlobal_s64)
1259{
1260 u64 * global = immediate (u64 *);
1261 * global = slot (u64);
1262
1263 nextOp ();
1264}
1265
1266#if d_m3HasFloat
1267d_m3Op (SetGlobal_f32)
1268{
1269 f32 * global = immediate (f32 *);
1270 * global = _fp0;
1271
1272 nextOp ();
1273}
1274
1275
1276d_m3Op (SetGlobal_f64)
1277{
1278 f64 * global = immediate (f64 *);
1279 * global = _fp0;
1280
1281 nextOp ();
1282}
1283#endif
1284
1285
1286#if d_m3SkipMemoryBoundsCheck
1287# define m3MemCheck(x) true
1288#else
1289# define m3MemCheck(x) M3_LIKELY(x)
1290#endif
1291
1292// memcpy here is to support non-aligned access on some platforms.
1293
1294#define d_m3Load(REG,DEST_TYPE,SRC_TYPE) \
1295d_m3Op(DEST_TYPE##_Load_##SRC_TYPE##_r) \
1296{ \
1297 d_m3TracePrepare \
1298 u32 offset = immediate (u32); \
1299 u64 operand = (u32) _r0; \
1300 operand += offset; \
1301 \
1302 if (m3MemCheck( \
1303 operand + sizeof (SRC_TYPE) <= _mem->length \
1304 )) { \
1305 u8* src8 = m3MemData(_mem) + operand; \
1306 SRC_TYPE value; \
1307 memcpy(&value, src8, sizeof(value)); \
1308 M3_BSWAP_##SRC_TYPE(value); \
1309 REG = (DEST_TYPE)value; \
1310 d_m3TraceLoad(DEST_TYPE, operand, REG); \
1311 nextOp (); \
1312 } else d_outOfBounds; \
1313} \
1314d_m3Op(DEST_TYPE##_Load_##SRC_TYPE##_s) \
1315{ \
1316 d_m3TracePrepare \
1317 u64 operand = slot (u32); \
1318 u32 offset = immediate (u32); \
1319 operand += offset; \
1320 \
1321 if (m3MemCheck( \
1322 operand + sizeof (SRC_TYPE) <= _mem->length \
1323 )) { \
1324 u8* src8 = m3MemData(_mem) + operand; \
1325 SRC_TYPE value; \
1326 memcpy(&value, src8, sizeof(value)); \
1327 M3_BSWAP_##SRC_TYPE(value); \
1328 REG = (DEST_TYPE)value; \
1329 d_m3TraceLoad(DEST_TYPE, operand, REG); \
1330 nextOp (); \
1331 } else d_outOfBounds; \
1332}
1333
1334// printf ("get: %d -> %d\n", operand + offset, (i64) REG);
1335
1336
1337#define d_m3Load_i(DEST_TYPE, SRC_TYPE) d_m3Load(_r0, DEST_TYPE, SRC_TYPE)
1338#define d_m3Load_f(DEST_TYPE, SRC_TYPE) d_m3Load(_fp0, DEST_TYPE, SRC_TYPE)
1339
1340#if d_m3HasFloat
1341d_m3Load_f (f32, f32);
1342d_m3Load_f (f64, f64);
1343#endif
1344
1345d_m3Load_i (i32, i8);
1346d_m3Load_i (i32, u8);
1347d_m3Load_i (i32, i16);
1348d_m3Load_i (i32, u16);
1349d_m3Load_i (i32, i32);
1350
1351d_m3Load_i (i64, i8);
1352d_m3Load_i (i64, u8);
1353d_m3Load_i (i64, i16);
1354d_m3Load_i (i64, u16);
1355d_m3Load_i (i64, i32);
1356d_m3Load_i (i64, u32);
1357d_m3Load_i (i64, i64);
1358
1359#define d_m3Store(REG, SRC_TYPE, DEST_TYPE) \
1360d_m3Op (SRC_TYPE##_Store_##DEST_TYPE##_rs) \
1361{ \
1362 d_m3TracePrepare \
1363 u64 operand = slot (u32); \
1364 u32 offset = immediate (u32); \
1365 operand += offset; \
1366 \
1367 if (m3MemCheck( \
1368 operand + sizeof (DEST_TYPE) <= _mem->length \
1369 )) { \
1370 d_m3TraceStore(SRC_TYPE, operand, REG); \
1371 u8* mem8 = m3MemData(_mem) + operand; \
1372 DEST_TYPE val = (DEST_TYPE) REG; \
1373 M3_BSWAP_##DEST_TYPE(val); \
1374 memcpy(mem8, &val, sizeof(val)); \
1375 nextOp (); \
1376 } else d_outOfBounds; \
1377} \
1378d_m3Op (SRC_TYPE##_Store_##DEST_TYPE##_sr) \
1379{ \
1380 d_m3TracePrepare \
1381 const SRC_TYPE value = slot (SRC_TYPE); \
1382 u64 operand = (u32) _r0; \
1383 u32 offset = immediate (u32); \
1384 operand += offset; \
1385 \
1386 if (m3MemCheck( \
1387 operand + sizeof (DEST_TYPE) <= _mem->length \
1388 )) { \
1389 d_m3TraceStore(SRC_TYPE, operand, value); \
1390 u8* mem8 = m3MemData(_mem) + operand; \
1391 DEST_TYPE val = (DEST_TYPE) value; \
1392 M3_BSWAP_##DEST_TYPE(val); \
1393 memcpy(mem8, &val, sizeof(val)); \
1394 nextOp (); \
1395 } else d_outOfBounds; \
1396} \
1397d_m3Op (SRC_TYPE##_Store_##DEST_TYPE##_ss) \
1398{ \
1399 d_m3TracePrepare \
1400 const SRC_TYPE value = slot (SRC_TYPE); \
1401 u64 operand = slot (u32); \
1402 u32 offset = immediate (u32); \
1403 operand += offset; \
1404 \
1405 if (m3MemCheck( \
1406 operand + sizeof (DEST_TYPE) <= _mem->length \
1407 )) { \
1408 d_m3TraceStore(SRC_TYPE, operand, value); \
1409 u8* mem8 = m3MemData(_mem) + operand; \
1410 DEST_TYPE val = (DEST_TYPE) value; \
1411 M3_BSWAP_##DEST_TYPE(val); \
1412 memcpy(mem8, &val, sizeof(val)); \
1413 nextOp (); \
1414 } else d_outOfBounds; \
1415}
1416
1417// both operands can be in regs when storing a float
1418#define d_m3StoreFp(REG, TYPE) \
1419d_m3Op (TYPE##_Store_##TYPE##_rr) \
1420{ \
1421 d_m3TracePrepare \
1422 u64 operand = (u32) _r0; \
1423 u32 offset = immediate (u32); \
1424 operand += offset; \
1425 \
1426 if (m3MemCheck( \
1427 operand + sizeof (TYPE) <= _mem->length \
1428 )) { \
1429 d_m3TraceStore(TYPE, operand, REG); \
1430 u8* mem8 = m3MemData(_mem) + operand; \
1431 TYPE val = (TYPE) REG; \
1432 M3_BSWAP_##TYPE(val); \
1433 memcpy(mem8, &val, sizeof(val)); \
1434 nextOp (); \
1435 } else d_outOfBounds; \
1436}
1437
1438
1439#define d_m3Store_i(SRC_TYPE, DEST_TYPE) d_m3Store(_r0, SRC_TYPE, DEST_TYPE)
1440#define d_m3Store_f(SRC_TYPE, DEST_TYPE) d_m3Store(_fp0, SRC_TYPE, DEST_TYPE) d_m3StoreFp (_fp0, SRC_TYPE);
1441
1442#if d_m3HasFloat
1443d_m3Store_f (f32, f32)
1444d_m3Store_f (f64, f64)
1445#endif
1446
1447d_m3Store_i (i32, u8)
1448d_m3Store_i (i32, i16)
1449d_m3Store_i (i32, i32)
1450
1451d_m3Store_i (i64, u8)
1452d_m3Store_i (i64, i16)
1453d_m3Store_i (i64, i32)
1454d_m3Store_i (i64, i64)
1455
1456#undef m3MemCheck
1457
1458
1459//---------------------------------------------------------------------------------------------------------------------
1460// debug/profiling
1461//---------------------------------------------------------------------------------------------------------------------
1462#if d_m3EnableOpTracing
1463d_m3RetSig debugOp (d_m3OpSig, cstr_t i_opcode)
1464{
1465 char name [100];
1466 strcpy (name, strstr (i_opcode, "op_") + 3);
1467 char * bracket = strstr (name, "(");
1468 if (bracket) {
1469 *bracket = 0;
1470 }
1471
1472 puts (name);
1473 nextOpDirect();
1474}
1475# endif
1476
1477# if d_m3EnableOpProfiling
1478d_m3RetSig profileOp (d_m3OpSig, cstr_t i_operationName)
1479{
1480 ProfileHit (i_operationName);
1481
1482 nextOpDirect();
1483}
1484# endif
1485
1486d_m3EndExternC
1487
1488#endif // m3_exec_h
1489