1/* Copyright (C) 2001-2020 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3
4 The GNU C Library is free software; you can redistribute it and/or
5 modify it under the terms of the GNU Lesser General Public
6 License as published by the Free Software Foundation; either
7 version 2.1 of the License, or (at your option) any later version.
8
9 The GNU C Library is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 Lesser General Public License for more details.
13
14 You should have received a copy of the GNU Lesser General Public
15 License along with the GNU C Library; if not, see
16 <https://www.gnu.org/licenses/>. */
17
18#ifndef _LINUX_X86_64_SYSDEP_H
19#define _LINUX_X86_64_SYSDEP_H 1
20
21/* There is some commonality. */
22#include <sysdeps/unix/sysv/linux/sysdep.h>
23#include <sysdeps/unix/x86_64/sysdep.h>
24#include <tls.h>
25
26/* Defines RTLD_PRIVATE_ERRNO. */
27#include <dl-sysdep.h>
28
29/* For Linux we can use the system call table in the header file
30 /usr/include/asm/unistd.h
31 of the kernel. But these symbols do not follow the SYS_* syntax
32 so we have to redefine the `SYS_ify' macro here. */
33#undef SYS_ify
34#define SYS_ify(syscall_name) __NR_##syscall_name
35
36#ifdef __ASSEMBLER__
37
38/* Linux uses a negative return value to indicate syscall errors,
39 unlike most Unices, which use the condition codes' carry flag.
40
41 Since version 2.1 the return value of a system call might be
42 negative even if the call succeeded. E.g., the `lseek' system call
43 might return a large offset. Therefore we must not anymore test
44 for < 0, but test for a real error by making sure the value in %eax
45 is a real error number. Linus said he will make sure the no syscall
46 returns a value in -1 .. -4095 as a valid result so we can savely
47 test with -4095. */
48
49/* We don't want the label for the error handle to be global when we define
50 it here. */
51# undef SYSCALL_ERROR_LABEL
52# ifdef PIC
53# undef SYSCALL_ERROR_LABEL
54# define SYSCALL_ERROR_LABEL 0f
55# else
56# undef SYSCALL_ERROR_LABEL
57# define SYSCALL_ERROR_LABEL syscall_error
58# endif
59
60/* PSEUDO and T_PSEUDO macros have 2 extra arguments for unsigned long
61 int arguments. */
62# define PSEUDOS_HAVE_ULONG_INDICES 1
63
64# ifndef SYSCALL_ULONG_ARG_1
65# define SYSCALL_ULONG_ARG_1 0
66# define SYSCALL_ULONG_ARG_2 0
67# endif
68
69# undef PSEUDO
70# if SYSCALL_ULONG_ARG_1
71# define PSEUDO(name, syscall_name, args, ulong_arg_1, ulong_arg_2) \
72 .text; \
73 ENTRY (name) \
74 DO_CALL (syscall_name, args, ulong_arg_1, ulong_arg_2); \
75 cmpq $-4095, %rax; \
76 jae SYSCALL_ERROR_LABEL
77# else
78# define PSEUDO(name, syscall_name, args) \
79 .text; \
80 ENTRY (name) \
81 DO_CALL (syscall_name, args, 0, 0); \
82 cmpq $-4095, %rax; \
83 jae SYSCALL_ERROR_LABEL
84# endif
85
86# undef PSEUDO_END
87# define PSEUDO_END(name) \
88 SYSCALL_ERROR_HANDLER \
89 END (name)
90
91# undef PSEUDO_NOERRNO
92# if SYSCALL_ULONG_ARG_1
93# define PSEUDO_NOERRNO(name, syscall_name, args, ulong_arg_1, ulong_arg_2) \
94 .text; \
95 ENTRY (name) \
96 DO_CALL (syscall_name, args, ulong_arg_1, ulong_arg_2)
97# else
98# define PSEUDO_NOERRNO(name, syscall_name, args) \
99 .text; \
100 ENTRY (name) \
101 DO_CALL (syscall_name, args, 0, 0)
102# endif
103
104# undef PSEUDO_END_NOERRNO
105# define PSEUDO_END_NOERRNO(name) \
106 END (name)
107
108# define ret_NOERRNO ret
109
110# undef PSEUDO_ERRVAL
111# if SYSCALL_ULONG_ARG_1
112# define PSEUDO_ERRVAL(name, syscall_name, args, ulong_arg_1, ulong_arg_2) \
113 .text; \
114 ENTRY (name) \
115 DO_CALL (syscall_name, args, ulong_arg_1, ulong_arg_2); \
116 negq %rax
117# else
118# define PSEUDO_ERRVAL(name, syscall_name, args) \
119 .text; \
120 ENTRY (name) \
121 DO_CALL (syscall_name, args, 0, 0); \
122 negq %rax
123# endif
124
125# undef PSEUDO_END_ERRVAL
126# define PSEUDO_END_ERRVAL(name) \
127 END (name)
128
129# define ret_ERRVAL ret
130
131# if defined PIC && RTLD_PRIVATE_ERRNO
132# define SYSCALL_SET_ERRNO \
133 lea rtld_errno(%rip), %RCX_LP; \
134 neg %eax; \
135 movl %eax, (%rcx)
136# else
137# if IS_IN (libc)
138# define SYSCALL_ERROR_ERRNO __libc_errno
139# else
140# define SYSCALL_ERROR_ERRNO errno
141# endif
142# define SYSCALL_SET_ERRNO \
143 movq SYSCALL_ERROR_ERRNO@GOTTPOFF(%rip), %rcx;\
144 neg %eax; \
145 movl %eax, %fs:(%rcx);
146# endif
147
148# ifndef PIC
149# define SYSCALL_ERROR_HANDLER /* Nothing here; code in sysdep.S is used. */
150# else
151# define SYSCALL_ERROR_HANDLER \
1520: \
153 SYSCALL_SET_ERRNO; \
154 or $-1, %RAX_LP; \
155 ret;
156# endif /* PIC */
157
158/* The Linux/x86-64 kernel expects the system call parameters in
159 registers according to the following table:
160
161 syscall number rax
162 arg 1 rdi
163 arg 2 rsi
164 arg 3 rdx
165 arg 4 r10
166 arg 5 r8
167 arg 6 r9
168
169 The Linux kernel uses and destroys internally these registers:
170 return address from
171 syscall rcx
172 eflags from syscall r11
173
174 Normal function call, including calls to the system call stub
175 functions in the libc, get the first six parameters passed in
176 registers and the seventh parameter and later on the stack. The
177 register use is as follows:
178
179 system call number in the DO_CALL macro
180 arg 1 rdi
181 arg 2 rsi
182 arg 3 rdx
183 arg 4 rcx
184 arg 5 r8
185 arg 6 r9
186
187 We have to take care that the stack is aligned to 16 bytes. When
188 called the stack is not aligned since the return address has just
189 been pushed.
190
191
192 Syscalls of more than 6 arguments are not supported. */
193
194# undef DO_CALL
195# define DO_CALL(syscall_name, args, ulong_arg_1, ulong_arg_2) \
196 DOARGS_##args \
197 ZERO_EXTEND_##ulong_arg_1 \
198 ZERO_EXTEND_##ulong_arg_2 \
199 movl $SYS_ify (syscall_name), %eax; \
200 syscall;
201
202# define DOARGS_0 /* nothing */
203# define DOARGS_1 /* nothing */
204# define DOARGS_2 /* nothing */
205# define DOARGS_3 /* nothing */
206# define DOARGS_4 movq %rcx, %r10;
207# define DOARGS_5 DOARGS_4
208# define DOARGS_6 DOARGS_5
209
210# define ZERO_EXTEND_0 /* nothing */
211# define ZERO_EXTEND_1 /* nothing */
212# define ZERO_EXTEND_2 /* nothing */
213# define ZERO_EXTEND_3 /* nothing */
214# define ZERO_EXTEND_4 /* nothing */
215# define ZERO_EXTEND_5 /* nothing */
216# define ZERO_EXTEND_6 /* nothing */
217
218#else /* !__ASSEMBLER__ */
219
220/* Registers clobbered by syscall. */
221# define REGISTERS_CLOBBERED_BY_SYSCALL "cc", "r11", "cx"
222
223/* NB: This also works when X is an array. For an array X, type of
224 (X) - (X) is ptrdiff_t, which is signed, since size of ptrdiff_t
225 == size of pointer, cast is a NOP. */
226#define TYPEFY1(X) __typeof__ ((X) - (X))
227/* Explicit cast the argument. */
228#define ARGIFY(X) ((TYPEFY1 (X)) (X))
229/* Create a variable 'name' based on type of variable 'X' to avoid
230 explicit types. */
231#define TYPEFY(X, name) __typeof__ (ARGIFY (X)) name
232
233#undef INTERNAL_SYSCALL
234#define INTERNAL_SYSCALL(name, nr, args...) \
235 internal_syscall##nr (SYS_ify (name), args)
236
237#undef INTERNAL_SYSCALL_NCS
238#define INTERNAL_SYSCALL_NCS(number, nr, args...) \
239 internal_syscall##nr (number, args)
240
241#undef internal_syscall0
242#define internal_syscall0(number, dummy...) \
243({ \
244 unsigned long int resultvar; \
245 asm volatile ( \
246 "syscall\n\t" \
247 : "=a" (resultvar) \
248 : "0" (number) \
249 : "memory", REGISTERS_CLOBBERED_BY_SYSCALL); \
250 (long int) resultvar; \
251})
252
253#undef internal_syscall1
254#define internal_syscall1(number, arg1) \
255({ \
256 unsigned long int resultvar; \
257 TYPEFY (arg1, __arg1) = ARGIFY (arg1); \
258 register TYPEFY (arg1, _a1) asm ("rdi") = __arg1; \
259 asm volatile ( \
260 "syscall\n\t" \
261 : "=a" (resultvar) \
262 : "0" (number), "r" (_a1) \
263 : "memory", REGISTERS_CLOBBERED_BY_SYSCALL); \
264 (long int) resultvar; \
265})
266
267#undef internal_syscall2
268#define internal_syscall2(number, arg1, arg2) \
269({ \
270 unsigned long int resultvar; \
271 TYPEFY (arg2, __arg2) = ARGIFY (arg2); \
272 TYPEFY (arg1, __arg1) = ARGIFY (arg1); \
273 register TYPEFY (arg2, _a2) asm ("rsi") = __arg2; \
274 register TYPEFY (arg1, _a1) asm ("rdi") = __arg1; \
275 asm volatile ( \
276 "syscall\n\t" \
277 : "=a" (resultvar) \
278 : "0" (number), "r" (_a1), "r" (_a2) \
279 : "memory", REGISTERS_CLOBBERED_BY_SYSCALL); \
280 (long int) resultvar; \
281})
282
283#undef internal_syscall3
284#define internal_syscall3(number, arg1, arg2, arg3) \
285({ \
286 unsigned long int resultvar; \
287 TYPEFY (arg3, __arg3) = ARGIFY (arg3); \
288 TYPEFY (arg2, __arg2) = ARGIFY (arg2); \
289 TYPEFY (arg1, __arg1) = ARGIFY (arg1); \
290 register TYPEFY (arg3, _a3) asm ("rdx") = __arg3; \
291 register TYPEFY (arg2, _a2) asm ("rsi") = __arg2; \
292 register TYPEFY (arg1, _a1) asm ("rdi") = __arg1; \
293 asm volatile ( \
294 "syscall\n\t" \
295 : "=a" (resultvar) \
296 : "0" (number), "r" (_a1), "r" (_a2), "r" (_a3) \
297 : "memory", REGISTERS_CLOBBERED_BY_SYSCALL); \
298 (long int) resultvar; \
299})
300
301#undef internal_syscall4
302#define internal_syscall4(number, arg1, arg2, arg3, arg4) \
303({ \
304 unsigned long int resultvar; \
305 TYPEFY (arg4, __arg4) = ARGIFY (arg4); \
306 TYPEFY (arg3, __arg3) = ARGIFY (arg3); \
307 TYPEFY (arg2, __arg2) = ARGIFY (arg2); \
308 TYPEFY (arg1, __arg1) = ARGIFY (arg1); \
309 register TYPEFY (arg4, _a4) asm ("r10") = __arg4; \
310 register TYPEFY (arg3, _a3) asm ("rdx") = __arg3; \
311 register TYPEFY (arg2, _a2) asm ("rsi") = __arg2; \
312 register TYPEFY (arg1, _a1) asm ("rdi") = __arg1; \
313 asm volatile ( \
314 "syscall\n\t" \
315 : "=a" (resultvar) \
316 : "0" (number), "r" (_a1), "r" (_a2), "r" (_a3), "r" (_a4) \
317 : "memory", REGISTERS_CLOBBERED_BY_SYSCALL); \
318 (long int) resultvar; \
319})
320
321#undef internal_syscall5
322#define internal_syscall5(number, arg1, arg2, arg3, arg4, arg5) \
323({ \
324 unsigned long int resultvar; \
325 TYPEFY (arg5, __arg5) = ARGIFY (arg5); \
326 TYPEFY (arg4, __arg4) = ARGIFY (arg4); \
327 TYPEFY (arg3, __arg3) = ARGIFY (arg3); \
328 TYPEFY (arg2, __arg2) = ARGIFY (arg2); \
329 TYPEFY (arg1, __arg1) = ARGIFY (arg1); \
330 register TYPEFY (arg5, _a5) asm ("r8") = __arg5; \
331 register TYPEFY (arg4, _a4) asm ("r10") = __arg4; \
332 register TYPEFY (arg3, _a3) asm ("rdx") = __arg3; \
333 register TYPEFY (arg2, _a2) asm ("rsi") = __arg2; \
334 register TYPEFY (arg1, _a1) asm ("rdi") = __arg1; \
335 asm volatile ( \
336 "syscall\n\t" \
337 : "=a" (resultvar) \
338 : "0" (number), "r" (_a1), "r" (_a2), "r" (_a3), "r" (_a4), \
339 "r" (_a5) \
340 : "memory", REGISTERS_CLOBBERED_BY_SYSCALL); \
341 (long int) resultvar; \
342})
343
344#undef internal_syscall6
345#define internal_syscall6(number, arg1, arg2, arg3, arg4, arg5, arg6) \
346({ \
347 unsigned long int resultvar; \
348 TYPEFY (arg6, __arg6) = ARGIFY (arg6); \
349 TYPEFY (arg5, __arg5) = ARGIFY (arg5); \
350 TYPEFY (arg4, __arg4) = ARGIFY (arg4); \
351 TYPEFY (arg3, __arg3) = ARGIFY (arg3); \
352 TYPEFY (arg2, __arg2) = ARGIFY (arg2); \
353 TYPEFY (arg1, __arg1) = ARGIFY (arg1); \
354 register TYPEFY (arg6, _a6) asm ("r9") = __arg6; \
355 register TYPEFY (arg5, _a5) asm ("r8") = __arg5; \
356 register TYPEFY (arg4, _a4) asm ("r10") = __arg4; \
357 register TYPEFY (arg3, _a3) asm ("rdx") = __arg3; \
358 register TYPEFY (arg2, _a2) asm ("rsi") = __arg2; \
359 register TYPEFY (arg1, _a1) asm ("rdi") = __arg1; \
360 asm volatile ( \
361 "syscall\n\t" \
362 : "=a" (resultvar) \
363 : "0" (number), "r" (_a1), "r" (_a2), "r" (_a3), "r" (_a4), \
364 "r" (_a5), "r" (_a6) \
365 : "memory", REGISTERS_CLOBBERED_BY_SYSCALL); \
366 (long int) resultvar; \
367})
368
369
370# define VDSO_NAME "LINUX_2.6"
371# define VDSO_HASH 61765110
372
373/* List of system calls which are supported as vsyscalls. */
374# define HAVE_CLOCK_GETTIME64_VSYSCALL "__vdso_clock_gettime"
375# define HAVE_GETTIMEOFDAY_VSYSCALL "__vdso_gettimeofday"
376# define HAVE_TIME_VSYSCALL "__vdso_time"
377# define HAVE_GETCPU_VSYSCALL "__vdso_getcpu"
378# define HAVE_CLOCK_GETRES64_VSYSCALL "__vdso_clock_getres"
379
380# define SINGLE_THREAD_BY_GLOBAL 1
381
382#endif /* __ASSEMBLER__ */
383
384
385/* Pointer mangling support. */
386#if IS_IN (rtld)
387/* We cannot use the thread descriptor because in ld.so we use setjmp
388 earlier than the descriptor is initialized. */
389# ifdef __ASSEMBLER__
390# define PTR_MANGLE(reg) xor __pointer_chk_guard_local(%rip), reg; \
391 rol $2*LP_SIZE+1, reg
392# define PTR_DEMANGLE(reg) ror $2*LP_SIZE+1, reg; \
393 xor __pointer_chk_guard_local(%rip), reg
394# else
395# define PTR_MANGLE(reg) asm ("xor __pointer_chk_guard_local(%%rip), %0\n" \
396 "rol $2*" LP_SIZE "+1, %0" \
397 : "=r" (reg) : "0" (reg))
398# define PTR_DEMANGLE(reg) asm ("ror $2*" LP_SIZE "+1, %0\n" \
399 "xor __pointer_chk_guard_local(%%rip), %0" \
400 : "=r" (reg) : "0" (reg))
401# endif
402#else
403# ifdef __ASSEMBLER__
404# define PTR_MANGLE(reg) xor %fs:POINTER_GUARD, reg; \
405 rol $2*LP_SIZE+1, reg
406# define PTR_DEMANGLE(reg) ror $2*LP_SIZE+1, reg; \
407 xor %fs:POINTER_GUARD, reg
408# else
409# define PTR_MANGLE(var) asm ("xor %%fs:%c2, %0\n" \
410 "rol $2*" LP_SIZE "+1, %0" \
411 : "=r" (var) \
412 : "0" (var), \
413 "i" (offsetof (tcbhead_t, \
414 pointer_guard)))
415# define PTR_DEMANGLE(var) asm ("ror $2*" LP_SIZE "+1, %0\n" \
416 "xor %%fs:%c2, %0" \
417 : "=r" (var) \
418 : "0" (var), \
419 "i" (offsetof (tcbhead_t, \
420 pointer_guard)))
421# endif
422#endif
423
424/* How to pass the off{64}_t argument on p{readv,writev}{64}. */
425#undef LO_HI_LONG
426#define LO_HI_LONG(val) (val), 0
427
428/* Each shadow stack slot takes 8 bytes. Assuming that each stack
429 frame takes 256 bytes, this is used to compute shadow stack size
430 from stack size. */
431#define STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT 5
432
433#endif /* linux/x86_64/sysdep.h */
434