1// Copyright 2016 The SwiftShader Authors. All Rights Reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#include "ExecutableMemory.hpp"
16
17#include "Debug.hpp"
18
19#if defined(_WIN32)
20 #ifndef WIN32_LEAN_AND_MEAN
21 #define WIN32_LEAN_AND_MEAN
22 #endif
23 #include <windows.h>
24 #include <intrin.h>
25#elif defined(__Fuchsia__)
26 #include <unistd.h>
27 #include <zircon/process.h>
28 #include <zircon/syscalls.h>
29#else
30 #include <errno.h>
31 #include <sys/mman.h>
32 #include <stdlib.h>
33 #include <unistd.h>
34#endif
35
36#include <memory.h>
37
38#undef allocate
39#undef deallocate
40
41#if (defined(__i386__) || defined(_M_IX86) || defined(__x86_64__) || defined (_M_X64)) && !defined(__x86__)
42#define __x86__
43#endif
44
45namespace rr
46{
47namespace
48{
49struct Allocation
50{
51// size_t bytes;
52 unsigned char *block;
53};
54
55void *allocateRaw(size_t bytes, size_t alignment)
56{
57 ASSERT((alignment & (alignment - 1)) == 0); // Power of 2 alignment.
58
59 #if defined(LINUX_ENABLE_NAMED_MMAP)
60 if(alignment < sizeof(void*))
61 {
62 return malloc(bytes);
63 }
64 else
65 {
66 void *allocation;
67 int result = posix_memalign(&allocation, alignment, bytes);
68 if(result != 0)
69 {
70 errno = result;
71 allocation = nullptr;
72 }
73 return allocation;
74 }
75 #else
76 unsigned char *block = new unsigned char[bytes + sizeof(Allocation) + alignment];
77 unsigned char *aligned = nullptr;
78
79 if(block)
80 {
81 aligned = (unsigned char*)((uintptr_t)(block + sizeof(Allocation) + alignment - 1) & -(intptr_t)alignment);
82 Allocation *allocation = (Allocation*)(aligned - sizeof(Allocation));
83
84 // allocation->bytes = bytes;
85 allocation->block = block;
86 }
87
88 return aligned;
89 #endif
90}
91
92#if defined(LINUX_ENABLE_NAMED_MMAP)
93// Create a file descriptor for anonymous memory with the given
94// name. Returns -1 on failure.
95// TODO: remove once libc wrapper exists.
96int memfd_create(const char* name, unsigned int flags)
97{
98 #if __aarch64__
99 #define __NR_memfd_create 279
100 #elif __arm__
101 #define __NR_memfd_create 279
102 #elif __powerpc64__
103 #define __NR_memfd_create 360
104 #elif __i386__
105 #define __NR_memfd_create 356
106 #elif __x86_64__
107 #define __NR_memfd_create 319
108 #endif /* __NR_memfd_create__ */
109 #ifdef __NR_memfd_create
110 // In the event of no system call this returns -1 with errno set
111 // as ENOSYS.
112 return syscall(__NR_memfd_create, name, flags);
113 #else
114 return -1;
115 #endif
116}
117
118// Returns a file descriptor for use with an anonymous mmap, if
119// memfd_create fails, -1 is returned. Note, the mappings should be
120// MAP_PRIVATE so that underlying pages aren't shared.
121int anonymousFd()
122{
123 static int fd = memfd_create("SwiftShader JIT", 0);
124 return fd;
125}
126
127// Ensure there is enough space in the "anonymous" fd for length.
128void ensureAnonFileSize(int anonFd, size_t length)
129{
130 static size_t fileSize = 0;
131 if(length > fileSize)
132 {
133 ftruncate(anonFd, length);
134 fileSize = length;
135 }
136}
137#endif // defined(LINUX_ENABLE_NAMED_MMAP)
138
139} // anonymous namespace
140
141size_t memoryPageSize()
142{
143 static int pageSize = 0;
144
145 if(pageSize == 0)
146 {
147 #if defined(_WIN32)
148 SYSTEM_INFO systemInfo;
149 GetSystemInfo(&systemInfo);
150 pageSize = systemInfo.dwPageSize;
151 #else
152 pageSize = sysconf(_SC_PAGESIZE);
153 #endif
154 }
155
156 return pageSize;
157}
158
159void *allocate(size_t bytes, size_t alignment)
160{
161 void *memory = allocateRaw(bytes, alignment);
162
163 if(memory)
164 {
165 memset(memory, 0, bytes);
166 }
167
168 return memory;
169}
170
171void deallocate(void *memory)
172{
173 #if defined(LINUX_ENABLE_NAMED_MMAP)
174 free(memory);
175 #else
176 if(memory)
177 {
178 unsigned char *aligned = (unsigned char*)memory;
179 Allocation *allocation = (Allocation*)(aligned - sizeof(Allocation));
180
181 delete[] allocation->block;
182 }
183 #endif
184}
185
186// Rounds |x| up to a multiple of |m|, where |m| is a power of 2.
187inline uintptr_t roundUp(uintptr_t x, uintptr_t m)
188{
189 ASSERT(m > 0 && (m & (m - 1)) == 0); // |m| must be a power of 2.
190 return (x + m - 1) & ~(m - 1);
191}
192
193void *allocateExecutable(size_t bytes)
194{
195 size_t pageSize = memoryPageSize();
196 size_t length = roundUp(bytes, pageSize);
197 void *mapping = nullptr;
198
199 #if defined(LINUX_ENABLE_NAMED_MMAP)
200 // Try to name the memory region for the executable code,
201 // to aid profilers.
202 int anonFd = anonymousFd();
203 if(anonFd == -1)
204 {
205 mapping = mmap(nullptr, length, PROT_READ | PROT_WRITE,
206 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
207 }
208 else
209 {
210 ensureAnonFileSize(anonFd, length);
211 mapping = mmap(nullptr, length, PROT_READ | PROT_WRITE,
212 MAP_PRIVATE, anonFd, 0);
213 }
214
215 if(mapping == MAP_FAILED)
216 {
217 mapping = nullptr;
218 }
219 #elif defined(__Fuchsia__)
220 zx_handle_t vmo;
221 if (zx_vmo_create(length, 0, &vmo) != ZX_OK) {
222 return nullptr;
223 }
224 if (zx_vmo_replace_as_executable(vmo, ZX_HANDLE_INVALID, &vmo) != ZX_OK) {
225 return nullptr;
226 }
227 zx_vaddr_t reservation;
228 zx_status_t status = zx_vmar_map(
229 zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE,
230 0, vmo, 0, length, &reservation);
231 zx_handle_close(vmo);
232 if (status != ZX_OK) {
233 return nullptr;
234 }
235
236 // zx_vmar_map() returns page-aligned address.
237 ASSERT(roundUp(reservation, pageSize) == reservation);
238
239 mapping = reinterpret_cast<void*>(reservation);
240 #elif defined(__APPLE__)
241 // On macOS 10.14 and higher, executables that are code signed with the
242 // "runtime" option cannot execute writable memory by default. They can opt
243 // into this capability by specifying the "com.apple.security.cs.allow-jit"
244 // code signing entitlement and allocating the region with the MAP_JIT flag.
245 mapping = mmap(nullptr, length, PROT_READ | PROT_WRITE,
246 MAP_PRIVATE | MAP_ANONYMOUS | MAP_JIT, -1, 0);
247
248 if(mapping == MAP_FAILED)
249 {
250 // Retry without MAP_JIT (for older macOS versions).
251 mapping = mmap(nullptr, length, PROT_READ | PROT_WRITE,
252 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
253 }
254
255 if(mapping == MAP_FAILED)
256 {
257 mapping = nullptr;
258 }
259 #else
260 mapping = allocate(length, pageSize);
261 #endif
262
263 return mapping;
264}
265
266void markExecutable(void *memory, size_t bytes)
267{
268 #if defined(_WIN32)
269 unsigned long oldProtection;
270 VirtualProtect(memory, bytes, PAGE_EXECUTE_READ, &oldProtection);
271 #elif defined(__Fuchsia__)
272 size_t pageSize = memoryPageSize();
273 size_t length = roundUp(bytes, pageSize);
274 zx_status_t status = zx_vmar_protect(
275 zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_EXECUTE,
276 reinterpret_cast<zx_vaddr_t>(memory), length);
277 ASSERT(status == ZX_OK);
278 #else
279 mprotect(memory, bytes, PROT_READ | PROT_EXEC);
280 #endif
281}
282
283void deallocateExecutable(void *memory, size_t bytes)
284{
285 #if defined(_WIN32)
286 unsigned long oldProtection;
287 VirtualProtect(memory, bytes, PAGE_READWRITE, &oldProtection);
288 deallocate(memory);
289 #elif defined(LINUX_ENABLE_NAMED_MMAP) || defined(__APPLE__)
290 size_t pageSize = memoryPageSize();
291 size_t length = (bytes + pageSize - 1) & ~(pageSize - 1);
292 munmap(memory, length);
293 #elif defined(__Fuchsia__)
294 size_t pageSize = memoryPageSize();
295 size_t length = roundUp(bytes, pageSize);
296 zx_status_t status = zx_vmar_unmap(
297 zx_vmar_root_self(), reinterpret_cast<zx_vaddr_t>(memory), length);
298 ASSERT(status == ZX_OK);
299 #else
300 mprotect(memory, bytes, PROT_READ | PROT_WRITE);
301 deallocate(memory);
302 #endif
303}
304}
305