1// Copyright 2009-2021 Intel Corporation
2// SPDX-License-Identifier: Apache-2.0
3
4#include "alloc.h"
5#include "intrinsics.h"
6#include "sysinfo.h"
7#include "mutex.h"
8
9////////////////////////////////////////////////////////////////////////////////
10/// All Platforms
11////////////////////////////////////////////////////////////////////////////////
12
13namespace embree
14{
15 void* alignedMalloc(size_t size, size_t align)
16 {
17 if (size == 0)
18 return nullptr;
19
20 assert((align & (align-1)) == 0);
21 void* ptr = _mm_malloc(size,align);
22
23 if (size != 0 && ptr == nullptr)
24 // -- GODOT start --
25 // throw std::bad_alloc();
26 abort();
27 // -- GODOT end --
28
29 return ptr;
30 }
31
32 void alignedFree(void* ptr)
33 {
34 if (ptr)
35 _mm_free(ptr);
36 }
37
38 static bool huge_pages_enabled = false;
39 static MutexSys os_init_mutex;
40
41 __forceinline bool isHugePageCandidate(const size_t bytes)
42 {
43 if (!huge_pages_enabled)
44 return false;
45
46 /* use huge pages only when memory overhead is low */
47 const size_t hbytes = (bytes+PAGE_SIZE_2M-1) & ~size_t(PAGE_SIZE_2M-1);
48 return 66*(hbytes-bytes) < bytes; // at most 1.5% overhead
49 }
50}
51
52////////////////////////////////////////////////////////////////////////////////
53/// Windows Platform
54////////////////////////////////////////////////////////////////////////////////
55
56#ifdef _WIN32
57
58#define WIN32_LEAN_AND_MEAN
59#include <windows.h>
60#include <malloc.h>
61
62namespace embree
63{
64 bool win_enable_selockmemoryprivilege (bool verbose)
65 {
66 HANDLE hToken;
67 if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES, &hToken)) {
68 if (verbose) std::cout << "WARNING: OpenProcessToken failed while trying to enable SeLockMemoryPrivilege: " << GetLastError() << std::endl;
69 return false;
70 }
71
72 TOKEN_PRIVILEGES tp;
73 tp.PrivilegeCount = 1;
74 tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
75
76 if (!LookupPrivilegeValueW(nullptr, L"SeLockMemoryPrivilege", &tp.Privileges[0].Luid)) {
77 if (verbose) std::cout << "WARNING: LookupPrivilegeValue failed while trying to enable SeLockMemoryPrivilege: " << GetLastError() << std::endl;
78 return false;
79 }
80
81 SetLastError(ERROR_SUCCESS);
82 if (!AdjustTokenPrivileges(hToken, FALSE, &tp, sizeof(tp), nullptr, 0)) {
83 if (verbose) std::cout << "WARNING: AdjustTokenPrivileges failed while trying to enable SeLockMemoryPrivilege" << std::endl;
84 return false;
85 }
86
87 if (GetLastError() == ERROR_NOT_ALL_ASSIGNED) {
88 if (verbose) std::cout << "WARNING: AdjustTokenPrivileges failed to enable SeLockMemoryPrivilege: Add SeLockMemoryPrivilege for current user and run process in elevated mode (Run as administrator)." << std::endl;
89 return false;
90 }
91
92 return true;
93 }
94
95 bool os_init(bool hugepages, bool verbose)
96 {
97 Lock<MutexSys> lock(os_init_mutex);
98
99 if (!hugepages) {
100 huge_pages_enabled = false;
101 return true;
102 }
103
104 if (GetLargePageMinimum() != PAGE_SIZE_2M) {
105 huge_pages_enabled = false;
106 return false;
107 }
108
109 huge_pages_enabled = true;
110 return true;
111 }
112
113 void* os_malloc(size_t bytes, bool& hugepages)
114 {
115 if (bytes == 0) {
116 hugepages = false;
117 return nullptr;
118 }
119
120 /* try direct huge page allocation first */
121 if (isHugePageCandidate(bytes))
122 {
123 int flags = MEM_COMMIT | MEM_RESERVE | MEM_LARGE_PAGES;
124 char* ptr = (char*) VirtualAlloc(nullptr,bytes,flags,PAGE_READWRITE);
125 if (ptr != nullptr) {
126 hugepages = true;
127 return ptr;
128 }
129 }
130
131 /* fall back to 4k pages */
132 int flags = MEM_COMMIT | MEM_RESERVE;
133 char* ptr = (char*) VirtualAlloc(nullptr,bytes,flags,PAGE_READWRITE);
134 // -- GODOT start --
135 // if (ptr == nullptr) throw std::bad_alloc();
136 if (ptr == nullptr) abort();
137 // -- GODOT end --
138 hugepages = false;
139 return ptr;
140 }
141
142 size_t os_shrink(void* ptr, size_t bytesNew, size_t bytesOld, bool hugepages)
143 {
144 if (hugepages) // decommitting huge pages seems not to work under Windows
145 return bytesOld;
146
147 const size_t pageSize = hugepages ? PAGE_SIZE_2M : PAGE_SIZE_4K;
148 bytesNew = (bytesNew+pageSize-1) & ~(pageSize-1);
149 bytesOld = (bytesOld+pageSize-1) & ~(pageSize-1);
150 if (bytesNew >= bytesOld)
151 return bytesOld;
152
153 if (!VirtualFree((char*)ptr+bytesNew,bytesOld-bytesNew,MEM_DECOMMIT))
154 // -- GODOT start --
155 // throw std::bad_alloc();
156 abort();
157 // -- GODOT end --
158
159 return bytesNew;
160 }
161
162 void os_free(void* ptr, size_t bytes, bool hugepages)
163 {
164 if (bytes == 0)
165 return;
166
167 if (!VirtualFree(ptr,0,MEM_RELEASE))
168 // -- GODOT start --
169 // throw std::bad_alloc();
170 abort();
171 // -- GODOT end --
172 }
173
174 void os_advise(void *ptr, size_t bytes)
175 {
176 }
177}
178
179#endif
180
181////////////////////////////////////////////////////////////////////////////////
182/// Unix Platform
183////////////////////////////////////////////////////////////////////////////////
184
185#if defined(__UNIX__)
186
187#include <sys/mman.h>
188#include <errno.h>
189#include <stdlib.h>
190#include <string.h>
191#include <sstream>
192
193#if defined(__MACOSX__)
194#include <mach/vm_statistics.h>
195#endif
196
197namespace embree
198{
199 bool os_init(bool hugepages, bool verbose)
200 {
201 Lock<MutexSys> lock(os_init_mutex);
202
203 if (!hugepages) {
204 huge_pages_enabled = false;
205 return true;
206 }
207
208#if defined(__LINUX__)
209
210 int hugepagesize = 0;
211
212 std::ifstream file;
213 file.open("/proc/meminfo",std::ios::in);
214 if (!file.is_open()) {
215 if (verbose) std::cout << "WARNING: Could not open /proc/meminfo. Huge page support cannot get enabled!" << std::endl;
216 huge_pages_enabled = false;
217 return false;
218 }
219
220 std::string line;
221 while (getline(file,line))
222 {
223 std::stringstream sline(line);
224 while (!sline.eof() && sline.peek() == ' ') sline.ignore();
225 std::string tag; getline(sline,tag,' ');
226 while (!sline.eof() && sline.peek() == ' ') sline.ignore();
227 std::string val; getline(sline,val,' ');
228 while (!sline.eof() && sline.peek() == ' ') sline.ignore();
229 std::string unit; getline(sline,unit,' ');
230 if (tag == "Hugepagesize:" && unit == "kB") {
231 hugepagesize = std::stoi(val)*1024;
232 break;
233 }
234 }
235
236 if (hugepagesize != PAGE_SIZE_2M)
237 {
238 if (verbose) std::cout << "WARNING: Only 2MB huge pages supported. Huge page support cannot get enabled!" << std::endl;
239 huge_pages_enabled = false;
240 return false;
241 }
242#endif
243
244 huge_pages_enabled = true;
245 return true;
246 }
247
248 void* os_malloc(size_t bytes, bool& hugepages)
249 {
250 if (bytes == 0) {
251 hugepages = false;
252 return nullptr;
253 }
254
255 /* try direct huge page allocation first */
256 if (isHugePageCandidate(bytes))
257 {
258#if defined(__MACOSX__)
259 void* ptr = mmap(0, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, VM_FLAGS_SUPERPAGE_SIZE_2MB, 0);
260 if (ptr != MAP_FAILED) {
261 hugepages = true;
262 return ptr;
263 }
264#elif defined(MAP_HUGETLB)
265 void* ptr = mmap(0, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_HUGETLB, -1, 0);
266 if (ptr != MAP_FAILED) {
267 hugepages = true;
268 return ptr;
269 }
270#endif
271 }
272
273 /* fallback to 4k pages */
274 void* ptr = (char*) mmap(0, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
275 // -- GODOT start --
276 // if (ptr == MAP_FAILED) throw std::bad_alloc();
277 if (ptr == MAP_FAILED) abort();
278 // -- GODOT end --
279 hugepages = false;
280
281 /* advise huge page hint for THP */
282 os_advise(ptr,bytes);
283 return ptr;
284 }
285
286 size_t os_shrink(void* ptr, size_t bytesNew, size_t bytesOld, bool hugepages)
287 {
288 const size_t pageSize = hugepages ? PAGE_SIZE_2M : PAGE_SIZE_4K;
289 bytesNew = (bytesNew+pageSize-1) & ~(pageSize-1);
290 bytesOld = (bytesOld+pageSize-1) & ~(pageSize-1);
291 if (bytesNew >= bytesOld)
292 return bytesOld;
293
294 if (munmap((char*)ptr+bytesNew,bytesOld-bytesNew) == -1)
295 // -- GODOT start --
296 // throw std::bad_alloc();
297 abort();
298 // -- GODOT end --
299
300 return bytesNew;
301 }
302
303 void os_free(void* ptr, size_t bytes, bool hugepages)
304 {
305 if (bytes == 0)
306 return;
307
308 /* for hugepages we need to also align the size */
309 const size_t pageSize = hugepages ? PAGE_SIZE_2M : PAGE_SIZE_4K;
310 bytes = (bytes+pageSize-1) & ~(pageSize-1);
311 if (munmap(ptr,bytes) == -1)
312 // -- GODOT start --
313 // throw std::bad_alloc();
314 abort();
315 // -- GODOT end --
316 }
317
318 /* hint for transparent huge pages (THP) */
319 void os_advise(void* pptr, size_t bytes)
320 {
321#if defined(MADV_HUGEPAGE)
322 madvise(pptr,bytes,MADV_HUGEPAGE);
323#endif
324 }
325}
326
327#endif
328