1// Licensed to the .NET Foundation under one or more agreements.
2// The .NET Foundation licenses this file to you under the MIT license.
3// See the LICENSE file in the project root for more information.
4
5// ===========================================================================
6// File: JITinterfaceCpu.CPP
7// ===========================================================================
8
9// This contains JITinterface routines that are specific to the
10// AMD64 platform. They are modeled after the X86 specific routines
11// found in JITinterfaceX86.cpp or JIThelp.asm
12
13
14#include "common.h"
15#include "jitinterface.h"
16#include "eeconfig.h"
17#include "excep.h"
18#include "threadsuspend.h"
19
20extern uint8_t* g_ephemeral_low;
21extern uint8_t* g_ephemeral_high;
22extern uint32_t* g_card_table;
23extern uint32_t* g_card_bundle_table;
24
25// Patch Labels for the various write barriers
26EXTERN_C void JIT_WriteBarrier_End();
27
28EXTERN_C void JIT_WriteBarrier_PreGrow64(Object **dst, Object *ref);
29EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_Lower();
30EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_CardTable();
31#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
32EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_CardBundleTable();
33#endif
34EXTERN_C void JIT_WriteBarrier_PreGrow64_End();
35
36EXTERN_C void JIT_WriteBarrier_PostGrow64(Object **dst, Object *ref);
37EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_Lower();
38EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_Upper();
39EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_CardTable();
40#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
41EXTERN_C void JIT_WriteBarrier_PostGrow64_Patch_Label_CardBundleTable();
42#endif
43EXTERN_C void JIT_WriteBarrier_PostGrow64_End();
44
45#ifdef FEATURE_SVR_GC
46EXTERN_C void JIT_WriteBarrier_SVR64(Object **dst, Object *ref);
47EXTERN_C void JIT_WriteBarrier_SVR64_PatchLabel_CardTable();
48#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
49EXTERN_C void JIT_WriteBarrier_SVR64_PatchLabel_CardBundleTable();
50#endif
51EXTERN_C void JIT_WriteBarrier_SVR64_End();
52#endif // FEATURE_SVR_GC
53
54#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
55EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64(Object **dst, Object *ref);
56EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_WriteWatchTable();
57EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_Lower();
58EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardTable();
59#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
60EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardBundleTable();
61#endif
62EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_End();
63
64EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64(Object **dst, Object *ref);
65EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_WriteWatchTable();
66EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_Lower();
67EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_Upper();
68EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_CardTable();
69#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
70EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_Patch_Label_CardBundleTable();
71#endif
72EXTERN_C void JIT_WriteBarrier_WriteWatch_PostGrow64_End();
73
74#ifdef FEATURE_SVR_GC
75EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64(Object **dst, Object *ref);
76EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_WriteWatchTable();
77EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardTable();
78#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
79EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_PatchLabel_CardBundleTable();
80#endif
81EXTERN_C void JIT_WriteBarrier_WriteWatch_SVR64_End();
82#endif // FEATURE_SVR_GC
83#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
84
85WriteBarrierManager g_WriteBarrierManager;
86
87// Use this somewhat hokey macro to concatenate the function start with the patch
88// label. This allows the code below to look relatively nice, but relies on the
89// naming convention which we have established for these helpers.
90#define CALC_PATCH_LOCATION(func,label,offset) CalculatePatchLocation((PVOID)func, (PVOID)func##_##label, offset)
91
92WriteBarrierManager::WriteBarrierManager() :
93 m_currentWriteBarrier(WRITE_BARRIER_UNINITIALIZED)
94{
95 LIMITED_METHOD_CONTRACT;
96}
97
98#ifndef CODECOVERAGE // Deactivate alignment validation for code coverage builds
99 // because the instrumentation tool will not preserve alignment
100 // constraints and we will fail.
101
102void WriteBarrierManager::Validate()
103{
104 CONTRACTL
105 {
106 MODE_ANY;
107 GC_NOTRIGGER;
108 NOTHROW;
109 }
110 CONTRACTL_END;
111
112 // we have an invariant that the addresses of all the values that we update in our write barrier
113 // helpers must be naturally aligned, this is so that the update can happen atomically since there
114 // are places where these values are updated while the EE is running
115 // NOTE: we can't call this from the ctor since our infrastructure isn't ready for assert dialogs
116
117 PBYTE pLowerBoundImmediate, pUpperBoundImmediate, pCardTableImmediate;
118
119#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
120 PBYTE pCardBundleTableImmediate;
121#endif
122
123 pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Lower, 2);
124 pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardTable, 2);
125
126 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pLowerBoundImmediate) & 0x7) == 0);
127 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardTableImmediate) & 0x7) == 0);
128
129#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
130 pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardBundleTable, 2);
131 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardBundleTableImmediate) & 0x7) == 0);
132#endif
133
134 pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Lower, 2);
135 pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Upper, 2);
136 pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardTable, 2);
137 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pLowerBoundImmediate) & 0x7) == 0);
138 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pUpperBoundImmediate) & 0x7) == 0);
139 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardTableImmediate) & 0x7) == 0);
140
141#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
142 pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardBundleTable, 2);
143 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardBundleTableImmediate) & 0x7) == 0);
144#endif
145
146#ifdef FEATURE_SVR_GC
147 pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardTable, 2);
148 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardTableImmediate) & 0x7) == 0);
149
150#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
151 pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardBundleTable, 2);
152 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardBundleTableImmediate) & 0x7) == 0);
153#endif // FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
154#endif // FEATURE_SVR_GC
155
156#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
157 PBYTE pWriteWatchTableImmediate;
158
159 pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_WriteWatchTable, 2);
160 pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_Lower, 2);
161 pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardTable, 2);
162
163 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pWriteWatchTableImmediate) & 0x7) == 0);
164 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pLowerBoundImmediate) & 0x7) == 0);
165 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardTableImmediate) & 0x7) == 0);
166
167#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
168 pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardBundleTable, 2);
169 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardBundleTableImmediate) & 0x7) == 0);
170#endif
171
172 pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_WriteWatchTable, 2);
173 pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Lower, 2);
174 pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Upper, 2);
175 pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardTable, 2);
176
177 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pWriteWatchTableImmediate) & 0x7) == 0);
178 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pLowerBoundImmediate) & 0x7) == 0);
179 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pUpperBoundImmediate) & 0x7) == 0);
180 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardTableImmediate) & 0x7) == 0);
181
182#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
183 pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardBundleTable, 2);
184 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardBundleTableImmediate) & 0x7) == 0);
185#endif
186
187#ifdef FEATURE_SVR_GC
188 pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_WriteWatchTable, 2);
189 pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardTable, 2);
190 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pWriteWatchTableImmediate) & 0x7) == 0);
191 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardTableImmediate) & 0x7) == 0);
192
193#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
194 pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardBundleTable, 2);
195 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (reinterpret_cast<UINT64>(pCardBundleTableImmediate) & 0x7) == 0);
196#endif // FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
197#endif // FEATURE_SVR_GC
198#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
199}
200
201#endif // CODECOVERAGE
202
203
204PCODE WriteBarrierManager::GetCurrentWriteBarrierCode()
205{
206 LIMITED_METHOD_CONTRACT;
207
208 switch (m_currentWriteBarrier)
209 {
210 case WRITE_BARRIER_PREGROW64:
211 return GetEEFuncEntryPoint(JIT_WriteBarrier_PreGrow64);
212 case WRITE_BARRIER_POSTGROW64:
213 return GetEEFuncEntryPoint(JIT_WriteBarrier_PostGrow64);
214#ifdef FEATURE_SVR_GC
215 case WRITE_BARRIER_SVR64:
216 return GetEEFuncEntryPoint(JIT_WriteBarrier_SVR64);
217#endif // FEATURE_SVR_GC
218#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
219 case WRITE_BARRIER_WRITE_WATCH_PREGROW64:
220 return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_PreGrow64);
221 case WRITE_BARRIER_WRITE_WATCH_POSTGROW64:
222 return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_PostGrow64);
223#ifdef FEATURE_SVR_GC
224 case WRITE_BARRIER_WRITE_WATCH_SVR64:
225 return GetEEFuncEntryPoint(JIT_WriteBarrier_WriteWatch_SVR64);
226#endif // FEATURE_SVR_GC
227#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
228 default:
229 UNREACHABLE_MSG("unexpected m_currentWriteBarrier!");
230 };
231}
232
233size_t WriteBarrierManager::GetSpecificWriteBarrierSize(WriteBarrierType writeBarrier)
234{
235// marked asm functions are those which use the LEAF_END_MARKED macro to end them which
236// creates a public Name_End label which can be used to figure out their size without
237// having to create unwind info.
238#define MARKED_FUNCTION_SIZE(pfn) (size_t)((LPBYTE)GetEEFuncEntryPoint(pfn##_End) - (LPBYTE)GetEEFuncEntryPoint(pfn))
239
240 switch (writeBarrier)
241 {
242 case WRITE_BARRIER_PREGROW64:
243 return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_PreGrow64);
244 case WRITE_BARRIER_POSTGROW64:
245 return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_PostGrow64);
246#ifdef FEATURE_SVR_GC
247 case WRITE_BARRIER_SVR64:
248 return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_SVR64);
249#endif // FEATURE_SVR_GC
250#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
251 case WRITE_BARRIER_WRITE_WATCH_PREGROW64:
252 return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_PreGrow64);
253 case WRITE_BARRIER_WRITE_WATCH_POSTGROW64:
254 return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_PostGrow64);
255#ifdef FEATURE_SVR_GC
256 case WRITE_BARRIER_WRITE_WATCH_SVR64:
257 return MARKED_FUNCTION_SIZE(JIT_WriteBarrier_WriteWatch_SVR64);
258#endif // FEATURE_SVR_GC
259#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
260 case WRITE_BARRIER_BUFFER:
261 return MARKED_FUNCTION_SIZE(JIT_WriteBarrier);
262 default:
263 UNREACHABLE_MSG("unexpected m_currentWriteBarrier!");
264 };
265#undef MARKED_FUNCTION_SIZE
266}
267
268size_t WriteBarrierManager::GetCurrentWriteBarrierSize()
269{
270 return GetSpecificWriteBarrierSize(m_currentWriteBarrier);
271}
272
273PBYTE WriteBarrierManager::CalculatePatchLocation(LPVOID base, LPVOID label, int offset)
274{
275 // the label should always come after the entrypoint for this funtion
276 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", (LPBYTE)label > (LPBYTE)base);
277
278 return ((LPBYTE)GetEEFuncEntryPoint(JIT_WriteBarrier) + ((LPBYTE)GetEEFuncEntryPoint(label) - (LPBYTE)GetEEFuncEntryPoint(base) + offset));
279}
280
281int WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier, bool isRuntimeSuspended)
282{
283 GCX_MAYBE_COOP_NO_THREAD_BROKEN((!isRuntimeSuspended && GetThread() != NULL));
284 int stompWBCompleteActions = SWB_PASS;
285 if (!isRuntimeSuspended && m_currentWriteBarrier != WRITE_BARRIER_UNINITIALIZED)
286 {
287 ThreadSuspend::SuspendEE(ThreadSuspend::SUSPEND_FOR_GC_PREP);
288 stompWBCompleteActions |= SWB_EE_RESTART;
289 }
290
291 _ASSERTE(m_currentWriteBarrier != newWriteBarrier);
292 m_currentWriteBarrier = newWriteBarrier;
293
294 // the memcpy must come before the switch statment because the asserts inside the switch
295 // are actually looking into the JIT_WriteBarrier buffer
296 memcpy((PVOID)JIT_WriteBarrier, (LPVOID)GetCurrentWriteBarrierCode(), GetCurrentWriteBarrierSize());
297
298 switch (newWriteBarrier)
299 {
300 case WRITE_BARRIER_PREGROW64:
301 {
302 m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Lower, 2);
303 m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardTable, 2);
304
305 // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0).
306 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate);
307 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate);
308
309#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
310 m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardBundleTable, 2);
311 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate);
312#endif
313 break;
314 }
315
316 case WRITE_BARRIER_POSTGROW64:
317 {
318 m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Lower, 2);
319 m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_Upper, 2);
320 m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardTable, 2);
321
322 // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0).
323 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate);
324 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate);
325 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate);
326
327#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
328 m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PostGrow64, Patch_Label_CardBundleTable, 2);
329 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate);
330#endif
331 break;
332 }
333
334#ifdef FEATURE_SVR_GC
335 case WRITE_BARRIER_SVR64:
336 {
337 m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardTable, 2);
338
339 // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0).
340 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate);
341
342#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
343 m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_SVR64, PatchLabel_CardBundleTable, 2);
344 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate);
345#endif
346 break;
347 }
348#endif // FEATURE_SVR_GC
349
350#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
351 case WRITE_BARRIER_WRITE_WATCH_PREGROW64:
352 {
353 m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_WriteWatchTable, 2);
354 m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_Lower, 2);
355 m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardTable, 2);
356
357 // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0).
358 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate);
359 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate);
360 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate);
361
362#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
363 m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardBundleTable, 2);
364 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate);
365#endif
366 break;
367 }
368
369 case WRITE_BARRIER_WRITE_WATCH_POSTGROW64:
370 {
371 m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_WriteWatchTable, 2);
372 m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Lower, 2);
373 m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_Upper, 2);
374 m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardTable, 2);
375
376 // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0).
377 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate);
378 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate);
379 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate);
380 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate);
381
382#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
383 m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PostGrow64, Patch_Label_CardBundleTable, 2);
384 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate);
385#endif
386 break;
387 }
388
389#ifdef FEATURE_SVR_GC
390 case WRITE_BARRIER_WRITE_WATCH_SVR64:
391 {
392 m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_WriteWatchTable, 2);
393 m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardTable, 2);
394
395 // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0).
396 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate);
397 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate);
398
399#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
400 m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_SVR64, PatchLabel_CardBundleTable, 2);
401 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", 0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardBundleTableImmediate);
402#endif
403 break;
404 }
405#endif // FEATURE_SVR_GC
406#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
407
408 default:
409 UNREACHABLE_MSG("unexpected write barrier type!");
410 }
411
412 stompWBCompleteActions |= UpdateEphemeralBounds(true);
413 stompWBCompleteActions |= UpdateWriteWatchAndCardTableLocations(true, false);
414
415 return stompWBCompleteActions;
416}
417
418#undef CALC_PATCH_LOCATION
419
420void WriteBarrierManager::Initialize()
421{
422 CONTRACTL
423 {
424 MODE_ANY;
425 GC_NOTRIGGER;
426 NOTHROW;
427 }
428 CONTRACTL_END;
429
430
431 // Ensure that the generic JIT_WriteBarrier function buffer is large enough to hold any of the more specific
432 // write barrier implementations.
433 size_t cbWriteBarrierBuffer = GetSpecificWriteBarrierSize(WRITE_BARRIER_BUFFER);
434
435 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_PREGROW64));
436 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_POSTGROW64));
437#ifdef FEATURE_SVR_GC
438 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_SVR64));
439#endif // FEATURE_SVR_GC
440#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
441 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_PREGROW64));
442 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_POSTGROW64));
443#ifdef FEATURE_SVR_GC
444 _ASSERTE_ALL_BUILDS("clr/src/VM/AMD64/JITinterfaceAMD64.cpp", cbWriteBarrierBuffer >= GetSpecificWriteBarrierSize(WRITE_BARRIER_WRITE_WATCH_SVR64));
445#endif // FEATURE_SVR_GC
446#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
447
448#if !defined(CODECOVERAGE)
449 Validate();
450#endif
451}
452
453bool WriteBarrierManager::NeedDifferentWriteBarrier(bool bReqUpperBoundsCheck, WriteBarrierType* pNewWriteBarrierType)
454{
455 // Init code for the JIT_WriteBarrier assembly routine. Since it will be bashed everytime the GC Heap
456 // changes size, we want to do most of the work just once.
457 //
458 // The actual JIT_WriteBarrier routine will only be called in free builds, but we keep this code (that
459 // modifies it) around in debug builds to check that it works (with assertions).
460
461
462 WriteBarrierType writeBarrierType = m_currentWriteBarrier;
463
464 for(;;)
465 {
466 switch (writeBarrierType)
467 {
468 case WRITE_BARRIER_UNINITIALIZED:
469#ifdef _DEBUG
470 // Use the default slow write barrier some of the time in debug builds because of of contains some good asserts
471 if ((g_pConfig->GetHeapVerifyLevel() & EEConfig::HEAPVERIFY_BARRIERCHECK) || DbgRandomOnExe(0.5)) {
472 break;
473 }
474#endif
475
476 writeBarrierType = GCHeapUtilities::IsServerHeap() ? WRITE_BARRIER_SVR64 : WRITE_BARRIER_PREGROW64;
477 continue;
478
479 case WRITE_BARRIER_PREGROW64:
480 if (bReqUpperBoundsCheck)
481 {
482 writeBarrierType = WRITE_BARRIER_POSTGROW64;
483 }
484 break;
485
486 case WRITE_BARRIER_POSTGROW64:
487 break;
488
489#ifdef FEATURE_SVR_GC
490 case WRITE_BARRIER_SVR64:
491 break;
492#endif // FEATURE_SVR_GC
493
494#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
495 case WRITE_BARRIER_WRITE_WATCH_PREGROW64:
496 if (bReqUpperBoundsCheck)
497 {
498 writeBarrierType = WRITE_BARRIER_WRITE_WATCH_POSTGROW64;
499 }
500 break;
501
502 case WRITE_BARRIER_WRITE_WATCH_POSTGROW64:
503 break;
504
505#ifdef FEATURE_SVR_GC
506 case WRITE_BARRIER_WRITE_WATCH_SVR64:
507 break;
508#endif // FEATURE_SVR_GC
509#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
510
511 default:
512 UNREACHABLE_MSG("unexpected write barrier type!");
513 }
514 break;
515 }
516
517 *pNewWriteBarrierType = writeBarrierType;
518 return m_currentWriteBarrier != writeBarrierType;
519}
520
521int WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended)
522{
523 WriteBarrierType newType;
524 if (NeedDifferentWriteBarrier(false, &newType))
525 {
526 return ChangeWriteBarrierTo(newType, isRuntimeSuspended);
527 }
528
529 int stompWBCompleteActions = SWB_PASS;
530
531#ifdef _DEBUG
532 // Using debug-only write barrier?
533 if (m_currentWriteBarrier == WRITE_BARRIER_UNINITIALIZED)
534 return stompWBCompleteActions;
535#endif
536
537 switch (m_currentWriteBarrier)
538 {
539 case WRITE_BARRIER_POSTGROW64:
540#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
541 case WRITE_BARRIER_WRITE_WATCH_POSTGROW64:
542#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
543 {
544 // Change immediate if different from new g_ephermeral_high.
545 if (*(UINT64*)m_pUpperBoundImmediate != (size_t)g_ephemeral_high)
546 {
547 *(UINT64*)m_pUpperBoundImmediate = (size_t)g_ephemeral_high;
548 stompWBCompleteActions |= SWB_ICACHE_FLUSH;
549 }
550 }
551 //
552 // INTENTIONAL FALL-THROUGH!
553 //
554 case WRITE_BARRIER_PREGROW64:
555#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
556 case WRITE_BARRIER_WRITE_WATCH_PREGROW64:
557#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
558 {
559 // Change immediate if different from new g_ephermeral_low.
560 if (*(UINT64*)m_pLowerBoundImmediate != (size_t)g_ephemeral_low)
561 {
562 *(UINT64*)m_pLowerBoundImmediate = (size_t)g_ephemeral_low;
563 stompWBCompleteActions |= SWB_ICACHE_FLUSH;
564 }
565 break;
566 }
567
568#ifdef FEATURE_SVR_GC
569 case WRITE_BARRIER_SVR64:
570#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
571 case WRITE_BARRIER_WRITE_WATCH_SVR64:
572#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
573 {
574 break;
575 }
576#endif // FEATURE_SVR_GC
577
578 default:
579 UNREACHABLE_MSG("unexpected m_currentWriteBarrier in UpdateEphemeralBounds");
580 }
581
582 return stompWBCompleteActions;
583}
584
585int WriteBarrierManager::UpdateWriteWatchAndCardTableLocations(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
586{
587 // If we are told that we require an upper bounds check (GC did some heap reshuffling),
588 // we need to switch to the WriteBarrier_PostGrow function for good.
589
590 WriteBarrierType newType;
591 if (NeedDifferentWriteBarrier(bReqUpperBoundsCheck, &newType))
592 {
593 return ChangeWriteBarrierTo(newType, isRuntimeSuspended);
594 }
595
596 int stompWBCompleteActions = SWB_PASS;
597
598#ifdef _DEBUG
599 // Using debug-only write barrier?
600 if (m_currentWriteBarrier == WRITE_BARRIER_UNINITIALIZED)
601 return stompWBCompleteActions;
602#endif
603
604#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
605 switch (m_currentWriteBarrier)
606 {
607 case WRITE_BARRIER_WRITE_WATCH_PREGROW64:
608 case WRITE_BARRIER_WRITE_WATCH_POSTGROW64:
609#ifdef FEATURE_SVR_GC
610 case WRITE_BARRIER_WRITE_WATCH_SVR64:
611#endif // FEATURE_SVR_GC
612 if (*(UINT64*)m_pWriteWatchTableImmediate != (size_t)g_sw_ww_table)
613 {
614 *(UINT64*)m_pWriteWatchTableImmediate = (size_t)g_sw_ww_table;
615 stompWBCompleteActions |= SWB_ICACHE_FLUSH;
616 }
617 break;
618
619 default:
620 break; // clang seems to require all enum values to be covered for some reason
621 }
622#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
623
624 if (*(UINT64*)m_pCardTableImmediate != (size_t)g_card_table)
625 {
626 *(UINT64*)m_pCardTableImmediate = (size_t)g_card_table;
627 stompWBCompleteActions |= SWB_ICACHE_FLUSH;
628 }
629
630#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
631 if (*(UINT64*)m_pCardBundleTableImmediate != (size_t)g_card_bundle_table)
632 {
633 *(UINT64*)m_pCardBundleTableImmediate = (size_t)g_card_bundle_table;
634 stompWBCompleteActions |= SWB_ICACHE_FLUSH;
635 }
636#endif
637
638 return stompWBCompleteActions;
639}
640
641#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
642int WriteBarrierManager::SwitchToWriteWatchBarrier(bool isRuntimeSuspended)
643{
644 WriteBarrierType newWriteBarrierType;
645 switch (m_currentWriteBarrier)
646 {
647 case WRITE_BARRIER_UNINITIALIZED:
648 // Using the debug-only write barrier
649 return SWB_PASS;
650
651 case WRITE_BARRIER_PREGROW64:
652 newWriteBarrierType = WRITE_BARRIER_WRITE_WATCH_PREGROW64;
653 break;
654
655 case WRITE_BARRIER_POSTGROW64:
656 newWriteBarrierType = WRITE_BARRIER_WRITE_WATCH_POSTGROW64;
657 break;
658
659#ifdef FEATURE_SVR_GC
660 case WRITE_BARRIER_SVR64:
661 newWriteBarrierType = WRITE_BARRIER_WRITE_WATCH_SVR64;
662 break;
663#endif // FEATURE_SVR_GC
664
665 default:
666 UNREACHABLE();
667 }
668
669 return ChangeWriteBarrierTo(newWriteBarrierType, isRuntimeSuspended);
670}
671
672int WriteBarrierManager::SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended)
673{
674 WriteBarrierType newWriteBarrierType;
675 switch (m_currentWriteBarrier)
676 {
677 case WRITE_BARRIER_UNINITIALIZED:
678 // Using the debug-only write barrier
679 return SWB_PASS;
680
681 case WRITE_BARRIER_WRITE_WATCH_PREGROW64:
682 newWriteBarrierType = WRITE_BARRIER_PREGROW64;
683 break;
684
685 case WRITE_BARRIER_WRITE_WATCH_POSTGROW64:
686 newWriteBarrierType = WRITE_BARRIER_POSTGROW64;
687 break;
688
689#ifdef FEATURE_SVR_GC
690 case WRITE_BARRIER_WRITE_WATCH_SVR64:
691 newWriteBarrierType = WRITE_BARRIER_SVR64;
692 break;
693#endif // FEATURE_SVR_GC
694
695 default:
696 UNREACHABLE();
697 }
698
699 return ChangeWriteBarrierTo(newWriteBarrierType, isRuntimeSuspended);
700}
701#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
702
703// This function bashes the super fast amd64 version of the JIT_WriteBarrier
704// helper. It should be called by the GC whenever the ephermeral region
705// bounds get changed, but still remain on the top of the GC Heap.
706int StompWriteBarrierEphemeral(bool isRuntimeSuspended)
707{
708 WRAPPER_NO_CONTRACT;
709
710 return g_WriteBarrierManager.UpdateEphemeralBounds(isRuntimeSuspended);
711}
712
713// This function bashes the super fast amd64 versions of the JIT_WriteBarrier
714// helpers. It should be called by the GC whenever the ephermeral region gets moved
715// from being at the top of the GC Heap, and/or when the cards table gets moved.
716int StompWriteBarrierResize(bool isRuntimeSuspended, bool bReqUpperBoundsCheck)
717{
718 WRAPPER_NO_CONTRACT;
719
720 return g_WriteBarrierManager.UpdateWriteWatchAndCardTableLocations(isRuntimeSuspended, bReqUpperBoundsCheck);
721}
722
723void FlushWriteBarrierInstructionCache()
724{
725 FlushInstructionCache(GetCurrentProcess(), (PVOID)JIT_WriteBarrier, g_WriteBarrierManager.GetCurrentWriteBarrierSize());
726}
727
728#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
729int SwitchToWriteWatchBarrier(bool isRuntimeSuspended)
730{
731 WRAPPER_NO_CONTRACT;
732
733 return g_WriteBarrierManager.SwitchToWriteWatchBarrier(isRuntimeSuspended);
734}
735
736int SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended)
737{
738 WRAPPER_NO_CONTRACT;
739
740 return g_WriteBarrierManager.SwitchToNonWriteWatchBarrier(isRuntimeSuspended);
741}
742#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
743