1// Licensed to the .NET Foundation under one or more agreements.
2// The .NET Foundation licenses this file to you under the MIT license.
3// See the LICENSE file in the project root for more information.
4
5#ifndef __SOFTWARE_WRITE_WATCH_H__
6#define __SOFTWARE_WRITE_WATCH_H__
7
8#include "gcinterface.h"
9#include "gc.h"
10
11#define WRITE_WATCH_UNIT_SIZE 0x1000
12
13#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
14#ifndef DACCESS_COMPILE
15
16extern "C"
17{
18 // Table containing the dirty state. This table is translated to exclude the lowest address it represents, see
19 // TranslateTableToExcludeHeapStartAddress.
20 extern uint8_t *g_gc_sw_ww_table;
21
22 // Write watch may be disabled when it is not needed (between GCs for instance). This indicates whether it is enabled.
23 extern bool g_gc_sw_ww_enabled_for_gc_heap;
24}
25
26class SoftwareWriteWatch
27{
28private:
29 // The granularity of dirty state in the table is one page. Dirtiness is tracked per byte of the table so that
30 // synchronization is not required when changing the dirty state. Shifting-right an address by the following value yields
31 // the byte index of the address into the write watch table. For instance,
32 // GetTable()[address >> AddressToTableByteIndexShift] is the byte that represents the region of memory for 'address'.
33 static const uint8_t AddressToTableByteIndexShift = SOFTWARE_WRITE_WATCH_AddressToTableByteIndexShift;
34
35private:
36 static void VerifyCreated();
37 static void VerifyMemoryRegion(void *baseAddress, size_t regionByteSize);
38 static void VerifyMemoryRegion(void *baseAddress, size_t regionByteSize, void *heapStartAddress, void *heapEndAddress);
39
40public:
41 static uint8_t *GetTable();
42private:
43 static uint8_t *GetUntranslatedTable();
44 static uint8_t *GetUntranslatedTable(uint8_t *table, void *heapStartAddress);
45 static uint8_t *GetUntranslatedTableEnd();
46 static uint8_t *GetUntranslatedTableEnd(uint8_t *table, void *heapEndAddress);
47public:
48 static void InitializeUntranslatedTable(uint8_t *untranslatedTable, void *heapStartAddress);
49private:
50 static void SetUntranslatedTable(uint8_t *untranslatedTable, void *heapStartAddress);
51public:
52 static void SetResizedUntranslatedTable(uint8_t *untranslatedTable, void *heapStartAddress, void *heapEndAddress);
53 static bool IsEnabledForGCHeap();
54 static void EnableForGCHeap();
55 static void DisableForGCHeap();
56private:
57 static void *GetHeapStartAddress();
58 static void *GetHeapEndAddress();
59
60public:
61 static void StaticClose();
62
63private:
64 static size_t GetTableByteIndex(void *address);
65 static void *GetPageAddress(size_t tableByteIndex);
66public:
67 static size_t GetTableByteSize(void *heapStartAddress, void *heapEndAddress);
68 static size_t GetTableStartByteOffset(size_t byteSizeBeforeTable);
69private:
70 static uint8_t *TranslateTableToExcludeHeapStartAddress(uint8_t *table, void *heapStartAddress);
71 static void TranslateToTableRegion(void *baseAddress, size_t regionByteSize, uint8_t **tableBaseAddressRef, size_t *tableRegionByteSizeRef);
72
73public:
74 static void ClearDirty(void *baseAddress, size_t regionByteSize);
75 static void SetDirty(void *address, size_t writeByteSize);
76 static void SetDirtyRegion(void *baseAddress, size_t regionByteSize);
77private:
78 static bool GetDirtyFromBlock(uint8_t *block, uint8_t *firstPageAddressInBlock, size_t startByteIndex, size_t endByteIndex, void **dirtyPages, size_t *dirtyPageIndexRef, size_t dirtyPageCount, bool clearDirty);
79public:
80 static void GetDirty(void *baseAddress, size_t regionByteSize, void **dirtyPages, size_t *dirtyPageCountRef, bool clearDirty, bool isRuntimeSuspended);
81};
82
83inline void SoftwareWriteWatch::VerifyCreated()
84{
85 assert(GetTable() != nullptr);
86 assert(GetHeapStartAddress() != nullptr);
87 assert(GetHeapEndAddress() != nullptr);
88 assert(GetHeapStartAddress() < GetHeapEndAddress());
89}
90
91inline void SoftwareWriteWatch::VerifyMemoryRegion(void *baseAddress, size_t regionByteSize)
92{
93 VerifyMemoryRegion(baseAddress, regionByteSize, GetHeapStartAddress(), GetHeapEndAddress());
94}
95
96inline void SoftwareWriteWatch::VerifyMemoryRegion(
97 void *baseAddress,
98 size_t regionByteSize,
99 void *heapStartAddress,
100 void *heapEndAddress)
101{
102 VerifyCreated();
103 assert(baseAddress != nullptr);
104 assert(heapStartAddress != nullptr);
105 assert(heapStartAddress >= GetHeapStartAddress());
106 assert(heapEndAddress != nullptr);
107 assert(heapEndAddress <= GetHeapEndAddress());
108 assert(baseAddress >= heapStartAddress);
109 assert(baseAddress < heapEndAddress);
110 assert(regionByteSize != 0);
111 assert(regionByteSize <= reinterpret_cast<size_t>(heapEndAddress) - reinterpret_cast<size_t>(baseAddress));
112}
113
114inline uint8_t *SoftwareWriteWatch::GetTable()
115{
116 return g_gc_sw_ww_table;
117}
118
119inline uint8_t *SoftwareWriteWatch::GetUntranslatedTable()
120{
121 VerifyCreated();
122 return GetUntranslatedTable(GetTable(), GetHeapStartAddress());
123}
124
125inline uint8_t *SoftwareWriteWatch::GetUntranslatedTable(uint8_t *table, void *heapStartAddress)
126{
127 assert(table != nullptr);
128 assert(heapStartAddress != nullptr);
129 assert(heapStartAddress >= GetHeapStartAddress());
130
131 uint8_t *untranslatedTable = table + GetTableByteIndex(heapStartAddress);
132 assert(ALIGN_DOWN(untranslatedTable, sizeof(size_t)) == untranslatedTable);
133 return untranslatedTable;
134}
135
136inline uint8_t *SoftwareWriteWatch::GetUntranslatedTableEnd()
137{
138 VerifyCreated();
139 return GetUntranslatedTableEnd(GetTable(), GetHeapEndAddress());
140}
141
142inline uint8_t *SoftwareWriteWatch::GetUntranslatedTableEnd(uint8_t *table, void *heapEndAddress)
143{
144 assert(table != nullptr);
145 assert(heapEndAddress != nullptr);
146 assert(heapEndAddress <= GetHeapEndAddress());
147
148 return ALIGN_UP(&table[GetTableByteIndex(reinterpret_cast<uint8_t *>(heapEndAddress) - 1) + 1], sizeof(size_t));
149}
150
151inline void SoftwareWriteWatch::InitializeUntranslatedTable(uint8_t *untranslatedTable, void *heapStartAddress)
152{
153 assert(GetTable() == nullptr);
154 SetUntranslatedTable(untranslatedTable, heapStartAddress);
155}
156
157inline void SoftwareWriteWatch::SetUntranslatedTable(uint8_t *untranslatedTable, void *heapStartAddress)
158{
159 assert(untranslatedTable != nullptr);
160 assert(ALIGN_DOWN(untranslatedTable, sizeof(size_t)) == untranslatedTable);
161 assert(heapStartAddress != nullptr);
162
163 g_gc_sw_ww_table = TranslateTableToExcludeHeapStartAddress(untranslatedTable, heapStartAddress);
164}
165
166inline void SoftwareWriteWatch::SetResizedUntranslatedTable(
167 uint8_t *untranslatedTable,
168 void *heapStartAddress,
169 void *heapEndAddress)
170{
171 // The runtime needs to be suspended during this call, and background GC threads need to synchronize calls to ClearDirty()
172 // and GetDirty() such that they are not called concurrently with this function
173
174 VerifyCreated();
175 assert(untranslatedTable != nullptr);
176 assert(ALIGN_DOWN(untranslatedTable, sizeof(size_t)) == untranslatedTable);
177 assert(heapStartAddress != nullptr);
178 assert(heapEndAddress != nullptr);
179 assert(heapStartAddress <= GetHeapStartAddress());
180 assert(heapEndAddress >= GetHeapEndAddress());
181 assert(heapStartAddress < GetHeapStartAddress() || heapEndAddress > GetHeapEndAddress());
182
183 uint8_t *oldUntranslatedTable = GetUntranslatedTable();
184 void *oldTableHeapStartAddress = GetHeapStartAddress();
185 size_t oldTableByteSize = GetTableByteSize(oldTableHeapStartAddress, GetHeapEndAddress());
186 SetUntranslatedTable(untranslatedTable, heapStartAddress);
187
188 uint8_t *tableRegionStart = &GetTable()[GetTableByteIndex(oldTableHeapStartAddress)];
189 memcpy(tableRegionStart, oldUntranslatedTable, oldTableByteSize);
190}
191
192inline bool SoftwareWriteWatch::IsEnabledForGCHeap()
193{
194 return g_gc_sw_ww_enabled_for_gc_heap;
195}
196
197inline void SoftwareWriteWatch::EnableForGCHeap()
198{
199 // The runtime needs to be suspended during this call. This is how it currently guarantees that GC heap writes from other
200 // threads between calls to EnableForGCHeap() and DisableForGCHeap() will be tracked.
201
202 VerifyCreated();
203 assert(!IsEnabledForGCHeap());
204 g_gc_sw_ww_enabled_for_gc_heap = true;
205
206 WriteBarrierParameters args = {};
207 args.operation = WriteBarrierOp::SwitchToWriteWatch;
208 args.write_watch_table = g_gc_sw_ww_table;
209 args.is_runtime_suspended = true;
210 GCToEEInterface::StompWriteBarrier(&args);
211}
212
213inline void SoftwareWriteWatch::DisableForGCHeap()
214{
215 // The runtime needs to be suspended during this call. This is how it currently guarantees that GC heap writes from other
216 // threads between calls to EnableForGCHeap() and DisableForGCHeap() will be tracked.
217
218 VerifyCreated();
219 assert(IsEnabledForGCHeap());
220 g_gc_sw_ww_enabled_for_gc_heap = false;
221
222 WriteBarrierParameters args = {};
223 args.operation = WriteBarrierOp::SwitchToNonWriteWatch;
224 args.is_runtime_suspended = true;
225 GCToEEInterface::StompWriteBarrier(&args);
226}
227
228inline void *SoftwareWriteWatch::GetHeapStartAddress()
229{
230 return g_gc_lowest_address;
231}
232
233inline void *SoftwareWriteWatch::GetHeapEndAddress()
234{
235 return g_gc_highest_address;
236}
237
238inline size_t SoftwareWriteWatch::GetTableByteIndex(void *address)
239{
240 assert(address != nullptr);
241
242 size_t tableByteIndex = reinterpret_cast<size_t>(address) >> AddressToTableByteIndexShift;
243 assert(tableByteIndex != 0);
244 return tableByteIndex;
245}
246
247inline void *SoftwareWriteWatch::GetPageAddress(size_t tableByteIndex)
248{
249 assert(tableByteIndex != 0);
250
251 void *pageAddress = reinterpret_cast<void *>(tableByteIndex << AddressToTableByteIndexShift);
252 assert(pageAddress >= GetHeapStartAddress());
253 assert(pageAddress < GetHeapEndAddress());
254 assert(ALIGN_DOWN(pageAddress, WRITE_WATCH_UNIT_SIZE) == pageAddress);
255 return pageAddress;
256}
257
258inline size_t SoftwareWriteWatch::GetTableByteSize(void *heapStartAddress, void *heapEndAddress)
259{
260 assert(heapStartAddress != nullptr);
261 assert(heapEndAddress != nullptr);
262 assert(heapStartAddress < heapEndAddress);
263
264 size_t tableByteSize =
265 GetTableByteIndex(reinterpret_cast<uint8_t *>(heapEndAddress) - 1) - GetTableByteIndex(heapStartAddress) + 1;
266 tableByteSize = ALIGN_UP(tableByteSize, sizeof(size_t));
267 return tableByteSize;
268}
269
270inline size_t SoftwareWriteWatch::GetTableStartByteOffset(size_t byteSizeBeforeTable)
271{
272 return ALIGN_UP(byteSizeBeforeTable, sizeof(size_t)); // start of the table needs to be aligned to size_t
273}
274
275inline uint8_t *SoftwareWriteWatch::TranslateTableToExcludeHeapStartAddress(uint8_t *table, void *heapStartAddress)
276{
277 assert(table != nullptr);
278 assert(heapStartAddress != nullptr);
279
280 // Exclude the table byte index corresponding to the heap start address from the table pointer, so that each lookup in the
281 // table by address does not have to calculate (address - heapStartAddress)
282 return table - GetTableByteIndex(heapStartAddress);
283}
284
285inline void SoftwareWriteWatch::TranslateToTableRegion(
286 void *baseAddress,
287 size_t regionByteSize,
288 uint8_t **tableBaseAddressRef,
289 size_t *tableRegionByteSizeRef)
290{
291 VerifyCreated();
292 VerifyMemoryRegion(baseAddress, regionByteSize);
293 assert(tableBaseAddressRef != nullptr);
294 assert(tableRegionByteSizeRef != nullptr);
295
296 size_t baseAddressTableByteIndex = GetTableByteIndex(baseAddress);
297 *tableBaseAddressRef = &GetTable()[baseAddressTableByteIndex];
298 *tableRegionByteSizeRef =
299 GetTableByteIndex(reinterpret_cast<uint8_t *>(baseAddress) + (regionByteSize - 1)) - baseAddressTableByteIndex + 1;
300}
301
302inline void SoftwareWriteWatch::ClearDirty(void *baseAddress, size_t regionByteSize)
303{
304 VerifyCreated();
305 VerifyMemoryRegion(baseAddress, regionByteSize);
306
307 uint8_t *tableBaseAddress;
308 size_t tableRegionByteSize;
309 TranslateToTableRegion(baseAddress, regionByteSize, &tableBaseAddress, &tableRegionByteSize);
310 memset(tableBaseAddress, 0, tableRegionByteSize);
311}
312
313inline void SoftwareWriteWatch::SetDirty(void *address, size_t writeByteSize)
314{
315 VerifyCreated();
316 VerifyMemoryRegion(address, writeByteSize);
317 assert(address != nullptr);
318 assert(writeByteSize <= sizeof(void *));
319
320 size_t tableByteIndex = GetTableByteIndex(address);
321 assert(GetTableByteIndex(reinterpret_cast<uint8_t *>(address) + (writeByteSize - 1)) == tableByteIndex);
322
323 uint8_t *tableByteAddress = &GetTable()[tableByteIndex];
324 if (*tableByteAddress == 0)
325 {
326 *tableByteAddress = 0xff;
327 }
328}
329
330inline void SoftwareWriteWatch::SetDirtyRegion(void *baseAddress, size_t regionByteSize)
331{
332 VerifyCreated();
333 VerifyMemoryRegion(baseAddress, regionByteSize);
334
335 uint8_t *tableBaseAddress;
336 size_t tableRegionByteSize;
337 TranslateToTableRegion(baseAddress, regionByteSize, &tableBaseAddress, &tableRegionByteSize);
338 memset(tableBaseAddress, ~0, tableRegionByteSize);
339}
340
341#endif // !DACCESS_COMPILE
342#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
343#endif // !__SOFTWARE_WRITE_WATCH_H__
344