1// Licensed to the .NET Foundation under one or more agreements.
2// The .NET Foundation licenses this file to you under the MIT license.
3// See the LICENSE file in the project root for more information.
4
5/*
6 * GCHELPERS.CPP
7 *
8 * GC Allocation and Write Barrier Helpers
9 *
10
11 *
12 */
13
14#include "common.h"
15#include "object.h"
16#include "threads.h"
17#include "eetwain.h"
18#include "eeconfig.h"
19#include "gcheaputilities.h"
20#include "corhost.h"
21#include "threads.h"
22#include "fieldmarshaler.h"
23#include "interoputil.h"
24#include "dynamicmethod.h"
25#include "stubhelpers.h"
26#include "eventtrace.h"
27
28#include "excep.h"
29
30#include "gchelpers.inl"
31#include "eeprofinterfaces.inl"
32
33#ifdef FEATURE_COMINTEROP
34#include "runtimecallablewrapper.h"
35#endif // FEATURE_COMINTEROP
36
37#include "rcwwalker.h"
38
39//========================================================================
40//
41// ALLOCATION HELPERS
42//
43//========================================================================
44
45#define ProfileTrackArrayAlloc(orObject) \
46 OBJECTREF objref = ObjectToOBJECTREF((Object*)orObject);\
47 GCPROTECT_BEGIN(objref);\
48 ProfilerObjectAllocatedCallback(objref, (ClassID) orObject->GetTypeHandle().AsPtr());\
49 GCPROTECT_END();\
50 orObject = (ArrayBase *) OBJECTREFToObject(objref);
51
52
53inline gc_alloc_context* GetThreadAllocContext()
54{
55 WRAPPER_NO_CONTRACT;
56
57 assert(GCHeapUtilities::UseThreadAllocationContexts());
58
59 return & GetThread()->m_alloc_context;
60}
61
62// When not using per-thread allocation contexts, we (the EE) need to take care that
63// no two threads are concurrently modifying the global allocation context. This lock
64// must be acquired before any sort of operations involving the global allocation context
65// can occur.
66//
67// This lock is acquired by all allocations when not using per-thread allocation contexts.
68// It is acquired in two kinds of places:
69// 1) JIT_TrialAllocFastSP (and related assembly alloc helpers), which attempt to
70// acquire it but move into an alloc slow path if acquiring fails
71// (but does not decrement the lock variable when doing so)
72// 2) Alloc and AllocAlign8 in gchelpers.cpp, which acquire the lock using
73// the Acquire and Release methods below.
74class GlobalAllocLock {
75 friend struct AsmOffsets;
76private:
77 // The lock variable. This field must always be first.
78 LONG m_lock;
79
80public:
81 // Creates a new GlobalAllocLock in the unlocked state.
82 GlobalAllocLock() : m_lock(-1) {}
83
84 // Copy and copy-assignment operators should never be invoked
85 // for this type
86 GlobalAllocLock(const GlobalAllocLock&) = delete;
87 GlobalAllocLock& operator=(const GlobalAllocLock&) = delete;
88
89 // Acquires the lock, spinning if necessary to do so. When this method
90 // returns, m_lock will be zero and the lock will be acquired.
91 void Acquire()
92 {
93 CONTRACTL {
94 NOTHROW;
95 GC_TRIGGERS; // switch to preemptive mode
96 MODE_COOPERATIVE;
97 } CONTRACTL_END;
98
99 DWORD spinCount = 0;
100 while(FastInterlockExchange(&m_lock, 0) != -1)
101 {
102 GCX_PREEMP();
103 __SwitchToThread(0, spinCount++);
104 }
105
106 assert(m_lock == 0);
107 }
108
109 // Releases the lock.
110 void Release()
111 {
112 LIMITED_METHOD_CONTRACT;
113
114 // the lock may not be exactly 0. This is because the
115 // assembly alloc routines increment the lock variable and
116 // jump if not zero to the slow alloc path, which eventually
117 // will try to acquire the lock again. At that point, it will
118 // spin in Acquire (since m_lock is some number that's not zero).
119 // When the thread that /does/ hold the lock releases it, the spinning
120 // thread will continue.
121 MemoryBarrier();
122 assert(m_lock >= 0);
123 m_lock = -1;
124 }
125
126 // Static helper to acquire a lock, for use with the Holder template.
127 static void AcquireLock(GlobalAllocLock *lock)
128 {
129 WRAPPER_NO_CONTRACT;
130 lock->Acquire();
131 }
132
133 // Static helper to release a lock, for use with the Holder template
134 static void ReleaseLock(GlobalAllocLock *lock)
135 {
136 WRAPPER_NO_CONTRACT;
137 lock->Release();
138 }
139
140 typedef Holder<GlobalAllocLock *, GlobalAllocLock::AcquireLock, GlobalAllocLock::ReleaseLock> Holder;
141};
142
143typedef GlobalAllocLock::Holder GlobalAllocLockHolder;
144
145struct AsmOffsets {
146 static_assert(offsetof(GlobalAllocLock, m_lock) == 0, "ASM code relies on this property");
147};
148
149// For single-proc machines, the global allocation context is protected
150// from concurrent modification by this lock.
151//
152// When not using per-thread allocation contexts, certain methods on IGCHeap
153// require that this lock be held before calling. These methods are documented
154// on the IGCHeap interface.
155extern "C"
156{
157 GlobalAllocLock g_global_alloc_lock;
158}
159
160
161// Checks to see if the given allocation size exceeds the
162// largest object size allowed - if it does, it throws
163// an OutOfMemoryException with a message indicating that
164// the OOM was not from memory pressure but from an object
165// being too large.
166inline void CheckObjectSize(size_t alloc_size)
167{
168 CONTRACTL {
169 THROWS;
170 GC_TRIGGERS;
171 } CONTRACTL_END;
172
173 size_t max_object_size;
174#ifdef BIT64
175 if (g_pConfig->GetGCAllowVeryLargeObjects())
176 {
177 max_object_size = (INT64_MAX - 7 - min_obj_size);
178 }
179 else
180#endif // BIT64
181 {
182 max_object_size = (INT32_MAX - 7 - min_obj_size);
183 }
184
185 if (alloc_size >= max_object_size)
186 {
187 if (g_pConfig->IsGCBreakOnOOMEnabled())
188 {
189 DebugBreak();
190 }
191
192 ThrowOutOfMemoryDimensionsExceeded();
193 }
194}
195
196
197// There are only three ways to get into allocate an object.
198// * Call optimized helpers that were generated on the fly. This is how JIT compiled code does most
199// allocations, however they fall back code:Alloc, when for all but the most common code paths. These
200// helpers are NOT used if profiler has asked to track GC allocation (see code:TrackAllocations)
201// * Call code:Alloc - When the jit helpers fall back, or we do allocations within the runtime code
202// itself, we ultimately call here.
203// * Call code:AllocLHeap - Used very rarely to force allocation to be on the large object heap.
204//
205// While this is a choke point into allocating an object, it is primitive (it does not want to know about
206// MethodTable and thus does not initialize that pointer. It also does not know if the object is finalizable
207// or contains pointers. Thus we quickly wrap this function in more user-friendly ones that know about
208// MethodTables etc. (see code:FastAllocatePrimitiveArray code:AllocateArrayEx code:AllocateObject)
209//
210// You can get an exhaustive list of code sites that allocate GC objects by finding all calls to
211// code:ProfilerObjectAllocatedCallback (since the profiler has to hook them all).
212inline Object* Alloc(size_t size, BOOL bFinalize, BOOL bContainsPointers )
213{
214 CONTRACTL {
215 THROWS;
216 GC_TRIGGERS;
217 MODE_COOPERATIVE; // returns an objref without pinning it => cooperative
218 } CONTRACTL_END;
219
220 _ASSERTE(!NingenEnabled() && "You cannot allocate managed objects inside the ngen compilation process.");
221
222#ifdef _DEBUG
223 if (g_pConfig->ShouldInjectFault(INJECTFAULT_GCHEAP))
224 {
225 char *a = new char;
226 delete a;
227 }
228#endif
229
230 DWORD flags = ((bContainsPointers ? GC_ALLOC_CONTAINS_REF : 0) |
231 (bFinalize ? GC_ALLOC_FINALIZE : 0));
232
233 Object *retVal = NULL;
234 CheckObjectSize(size);
235
236 // We don't want to throw an SO during the GC, so make sure we have plenty
237 // of stack before calling in.
238 INTERIOR_STACK_PROBE_FOR(GetThread(), static_cast<unsigned>(DEFAULT_ENTRY_PROBE_AMOUNT * 1.5));
239 if (GCHeapUtilities::UseThreadAllocationContexts())
240 {
241 gc_alloc_context *threadContext = GetThreadAllocContext();
242 GCStress<gc_on_alloc>::MaybeTrigger(threadContext);
243 retVal = GCHeapUtilities::GetGCHeap()->Alloc(threadContext, size, flags);
244 }
245 else
246 {
247 GlobalAllocLockHolder holder(&g_global_alloc_lock);
248 gc_alloc_context *globalContext = &g_global_alloc_context;
249 GCStress<gc_on_alloc>::MaybeTrigger(globalContext);
250 retVal = GCHeapUtilities::GetGCHeap()->Alloc(globalContext, size, flags);
251 }
252
253
254 if (!retVal)
255 {
256 ThrowOutOfMemory();
257 }
258
259 END_INTERIOR_STACK_PROBE;
260 return retVal;
261}
262
263#ifdef FEATURE_64BIT_ALIGNMENT
264// Helper for allocating 8-byte aligned objects (on platforms where this doesn't happen naturally, e.g. 32-bit
265// platforms).
266inline Object* AllocAlign8(size_t size, BOOL bFinalize, BOOL bContainsPointers, BOOL bAlignBias)
267{
268 CONTRACTL {
269 THROWS;
270 GC_TRIGGERS;
271 MODE_COOPERATIVE; // returns an objref without pinning it => cooperative
272 } CONTRACTL_END;
273
274 DWORD flags = ((bContainsPointers ? GC_ALLOC_CONTAINS_REF : 0) |
275 (bFinalize ? GC_ALLOC_FINALIZE : 0) |
276 (bAlignBias ? GC_ALLOC_ALIGN8_BIAS : 0));
277
278 Object *retVal = NULL;
279 CheckObjectSize(size);
280
281 // We don't want to throw an SO during the GC, so make sure we have plenty
282 // of stack before calling in.
283 INTERIOR_STACK_PROBE_FOR(GetThread(), static_cast<unsigned>(DEFAULT_ENTRY_PROBE_AMOUNT * 1.5));
284 if (GCHeapUtilities::UseThreadAllocationContexts())
285 {
286 gc_alloc_context *threadContext = GetThreadAllocContext();
287 GCStress<gc_on_alloc>::MaybeTrigger(threadContext);
288 retVal = GCHeapUtilities::GetGCHeap()->AllocAlign8(threadContext, size, flags);
289 }
290 else
291 {
292 GlobalAllocLockHolder holder(&g_global_alloc_lock);
293 gc_alloc_context *globalContext = &g_global_alloc_context;
294 GCStress<gc_on_alloc>::MaybeTrigger(globalContext);
295 retVal = GCHeapUtilities::GetGCHeap()->AllocAlign8(globalContext, size, flags);
296 }
297
298 if (!retVal)
299 {
300 ThrowOutOfMemory();
301 }
302
303 END_INTERIOR_STACK_PROBE;
304 return retVal;
305}
306#endif // FEATURE_64BIT_ALIGNMENT
307
308// This is one of three ways of allocating an object (see code:Alloc for more). This variation is used in the
309// rare circumstance when you want to allocate an object on the large object heap but the object is not big
310// enough to naturally go there.
311//
312// One (and only?) example of where this is needed is 8 byte aligning of arrays of doubles. See
313// code:EEConfig.GetDoubleArrayToLargeObjectHeapThreshold and code:CORINFO_HELP_NEWARR_1_ALIGN8 for more.
314inline Object* AllocLHeap(size_t size, BOOL bFinalize, BOOL bContainsPointers )
315{
316 CONTRACTL {
317 THROWS;
318 GC_TRIGGERS;
319 MODE_COOPERATIVE; // returns an objref without pinning it => cooperative (don't assume large heap doesn't compact!)
320 } CONTRACTL_END;
321
322
323 _ASSERTE(!NingenEnabled() && "You cannot allocate managed objects inside the ngen compilation process.");
324
325#ifdef _DEBUG
326 if (g_pConfig->ShouldInjectFault(INJECTFAULT_GCHEAP))
327 {
328 char *a = new char;
329 delete a;
330 }
331#endif
332
333 DWORD flags = ((bContainsPointers ? GC_ALLOC_CONTAINS_REF : 0) |
334 (bFinalize ? GC_ALLOC_FINALIZE : 0));
335
336 Object *retVal = NULL;
337 CheckObjectSize(size);
338
339 // We don't want to throw an SO during the GC, so make sure we have plenty
340 // of stack before calling in.
341 INTERIOR_STACK_PROBE_FOR(GetThread(), static_cast<unsigned>(DEFAULT_ENTRY_PROBE_AMOUNT * 1.5));
342 retVal = GCHeapUtilities::GetGCHeap()->AllocLHeap(size, flags);
343
344 if (!retVal)
345 {
346 ThrowOutOfMemory();
347 }
348
349 END_INTERIOR_STACK_PROBE;
350 return retVal;
351}
352
353
354#ifdef _LOGALLOC
355int g_iNumAllocs = 0;
356
357bool ToLogOrNotToLog(size_t size, const char *typeName)
358{
359 WRAPPER_NO_CONTRACT;
360
361 g_iNumAllocs++;
362
363 if (g_iNumAllocs > g_pConfig->AllocNumThreshold())
364 return true;
365
366 if (size > (size_t)g_pConfig->AllocSizeThreshold())
367 return true;
368
369 if (g_pConfig->ShouldLogAlloc(typeName))
370 return true;
371
372 return false;
373
374}
375
376// READ THIS!!!!!
377// this function is called on managed allocation path with unprotected Object*
378// as a result LogAlloc cannot call anything that would toggle the GC mode else
379// you'll introduce several GC holes!
380inline void LogAlloc(size_t size, MethodTable *pMT, Object* object)
381{
382 CONTRACTL
383 {
384 NOTHROW;
385 GC_NOTRIGGER;
386 MODE_COOPERATIVE;
387 }
388 CONTRACTL_END;
389
390#ifdef LOGGING
391 if (LoggingOn(LF_GCALLOC, LL_INFO10))
392 {
393 LogSpewAlways("Allocated %5d bytes for %s_TYPE" FMT_ADDR FMT_CLASS "\n",
394 size,
395 pMT->IsValueType() ? "VAL" : "REF",
396 DBG_ADDR(object),
397 DBG_CLASS_NAME_MT(pMT));
398
399 if (LoggingOn(LF_GCALLOC, LL_INFO1000000) ||
400 (LoggingOn(LF_GCALLOC, LL_INFO100) &&
401 ToLogOrNotToLog(size, DBG_CLASS_NAME_MT(pMT))))
402 {
403 void LogStackTrace();
404 LogStackTrace();
405 }
406 }
407#endif
408}
409#else
410#define LogAlloc(size, pMT, object)
411#endif
412
413
414inline SIZE_T MaxArrayLength(SIZE_T componentSize)
415{
416 // Impose limits on maximum array length in each dimension to allow efficient
417 // implementation of advanced range check elimination in future. We have to allow
418 // higher limit for array of bytes (or one byte structs) for backward compatibility.
419 // Keep in sync with Array.MaxArrayLength in BCL.
420 return (componentSize == 1) ? 0X7FFFFFC7 : 0X7FEFFFFF;
421}
422
423OBJECTREF AllocateValueSzArray(TypeHandle elementType, INT32 length)
424{
425 CONTRACTL {
426 THROWS;
427 GC_TRIGGERS;
428 MODE_COOPERATIVE; // returns an objref without pinning it => cooperative
429 } CONTRACTL_END;
430
431 return AllocateArrayEx(elementType.MakeSZArray(), &length, 1);
432}
433
434void ThrowOutOfMemoryDimensionsExceeded()
435{
436 CONTRACTL {
437 THROWS;
438 } CONTRACTL_END;
439
440#ifdef _WIN64
441 EX_THROW(EEMessageException, (kOutOfMemoryException, IDS_EE_ARRAY_DIMENSIONS_EXCEEDED));
442#else
443 ThrowOutOfMemory();
444#endif
445}
446
447//
448// Handles arrays of arbitrary dimensions
449//
450// This is wrapper overload to handle TypeHandle arrayType
451//
452OBJECTREF AllocateArrayEx(TypeHandle arrayType, INT32 *pArgs, DWORD dwNumArgs, BOOL bAllocateInLargeHeap
453 DEBUG_ARG(BOOL bDontSetAppDomain))
454{
455 CONTRACTL
456 {
457 WRAPPER_NO_CONTRACT;
458 } CONTRACTL_END;
459
460 ArrayTypeDesc* arrayDesc = arrayType.AsArray();
461 MethodTable* pArrayMT = arrayDesc->GetMethodTable();
462
463 return AllocateArrayEx(pArrayMT, pArgs, dwNumArgs, bAllocateInLargeHeap
464 DEBUG_ARG(bDontSetAppDomain));
465}
466
467//
468// Handles arrays of arbitrary dimensions
469//
470// If dwNumArgs is set to greater than 1 for a SZARRAY this function will recursively
471// allocate sub-arrays and fill them in.
472//
473// For arrays with lower bounds, pBounds is <lower bound 1>, <count 1>, <lower bound 2>, ...
474OBJECTREF AllocateArrayEx(MethodTable *pArrayMT, INT32 *pArgs, DWORD dwNumArgs, BOOL bAllocateInLargeHeap
475 DEBUG_ARG(BOOL bDontSetAppDomain))
476{
477 CONTRACTL {
478 THROWS;
479 GC_TRIGGERS;
480 MODE_COOPERATIVE; // returns an objref without pinning it => cooperative
481 PRECONDITION(CheckPointer(pArgs));
482 PRECONDITION(dwNumArgs > 0);
483 } CONTRACTL_END;
484
485 ArrayBase * orArray = NULL;
486
487#ifdef _DEBUG
488 if (g_pConfig->ShouldInjectFault(INJECTFAULT_GCHEAP))
489 {
490 char *a = new char;
491 delete a;
492 }
493#endif
494
495 _ASSERTE(pArrayMT->CheckInstanceActivated());
496 PREFIX_ASSUME(pArrayMT != NULL);
497 CorElementType kind = pArrayMT->GetInternalCorElementType();
498 _ASSERTE(kind == ELEMENT_TYPE_ARRAY || kind == ELEMENT_TYPE_SZARRAY);
499
500 CorElementType elemType = pArrayMT->GetArrayElementType();
501 // Disallow the creation of void[,] (a multi-dim array of System.Void)
502 if (elemType == ELEMENT_TYPE_VOID)
503 COMPlusThrow(kArgumentException);
504
505 // Calculate the total number of elements in the array
506 UINT32 cElements;
507
508 // IBC Log MethodTable access
509 g_IBCLogger.LogMethodTableAccess(pArrayMT);
510 SetTypeHandleOnThreadForAlloc(TypeHandle(pArrayMT));
511
512 SIZE_T componentSize = pArrayMT->GetComponentSize();
513 bool maxArrayDimensionLengthOverflow = false;
514 bool providedLowerBounds = false;
515
516 if (kind == ELEMENT_TYPE_ARRAY)
517 {
518 unsigned rank = pArrayMT->GetRank();
519 _ASSERTE(dwNumArgs == rank || dwNumArgs == 2*rank);
520
521 // Morph a ARRAY rank 1 with 0 lower bound into an SZARRAY
522 if (rank == 1 && (dwNumArgs == 1 || pArgs[0] == 0))
523 { // lower bound is zero
524
525 // This recursive call doesn't go any farther, because the dwNumArgs will be 1,
526 // so don't bother with stack probe.
527 TypeHandle szArrayType = ClassLoader::LoadArrayTypeThrowing(pArrayMT->GetApproxArrayElementTypeHandle(), ELEMENT_TYPE_SZARRAY, 1);
528 return AllocateArrayEx(szArrayType, &pArgs[dwNumArgs - 1], 1, bAllocateInLargeHeap DEBUG_ARG(bDontSetAppDomain));
529 }
530
531 providedLowerBounds = (dwNumArgs == 2*rank);
532
533 S_UINT32 safeTotalElements = S_UINT32(1);
534
535 for (unsigned i = 0; i < dwNumArgs; i++)
536 {
537 int lowerBound = 0;
538 if (providedLowerBounds)
539 {
540 lowerBound = pArgs[i];
541 i++;
542 }
543 int length = pArgs[i];
544 if (length < 0)
545 COMPlusThrow(kOverflowException);
546 if ((SIZE_T)length > MaxArrayLength(componentSize))
547 maxArrayDimensionLengthOverflow = true;
548 if ((length > 0) && (lowerBound + (length - 1) < lowerBound))
549 COMPlusThrow(kArgumentOutOfRangeException, W("ArgumentOutOfRange_ArrayLBAndLength"));
550 safeTotalElements = safeTotalElements * S_UINT32(length);
551 if (safeTotalElements.IsOverflow())
552 ThrowOutOfMemoryDimensionsExceeded();
553 }
554
555 cElements = safeTotalElements.Value();
556 }
557 else
558 {
559 int length = pArgs[0];
560 if (length < 0)
561 COMPlusThrow(kOverflowException);
562 if ((SIZE_T)length > MaxArrayLength(componentSize))
563 maxArrayDimensionLengthOverflow = true;
564 cElements = length;
565 }
566
567 // Throw this exception only after everything else was validated for backward compatibility.
568 if (maxArrayDimensionLengthOverflow)
569 ThrowOutOfMemoryDimensionsExceeded();
570
571 // Allocate the space from the GC heap
572 S_SIZE_T safeTotalSize = S_SIZE_T(cElements) * S_SIZE_T(componentSize) + S_SIZE_T(pArrayMT->GetBaseSize());
573 if (safeTotalSize.IsOverflow())
574 ThrowOutOfMemoryDimensionsExceeded();
575
576 size_t totalSize = safeTotalSize.Value();
577
578#ifdef FEATURE_DOUBLE_ALIGNMENT_HINT
579 if ((elemType == ELEMENT_TYPE_R8) &&
580 (cElements >= g_pConfig->GetDoubleArrayToLargeObjectHeapThreshold()))
581 {
582 STRESS_LOG2(LF_GC, LL_INFO10, "Allocating double MD array of size %d and length %d to large object heap\n", totalSize, cElements);
583 bAllocateInLargeHeap = TRUE;
584 }
585#endif
586
587 if (bAllocateInLargeHeap)
588 {
589 orArray = (ArrayBase *) AllocLHeap(totalSize, FALSE, pArrayMT->ContainsPointers());
590 orArray->SetArrayMethodTableForLargeObject(pArrayMT);
591 }
592 else
593 {
594#ifdef FEATURE_64BIT_ALIGNMENT
595 MethodTable *pElementMT = pArrayMT->GetApproxArrayElementTypeHandle().GetMethodTable();
596 if (pElementMT->RequiresAlign8() && pElementMT->IsValueType())
597 {
598 // This platform requires that certain fields are 8-byte aligned (and the runtime doesn't provide
599 // this guarantee implicitly, e.g. on 32-bit platforms). Since it's the array payload, not the
600 // header that requires alignment we need to be careful. However it just so happens that all the
601 // cases we care about (single and multi-dim arrays of value types) have an even number of DWORDs
602 // in their headers so the alignment requirements for the header and the payload are the same.
603 _ASSERTE(((pArrayMT->GetBaseSize() - SIZEOF_OBJHEADER) & 7) == 0);
604 orArray = (ArrayBase *) AllocAlign8(totalSize, FALSE, pArrayMT->ContainsPointers(), FALSE);
605 }
606 else
607#endif
608 {
609 orArray = (ArrayBase *) Alloc(totalSize, FALSE, pArrayMT->ContainsPointers());
610 }
611 orArray->SetArrayMethodTable(pArrayMT);
612 }
613
614 // Initialize Object
615 orArray->m_NumComponents = cElements;
616
617 if (bAllocateInLargeHeap ||
618 (totalSize >= g_pConfig->GetGCLOHThreshold()))
619 {
620 GCHeapUtilities::GetGCHeap()->PublishObject((BYTE*)orArray);
621 }
622
623#ifdef _LOGALLOC
624 LogAlloc(totalSize, pArrayMT, orArray);
625#endif // _LOGALLOC
626
627#ifdef _DEBUG
628 // Ensure the typehandle has been interned prior to allocation.
629 // This is important for OOM reliability.
630 OBJECTREF objref = ObjectToOBJECTREF((Object *) orArray);
631 GCPROTECT_BEGIN(objref);
632
633 orArray->GetTypeHandle();
634
635 GCPROTECT_END();
636 orArray = (ArrayBase *) OBJECTREFToObject(objref);
637#endif
638
639 if (kind == ELEMENT_TYPE_ARRAY)
640 {
641 INT32 *pCountsPtr = (INT32 *) orArray->GetBoundsPtr();
642 INT32 *pLowerBoundsPtr = (INT32 *) orArray->GetLowerBoundsPtr();
643 for (unsigned i = 0; i < dwNumArgs; i++)
644 {
645 if (providedLowerBounds)
646 *pLowerBoundsPtr++ = pArgs[i++]; // if not stated, lower bound becomes 0
647 *pCountsPtr++ = pArgs[i];
648 }
649 }
650
651 // Notify the profiler of the allocation
652 // do this after initializing bounds so callback has size information
653 if (TrackAllocations())
654 {
655 ProfileTrackArrayAlloc(orArray);
656 }
657
658#ifdef FEATURE_EVENT_TRACE
659 // Send ETW event for allocation
660 if(ETW::TypeSystemLog::IsHeapAllocEventEnabled())
661 {
662 ETW::TypeSystemLog::SendObjectAllocatedEvent(orArray);
663 }
664#endif // FEATURE_EVENT_TRACE
665
666 if (kind != ELEMENT_TYPE_ARRAY)
667 {
668 // Handle allocating multiple jagged array dimensions at once
669 if (dwNumArgs > 1)
670 {
671 PTRARRAYREF outerArray = (PTRARRAYREF) ObjectToOBJECTREF((Object *) orArray);
672 GCPROTECT_BEGIN(outerArray);
673
674 // Turn off GC stress, it is of little value here
675 {
676 GCStressPolicy::InhibitHolder iholder;
677
678 // Allocate dwProvidedBounds arrays
679 if (!pArrayMT->GetApproxArrayElementTypeHandle().IsArray())
680 {
681 orArray = NULL;
682 }
683 else
684 {
685 // Since we're about to *really* recurse, probe for stack.
686 // @todo: is the default amount really correct?
687 _ASSERTE(GetThread());
688 INTERIOR_STACK_PROBE(GetThread());
689
690 TypeHandle subArrayType = pArrayMT->GetApproxArrayElementTypeHandle();
691 for (UINT32 i = 0; i < cElements; i++)
692 {
693 OBJECTREF obj = AllocateArrayEx(subArrayType, &pArgs[1], dwNumArgs-1, bAllocateInLargeHeap DEBUG_ARG(bDontSetAppDomain));
694 outerArray->SetAt(i, obj);
695 }
696
697 iholder.Release();
698
699 END_INTERIOR_STACK_PROBE
700
701 orArray = (ArrayBase *) OBJECTREFToObject(outerArray);
702 }
703 } // GcStressPolicy::~InhibitHolder()
704
705 GCPROTECT_END();
706 }
707 }
708
709 return ObjectToOBJECTREF((Object *) orArray);
710}
711
712/*
713 * Allocates a single dimensional array of primitive types.
714 */
715OBJECTREF AllocatePrimitiveArray(CorElementType type, DWORD cElements, BOOL bAllocateInLargeHeap)
716{
717 CONTRACTL
718 {
719 THROWS;
720 GC_TRIGGERS;
721 INJECT_FAULT(COMPlusThrowOM());
722 MODE_COOPERATIVE; // returns an objref without pinning it => cooperative
723 }
724 CONTRACTL_END
725
726
727 // Allocating simple primite arrays is done in various places as internal storage.
728 // Because this is unlikely to result in any bad recursions, we will override the type limit
729 // here rather forever chase down all the callers.
730 OVERRIDE_TYPE_LOAD_LEVEL_LIMIT(CLASS_LOADED);
731
732 _ASSERTE(CorTypeInfo::IsPrimitiveType(type));
733
734 // Fetch the proper array type
735 if (g_pPredefinedArrayTypes[type] == NULL)
736 {
737 TypeHandle elemType = TypeHandle(MscorlibBinder::GetElementType(type));
738 TypeHandle typHnd = ClassLoader::LoadArrayTypeThrowing(elemType, ELEMENT_TYPE_SZARRAY, 0);
739 g_pPredefinedArrayTypes[type] = typHnd.AsArray();
740 }
741 return FastAllocatePrimitiveArray(g_pPredefinedArrayTypes[type]->GetMethodTable(), cElements, bAllocateInLargeHeap);
742}
743
744/*
745 * Allocates a single dimensional array of primitive types.
746 */
747
748OBJECTREF FastAllocatePrimitiveArray(MethodTable* pMT, DWORD cElements, BOOL bAllocateInLargeHeap)
749{
750 CONTRACTL {
751 THROWS;
752 GC_TRIGGERS;
753 MODE_COOPERATIVE; // returns an objref without pinning it => cooperative
754 PRECONDITION(pMT->CheckInstanceActivated());
755 } CONTRACTL_END;
756
757#ifdef _DEBUG
758 if (g_pConfig->ShouldInjectFault(INJECTFAULT_GCHEAP))
759 {
760 char *a = new char;
761 delete a;
762 }
763#endif
764
765 _ASSERTE(pMT && pMT->IsArray());
766 _ASSERTE(pMT->IsRestored_NoLogging());
767 _ASSERTE(CorTypeInfo::IsPrimitiveType(pMT->GetArrayElementType()) &&
768 g_pPredefinedArrayTypes[pMT->GetArrayElementType()] != NULL);
769
770 g_IBCLogger.LogMethodTableAccess(pMT);
771 SetTypeHandleOnThreadForAlloc(TypeHandle(pMT));
772
773 SIZE_T componentSize = pMT->GetComponentSize();
774 if (cElements > MaxArrayLength(componentSize))
775 ThrowOutOfMemory();
776
777 S_SIZE_T safeTotalSize = S_SIZE_T(cElements) * S_SIZE_T(componentSize) + S_SIZE_T(pMT->GetBaseSize());
778 if (safeTotalSize.IsOverflow())
779 ThrowOutOfMemory();
780
781 size_t totalSize = safeTotalSize.Value();
782
783 BOOL bPublish = bAllocateInLargeHeap;
784
785 ArrayBase* orObject;
786 if (bAllocateInLargeHeap)
787 {
788 orObject = (ArrayBase*) AllocLHeap(totalSize, FALSE, FALSE);
789 }
790 else
791 {
792 ArrayTypeDesc *pArrayR8TypeDesc = g_pPredefinedArrayTypes[ELEMENT_TYPE_R8];
793 if (DATA_ALIGNMENT < sizeof(double) && pArrayR8TypeDesc != NULL && pMT == pArrayR8TypeDesc->GetMethodTable() &&
794 (totalSize < g_pConfig->GetGCLOHThreshold() - MIN_OBJECT_SIZE))
795 {
796 // Creation of an array of doubles, not in the large object heap.
797 // We want to align the doubles to 8 byte boundaries, but the GC gives us pointers aligned
798 // to 4 bytes only (on 32 bit platforms). To align, we ask for 12 bytes more to fill with a
799 // dummy object.
800 // If the GC gives us a 8 byte aligned address, we use it for the array and place the dummy
801 // object after the array, otherwise we put the dummy object first, shifting the base of
802 // the array to an 8 byte aligned address.
803 // Note: on 64 bit platforms, the GC always returns 8 byte aligned addresses, and we don't
804 // execute this code because DATA_ALIGNMENT < sizeof(double) is false.
805
806 _ASSERTE(DATA_ALIGNMENT == sizeof(double)/2);
807 _ASSERTE((MIN_OBJECT_SIZE % sizeof(double)) == DATA_ALIGNMENT); // used to change alignment
808 _ASSERTE(pMT->GetComponentSize() == sizeof(double));
809 _ASSERTE(g_pObjectClass->GetBaseSize() == MIN_OBJECT_SIZE);
810 _ASSERTE(totalSize < totalSize + MIN_OBJECT_SIZE);
811 orObject = (ArrayBase*) Alloc(totalSize + MIN_OBJECT_SIZE, FALSE, FALSE);
812
813 Object *orDummyObject;
814 if((size_t)orObject % sizeof(double))
815 {
816 orDummyObject = orObject;
817 orObject = (ArrayBase*) ((size_t)orObject + MIN_OBJECT_SIZE);
818 }
819 else
820 {
821 orDummyObject = (Object*) ((size_t)orObject + totalSize);
822 }
823 _ASSERTE(((size_t)orObject % sizeof(double)) == 0);
824 orDummyObject->SetMethodTable(g_pObjectClass);
825 }
826 else
827 {
828 orObject = (ArrayBase*) Alloc(totalSize, FALSE, FALSE);
829 bPublish = (totalSize >= g_pConfig->GetGCLOHThreshold());
830 }
831 }
832
833 // Initialize Object
834 orObject->SetArrayMethodTable( pMT );
835 _ASSERTE(orObject->GetMethodTable() != NULL);
836 orObject->m_NumComponents = cElements;
837
838 if (bPublish)
839 {
840 GCHeapUtilities::GetGCHeap()->PublishObject((BYTE*)orObject);
841 }
842
843 // Notify the profiler of the allocation
844 if (TrackAllocations())
845 {
846 OBJECTREF objref = ObjectToOBJECTREF((Object*)orObject);
847 GCPROTECT_BEGIN(objref);
848 ProfilerObjectAllocatedCallback(objref, (ClassID) orObject->GetTypeHandle().AsPtr());
849 GCPROTECT_END();
850
851 orObject = (ArrayBase *) OBJECTREFToObject(objref);
852 }
853
854#ifdef FEATURE_EVENT_TRACE
855 // Send ETW event for allocation
856 if(ETW::TypeSystemLog::IsHeapAllocEventEnabled())
857 {
858 ETW::TypeSystemLog::SendObjectAllocatedEvent(orObject);
859 }
860#endif // FEATURE_EVENT_TRACE
861
862 // IBC Log MethodTable access
863 g_IBCLogger.LogMethodTableAccess(pMT);
864
865 LogAlloc(totalSize, pMT, orObject);
866
867 return( ObjectToOBJECTREF((Object*)orObject) );
868}
869
870//
871// Allocate an array which is the same size as pRef. However, do not zero out the array.
872//
873OBJECTREF DupArrayForCloning(BASEARRAYREF pRef, BOOL bAllocateInLargeHeap)
874{
875 CONTRACTL {
876 THROWS;
877 GC_TRIGGERS;
878 MODE_COOPERATIVE; // returns an objref without pinning it => cooperative
879 } CONTRACTL_END;
880
881 ArrayTypeDesc arrayType(pRef->GetMethodTable(), pRef->GetArrayElementTypeHandle());
882 unsigned rank = arrayType.GetRank();
883
884 DWORD numArgs = rank*2;
885 INT32* args = (INT32*) _alloca(sizeof(INT32)*numArgs);
886
887 if (arrayType.GetInternalCorElementType() == ELEMENT_TYPE_ARRAY)
888 {
889 const INT32* bounds = pRef->GetBoundsPtr();
890 const INT32* lowerBounds = pRef->GetLowerBoundsPtr();
891 for(unsigned int i=0; i < rank; i++)
892 {
893 args[2*i] = lowerBounds[i];
894 args[2*i+1] = bounds[i];
895 }
896 }
897 else
898 {
899 numArgs = 1;
900 args[0] = pRef->GetNumComponents();
901 }
902 return AllocateArrayEx(TypeHandle(&arrayType), args, numArgs, bAllocateInLargeHeap DEBUG_ARG(FALSE));
903}
904
905#if defined(_TARGET_X86_)
906
907// The fast version always allocates in the normal heap
908OBJECTREF AllocatePrimitiveArray(CorElementType type, DWORD cElements)
909{
910 CONTRACTL {
911 THROWS;
912 GC_TRIGGERS;
913 MODE_COOPERATIVE; // returns an objref without pinning it => cooperative
914 } CONTRACTL_END;
915
916#ifdef _DEBUG
917 // fastPrimitiveArrayAllocator is called by VM and managed code. If called from managed code, we
918 // make sure that the thread is in SOTolerantState.
919#ifdef FEATURE_STACK_PROBE
920 Thread::DisableSOCheckInHCALL disableSOCheckInHCALL;
921#endif // FEATURE_STACK_PROBE
922#endif // _DEBUG
923 return OBJECTREF( HCCALL2(fastPrimitiveArrayAllocator, type, cElements) );
924}
925
926// The fast version always allocates in the normal heap
927OBJECTREF AllocateObjectArray(DWORD cElements, TypeHandle ElementType)
928{
929 CONTRACTL {
930 THROWS;
931 GC_TRIGGERS;
932 MODE_COOPERATIVE; // returns an objref without pinning it => cooperative
933 } CONTRACTL_END;
934
935
936 OVERRIDE_TYPE_LOAD_LEVEL_LIMIT(CLASS_LOADED);
937
938 // We must call this here to ensure the typehandle for this object is
939 // interned before the object is allocated. As soon as the object is allocated,
940 // the profiler could do a heapwalk and it expects to find an interned
941 // typehandle for every object in the heap.
942 TypeHandle ArrayType = ClassLoader::LoadArrayTypeThrowing(ElementType);
943
944#ifdef _DEBUG
945 // fastObjectArrayAllocator is called by VM and managed code. If called from managed code, we
946 // make sure that the thread is in SOTolerantState.
947#ifdef FEATURE_STACK_PROBE
948 Thread::DisableSOCheckInHCALL disableSOCheckInHCALL;
949#endif // FEATURE_STACK_PROBE
950#endif // _DEBUG
951 return OBJECTREF( HCCALL2(fastObjectArrayAllocator, ArrayType.AsArray()->GetTemplateMethodTable(), cElements));
952}
953
954STRINGREF AllocateString( DWORD cchStringLength )
955{
956 CONTRACTL {
957 THROWS;
958 GC_TRIGGERS;
959 MODE_COOPERATIVE; // returns an objref without pinning it => cooperative
960 } CONTRACTL_END;
961
962#ifdef _DEBUG
963 // fastStringAllocator is called by VM and managed code. If called from managed code, we
964 // make sure that the thread is in SOTolerantState.
965#ifdef FEATURE_STACK_PROBE
966 Thread::DisableSOCheckInHCALL disableSOCheckInHCALL;
967#endif // FEATURE_STACK_PROBE
968#endif // _DEBUG
969 return STRINGREF(HCCALL1(fastStringAllocator, cchStringLength));
970}
971
972#endif
973
974//
975// Helper for parts of the EE which are allocating arrays
976//
977OBJECTREF AllocateObjectArray(DWORD cElements, TypeHandle elementType, BOOL bAllocateInLargeHeap)
978{
979 CONTRACTL {
980 THROWS;
981 GC_TRIGGERS;
982 MODE_COOPERATIVE; // returns an objref without pinning it => cooperative
983 } CONTRACTL_END;
984
985 OVERRIDE_TYPE_LOAD_LEVEL_LIMIT(CLASS_LOADED);
986
987 // The object array class is loaded at startup.
988 _ASSERTE(g_pPredefinedArrayTypes[ELEMENT_TYPE_OBJECT] != NULL);
989
990#ifdef _DEBUG
991 ArrayTypeDesc arrayType(g_pPredefinedArrayTypes[ELEMENT_TYPE_OBJECT]->GetMethodTable(), elementType);
992 _ASSERTE(arrayType.GetRank() == 1);
993 _ASSERTE(arrayType.GetInternalCorElementType() == ELEMENT_TYPE_SZARRAY);
994#endif //_DEBUG
995
996 return AllocateArrayEx(ClassLoader::LoadArrayTypeThrowing(elementType),
997 (INT32 *)(&cElements),
998 1,
999 bAllocateInLargeHeap
1000 DEBUG_ARG(FALSE));
1001}
1002
1003
1004STRINGREF SlowAllocateString( DWORD cchStringLength )
1005{
1006 CONTRACTL {
1007 THROWS;
1008 GC_TRIGGERS;
1009 MODE_COOPERATIVE; // returns an objref without pinning it => cooperative
1010 } CONTRACTL_END;
1011
1012 StringObject *orObject = NULL;
1013
1014#ifdef _DEBUG
1015 if (g_pConfig->ShouldInjectFault(INJECTFAULT_GCHEAP))
1016 {
1017 char *a = new char;
1018 delete a;
1019 }
1020#endif
1021
1022 // Limit the maximum string size to <2GB to mitigate risk of security issues caused by 32-bit integer
1023 // overflows in buffer size calculations.
1024 if (cchStringLength > 0x3FFFFFDF)
1025 ThrowOutOfMemory();
1026
1027 SIZE_T ObjectSize = PtrAlign(StringObject::GetSize(cchStringLength));
1028 _ASSERTE(ObjectSize > cchStringLength);
1029
1030 SetTypeHandleOnThreadForAlloc(TypeHandle(g_pStringClass));
1031
1032 orObject = (StringObject *)Alloc( ObjectSize, FALSE, FALSE );
1033
1034 // Object is zero-init already
1035 _ASSERTE( orObject->HasEmptySyncBlockInfo() );
1036
1037 // Initialize Object
1038 //<TODO>@TODO need to build a LARGE g_pStringMethodTable before</TODO>
1039 orObject->SetMethodTable( g_pStringClass );
1040 orObject->SetStringLength( cchStringLength );
1041
1042 if (ObjectSize >= g_pConfig->GetGCLOHThreshold())
1043 {
1044 GCHeapUtilities::GetGCHeap()->PublishObject((BYTE*)orObject);
1045 }
1046
1047 // Notify the profiler of the allocation
1048 if (TrackAllocations())
1049 {
1050 OBJECTREF objref = ObjectToOBJECTREF((Object*)orObject);
1051 GCPROTECT_BEGIN(objref);
1052 ProfilerObjectAllocatedCallback(objref, (ClassID) orObject->GetTypeHandle().AsPtr());
1053 GCPROTECT_END();
1054
1055 orObject = (StringObject *) OBJECTREFToObject(objref);
1056 }
1057
1058#ifdef FEATURE_EVENT_TRACE
1059 // Send ETW event for allocation
1060 if(ETW::TypeSystemLog::IsHeapAllocEventEnabled())
1061 {
1062 ETW::TypeSystemLog::SendObjectAllocatedEvent(orObject);
1063 }
1064#endif // FEATURE_EVENT_TRACE
1065
1066 LogAlloc(ObjectSize, g_pStringClass, orObject);
1067
1068 return( ObjectToSTRINGREF(orObject) );
1069}
1070
1071#ifdef FEATURE_COMINTEROP_UNMANAGED_ACTIVATION
1072// OBJECTREF AllocateComClassObject(ComClassFactory* pComClsFac)
1073void AllocateComClassObject(ComClassFactory* pComClsFac, OBJECTREF* ppRefClass)
1074{
1075 CONTRACTL {
1076 THROWS;
1077 GC_TRIGGERS;
1078 MODE_COOPERATIVE; // returns an objref (out param) without pinning it => cooperative
1079 PRECONDITION(CheckPointer(pComClsFac));
1080 PRECONDITION(CheckPointer(ppRefClass));
1081 } CONTRACTL_END;
1082
1083 // Create a COM+ Class object.
1084 MethodTable *pMT = g_pRuntimeTypeClass;
1085 _ASSERTE(pMT != NULL);
1086 *ppRefClass= AllocateObject(pMT);
1087
1088 if (*ppRefClass != NULL)
1089 {
1090 SyncBlock* pSyncBlock = (*((REFLECTCLASSBASEREF*) ppRefClass))->GetSyncBlock();
1091
1092 // <TODO> This needs to support a COM version of ReflectClass. Right now we
1093 // still work as we used to <darylo> </TODO>
1094 MethodTable* pComMT = g_pBaseCOMObject;
1095 _ASSERTE(pComMT != NULL);
1096
1097 // class for ComObject
1098 (*((REFLECTCLASSBASEREF*) ppRefClass))->SetType(TypeHandle(pComMT));
1099
1100 pSyncBlock->GetInteropInfo()->SetComClassFactory(pComClsFac);
1101 }
1102}
1103#endif // FEATURE_COMINTEROP_UNMANAGED_ACTIVATION
1104
1105// AllocateObject will throw OutOfMemoryException so don't need to check
1106// for NULL return value from it.
1107OBJECTREF AllocateObject(MethodTable *pMT
1108#ifdef FEATURE_COMINTEROP
1109 , bool fHandleCom
1110#endif
1111 )
1112{
1113 CONTRACTL {
1114 THROWS;
1115 GC_TRIGGERS;
1116 MODE_COOPERATIVE; // returns an objref without pinning it => cooperative
1117 PRECONDITION(CheckPointer(pMT));
1118 PRECONDITION(pMT->CheckInstanceActivated());
1119 } CONTRACTL_END;
1120
1121 Object *orObject = NULL;
1122 // use unchecked oref here to avoid triggering assert in Validate that the AD is
1123 // not set becuase it isn't until near the end of the fcn at which point we can allow
1124 // the check.
1125 _UNCHECKED_OBJECTREF oref;
1126
1127 g_IBCLogger.LogMethodTableAccess(pMT);
1128 SetTypeHandleOnThreadForAlloc(TypeHandle(pMT));
1129
1130
1131#ifdef FEATURE_COMINTEROP
1132#ifdef FEATURE_COMINTEROP_UNMANAGED_ACTIVATION
1133 if (fHandleCom && pMT->IsComObjectType() && !pMT->IsWinRTObjectType())
1134 {
1135 // Create a instance of __ComObject here is not allowed as we don't know what COM object to create
1136 if (pMT == g_pBaseCOMObject)
1137 COMPlusThrow(kInvalidComObjectException, IDS_EE_NO_BACKING_CLASS_FACTORY);
1138
1139 oref = OBJECTREF_TO_UNCHECKED_OBJECTREF(AllocateComObject_ForManaged(pMT));
1140 }
1141 else
1142#endif // FEATURE_COMINTEROP_UNMANAGED_ACTIVATION
1143#endif // FEATURE_COMINTEROP
1144 {
1145 DWORD baseSize = pMT->GetBaseSize();
1146
1147#ifdef FEATURE_64BIT_ALIGNMENT
1148 if (pMT->RequiresAlign8())
1149 {
1150 // The last argument to the allocation, indicates whether the alignment should be "biased". This
1151 // means that the object is allocated so that its header lies exactly between two 8-byte
1152 // boundaries. This is required in cases where we need to mis-align the header in order to align
1153 // the actual payload. Currently this is false for classes (where we apply padding to ensure the
1154 // first field is aligned relative to the header) and true for boxed value types (where we can't
1155 // do the same padding without introducing more complexity in type layout and unboxing stubs).
1156 _ASSERTE(sizeof(Object) == 4);
1157 orObject = (Object *) AllocAlign8(baseSize,
1158 pMT->HasFinalizer(),
1159 pMT->ContainsPointers(),
1160 pMT->IsValueType());
1161 }
1162 else
1163#endif // FEATURE_64BIT_ALIGNMENT
1164 {
1165 orObject = (Object *) Alloc(baseSize,
1166 pMT->HasFinalizer(),
1167 pMT->ContainsPointers());
1168 }
1169
1170 // verify zero'd memory (at least for sync block)
1171 _ASSERTE( orObject->HasEmptySyncBlockInfo() );
1172
1173 if ((baseSize >= g_pConfig->GetGCLOHThreshold()))
1174 {
1175 orObject->SetMethodTableForLargeObject(pMT);
1176 GCHeapUtilities::GetGCHeap()->PublishObject((BYTE*)orObject);
1177 }
1178 else
1179 {
1180 orObject->SetMethodTable(pMT);
1181 }
1182
1183 if (pMT->HasFinalizer())
1184 orObject->SetAppDomain();
1185
1186 // Notify the profiler of the allocation
1187 if (TrackAllocations())
1188 {
1189 OBJECTREF objref = ObjectToOBJECTREF((Object*)orObject);
1190 GCPROTECT_BEGIN(objref);
1191 ProfilerObjectAllocatedCallback(objref, (ClassID) orObject->GetTypeHandle().AsPtr());
1192 GCPROTECT_END();
1193
1194 orObject = (Object *) OBJECTREFToObject(objref);
1195 }
1196
1197#ifdef FEATURE_EVENT_TRACE
1198 // Send ETW event for allocation
1199 if(ETW::TypeSystemLog::IsHeapAllocEventEnabled())
1200 {
1201 ETW::TypeSystemLog::SendObjectAllocatedEvent(orObject);
1202 }
1203#endif // FEATURE_EVENT_TRACE
1204
1205 LogAlloc(pMT->GetBaseSize(), pMT, orObject);
1206
1207 oref = OBJECTREF_TO_UNCHECKED_OBJECTREF(orObject);
1208 }
1209
1210 return UNCHECKED_OBJECTREF_TO_OBJECTREF(oref);
1211}
1212
1213//========================================================================
1214//
1215// WRITE BARRIER HELPERS
1216//
1217//========================================================================
1218
1219
1220#define card_byte(addr) (((size_t)(addr)) >> card_byte_shift)
1221#define card_bit(addr) (1 << ((((size_t)(addr)) >> (card_byte_shift - 3)) & 7))
1222
1223#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
1224#define card_bundle_byte(addr) (((size_t)(addr)) >> card_bundle_byte_shift)
1225
1226static void SetCardBundleByte(BYTE* addr)
1227{
1228 BYTE* cbByte = (BYTE *)VolatileLoadWithoutBarrier(&g_card_bundle_table) + card_bundle_byte(addr);
1229 if (*cbByte != 0xFF)
1230 {
1231 *cbByte = 0xFF;
1232 }
1233}
1234#endif
1235
1236#ifdef FEATURE_USE_ASM_GC_WRITE_BARRIERS
1237
1238// implemented in assembly
1239// extern "C" HCIMPL2_RAW(VOID, JIT_CheckedWriteBarrier, Object **dst, Object *refUNSAFE)
1240// extern "C" HCIMPL2_RAW(VOID, JIT_WriteBarrier, Object **dst, Object *refUNSAFE)
1241
1242#else // FEATURE_USE_ASM_GC_WRITE_BARRIERS
1243
1244// NOTE: non-ASM write barriers only work with Workstation GC.
1245
1246#ifdef FEATURE_COUNT_GC_WRITE_BARRIERS
1247static UINT64 CheckedBarrierCount = 0;
1248static UINT64 CheckedBarrierRetBufCount = 0;
1249static UINT64 CheckedBarrierByrefArgCount = 0;
1250static UINT64 CheckedBarrierByrefOtherLocalCount = 0;
1251static UINT64 CheckedBarrierAddrOfLocalCount = 0;
1252static UINT64 UncheckedBarrierCount = 0;
1253static UINT64 CheckedAfterHeapFilter = 0;
1254static UINT64 CheckedAfterRefInEphemFilter = 0;
1255static UINT64 CheckedAfterAlreadyDirtyFilter = 0;
1256static UINT64 CheckedDestInEphem = 0;
1257static UINT64 UncheckedAfterRefInEphemFilter = 0;
1258static UINT64 UncheckedAfterAlreadyDirtyFilter = 0;
1259static UINT64 UncheckedDestInEphem = 0;
1260
1261const unsigned BarrierCountPrintInterval = 1000000;
1262static unsigned CheckedBarrierInterval = BarrierCountPrintInterval;
1263static unsigned UncheckedBarrierInterval = BarrierCountPrintInterval;
1264
1265
1266void IncCheckedBarrierCount()
1267{
1268 ++CheckedBarrierCount;
1269 if (--CheckedBarrierInterval == 0)
1270 {
1271 CheckedBarrierInterval = BarrierCountPrintInterval;
1272 printf("GC write barrier counts: checked = %lld, unchecked = %lld, total = %lld.\n",
1273 CheckedBarrierCount, UncheckedBarrierCount, (CheckedBarrierCount + UncheckedBarrierCount));
1274 printf(" [Checked: %lld after heap check, %lld after ephem check, %lld after already dirty check.]\n",
1275 CheckedAfterHeapFilter, CheckedAfterRefInEphemFilter, CheckedAfterAlreadyDirtyFilter);
1276 printf(" [Unchecked: %lld after ephem check, %lld after already dirty check.]\n",
1277 UncheckedAfterRefInEphemFilter, UncheckedAfterAlreadyDirtyFilter);
1278 printf(" [Dest in ephem: checked = %lld, unchecked = %lld.]\n",
1279 CheckedDestInEphem, UncheckedDestInEphem);
1280 printf(" [Checked: %lld are stores to fields of ret buff, %lld via byref args,\n",
1281 CheckedBarrierRetBufCount, CheckedBarrierByrefArgCount);
1282 printf(" %lld via other locals, %lld via addr of local.]\n",
1283 CheckedBarrierByrefOtherLocalCount, CheckedBarrierAddrOfLocalCount);
1284 }
1285}
1286
1287void IncUncheckedBarrierCount()
1288{
1289 ++UncheckedBarrierCount;
1290 if (--UncheckedBarrierInterval == 0)
1291 {
1292 printf("GC write barrier counts: checked = %lld, unchecked = %lld, total = %lld.\n",
1293 CheckedBarrierCount, UncheckedBarrierCount, (CheckedBarrierCount + UncheckedBarrierCount));
1294 UncheckedBarrierInterval = BarrierCountPrintInterval;
1295 }
1296}
1297#endif // FEATURE_COUNT_GC_WRITE_BARRIERS
1298
1299#ifdef FEATURE_COUNT_GC_WRITE_BARRIERS
1300// (We ignore the advice below on using a _RAW macro for this performance diagnostic mode, which need not function properly in
1301// all situations...)
1302extern "C" HCIMPL3(VOID, JIT_CheckedWriteBarrier, Object **dst, Object *ref, CheckedWriteBarrierKinds kind)
1303#else
1304
1305// This function is a JIT helper, but it must NOT use HCIMPL2 because it
1306// modifies Thread state that will not be restored if an exception occurs
1307// inside of memset. A normal EH unwind will not occur.
1308extern "C" HCIMPL2_RAW(VOID, JIT_CheckedWriteBarrier, Object **dst, Object *ref)
1309#endif
1310{
1311 // Must use static contract here, because if an AV occurs, a normal EH
1312 // unwind will not occur, and destructors will not run.
1313 STATIC_CONTRACT_MODE_COOPERATIVE;
1314 STATIC_CONTRACT_THROWS;
1315 STATIC_CONTRACT_GC_NOTRIGGER;
1316
1317#ifdef FEATURE_COUNT_GC_WRITE_BARRIERS
1318 IncCheckedBarrierCount();
1319 switch (kind)
1320 {
1321 case CWBKind_RetBuf:
1322 CheckedBarrierRetBufCount++;
1323 break;
1324 case CWBKind_ByRefArg:
1325 CheckedBarrierByrefArgCount++;
1326 break;
1327 case CWBKind_OtherByRefLocal:
1328 CheckedBarrierByrefOtherLocalCount++;
1329 break;
1330 case CWBKind_AddrOfLocal:
1331 CheckedBarrierAddrOfLocalCount++;
1332 break;
1333 case CWBKind_Unclassified:
1334 break;
1335 default:
1336 // It should be some member of the enumeration.
1337 _ASSERTE_ALL_BUILDS(__FILE__, false);
1338 break;
1339 }
1340#endif // FEATURE_COUNT_GC_WRITE_BARRIERS
1341
1342 // no HELPER_METHOD_FRAME because we are MODE_COOPERATIVE, GC_NOTRIGGER
1343
1344 *dst = ref;
1345
1346 // if the dst is outside of the heap (unboxed value classes) then we
1347 // simply exit
1348 if (((BYTE*)dst < g_lowest_address) || ((BYTE*)dst >= g_highest_address))
1349 return;
1350
1351#ifdef FEATURE_COUNT_GC_WRITE_BARRIERS
1352 CheckedAfterHeapFilter++;
1353#endif
1354
1355#ifdef WRITE_BARRIER_CHECK
1356 updateGCShadow(dst, ref); // support debugging write barrier
1357#endif
1358
1359#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1360 if (GCHeapUtilities::SoftwareWriteWatchIsEnabled())
1361 {
1362 GCHeapUtilities::SoftwareWriteWatchSetDirty(dst, sizeof(*dst));
1363 }
1364#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1365
1366#ifdef FEATURE_COUNT_GC_WRITE_BARRIERS
1367 if((BYTE*) dst >= g_ephemeral_low && (BYTE*) dst < g_ephemeral_high)
1368 {
1369 CheckedDestInEphem++;
1370 }
1371#endif
1372 if((BYTE*) ref >= g_ephemeral_low && (BYTE*) ref < g_ephemeral_high)
1373 {
1374#ifdef FEATURE_COUNT_GC_WRITE_BARRIERS
1375 CheckedAfterRefInEphemFilter++;
1376#endif
1377 // VolatileLoadWithoutBarrier() is used here to prevent fetch of g_card_table from being reordered
1378 // with g_lowest/highest_address check above. See comment in code:gc_heap::grow_brick_card_tables.
1379 BYTE* pCardByte = (BYTE *)VolatileLoadWithoutBarrier(&g_card_table) + card_byte((BYTE *)dst);
1380 if(*pCardByte != 0xFF)
1381 {
1382#ifdef FEATURE_COUNT_GC_WRITE_BARRIERS
1383 CheckedAfterAlreadyDirtyFilter++;
1384#endif
1385 *pCardByte = 0xFF;
1386
1387#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
1388 SetCardBundleByte((BYTE*)dst);
1389#endif
1390 }
1391 }
1392}
1393HCIMPLEND_RAW
1394
1395// This function is a JIT helper, but it must NOT use HCIMPL2 because it
1396// modifies Thread state that will not be restored if an exception occurs
1397// inside of memset. A normal EH unwind will not occur.
1398extern "C" HCIMPL2_RAW(VOID, JIT_WriteBarrier, Object **dst, Object *ref)
1399{
1400 // Must use static contract here, because if an AV occurs, a normal EH
1401 // unwind will not occur, and destructors will not run.
1402 STATIC_CONTRACT_MODE_COOPERATIVE;
1403 STATIC_CONTRACT_THROWS;
1404 STATIC_CONTRACT_GC_NOTRIGGER;
1405
1406#ifdef FEATURE_COUNT_GC_WRITE_BARRIERS
1407 IncUncheckedBarrierCount();
1408#endif
1409 // no HELPER_METHOD_FRAME because we are MODE_COOPERATIVE, GC_NOTRIGGER
1410
1411 *dst = ref;
1412
1413 // If the store above succeeded, "dst" should be in the heap.
1414 assert(GCHeapUtilities::GetGCHeap()->IsHeapPointer((void*)dst));
1415
1416#ifdef WRITE_BARRIER_CHECK
1417 updateGCShadow(dst, ref); // support debugging write barrier
1418#endif
1419
1420#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1421 if (GCHeapUtilities::SoftwareWriteWatchIsEnabled())
1422 {
1423 GCHeapUtilities::SoftwareWriteWatchSetDirty(dst, sizeof(*dst));
1424 }
1425#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1426
1427#ifdef FEATURE_COUNT_GC_WRITE_BARRIERS
1428 if((BYTE*) dst >= g_ephemeral_low && (BYTE*) dst < g_ephemeral_high)
1429 {
1430 UncheckedDestInEphem++;
1431 }
1432#endif
1433 if((BYTE*) ref >= g_ephemeral_low && (BYTE*) ref < g_ephemeral_high)
1434 {
1435#ifdef FEATURE_COUNT_GC_WRITE_BARRIERS
1436 UncheckedAfterRefInEphemFilter++;
1437#endif
1438 BYTE* pCardByte = (BYTE *)VolatileLoadWithoutBarrier(&g_card_table) + card_byte((BYTE *)dst);
1439 if(*pCardByte != 0xFF)
1440 {
1441#ifdef FEATURE_COUNT_GC_WRITE_BARRIERS
1442 UncheckedAfterAlreadyDirtyFilter++;
1443#endif
1444 *pCardByte = 0xFF;
1445
1446#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
1447 SetCardBundleByte((BYTE*)dst);
1448#endif
1449
1450 }
1451 }
1452}
1453HCIMPLEND_RAW
1454
1455#endif // FEATURE_USE_ASM_GC_WRITE_BARRIERS
1456
1457extern "C" HCIMPL2_RAW(VOID, JIT_WriteBarrierEnsureNonHeapTarget, Object **dst, Object *ref)
1458{
1459 // Must use static contract here, because if an AV occurs, a normal EH
1460 // unwind will not occur, and destructors will not run.
1461 STATIC_CONTRACT_MODE_COOPERATIVE;
1462 STATIC_CONTRACT_THROWS;
1463 STATIC_CONTRACT_GC_NOTRIGGER;
1464
1465 assert(!GCHeapUtilities::GetGCHeap()->IsHeapPointer((void*)dst));
1466
1467 // no HELPER_METHOD_FRAME because we are MODE_COOPERATIVE, GC_NOTRIGGER
1468
1469 *dst = ref;
1470}
1471HCIMPLEND_RAW
1472
1473// This function sets the card table with the granularity of 1 byte, to avoid ghost updates
1474// that could occur if multiple threads were trying to set different bits in the same card.
1475
1476#include <optsmallperfcritical.h>
1477void ErectWriteBarrier(OBJECTREF *dst, OBJECTREF ref)
1478{
1479 STATIC_CONTRACT_MODE_COOPERATIVE;
1480 STATIC_CONTRACT_NOTHROW;
1481 STATIC_CONTRACT_GC_NOTRIGGER;
1482 STATIC_CONTRACT_SO_TOLERANT;
1483
1484 // if the dst is outside of the heap (unboxed value classes) then we
1485 // simply exit
1486 if (((BYTE*)dst < g_lowest_address) || ((BYTE*)dst >= g_highest_address))
1487 return;
1488
1489#ifdef WRITE_BARRIER_CHECK
1490 updateGCShadow((Object**) dst, OBJECTREFToObject(ref)); // support debugging write barrier
1491#endif
1492
1493#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1494 if (GCHeapUtilities::SoftwareWriteWatchIsEnabled())
1495 {
1496 GCHeapUtilities::SoftwareWriteWatchSetDirty(dst, sizeof(*dst));
1497 }
1498#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1499
1500 if ((BYTE*) OBJECTREFToObject(ref) >= g_ephemeral_low && (BYTE*) OBJECTREFToObject(ref) < g_ephemeral_high)
1501 {
1502 // VolatileLoadWithoutBarrier() is used here to prevent fetch of g_card_table from being reordered
1503 // with g_lowest/highest_address check above. See comment in code:gc_heap::grow_brick_card_tables.
1504 BYTE* pCardByte = (BYTE *)VolatileLoadWithoutBarrier(&g_card_table) + card_byte((BYTE *)dst);
1505 if (*pCardByte != 0xFF)
1506 {
1507 *pCardByte = 0xFF;
1508
1509#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
1510 SetCardBundleByte((BYTE*)dst);
1511#endif
1512
1513 }
1514 }
1515}
1516#include <optdefault.h>
1517
1518void ErectWriteBarrierForMT(MethodTable **dst, MethodTable *ref)
1519{
1520 STATIC_CONTRACT_MODE_COOPERATIVE;
1521 STATIC_CONTRACT_NOTHROW;
1522 STATIC_CONTRACT_GC_NOTRIGGER;
1523 STATIC_CONTRACT_SO_TOLERANT;
1524
1525 *dst = ref;
1526
1527#ifdef WRITE_BARRIER_CHECK
1528 updateGCShadow((Object **)dst, (Object *)ref); // support debugging write barrier, updateGCShadow only cares that these are pointers
1529#endif
1530
1531 if (ref->Collectible())
1532 {
1533#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1534 if (GCHeapUtilities::SoftwareWriteWatchIsEnabled())
1535 {
1536 GCHeapUtilities::SoftwareWriteWatchSetDirty(dst, sizeof(*dst));
1537 }
1538
1539#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1540
1541 BYTE *refObject = *(BYTE **)((MethodTable*)ref)->GetLoaderAllocatorObjectHandle();
1542 if((BYTE*) refObject >= g_ephemeral_low && (BYTE*) refObject < g_ephemeral_high)
1543 {
1544 // See comment above
1545 BYTE* pCardByte = (BYTE *)VolatileLoadWithoutBarrier(&g_card_table) + card_byte((BYTE *)dst);
1546 if( !((*pCardByte) & card_bit((BYTE *)dst)) )
1547 {
1548 *pCardByte = 0xFF;
1549
1550#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
1551 SetCardBundleByte((BYTE*)dst);
1552#endif
1553
1554 }
1555 }
1556 }
1557}
1558
1559//----------------------------------------------------------------------------
1560//
1561// Write Barrier Support for bulk copy ("Clone") operations
1562//
1563// StartPoint is the target bulk copy start point
1564// len is the length of the bulk copy (in bytes)
1565//
1566//
1567// Performance Note:
1568//
1569// This is implemented somewhat "conservatively", that is we
1570// assume that all the contents of the bulk copy are object
1571// references. If they are not, and the value lies in the
1572// ephemeral range, we will set false positives in the card table.
1573//
1574// We could use the pointer maps and do this more accurately if necessary
1575
1576#if defined(_MSC_VER) && defined(_TARGET_X86_)
1577#pragma optimize("y", on) // Small critical routines, don't put in EBP frame
1578#endif //_MSC_VER && _TARGET_X86_
1579
1580void
1581SetCardsAfterBulkCopy(Object **start, size_t len)
1582{
1583 // If the size is smaller than a pointer, no write barrier is required.
1584 if (len >= sizeof(uintptr_t))
1585 {
1586 InlinedSetCardsAfterBulkCopyHelper(start, len);
1587 }
1588}
1589
1590#if defined(_MSC_VER) && defined(_TARGET_X86_)
1591#pragma optimize("", on) // Go back to command line default optimizations
1592#endif //_MSC_VER && _TARGET_X86_
1593