1// Protocol Buffers - Google's data interchange format
2// Copyright 2008 Google Inc. All rights reserved.
3// https://developers.google.com/protocol-buffers/
4//
5// Redistribution and use in source and binary forms, with or without
6// modification, are permitted provided that the following conditions are
7// met:
8//
9// * Redistributions of source code must retain the above copyright
10// notice, this list of conditions and the following disclaimer.
11// * Redistributions in binary form must reproduce the above
12// copyright notice, this list of conditions and the following disclaimer
13// in the documentation and/or other materials provided with the
14// distribution.
15// * Neither the name of Google Inc. nor the names of its
16// contributors may be used to endorse or promote products derived from
17// this software without specific prior written permission.
18//
19// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31// This file defines an Arena allocator for better allocation performance.
32
33#ifndef GOOGLE_PROTOBUF_ARENA_H__
34#define GOOGLE_PROTOBUF_ARENA_H__
35
36
37#include <limits>
38#include <type_traits>
39#include <utility>
40#if defined(_MSC_VER) && !defined(_LIBCPP_STD_VER) && !_HAS_EXCEPTIONS
41// Work around bugs in MSVC <typeinfo> header when _HAS_EXCEPTIONS=0.
42#include <exception>
43#include <typeinfo>
44namespace std {
45using type_info = ::type_info;
46}
47#else
48#include <typeinfo>
49#endif
50
51#include <type_traits>
52#include <google/protobuf/arena_impl.h>
53#include <google/protobuf/port.h>
54
55// Must be included last.
56#include <google/protobuf/port_def.inc>
57
58#ifdef SWIG
59#error "You cannot SWIG proto headers"
60#endif
61
62namespace google {
63namespace protobuf {
64
65struct ArenaOptions; // defined below
66class Arena; // defined below
67class Message; // defined in message.h
68class MessageLite;
69template <typename Key, typename T>
70class Map;
71
72namespace arena_metrics {
73
74void EnableArenaMetrics(ArenaOptions* options);
75
76} // namespace arena_metrics
77
78namespace TestUtil {
79class ReflectionTester; // defined in test_util.h
80} // namespace TestUtil
81
82namespace internal {
83
84struct ArenaTestPeer; // defined in arena_test_util.h
85class InternalMetadata; // defined in metadata_lite.h
86class LazyField; // defined in lazy_field.h
87class EpsCopyInputStream; // defined in parse_context.h
88class RepeatedPtrFieldBase; // defined in repeated_ptr_field.h
89
90template <typename Type>
91class GenericTypeHandler; // defined in repeated_field.h
92
93inline PROTOBUF_ALWAYS_INLINE
94void* AlignTo(void* ptr, size_t align) {
95 return reinterpret_cast<void*>(
96 (reinterpret_cast<uintptr_t>(ptr) + align - 1) & (~align + 1));
97}
98
99// Templated cleanup methods.
100template <typename T>
101void arena_destruct_object(void* object) {
102 reinterpret_cast<T*>(object)->~T();
103}
104
105template <bool destructor_skippable, typename T>
106struct ObjectDestructor {
107 constexpr static void (*destructor)(void*) = &arena_destruct_object<T>;
108};
109
110template <typename T>
111struct ObjectDestructor<true, T> {
112 constexpr static void (*destructor)(void*) = nullptr;
113};
114
115template <typename T>
116void arena_delete_object(void* object) {
117 delete reinterpret_cast<T*>(object);
118}
119} // namespace internal
120
121// ArenaOptions provides optional additional parameters to arena construction
122// that control its block-allocation behavior.
123struct ArenaOptions {
124 // This defines the size of the first block requested from the system malloc.
125 // Subsequent block sizes will increase in a geometric series up to a maximum.
126 size_t start_block_size;
127
128 // This defines the maximum block size requested from system malloc (unless an
129 // individual arena allocation request occurs with a size larger than this
130 // maximum). Requested block sizes increase up to this value, then remain
131 // here.
132 size_t max_block_size;
133
134 // An initial block of memory for the arena to use, or NULL for none. If
135 // provided, the block must live at least as long as the arena itself. The
136 // creator of the Arena retains ownership of the block after the Arena is
137 // destroyed.
138 char* initial_block;
139
140 // The size of the initial block, if provided.
141 size_t initial_block_size;
142
143 // A function pointer to an alloc method that returns memory blocks of size
144 // requested. By default, it contains a ptr to the malloc function.
145 //
146 // NOTE: block_alloc and dealloc functions are expected to behave like
147 // malloc and free, including Asan poisoning.
148 void* (*block_alloc)(size_t);
149 // A function pointer to a dealloc method that takes ownership of the blocks
150 // from the arena. By default, it contains a ptr to a wrapper function that
151 // calls free.
152 void (*block_dealloc)(void*, size_t);
153
154 ArenaOptions()
155 : start_block_size(internal::AllocationPolicy::kDefaultStartBlockSize),
156 max_block_size(internal::AllocationPolicy::kDefaultMaxBlockSize),
157 initial_block(NULL),
158 initial_block_size(0),
159 block_alloc(nullptr),
160 block_dealloc(nullptr),
161 make_metrics_collector(nullptr) {}
162
163 private:
164 // If make_metrics_collector is not nullptr, it will be called at Arena init
165 // time. It may return a pointer to a collector instance that will be notified
166 // of interesting events related to the arena.
167 internal::ArenaMetricsCollector* (*make_metrics_collector)();
168
169 internal::ArenaMetricsCollector* MetricsCollector() const {
170 return make_metrics_collector ? (*make_metrics_collector)() : nullptr;
171 }
172
173 internal::AllocationPolicy AllocationPolicy() const {
174 internal::AllocationPolicy res;
175 res.start_block_size = start_block_size;
176 res.max_block_size = max_block_size;
177 res.block_alloc = block_alloc;
178 res.block_dealloc = block_dealloc;
179 res.metrics_collector = MetricsCollector();
180 return res;
181 }
182
183 friend void arena_metrics::EnableArenaMetrics(ArenaOptions*);
184
185 friend class Arena;
186 friend class ArenaOptionsTestFriend;
187};
188
189// Support for non-RTTI environments. (The metrics hooks API uses type
190// information.)
191#if PROTOBUF_RTTI
192#define RTTI_TYPE_ID(type) (&typeid(type))
193#else
194#define RTTI_TYPE_ID(type) (NULL)
195#endif
196
197// Arena allocator. Arena allocation replaces ordinary (heap-based) allocation
198// with new/delete, and improves performance by aggregating allocations into
199// larger blocks and freeing allocations all at once. Protocol messages are
200// allocated on an arena by using Arena::CreateMessage<T>(Arena*), below, and
201// are automatically freed when the arena is destroyed.
202//
203// This is a thread-safe implementation: multiple threads may allocate from the
204// arena concurrently. Destruction is not thread-safe and the destructing
205// thread must synchronize with users of the arena first.
206//
207// An arena provides two allocation interfaces: CreateMessage<T>, which works
208// for arena-enabled proto2 message types as well as other types that satisfy
209// the appropriate protocol (described below), and Create<T>, which works for
210// any arbitrary type T. CreateMessage<T> is better when the type T supports it,
211// because this interface (i) passes the arena pointer to the created object so
212// that its sub-objects and internal allocations can use the arena too, and (ii)
213// elides the object's destructor call when possible. Create<T> does not place
214// any special requirements on the type T, and will invoke the object's
215// destructor when the arena is destroyed.
216//
217// The arena message allocation protocol, required by
218// CreateMessage<T>(Arena* arena, Args&&... args), is as follows:
219//
220// - The type T must have (at least) two constructors: a constructor callable
221// with `args` (without `arena`), called when a T is allocated on the heap;
222// and a constructor callable with `Arena* arena, Args&&... args`, called when
223// a T is allocated on an arena. If the second constructor is called with a
224// NULL arena pointer, it must be equivalent to invoking the first
225// (`args`-only) constructor.
226//
227// - The type T must have a particular type trait: a nested type
228// |InternalArenaConstructable_|. This is usually a typedef to |void|. If no
229// such type trait exists, then the instantiation CreateMessage<T> will fail
230// to compile.
231//
232// - The type T *may* have the type trait |DestructorSkippable_|. If this type
233// trait is present in the type, then its destructor will not be called if and
234// only if it was passed a non-NULL arena pointer. If this type trait is not
235// present on the type, then its destructor is always called when the
236// containing arena is destroyed.
237//
238// This protocol is implemented by all arena-enabled proto2 message classes as
239// well as protobuf container types like RepeatedPtrField and Map. The protocol
240// is internal to protobuf and is not guaranteed to be stable. Non-proto types
241// should not rely on this protocol.
242class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
243 public:
244 // Default constructor with sensible default options, tuned for average
245 // use-cases.
246 inline Arena() : impl_() {}
247
248 // Construct an arena with default options, except for the supplied
249 // initial block. It is more efficient to use this constructor
250 // instead of passing ArenaOptions if the only configuration needed
251 // by the caller is supplying an initial block.
252 inline Arena(char* initial_block, size_t initial_block_size)
253 : impl_(initial_block, initial_block_size) {}
254
255 // Arena constructor taking custom options. See ArenaOptions above for
256 // descriptions of the options available.
257 explicit Arena(const ArenaOptions& options)
258 : impl_(options.initial_block, options.initial_block_size,
259 options.AllocationPolicy()) {}
260
261 // Block overhead. Use this as a guide for how much to over-allocate the
262 // initial block if you want an allocation of size N to fit inside it.
263 //
264 // WARNING: if you allocate multiple objects, it is difficult to guarantee
265 // that a series of allocations will fit in the initial block, especially if
266 // Arena changes its alignment guarantees in the future!
267 static const size_t kBlockOverhead =
268 internal::ThreadSafeArena::kBlockHeaderSize +
269 internal::ThreadSafeArena::kSerialArenaSize;
270
271 inline ~Arena() {}
272
273 // TODO(protobuf-team): Fix callers to use constructor and delete this method.
274 void Init(const ArenaOptions&) {}
275
276 // API to create proto2 message objects on the arena. If the arena passed in
277 // is NULL, then a heap allocated object is returned. Type T must be a message
278 // defined in a .proto file with cc_enable_arenas set to true, otherwise a
279 // compilation error will occur.
280 //
281 // RepeatedField and RepeatedPtrField may also be instantiated directly on an
282 // arena with this method.
283 //
284 // This function also accepts any type T that satisfies the arena message
285 // allocation protocol, documented above.
286 template <typename T, typename... Args>
287 PROTOBUF_ALWAYS_INLINE static T* CreateMessage(Arena* arena, Args&&... args) {
288 static_assert(
289 InternalHelper<T>::is_arena_constructable::value,
290 "CreateMessage can only construct types that are ArenaConstructable");
291 // We must delegate to CreateMaybeMessage() and NOT CreateMessageInternal()
292 // because protobuf generated classes specialize CreateMaybeMessage() and we
293 // need to use that specialization for code size reasons.
294 return Arena::CreateMaybeMessage<T>(arena, static_cast<Args&&>(args)...);
295 }
296
297 // API to create any objects on the arena. Note that only the object will
298 // be created on the arena; the underlying ptrs (in case of a proto2 message)
299 // will be still heap allocated. Proto messages should usually be allocated
300 // with CreateMessage<T>() instead.
301 //
302 // Note that even if T satisfies the arena message construction protocol
303 // (InternalArenaConstructable_ trait and optional DestructorSkippable_
304 // trait), as described above, this function does not follow the protocol;
305 // instead, it treats T as a black-box type, just as if it did not have these
306 // traits. Specifically, T's constructor arguments will always be only those
307 // passed to Create<T>() -- no additional arena pointer is implicitly added.
308 // Furthermore, the destructor will always be called at arena destruction time
309 // (unless the destructor is trivial). Hence, from T's point of view, it is as
310 // if the object were allocated on the heap (except that the underlying memory
311 // is obtained from the arena).
312 template <typename T, typename... Args>
313 PROTOBUF_NDEBUG_INLINE static T* Create(Arena* arena, Args&&... args) {
314 return CreateInternal<T>(arena, std::is_convertible<T*, MessageLite*>(),
315 static_cast<Args&&>(args)...);
316 }
317
318 // Allocates memory with the specific size and alignment.
319 void* AllocateAligned(size_t size, size_t align = 8) {
320 if (align <= 8) {
321 return AllocateAlignedNoHook(n: internal::AlignUpTo8(n: size));
322 } else {
323 // We are wasting space by over allocating align - 8 bytes. Compared
324 // to a dedicated function that takes current alignment in consideration.
325 // Such a scheme would only waste (align - 8)/2 bytes on average, but
326 // requires a dedicated function in the outline arena allocation
327 // functions. Possibly re-evaluate tradeoffs later.
328 return internal::AlignTo(ptr: AllocateAlignedNoHook(n: size + align - 8), align);
329 }
330 }
331
332 // Create an array of object type T on the arena *without* invoking the
333 // constructor of T. If `arena` is null, then the return value should be freed
334 // with `delete[] x;` (or `::operator delete[](x);`).
335 // To ensure safe uses, this function checks at compile time
336 // (when compiled as C++11) that T is trivially default-constructible and
337 // trivially destructible.
338 template <typename T>
339 PROTOBUF_NDEBUG_INLINE static T* CreateArray(Arena* arena,
340 size_t num_elements) {
341 static_assert(std::is_trivial<T>::value,
342 "CreateArray requires a trivially constructible type");
343 static_assert(std::is_trivially_destructible<T>::value,
344 "CreateArray requires a trivially destructible type");
345 GOOGLE_CHECK_LE(num_elements, std::numeric_limits<size_t>::max() / sizeof(T))
346 << "Requested size is too large to fit into size_t.";
347 if (arena == NULL) {
348 return static_cast<T*>(::operator new[](num_elements * sizeof(T)));
349 } else {
350 return arena->CreateInternalRawArray<T>(num_elements);
351 }
352 }
353
354 // The following are routines are for monitoring. They will approximate the
355 // total sum allocated and used memory, but the exact value is an
356 // implementation deal. For instance allocated space depends on growth
357 // policies. Do not use these in unit tests.
358 // Returns the total space allocated by the arena, which is the sum of the
359 // sizes of the underlying blocks.
360 uint64_t SpaceAllocated() const { return impl_.SpaceAllocated(); }
361 // Returns the total space used by the arena. Similar to SpaceAllocated but
362 // does not include free space and block overhead. The total space returned
363 // may not include space used by other threads executing concurrently with
364 // the call to this method.
365 uint64_t SpaceUsed() const { return impl_.SpaceUsed(); }
366
367 // Frees all storage allocated by this arena after calling destructors
368 // registered with OwnDestructor() and freeing objects registered with Own().
369 // Any objects allocated on this arena are unusable after this call. It also
370 // returns the total space used by the arena which is the sums of the sizes
371 // of the allocated blocks. This method is not thread-safe.
372 uint64_t Reset() { return impl_.Reset(); }
373
374 // Adds |object| to a list of heap-allocated objects to be freed with |delete|
375 // when the arena is destroyed or reset.
376 template <typename T>
377 PROTOBUF_ALWAYS_INLINE void Own(T* object) {
378 OwnInternal(object, std::is_convertible<T*, MessageLite*>());
379 }
380
381 // Adds |object| to a list of objects whose destructors will be manually
382 // called when the arena is destroyed or reset. This differs from Own() in
383 // that it does not free the underlying memory with |delete|; hence, it is
384 // normally only used for objects that are placement-newed into
385 // arena-allocated memory.
386 template <typename T>
387 PROTOBUF_ALWAYS_INLINE void OwnDestructor(T* object) {
388 if (object != NULL) {
389 impl_.AddCleanup(elem: object, cleanup: &internal::arena_destruct_object<T>);
390 }
391 }
392
393 // Adds a custom member function on an object to the list of destructors that
394 // will be manually called when the arena is destroyed or reset. This differs
395 // from OwnDestructor() in that any member function may be specified, not only
396 // the class destructor.
397 PROTOBUF_ALWAYS_INLINE void OwnCustomDestructor(void* object,
398 void (*destruct)(void*)) {
399 impl_.AddCleanup(elem: object, cleanup: destruct);
400 }
401
402 // Retrieves the arena associated with |value| if |value| is an arena-capable
403 // message, or NULL otherwise. If possible, the call resolves at compile time.
404 // Note that we can often devirtualize calls to `value->GetArena()` so usually
405 // calling this method is unnecessary.
406 template <typename T>
407 PROTOBUF_ALWAYS_INLINE static Arena* GetArena(const T* value) {
408 return GetArenaInternal(value);
409 }
410
411 template <typename T>
412 class InternalHelper {
413 private:
414 // Provides access to protected GetOwningArena to generated messages.
415 static Arena* GetOwningArena(const T* p) { return p->GetOwningArena(); }
416
417 static void InternalSwap(T* a, T* b) { a->InternalSwap(b); }
418
419 static Arena* GetArenaForAllocationInternal(
420 const T* p, std::true_type /*is_derived_from<MessageLite>*/) {
421 return p->GetArenaForAllocation();
422 }
423
424 static Arena* GetArenaForAllocationInternal(
425 const T* p, std::false_type /*is_derived_from<MessageLite>*/) {
426 return GetArenaForAllocationForNonMessage(
427 p, typename is_arena_constructable::type());
428 }
429
430 static Arena* GetArenaForAllocationForNonMessage(
431 const T* p, std::true_type /*is_arena_constructible*/) {
432 return p->GetArena();
433 }
434
435 static Arena* GetArenaForAllocationForNonMessage(
436 const T* p, std::false_type /*is_arena_constructible*/) {
437 return GetArenaForAllocationForNonMessageNonArenaConstructible(
438 p, typename has_get_arena::type());
439 }
440
441 static Arena* GetArenaForAllocationForNonMessageNonArenaConstructible(
442 const T* p, std::true_type /*has_get_arena*/) {
443 return p->GetArena();
444 }
445
446 static Arena* GetArenaForAllocationForNonMessageNonArenaConstructible(
447 const T* /* p */, std::false_type /*has_get_arena*/) {
448 return nullptr;
449 }
450
451 template <typename U>
452 static char DestructorSkippable(const typename U::DestructorSkippable_*);
453 template <typename U>
454 static double DestructorSkippable(...);
455
456 typedef std::integral_constant<
457 bool, sizeof(DestructorSkippable<T>(static_cast<const T*>(0))) ==
458 sizeof(char) ||
459 std::is_trivially_destructible<T>::value>
460 is_destructor_skippable;
461
462 template <typename U>
463 static char ArenaConstructable(
464 const typename U::InternalArenaConstructable_*);
465 template <typename U>
466 static double ArenaConstructable(...);
467
468 typedef std::integral_constant<bool, sizeof(ArenaConstructable<T>(
469 static_cast<const T*>(0))) ==
470 sizeof(char)>
471 is_arena_constructable;
472
473 template <typename U,
474 typename std::enable_if<
475 std::is_same<Arena*, decltype(std::declval<const U>()
476 .GetArena())>::value,
477 int>::type = 0>
478 static char HasGetArena(decltype(&U::GetArena));
479 template <typename U>
480 static double HasGetArena(...);
481
482 typedef std::integral_constant<bool, sizeof(HasGetArena<T>(nullptr)) ==
483 sizeof(char)>
484 has_get_arena;
485
486 template <typename... Args>
487 static T* Construct(void* ptr, Args&&... args) {
488 return new (ptr) T(static_cast<Args&&>(args)...);
489 }
490
491 static inline PROTOBUF_ALWAYS_INLINE T* New() {
492 return new T(nullptr);
493 }
494
495 static Arena* GetArena(const T* p) { return p->GetArena(); }
496
497 friend class Arena;
498 friend class TestUtil::ReflectionTester;
499 };
500
501 // Provides access to protected GetOwningArena to generated messages. For
502 // internal use only.
503 template <typename T>
504 static Arena* InternalGetOwningArena(const T* p) {
505 return InternalHelper<T>::GetOwningArena(p);
506 }
507
508 // Provides access to protected GetArenaForAllocation to generated messages.
509 // For internal use only.
510 template <typename T>
511 static Arena* InternalGetArenaForAllocation(const T* p) {
512 return InternalHelper<T>::GetArenaForAllocationInternal(
513 p, std::is_convertible<T*, MessageLite*>());
514 }
515
516 // Creates message-owned arena. For internal use only.
517 static Arena* InternalCreateMessageOwnedArena() {
518 return new Arena(internal::MessageOwned{});
519 }
520
521 // Checks whether this arena is message-owned. For internal use only.
522 bool InternalIsMessageOwnedArena() { return IsMessageOwned(); }
523
524 // Helper typetraits that indicates support for arenas in a type T at compile
525 // time. This is public only to allow construction of higher-level templated
526 // utilities.
527 //
528 // is_arena_constructable<T>::value is true if the message type T has arena
529 // support enabled, and false otherwise.
530 //
531 // is_destructor_skippable<T>::value is true if the message type T has told
532 // the arena that it is safe to skip the destructor, and false otherwise.
533 //
534 // This is inside Arena because only Arena has the friend relationships
535 // necessary to see the underlying generated code traits.
536 template <typename T>
537 struct is_arena_constructable : InternalHelper<T>::is_arena_constructable {};
538 template <typename T>
539 struct is_destructor_skippable : InternalHelper<T>::is_destructor_skippable {
540 };
541
542 private:
543 internal::ThreadSafeArena impl_;
544
545 template <typename T>
546 struct has_get_arena : InternalHelper<T>::has_get_arena {};
547
548 // Constructor solely used by message-owned arena.
549 inline Arena(internal::MessageOwned) : impl_(internal::MessageOwned{}) {}
550
551 // Checks whether this arena is message-owned.
552 PROTOBUF_ALWAYS_INLINE bool IsMessageOwned() const {
553 return impl_.IsMessageOwned();
554 }
555
556 void ReturnArrayMemory(void* p, size_t size) {
557 impl_.ReturnArrayMemory(p, size);
558 }
559
560 template <typename T, typename... Args>
561 PROTOBUF_NDEBUG_INLINE static T* CreateMessageInternal(Arena* arena,
562 Args&&... args) {
563 static_assert(
564 InternalHelper<T>::is_arena_constructable::value,
565 "CreateMessage can only construct types that are ArenaConstructable");
566 if (arena == NULL) {
567 return new T(nullptr, static_cast<Args&&>(args)...);
568 } else {
569 return arena->DoCreateMessage<T>(static_cast<Args&&>(args)...);
570 }
571 }
572
573 // This specialization for no arguments is necessary, because its behavior is
574 // slightly different. When the arena pointer is nullptr, it calls T()
575 // instead of T(nullptr).
576 template <typename T>
577 PROTOBUF_NDEBUG_INLINE static T* CreateMessageInternal(Arena* arena) {
578 static_assert(
579 InternalHelper<T>::is_arena_constructable::value,
580 "CreateMessage can only construct types that are ArenaConstructable");
581 if (arena == NULL) {
582 // Generated arena constructor T(Arena*) is protected. Call via
583 // InternalHelper.
584 return InternalHelper<T>::New();
585 } else {
586 return arena->DoCreateMessage<T>();
587 }
588 }
589
590 // Allocate and also optionally call collector with the allocated type info
591 // when allocation recording is enabled.
592 PROTOBUF_NDEBUG_INLINE void* AllocateInternal(size_t size, size_t align,
593 void (*destructor)(void*),
594 const std::type_info* type) {
595 // Monitor allocation if needed.
596 if (destructor == nullptr) {
597 return AllocateAlignedWithHook(n: size, align, type);
598 } else {
599 if (align <= 8) {
600 auto res = AllocateAlignedWithCleanup(n: internal::AlignUpTo8(n: size), type);
601 res.second->elem = res.first;
602 res.second->cleanup = destructor;
603 return res.first;
604 } else {
605 auto res = AllocateAlignedWithCleanup(n: size + align - 8, type);
606 auto ptr = internal::AlignTo(ptr: res.first, align);
607 res.second->elem = ptr;
608 res.second->cleanup = destructor;
609 return ptr;
610 }
611 }
612 }
613
614 // CreateMessage<T> requires that T supports arenas, but this private method
615 // works whether or not T supports arenas. These are not exposed to user code
616 // as it can cause confusing API usages, and end up having double free in
617 // user code. These are used only internally from LazyField and Repeated
618 // fields, since they are designed to work in all mode combinations.
619 template <typename Msg, typename... Args>
620 PROTOBUF_ALWAYS_INLINE static Msg* DoCreateMaybeMessage(Arena* arena,
621 std::true_type,
622 Args&&... args) {
623 return CreateMessageInternal<Msg>(arena, std::forward<Args>(args)...);
624 }
625
626 template <typename T, typename... Args>
627 PROTOBUF_ALWAYS_INLINE static T* DoCreateMaybeMessage(Arena* arena,
628 std::false_type,
629 Args&&... args) {
630 return Create<T>(arena, std::forward<Args>(args)...);
631 }
632
633 template <typename T, typename... Args>
634 PROTOBUF_ALWAYS_INLINE static T* CreateMaybeMessage(Arena* arena,
635 Args&&... args) {
636 return DoCreateMaybeMessage<T>(arena, is_arena_constructable<T>(),
637 std::forward<Args>(args)...);
638 }
639
640 // Just allocate the required size for the given type assuming the
641 // type has a trivial constructor.
642 template <typename T>
643 PROTOBUF_NDEBUG_INLINE T* CreateInternalRawArray(size_t num_elements) {
644 GOOGLE_CHECK_LE(num_elements, std::numeric_limits<size_t>::max() / sizeof(T))
645 << "Requested size is too large to fit into size_t.";
646 // We count on compiler to realize that if sizeof(T) is a multiple of
647 // 8 AlignUpTo can be elided.
648 const size_t n = sizeof(T) * num_elements;
649 return static_cast<T*>(
650 AllocateAlignedWithHookForArray(n, align: alignof(T), RTTI_TYPE_ID(T)));
651 }
652
653 template <typename T, typename... Args>
654 PROTOBUF_NDEBUG_INLINE T* DoCreateMessage(Args&&... args) {
655 return InternalHelper<T>::Construct(
656 AllocateInternal(size: sizeof(T), align: alignof(T),
657 destructor: internal::ObjectDestructor<
658 InternalHelper<T>::is_destructor_skippable::value,
659 T>::destructor,
660 RTTI_TYPE_ID(T)),
661 this, std::forward<Args>(args)...);
662 }
663
664 // CreateInArenaStorage is used to implement map field. Without it,
665 // Map need to call generated message's protected arena constructor,
666 // which needs to declare Map as friend of generated message.
667 template <typename T, typename... Args>
668 static void CreateInArenaStorage(T* ptr, Arena* arena, Args&&... args) {
669 CreateInArenaStorageInternal(ptr, arena,
670 typename is_arena_constructable<T>::type(),
671 std::forward<Args>(args)...);
672 if (arena != nullptr) {
673 RegisterDestructorInternal(
674 ptr, arena,
675 typename InternalHelper<T>::is_destructor_skippable::type());
676 }
677 }
678
679 template <typename T, typename... Args>
680 static void CreateInArenaStorageInternal(T* ptr, Arena* arena,
681 std::true_type, Args&&... args) {
682 InternalHelper<T>::Construct(ptr, arena, std::forward<Args>(args)...);
683 }
684 template <typename T, typename... Args>
685 static void CreateInArenaStorageInternal(T* ptr, Arena* /* arena */,
686 std::false_type, Args&&... args) {
687 new (ptr) T(std::forward<Args>(args)...);
688 }
689
690 template <typename T>
691 static void RegisterDestructorInternal(T* /* ptr */, Arena* /* arena */,
692 std::true_type) {}
693 template <typename T>
694 static void RegisterDestructorInternal(T* ptr, Arena* arena,
695 std::false_type) {
696 arena->OwnDestructor(ptr);
697 }
698
699 // These implement Create(). The second parameter has type 'true_type' if T is
700 // a subtype of Message and 'false_type' otherwise.
701 template <typename T, typename... Args>
702 PROTOBUF_ALWAYS_INLINE static T* CreateInternal(Arena* arena, std::true_type,
703 Args&&... args) {
704 if (arena == nullptr) {
705 return new T(std::forward<Args>(args)...);
706 } else {
707 auto destructor =
708 internal::ObjectDestructor<std::is_trivially_destructible<T>::value,
709 T>::destructor;
710 T* result =
711 new (arena->AllocateInternal(size: sizeof(T), align: alignof(T), destructor,
712 RTTI_TYPE_ID(T)))
713 T(std::forward<Args>(args)...);
714 return result;
715 }
716 }
717 template <typename T, typename... Args>
718 PROTOBUF_ALWAYS_INLINE static T* CreateInternal(Arena* arena, std::false_type,
719 Args&&... args) {
720 if (arena == nullptr) {
721 return new T(std::forward<Args>(args)...);
722 } else {
723 auto destructor =
724 internal::ObjectDestructor<std::is_trivially_destructible<T>::value,
725 T>::destructor;
726 return new (arena->AllocateInternal(size: sizeof(T), align: alignof(T), destructor,
727 RTTI_TYPE_ID(T)))
728 T(std::forward<Args>(args)...);
729 }
730 }
731
732 // These implement Own(), which registers an object for deletion (destructor
733 // call and operator delete()). The second parameter has type 'true_type' if T
734 // is a subtype of Message and 'false_type' otherwise. Collapsing
735 // all template instantiations to one for generic Message reduces code size,
736 // using the virtual destructor instead.
737 template <typename T>
738 PROTOBUF_ALWAYS_INLINE void OwnInternal(T* object, std::true_type) {
739 if (object != NULL) {
740 impl_.AddCleanup(elem: object, cleanup: &internal::arena_delete_object<MessageLite>);
741 }
742 }
743 template <typename T>
744 PROTOBUF_ALWAYS_INLINE void OwnInternal(T* object, std::false_type) {
745 if (object != NULL) {
746 impl_.AddCleanup(elem: object, cleanup: &internal::arena_delete_object<T>);
747 }
748 }
749
750 // Implementation for GetArena(). Only message objects with
751 // InternalArenaConstructable_ tags can be associated with an arena, and such
752 // objects must implement a GetArena() method.
753 template <typename T, typename std::enable_if<
754 is_arena_constructable<T>::value, int>::type = 0>
755 PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) {
756 return InternalHelper<T>::GetArena(value);
757 }
758 template <typename T,
759 typename std::enable_if<!is_arena_constructable<T>::value &&
760 has_get_arena<T>::value,
761 int>::type = 0>
762 PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) {
763 return value->GetArena();
764 }
765 template <typename T,
766 typename std::enable_if<!is_arena_constructable<T>::value &&
767 !has_get_arena<T>::value,
768 int>::type = 0>
769 PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) {
770 (void)value;
771 return nullptr;
772 }
773
774 template <typename T>
775 PROTOBUF_ALWAYS_INLINE static Arena* GetOwningArena(const T* value) {
776 return GetOwningArenaInternal(
777 value, std::is_convertible<T*, MessageLite*>());
778 }
779
780 // Implementation for GetOwningArena(). All and only message objects have
781 // GetOwningArena() method.
782 template <typename T>
783 PROTOBUF_ALWAYS_INLINE static Arena* GetOwningArenaInternal(
784 const T* value, std::true_type) {
785 return InternalHelper<T>::GetOwningArena(value);
786 }
787 template <typename T>
788 PROTOBUF_ALWAYS_INLINE static Arena* GetOwningArenaInternal(
789 const T* /* value */, std::false_type) {
790 return nullptr;
791 }
792
793 void* AllocateAlignedWithHookForArray(size_t n, size_t align,
794 const std::type_info* type) {
795 if (align <= 8) {
796 return AllocateAlignedWithHookForArray(n: internal::AlignUpTo8(n), type);
797 } else {
798 // We are wasting space by over allocating align - 8 bytes. Compared
799 // to a dedicated function that takes current alignment in consideration.
800 // Such a scheme would only waste (align - 8)/2 bytes on average, but
801 // requires a dedicated function in the outline arena allocation
802 // functions. Possibly re-evaluate tradeoffs later.
803 return internal::AlignTo(
804 ptr: AllocateAlignedWithHookForArray(n: n + align - 8, type), align);
805 }
806 }
807
808 void* AllocateAlignedWithHook(size_t n, size_t align,
809 const std::type_info* type) {
810 if (align <= 8) {
811 return AllocateAlignedWithHook(n: internal::AlignUpTo8(n), type);
812 } else {
813 // We are wasting space by over allocating align - 8 bytes. Compared
814 // to a dedicated function that takes current alignment in consideration.
815 // Such a scheme would only waste (align - 8)/2 bytes on average, but
816 // requires a dedicated function in the outline arena allocation
817 // functions. Possibly re-evaluate tradeoffs later.
818 return internal::AlignTo(ptr: AllocateAlignedWithHook(n: n + align - 8, type),
819 align);
820 }
821 }
822
823 void* AllocateAlignedNoHook(size_t n);
824 void* AllocateAlignedWithHook(size_t n, const std::type_info* type);
825 void* AllocateAlignedWithHookForArray(size_t n, const std::type_info* type);
826 std::pair<void*, internal::SerialArena::CleanupNode*>
827 AllocateAlignedWithCleanup(size_t n, const std::type_info* type);
828
829 template <typename Type>
830 friend class internal::GenericTypeHandler;
831 friend class internal::InternalMetadata; // For user_arena().
832 friend class internal::LazyField; // For CreateMaybeMessage.
833 friend class internal::EpsCopyInputStream; // For parser performance
834 friend class MessageLite;
835 template <typename Key, typename T>
836 friend class Map;
837 template <typename>
838 friend class RepeatedField; // For ReturnArrayMemory
839 friend class internal::RepeatedPtrFieldBase; // For ReturnArrayMemory
840 friend struct internal::ArenaTestPeer;
841};
842
843// Defined above for supporting environments without RTTI.
844#undef RTTI_TYPE_ID
845
846} // namespace protobuf
847} // namespace google
848
849#include <google/protobuf/port_undef.inc>
850
851#endif // GOOGLE_PROTOBUF_ARENA_H__
852