1// Copyright (c) Microsoft Corporation. All rights reserved.
2// Licensed under the MIT license.
3
4#include <cassert>
5#include <cstdlib>
6
7#include "alloc.h"
8#include "auto_ptr.h"
9#include "lss_allocator.h"
10#include "thread.h"
11
12namespace FASTER {
13namespace core {
14
15#define thread_index_ Thread::id()
16
17LssAllocator lss_allocator{};
18
19namespace lss_memory {
20
21static_assert(sizeof(Header) < kBaseAlignment, "Unexpected header size!");
22
23void SegmentAllocator::Free(void* bytes) {
24#ifdef _DEBUG
25 Header* header = reinterpret_cast<Header*>(bytes) - 1;
26 assert(header->offset < kSegmentSize);
27 assert(header->offset + header->size <= kSegmentSize);
28 // - 0xDA - freed.
29 ::memset(header + 1, 0xDA, header->size);
30#endif
31 Free();
32}
33
34void SegmentAllocator::Seal(uint32_t allocations) {
35 SegmentState delta_state{ allocations, 1 };
36 SegmentState old_state{ state.control.fetch_add(delta_state.control) };
37 assert(old_state.allocations == 0);
38 assert(old_state.frees < allocations);
39 if(allocations == old_state.frees + 1) {
40 // We were the last to free a block inside this segment, so we must free it.
41 this->~SegmentAllocator();
42 aligned_free(this);
43 }
44}
45
46void SegmentAllocator::Free() {
47 SegmentState delta_state{ 0, 1 };
48 SegmentState old_state{ state.control.fetch_add(delta_state.control) };
49 assert(old_state.allocations == 0 || old_state.frees < old_state.allocations);
50 if(old_state.allocations == old_state.frees + 1) {
51 // We were the last to free a block inside this segment, so we must free it.
52 this->~SegmentAllocator();
53 aligned_free(this);
54 }
55}
56
57void* ThreadAllocator::Allocate(uint32_t size) {
58 if(!segment_allocator_) {
59 segment_allocator_ = reinterpret_cast<SegmentAllocator*>(aligned_alloc(kCacheLineSize,
60 sizeof(SegmentAllocator)));
61 if(!segment_allocator_) {
62 return nullptr;
63 }
64 new(segment_allocator_) SegmentAllocator{};
65 }
66 // Block is 16-byte aligned, after a 2-byte (8-byte in _DEBUG mode) header.
67 uint32_t block_size = static_cast<uint32_t>(pad_alignment(size + sizeof(Header),
68 kBaseAlignment));
69 uint32_t offset = Reserve(block_size);
70 if(segment_offset_ <= kSegmentSize) {
71 // The allocation succeeded inside the active segment.
72 uint8_t* buffer = segment_allocator_->buffer;
73#ifdef _DEBUG
74 // - 0xCA - allocated.
75 ::memset(&buffer[offset], 0xCA, block_size);
76#endif
77 Header* header = reinterpret_cast<Header*>(&buffer[offset]);
78#ifdef _DEBUG
79 new(header) Header(size, offset);
80#else
81 new(header) Header(offset);
82#endif
83 return header + 1;
84 } else {
85 // We filled the active segment; seal it.
86 segment_allocator_->Seal(allocations_);
87 segment_allocator_ = nullptr;
88 allocations_ = 0;
89 segment_offset_ = 0;
90 // Call self recursively, to allocate inside a new segment.
91 return Allocate(size);
92 }
93}
94
95void* ThreadAllocator::AllocateAligned(uint32_t size, uint32_t alignment) {
96 if(!segment_allocator_) {
97 segment_allocator_ = reinterpret_cast<SegmentAllocator*>(aligned_alloc(kCacheLineSize,
98 sizeof(SegmentAllocator)));
99 if(!segment_allocator_) {
100 return nullptr;
101 }
102 new(segment_allocator_) SegmentAllocator{};
103 }
104 // Alignment must be >= base alignment, and a power of 2.
105 assert(alignment >= kBaseAlignment);
106 assert((alignment & (alignment - 1)) == 0);
107 // Block needs to be large enough to hold the user block, the header, and the align land fill.
108 // Max align land fill size is (alignment - kBaseAlignment).
109 uint32_t block_size = static_cast<uint32_t>(pad_alignment(
110 size + sizeof(Header) + alignment - kBaseAlignment,
111 kBaseAlignment));
112 uint32_t block_offset = Reserve(block_size);
113 if(segment_offset_ <= kSegmentSize) {
114 // The allocation succeeded inside the active segment.
115 uint8_t* buffer = segment_allocator_->buffer;
116#ifdef _DEBUG
117 // - 0xEA - align land fill.
118 ::memset(&buffer[block_offset], 0xEA, block_size);
119#endif
120 // Align the user block.
121 uint32_t user_offset = static_cast<uint32_t>(pad_alignment(reinterpret_cast<size_t>(
122 &buffer[block_offset]) + sizeof(Header), alignment) -
123 reinterpret_cast<size_t>(&buffer[block_offset]) - sizeof(Header));
124 assert(user_offset + sizeof(Header) + size <= block_size);
125 uint32_t offset = block_offset + user_offset;
126#ifdef _DEBUG
127 // - 0xCA - allocated.
128 ::memset(&buffer[offset], 0xCA, size + sizeof(Header));
129#endif
130 Header* header = reinterpret_cast<Header*>(&buffer[offset]);
131#ifdef _DEBUG
132 new(header) Header(size, offset);
133#else
134 new(header) Header(offset);
135#endif
136 return header + 1;
137 } else {
138 // We filled the active segment; seal it.
139 segment_allocator_->Seal(allocations_);
140 segment_allocator_ = nullptr;
141 allocations_ = 0;
142 segment_offset_ = 0;
143 // Call self recursively, to allocate inside a new segment.
144 return AllocateAligned(size, alignment);
145 }
146}
147} // namespace lss_memory
148
149void* LssAllocator::Allocate(uint32_t size) {
150 return thread_allocators_[thread_index_].Allocate(size);
151}
152
153void* LssAllocator::AllocateAligned(uint32_t size, uint32_t alignment) {
154 return thread_allocators_[thread_index_].AllocateAligned(size, alignment);
155}
156
157void LssAllocator::Free(void* bytes) {
158 lss_memory::Header* header = reinterpret_cast<lss_memory::Header*>(bytes) - 1;
159 uint8_t* block = reinterpret_cast<uint8_t*>(header);
160 uint32_t offset = header->offset + lss_memory::SegmentAllocator::kBufferOffset;
161 lss_memory::SegmentAllocator* segment_allocator =
162 reinterpret_cast<lss_memory::SegmentAllocator*>(block - offset);
163 segment_allocator->Free(bytes);
164}
165
166#undef thread_index_
167
168}
169} // namespace FASTER::core
170