1 | /* |
2 | * Copyright 2013-present Facebook, Inc. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | |
17 | #pragma once |
18 | |
19 | #include <glog/logging.h> |
20 | #include <atomic> |
21 | #include <cassert> |
22 | #include <cinttypes> |
23 | #include <cstddef> |
24 | #include <cstring> |
25 | #include <iterator> |
26 | #include <limits> |
27 | #include <memory> |
28 | #include <type_traits> |
29 | |
30 | #include <folly/FBString.h> |
31 | #include <folly/FBVector.h> |
32 | #include <folly/Portability.h> |
33 | #include <folly/Range.h> |
34 | #include <folly/detail/Iterators.h> |
35 | #include <folly/lang/Ordering.h> |
36 | #include <folly/portability/SysUio.h> |
37 | |
38 | // Ignore shadowing warnings within this file, so includers can use -Wshadow. |
39 | FOLLY_PUSH_WARNING |
40 | FOLLY_GNU_DISABLE_WARNING("-Wshadow" ) |
41 | |
42 | namespace folly { |
43 | |
44 | /** |
45 | * An IOBuf is a pointer to a buffer of data. |
46 | * |
47 | * IOBuf objects are intended to be used primarily for networking code, and are |
48 | * modelled somewhat after FreeBSD's mbuf data structure, and Linux's sk_buff |
49 | * structure. |
50 | * |
51 | * IOBuf objects facilitate zero-copy network programming, by allowing multiple |
52 | * IOBuf objects to point to the same underlying buffer of data, using a |
53 | * reference count to track when the buffer is no longer needed and can be |
54 | * freed. |
55 | * |
56 | * |
57 | * Data Layout |
58 | * ----------- |
59 | * |
60 | * The IOBuf itself is a small object containing a pointer to the buffer and |
61 | * information about which segment of the buffer contains valid data. |
62 | * |
63 | * The data layout looks like this: |
64 | * |
65 | * +-------+ |
66 | * | IOBuf | |
67 | * +-------+ |
68 | * / |
69 | * | |
70 | * v |
71 | * +------------+--------------------+-----------+ |
72 | * | headroom | data | tailroom | |
73 | * +------------+--------------------+-----------+ |
74 | * ^ ^ ^ ^ |
75 | * buffer() data() tail() bufferEnd() |
76 | * |
77 | * The length() method returns the length of the valid data; capacity() |
78 | * returns the entire capacity of the buffer (from buffer() to bufferEnd()). |
79 | * The headroom() and tailroom() methods return the amount of unused capacity |
80 | * available before and after the data. |
81 | * |
82 | * |
83 | * Buffer Sharing |
84 | * -------------- |
85 | * |
86 | * The buffer itself is reference counted, and multiple IOBuf objects may point |
87 | * to the same buffer. Each IOBuf may point to a different section of valid |
88 | * data within the underlying buffer. For example, if multiple protocol |
89 | * requests are read from the network into a single buffer, a separate IOBuf |
90 | * may be created for each request, all sharing the same underlying buffer. |
91 | * |
92 | * In other words, when multiple IOBufs share the same underlying buffer, the |
93 | * data() and tail() methods on each IOBuf may point to a different segment of |
94 | * the data. However, the buffer() and bufferEnd() methods will point to the |
95 | * same location for all IOBufs sharing the same underlying buffer. |
96 | * |
97 | * +-----------+ +---------+ |
98 | * | IOBuf 1 | | IOBuf 2 | |
99 | * +-----------+ +---------+ |
100 | * | | _____/ | |
101 | * data | tail |/ data | tail |
102 | * v v v |
103 | * +-------------------------------------+ |
104 | * | | | | | |
105 | * +-------------------------------------+ |
106 | * |
107 | * If you only read data from an IOBuf, you don't need to worry about other |
108 | * IOBuf objects possibly sharing the same underlying buffer. However, if you |
109 | * ever write to the buffer you need to first ensure that no other IOBufs point |
110 | * to the same buffer. The unshare() method may be used to ensure that you |
111 | * have an unshared buffer. |
112 | * |
113 | * |
114 | * IOBuf Chains |
115 | * ------------ |
116 | * |
117 | * IOBuf objects also contain pointers to next and previous IOBuf objects. |
118 | * This can be used to represent a single logical piece of data that its stored |
119 | * in non-contiguous chunks in separate buffers. |
120 | * |
121 | * A single IOBuf object can only belong to one chain at a time. |
122 | * |
123 | * IOBuf chains are always circular. The "prev" pointer in the head of the |
124 | * chain points to the tail of the chain. However, it is up to the user to |
125 | * decide which IOBuf is the head. Internally the IOBuf code does not care |
126 | * which element is the head. |
127 | * |
128 | * The lifetime of all IOBufs in the chain are linked: when one element in the |
129 | * chain is deleted, all other chained elements are also deleted. Conceptually |
130 | * it is simplest to treat this as if the head of the chain owns all other |
131 | * IOBufs in the chain. When you delete the head of the chain, it will delete |
132 | * the other elements as well. For this reason, prependChain() and |
133 | * appendChain() take ownership of of the new elements being added to this |
134 | * chain. |
135 | * |
136 | * When the coalesce() method is used to coalesce an entire IOBuf chain into a |
137 | * single IOBuf, all other IOBufs in the chain are eliminated and automatically |
138 | * deleted. The unshare() method may coalesce the chain; if it does it will |
139 | * similarly delete all IOBufs eliminated from the chain. |
140 | * |
141 | * As discussed in the following section, it is up to the user to maintain a |
142 | * lock around the entire IOBuf chain if multiple threads need to access the |
143 | * chain. IOBuf does not provide any internal locking. |
144 | * |
145 | * |
146 | * Synchronization |
147 | * --------------- |
148 | * |
149 | * When used in multithread programs, a single IOBuf object should only be used |
150 | * in a single thread at a time. If a caller uses a single IOBuf across |
151 | * multiple threads the caller is responsible for using an external lock to |
152 | * synchronize access to the IOBuf. |
153 | * |
154 | * Two separate IOBuf objects may be accessed concurrently in separate threads |
155 | * without locking, even if they point to the same underlying buffer. The |
156 | * buffer reference count is always accessed atomically, and no other |
157 | * operations should affect other IOBufs that point to the same data segment. |
158 | * The caller is responsible for using unshare() to ensure that the data buffer |
159 | * is not shared by other IOBufs before writing to it, and this ensures that |
160 | * the data itself is not modified in one thread while also being accessed from |
161 | * another thread. |
162 | * |
163 | * For IOBuf chains, no two IOBufs in the same chain should be accessed |
164 | * simultaneously in separate threads. The caller must maintain a lock around |
165 | * the entire chain if the chain, or individual IOBufs in the chain, may be |
166 | * accessed by multiple threads. |
167 | * |
168 | * |
169 | * IOBuf Object Allocation |
170 | * ----------------------- |
171 | * |
172 | * IOBuf objects themselves exist separately from the data buffer they point |
173 | * to. Therefore one must also consider how to allocate and manage the IOBuf |
174 | * objects. |
175 | * |
176 | * It is more common to allocate IOBuf objects on the heap, using the create(), |
177 | * takeOwnership(), or wrapBuffer() factory functions. The clone()/cloneOne() |
178 | * functions also return new heap-allocated IOBufs. The createCombined() |
179 | * function allocates the IOBuf object and data storage space together, in a |
180 | * single memory allocation. This can improve performance, particularly if you |
181 | * know that the data buffer and the IOBuf itself will have similar lifetimes. |
182 | * |
183 | * That said, it is also possible to allocate IOBufs on the stack or inline |
184 | * inside another object as well. This is useful for cases where the IOBuf is |
185 | * short-lived, or when the overhead of allocating the IOBuf on the heap is |
186 | * undesirable. |
187 | * |
188 | * However, note that stack-allocated IOBufs may only be used as the head of a |
189 | * chain (or standalone as the only IOBuf in a chain). All non-head members of |
190 | * an IOBuf chain must be heap allocated. (All functions to add nodes to a |
191 | * chain require a std::unique_ptr<IOBuf>, which enforces this requrement.) |
192 | * |
193 | * Copying IOBufs is only meaningful for the head of a chain. The entire chain |
194 | * is cloned; the IOBufs will become shared, and the old and new IOBufs will |
195 | * refer to the same underlying memory. |
196 | * |
197 | * IOBuf Sharing |
198 | * ------------- |
199 | * |
200 | * The IOBuf class manages sharing of the underlying buffer that it points to, |
201 | * maintaining a reference count if multiple IOBufs are pointing at the same |
202 | * buffer. |
203 | * |
204 | * However, it is the callers responsibility to manage sharing and ownership of |
205 | * IOBuf objects themselves. The IOBuf structure does not provide room for an |
206 | * intrusive refcount on the IOBuf object itself, only the underlying data |
207 | * buffer is reference counted. If users want to share the same IOBuf object |
208 | * between multiple parts of the code, they are responsible for managing this |
209 | * sharing on their own. (For example, by using a shared_ptr. Alternatively, |
210 | * users always have the option of using clone() to create a second IOBuf that |
211 | * points to the same underlying buffer.) |
212 | */ |
213 | namespace detail { |
214 | // Is T a unique_ptr<> to a standard-layout type? |
215 | template <typename T> |
216 | struct IsUniquePtrToSL : std::false_type {}; |
217 | template <typename T, typename D> |
218 | struct IsUniquePtrToSL<std::unique_ptr<T, D>> : std::is_standard_layout<T> {}; |
219 | } // namespace detail |
220 | |
221 | class IOBuf { |
222 | public: |
223 | class Iterator; |
224 | |
225 | enum CreateOp { CREATE }; |
226 | enum WrapBufferOp { WRAP_BUFFER }; |
227 | enum TakeOwnershipOp { TAKE_OWNERSHIP }; |
228 | enum CopyBufferOp { COPY_BUFFER }; |
229 | |
230 | typedef ByteRange value_type; |
231 | typedef Iterator iterator; |
232 | typedef Iterator const_iterator; |
233 | |
234 | typedef void (*FreeFunction)(void* buf, void* userData); |
235 | |
236 | /** |
237 | * Allocate a new IOBuf object with the requested capacity. |
238 | * |
239 | * Returns a new IOBuf object that must be (eventually) deleted by the |
240 | * caller. The returned IOBuf may actually have slightly more capacity than |
241 | * requested. |
242 | * |
243 | * The data pointer will initially point to the start of the newly allocated |
244 | * buffer, and will have a data length of 0. |
245 | * |
246 | * Throws std::bad_alloc on error. |
247 | */ |
248 | static std::unique_ptr<IOBuf> create(std::size_t capacity); |
249 | IOBuf(CreateOp, std::size_t capacity); |
250 | |
251 | /** |
252 | * Create a new IOBuf, using a single memory allocation to allocate space |
253 | * for both the IOBuf object and the data storage space. |
254 | * |
255 | * This saves one memory allocation. However, it can be wasteful if you |
256 | * later need to grow the buffer using reserve(). If the buffer needs to be |
257 | * reallocated, the space originally allocated will not be freed() until the |
258 | * IOBuf object itself is also freed. (It can also be slightly wasteful in |
259 | * some cases where you clone this IOBuf and then free the original IOBuf.) |
260 | */ |
261 | static std::unique_ptr<IOBuf> createCombined(std::size_t capacity); |
262 | |
263 | /** |
264 | * Create a new IOBuf, using separate memory allocations for the IOBuf object |
265 | * for the IOBuf and the data storage space. |
266 | * |
267 | * This requires two memory allocations, but saves space in the long run |
268 | * if you know that you will need to reallocate the data buffer later. |
269 | */ |
270 | static std::unique_ptr<IOBuf> createSeparate(std::size_t capacity); |
271 | |
272 | /** |
273 | * Allocate a new IOBuf chain with the requested total capacity, allocating |
274 | * no more than maxBufCapacity to each buffer. |
275 | */ |
276 | static std::unique_ptr<IOBuf> createChain( |
277 | size_t totalCapacity, |
278 | std::size_t maxBufCapacity); |
279 | |
280 | /** |
281 | * Create a new IOBuf pointing to an existing data buffer. |
282 | * |
283 | * The new IOBuffer will assume ownership of the buffer, and free it by |
284 | * calling the specified FreeFunction when the last IOBuf pointing to this |
285 | * buffer is destroyed. The function will be called with a pointer to the |
286 | * buffer as the first argument, and the supplied userData value as the |
287 | * second argument. The free function must never throw exceptions. |
288 | * |
289 | * If no FreeFunction is specified, the buffer will be freed using free() |
290 | * which will result in undefined behavior if the memory was allocated |
291 | * using 'new'. |
292 | * |
293 | * The IOBuf data pointer will initially point to the start of the buffer, |
294 | * |
295 | * In the first version of this function, the length of data is unspecified |
296 | * and is initialized to the capacity of the buffer |
297 | * |
298 | * In the second version, the user specifies the valid length of data |
299 | * in the buffer |
300 | * |
301 | * On error, std::bad_alloc will be thrown. If freeOnError is true (the |
302 | * default) the buffer will be freed before throwing the error. |
303 | */ |
304 | static std::unique_ptr<IOBuf> takeOwnership( |
305 | void* buf, |
306 | std::size_t capacity, |
307 | FreeFunction freeFn = nullptr, |
308 | void* userData = nullptr, |
309 | bool freeOnError = true) { |
310 | return takeOwnership( |
311 | buf, capacity, capacity, freeFn, userData, freeOnError); |
312 | } |
313 | IOBuf( |
314 | TakeOwnershipOp op, |
315 | void* buf, |
316 | std::size_t capacity, |
317 | FreeFunction freeFn = nullptr, |
318 | void* userData = nullptr, |
319 | bool freeOnError = true) |
320 | : IOBuf(op, buf, capacity, capacity, freeFn, userData, freeOnError) {} |
321 | |
322 | static std::unique_ptr<IOBuf> takeOwnership( |
323 | void* buf, |
324 | std::size_t capacity, |
325 | std::size_t length, |
326 | FreeFunction freeFn = nullptr, |
327 | void* userData = nullptr, |
328 | bool freeOnError = true); |
329 | IOBuf( |
330 | TakeOwnershipOp, |
331 | void* buf, |
332 | std::size_t capacity, |
333 | std::size_t length, |
334 | FreeFunction freeFn = nullptr, |
335 | void* userData = nullptr, |
336 | bool freeOnError = true); |
337 | |
338 | /** |
339 | * Create a new IOBuf pointing to an existing data buffer made up of |
340 | * count objects of a given standard-layout type. |
341 | * |
342 | * This is dangerous -- it is essentially equivalent to doing |
343 | * reinterpret_cast<unsigned char*> on your data -- but it's often useful |
344 | * for serialization / deserialization. |
345 | * |
346 | * The new IOBuffer will assume ownership of the buffer, and free it |
347 | * appropriately (by calling the UniquePtr's custom deleter, or by calling |
348 | * delete or delete[] appropriately if there is no custom deleter) |
349 | * when the buffer is destroyed. The custom deleter, if any, must never |
350 | * throw exceptions. |
351 | * |
352 | * The IOBuf data pointer will initially point to the start of the buffer, |
353 | * and the length will be the full capacity of the buffer (count * |
354 | * sizeof(T)). |
355 | * |
356 | * On error, std::bad_alloc will be thrown, and the buffer will be freed |
357 | * before throwing the error. |
358 | */ |
359 | template <class UniquePtr> |
360 | static typename std::enable_if< |
361 | detail::IsUniquePtrToSL<UniquePtr>::value, |
362 | std::unique_ptr<IOBuf>>::type |
363 | takeOwnership(UniquePtr&& buf, size_t count = 1); |
364 | |
365 | /** |
366 | * Create a new IOBuf object that points to an existing user-owned buffer. |
367 | * |
368 | * This should only be used when the caller knows the lifetime of the IOBuf |
369 | * object ahead of time and can ensure that all IOBuf objects that will point |
370 | * to this buffer will be destroyed before the buffer itself is destroyed. |
371 | * |
372 | * This buffer will not be freed automatically when the last IOBuf |
373 | * referencing it is destroyed. It is the caller's responsibility to free |
374 | * the buffer after the last IOBuf has been destroyed. |
375 | * |
376 | * The IOBuf data pointer will initially point to the start of the buffer, |
377 | * and the length will be the full capacity of the buffer. |
378 | * |
379 | * An IOBuf created using wrapBuffer() will always be reported as shared. |
380 | * unshare() may be used to create a writable copy of the buffer. |
381 | * |
382 | * On error, std::bad_alloc will be thrown. |
383 | */ |
384 | static std::unique_ptr<IOBuf> wrapBuffer( |
385 | const void* buf, |
386 | std::size_t capacity); |
387 | static std::unique_ptr<IOBuf> wrapBuffer(ByteRange br) { |
388 | return wrapBuffer(br.data(), br.size()); |
389 | } |
390 | |
391 | /** |
392 | * Similar to wrapBuffer(), but returns IOBuf by value rather than |
393 | * heap-allocating it. |
394 | */ |
395 | static IOBuf wrapBufferAsValue( |
396 | const void* buf, |
397 | std::size_t capacity) noexcept; |
398 | static IOBuf wrapBufferAsValue(ByteRange br) noexcept { |
399 | return wrapBufferAsValue(br.data(), br.size()); |
400 | } |
401 | |
402 | IOBuf(WrapBufferOp op, const void* buf, std::size_t capacity) noexcept; |
403 | IOBuf(WrapBufferOp op, ByteRange br) noexcept; |
404 | |
405 | /** |
406 | * Convenience function to create a new IOBuf object that copies data from a |
407 | * user-supplied buffer, optionally allocating a given amount of |
408 | * headroom and tailroom. |
409 | */ |
410 | static std::unique_ptr<IOBuf> copyBuffer( |
411 | const void* buf, |
412 | std::size_t size, |
413 | std::size_t headroom = 0, |
414 | std::size_t minTailroom = 0); |
415 | static std::unique_ptr<IOBuf> copyBuffer( |
416 | ByteRange br, |
417 | std::size_t headroom = 0, |
418 | std::size_t minTailroom = 0) { |
419 | return copyBuffer(br.data(), br.size(), headroom, minTailroom); |
420 | } |
421 | IOBuf( |
422 | CopyBufferOp op, |
423 | const void* buf, |
424 | std::size_t size, |
425 | std::size_t headroom = 0, |
426 | std::size_t minTailroom = 0); |
427 | IOBuf( |
428 | CopyBufferOp op, |
429 | ByteRange br, |
430 | std::size_t headroom = 0, |
431 | std::size_t minTailroom = 0); |
432 | |
433 | /** |
434 | * Convenience function to create a new IOBuf object that copies data from a |
435 | * user-supplied string, optionally allocating a given amount of |
436 | * headroom and tailroom. |
437 | * |
438 | * Beware when attempting to invoke this function with a constant string |
439 | * literal and a headroom argument: you will likely end up invoking the |
440 | * version of copyBuffer() above. IOBuf::copyBuffer("hello", 3) will treat |
441 | * the first argument as a const void*, and will invoke the version of |
442 | * copyBuffer() above, with the size argument of 3. |
443 | */ |
444 | static std::unique_ptr<IOBuf> copyBuffer( |
445 | const std::string& buf, |
446 | std::size_t headroom = 0, |
447 | std::size_t minTailroom = 0); |
448 | IOBuf( |
449 | CopyBufferOp op, |
450 | const std::string& buf, |
451 | std::size_t headroom = 0, |
452 | std::size_t minTailroom = 0) |
453 | : IOBuf(op, buf.data(), buf.size(), headroom, minTailroom) {} |
454 | |
455 | /** |
456 | * A version of copyBuffer() that returns a null pointer if the input string |
457 | * is empty. |
458 | */ |
459 | static std::unique_ptr<IOBuf> maybeCopyBuffer( |
460 | const std::string& buf, |
461 | std::size_t headroom = 0, |
462 | std::size_t minTailroom = 0); |
463 | |
464 | /** |
465 | * Convenience function to free a chain of IOBufs held by a unique_ptr. |
466 | */ |
467 | static void destroy(std::unique_ptr<IOBuf>&& data) { |
468 | auto destroyer = std::move(data); |
469 | } |
470 | |
471 | /** |
472 | * Destroy this IOBuf. |
473 | * |
474 | * Deleting an IOBuf will automatically destroy all IOBufs in the chain. |
475 | * (See the comments above regarding the ownership model of IOBuf chains. |
476 | * All subsequent IOBufs in the chain are considered to be owned by the head |
477 | * of the chain. Users should only explicitly delete the head of a chain.) |
478 | * |
479 | * When each individual IOBuf is destroyed, it will release its reference |
480 | * count on the underlying buffer. If it was the last user of the buffer, |
481 | * the buffer will be freed. |
482 | */ |
483 | ~IOBuf(); |
484 | |
485 | /** |
486 | * Check whether the chain is empty (i.e., whether the IOBufs in the |
487 | * chain have a total data length of zero). |
488 | * |
489 | * This method is semantically equivalent to |
490 | * i->computeChainDataLength()==0 |
491 | * but may run faster because it can short-circuit as soon as it |
492 | * encounters a buffer with length()!=0 |
493 | */ |
494 | bool empty() const; |
495 | |
496 | /** |
497 | * Get the pointer to the start of the data. |
498 | */ |
499 | const uint8_t* data() const { |
500 | return data_; |
501 | } |
502 | |
503 | /** |
504 | * Get a writable pointer to the start of the data. |
505 | * |
506 | * The caller is responsible for calling unshare() first to ensure that it is |
507 | * actually safe to write to the buffer. |
508 | */ |
509 | uint8_t* writableData() { |
510 | return data_; |
511 | } |
512 | |
513 | /** |
514 | * Get the pointer to the end of the data. |
515 | */ |
516 | const uint8_t* tail() const { |
517 | return data_ + length_; |
518 | } |
519 | |
520 | /** |
521 | * Get a writable pointer to the end of the data. |
522 | * |
523 | * The caller is responsible for calling unshare() first to ensure that it is |
524 | * actually safe to write to the buffer. |
525 | */ |
526 | uint8_t* writableTail() { |
527 | return data_ + length_; |
528 | } |
529 | |
530 | /** |
531 | * Get the data length. |
532 | */ |
533 | std::size_t length() const { |
534 | return length_; |
535 | } |
536 | |
537 | /** |
538 | * Get the amount of head room. |
539 | * |
540 | * Returns the number of bytes in the buffer before the start of the data. |
541 | */ |
542 | std::size_t headroom() const { |
543 | return std::size_t(data_ - buffer()); |
544 | } |
545 | |
546 | /** |
547 | * Get the amount of tail room. |
548 | * |
549 | * Returns the number of bytes in the buffer after the end of the data. |
550 | */ |
551 | std::size_t tailroom() const { |
552 | return std::size_t(bufferEnd() - tail()); |
553 | } |
554 | |
555 | /** |
556 | * Get the pointer to the start of the buffer. |
557 | * |
558 | * Note that this is the pointer to the very beginning of the usable buffer, |
559 | * not the start of valid data within the buffer. Use the data() method to |
560 | * get a pointer to the start of the data within the buffer. |
561 | */ |
562 | const uint8_t* buffer() const { |
563 | return buf_; |
564 | } |
565 | |
566 | /** |
567 | * Get a writable pointer to the start of the buffer. |
568 | * |
569 | * The caller is responsible for calling unshare() first to ensure that it is |
570 | * actually safe to write to the buffer. |
571 | */ |
572 | uint8_t* writableBuffer() { |
573 | return buf_; |
574 | } |
575 | |
576 | /** |
577 | * Get the pointer to the end of the buffer. |
578 | * |
579 | * Note that this is the pointer to the very end of the usable buffer, |
580 | * not the end of valid data within the buffer. Use the tail() method to |
581 | * get a pointer to the end of the data within the buffer. |
582 | */ |
583 | const uint8_t* bufferEnd() const { |
584 | return buf_ + capacity_; |
585 | } |
586 | |
587 | /** |
588 | * Get the total size of the buffer. |
589 | * |
590 | * This returns the total usable length of the buffer. Use the length() |
591 | * method to get the length of the actual valid data in this IOBuf. |
592 | */ |
593 | std::size_t capacity() const { |
594 | return capacity_; |
595 | } |
596 | |
597 | /** |
598 | * Get a pointer to the next IOBuf in this chain. |
599 | */ |
600 | IOBuf* next() { |
601 | return next_; |
602 | } |
603 | const IOBuf* next() const { |
604 | return next_; |
605 | } |
606 | |
607 | /** |
608 | * Get a pointer to the previous IOBuf in this chain. |
609 | */ |
610 | IOBuf* prev() { |
611 | return prev_; |
612 | } |
613 | const IOBuf* prev() const { |
614 | return prev_; |
615 | } |
616 | |
617 | /** |
618 | * Shift the data forwards in the buffer. |
619 | * |
620 | * This shifts the data pointer forwards in the buffer to increase the |
621 | * headroom. This is commonly used to increase the headroom in a newly |
622 | * allocated buffer. |
623 | * |
624 | * The caller is responsible for ensuring that there is sufficient |
625 | * tailroom in the buffer before calling advance(). |
626 | * |
627 | * If there is a non-zero data length, advance() will use memmove() to shift |
628 | * the data forwards in the buffer. In this case, the caller is responsible |
629 | * for making sure the buffer is unshared, so it will not affect other IOBufs |
630 | * that may be sharing the same underlying buffer. |
631 | */ |
632 | void advance(std::size_t amount) { |
633 | // In debug builds, assert if there is a problem. |
634 | assert(amount <= tailroom()); |
635 | |
636 | if (length_ > 0) { |
637 | memmove(data_ + amount, data_, length_); |
638 | } |
639 | data_ += amount; |
640 | } |
641 | |
642 | /** |
643 | * Shift the data backwards in the buffer. |
644 | * |
645 | * The caller is responsible for ensuring that there is sufficient headroom |
646 | * in the buffer before calling retreat(). |
647 | * |
648 | * If there is a non-zero data length, retreat() will use memmove() to shift |
649 | * the data backwards in the buffer. In this case, the caller is responsible |
650 | * for making sure the buffer is unshared, so it will not affect other IOBufs |
651 | * that may be sharing the same underlying buffer. |
652 | */ |
653 | void retreat(std::size_t amount) { |
654 | // In debug builds, assert if there is a problem. |
655 | assert(amount <= headroom()); |
656 | |
657 | if (length_ > 0) { |
658 | memmove(data_ - amount, data_, length_); |
659 | } |
660 | data_ -= amount; |
661 | } |
662 | |
663 | /** |
664 | * Adjust the data pointer to include more valid data at the beginning. |
665 | * |
666 | * This moves the data pointer backwards to include more of the available |
667 | * buffer. The caller is responsible for ensuring that there is sufficient |
668 | * headroom for the new data. The caller is also responsible for populating |
669 | * this section with valid data. |
670 | * |
671 | * This does not modify any actual data in the buffer. |
672 | */ |
673 | void prepend(std::size_t amount) { |
674 | DCHECK_LE(amount, headroom()); |
675 | data_ -= amount; |
676 | length_ += amount; |
677 | } |
678 | |
679 | /** |
680 | * Adjust the tail pointer to include more valid data at the end. |
681 | * |
682 | * This moves the tail pointer forwards to include more of the available |
683 | * buffer. The caller is responsible for ensuring that there is sufficient |
684 | * tailroom for the new data. The caller is also responsible for populating |
685 | * this section with valid data. |
686 | * |
687 | * This does not modify any actual data in the buffer. |
688 | */ |
689 | void append(std::size_t amount) { |
690 | DCHECK_LE(amount, tailroom()); |
691 | length_ += amount; |
692 | } |
693 | |
694 | /** |
695 | * Adjust the data pointer forwards to include less valid data. |
696 | * |
697 | * This moves the data pointer forwards so that the first amount bytes are no |
698 | * longer considered valid data. The caller is responsible for ensuring that |
699 | * amount is less than or equal to the actual data length. |
700 | * |
701 | * This does not modify any actual data in the buffer. |
702 | */ |
703 | void trimStart(std::size_t amount) { |
704 | DCHECK_LE(amount, length_); |
705 | data_ += amount; |
706 | length_ -= amount; |
707 | } |
708 | |
709 | /** |
710 | * Adjust the tail pointer backwards to include less valid data. |
711 | * |
712 | * This moves the tail pointer backwards so that the last amount bytes are no |
713 | * longer considered valid data. The caller is responsible for ensuring that |
714 | * amount is less than or equal to the actual data length. |
715 | * |
716 | * This does not modify any actual data in the buffer. |
717 | */ |
718 | void trimEnd(std::size_t amount) { |
719 | DCHECK_LE(amount, length_); |
720 | length_ -= amount; |
721 | } |
722 | |
723 | /** |
724 | * Clear the buffer. |
725 | * |
726 | * Postcondition: headroom() == 0, length() == 0, tailroom() == capacity() |
727 | */ |
728 | void clear() { |
729 | data_ = writableBuffer(); |
730 | length_ = 0; |
731 | } |
732 | |
733 | /** |
734 | * Ensure that this buffer has at least minHeadroom headroom bytes and at |
735 | * least minTailroom tailroom bytes. The buffer must be writable |
736 | * (you must call unshare() before this, if necessary). |
737 | * |
738 | * Postcondition: headroom() >= minHeadroom, tailroom() >= minTailroom, |
739 | * the data (between data() and data() + length()) is preserved. |
740 | */ |
741 | void reserve(std::size_t minHeadroom, std::size_t minTailroom) { |
742 | // Maybe we don't need to do anything. |
743 | if (headroom() >= minHeadroom && tailroom() >= minTailroom) { |
744 | return; |
745 | } |
746 | // If the buffer is empty but we have enough total room (head + tail), |
747 | // move the data_ pointer around. |
748 | if (length() == 0 && headroom() + tailroom() >= minHeadroom + minTailroom) { |
749 | data_ = writableBuffer() + minHeadroom; |
750 | return; |
751 | } |
752 | // Bah, we have to do actual work. |
753 | reserveSlow(minHeadroom, minTailroom); |
754 | } |
755 | |
756 | /** |
757 | * Return true if this IOBuf is part of a chain of multiple IOBufs, or false |
758 | * if this is the only IOBuf in its chain. |
759 | */ |
760 | bool isChained() const { |
761 | assert((next_ == this) == (prev_ == this)); |
762 | return next_ != this; |
763 | } |
764 | |
765 | /** |
766 | * Get the number of IOBufs in this chain. |
767 | * |
768 | * Beware that this method has to walk the entire chain. |
769 | * Use isChained() if you just want to check if this IOBuf is part of a chain |
770 | * or not. |
771 | */ |
772 | size_t countChainElements() const; |
773 | |
774 | /** |
775 | * Get the length of all the data in this IOBuf chain. |
776 | * |
777 | * Beware that this method has to walk the entire chain. |
778 | */ |
779 | std::size_t computeChainDataLength() const; |
780 | |
781 | /** |
782 | * Insert another IOBuf chain immediately before this IOBuf. |
783 | * |
784 | * For example, if there are two IOBuf chains (A, B, C) and (D, E, F), |
785 | * and B->prependChain(D) is called, the (D, E, F) chain will be subsumed |
786 | * and become part of the chain starting at A, which will now look like |
787 | * (A, D, E, F, B, C) |
788 | * |
789 | * Note that since IOBuf chains are circular, head->prependChain(other) can |
790 | * be used to append the other chain at the very end of the chain pointed to |
791 | * by head. For example, if there are two IOBuf chains (A, B, C) and |
792 | * (D, E, F), and A->prependChain(D) is called, the chain starting at A will |
793 | * now consist of (A, B, C, D, E, F) |
794 | * |
795 | * The elements in the specified IOBuf chain will become part of this chain, |
796 | * and will be owned by the head of this chain. When this chain is |
797 | * destroyed, all elements in the supplied chain will also be destroyed. |
798 | * |
799 | * For this reason, appendChain() only accepts an rvalue-reference to a |
800 | * unique_ptr(), to make it clear that it is taking ownership of the supplied |
801 | * chain. If you have a raw pointer, you can pass in a new temporary |
802 | * unique_ptr around the raw pointer. If you have an existing, |
803 | * non-temporary unique_ptr, you must call std::move(ptr) to make it clear |
804 | * that you are destroying the original pointer. |
805 | */ |
806 | void prependChain(std::unique_ptr<IOBuf>&& iobuf); |
807 | |
808 | /** |
809 | * Append another IOBuf chain immediately after this IOBuf. |
810 | * |
811 | * For example, if there are two IOBuf chains (A, B, C) and (D, E, F), |
812 | * and B->appendChain(D) is called, the (D, E, F) chain will be subsumed |
813 | * and become part of the chain starting at A, which will now look like |
814 | * (A, B, D, E, F, C) |
815 | * |
816 | * The elements in the specified IOBuf chain will become part of this chain, |
817 | * and will be owned by the head of this chain. When this chain is |
818 | * destroyed, all elements in the supplied chain will also be destroyed. |
819 | * |
820 | * For this reason, appendChain() only accepts an rvalue-reference to a |
821 | * unique_ptr(), to make it clear that it is taking ownership of the supplied |
822 | * chain. If you have a raw pointer, you can pass in a new temporary |
823 | * unique_ptr around the raw pointer. If you have an existing, |
824 | * non-temporary unique_ptr, you must call std::move(ptr) to make it clear |
825 | * that you are destroying the original pointer. |
826 | */ |
827 | void appendChain(std::unique_ptr<IOBuf>&& iobuf) { |
828 | // Just use prependChain() on the next element in our chain |
829 | next_->prependChain(std::move(iobuf)); |
830 | } |
831 | |
832 | /** |
833 | * Remove this IOBuf from its current chain. |
834 | * |
835 | * Since ownership of all elements an IOBuf chain is normally maintained by |
836 | * the head of the chain, unlink() transfers ownership of this IOBuf from the |
837 | * chain and gives it to the caller. A new unique_ptr to the IOBuf is |
838 | * returned to the caller. The caller must store the returned unique_ptr (or |
839 | * call release() on it) to take ownership, otherwise the IOBuf will be |
840 | * immediately destroyed. |
841 | * |
842 | * Since unlink transfers ownership of the IOBuf to the caller, be careful |
843 | * not to call unlink() on the head of a chain if you already maintain |
844 | * ownership on the head of the chain via other means. The pop() method |
845 | * is a better choice for that situation. |
846 | */ |
847 | std::unique_ptr<IOBuf> unlink() { |
848 | next_->prev_ = prev_; |
849 | prev_->next_ = next_; |
850 | prev_ = this; |
851 | next_ = this; |
852 | return std::unique_ptr<IOBuf>(this); |
853 | } |
854 | |
855 | /** |
856 | * Remove this IOBuf from its current chain and return a unique_ptr to |
857 | * the IOBuf that formerly followed it in the chain. |
858 | */ |
859 | std::unique_ptr<IOBuf> pop() { |
860 | IOBuf* next = next_; |
861 | next_->prev_ = prev_; |
862 | prev_->next_ = next_; |
863 | prev_ = this; |
864 | next_ = this; |
865 | return std::unique_ptr<IOBuf>((next == this) ? nullptr : next); |
866 | } |
867 | |
868 | /** |
869 | * Remove a subchain from this chain. |
870 | * |
871 | * Remove the subchain starting at head and ending at tail from this chain. |
872 | * |
873 | * Returns a unique_ptr pointing to head. (In other words, ownership of the |
874 | * head of the subchain is transferred to the caller.) If the caller ignores |
875 | * the return value and lets the unique_ptr be destroyed, the subchain will |
876 | * be immediately destroyed. |
877 | * |
878 | * The subchain referenced by the specified head and tail must be part of the |
879 | * same chain as the current IOBuf, but must not contain the current IOBuf. |
880 | * However, the specified head and tail may be equal to each other (i.e., |
881 | * they may be a subchain of length 1). |
882 | */ |
883 | std::unique_ptr<IOBuf> separateChain(IOBuf* head, IOBuf* tail) { |
884 | assert(head != this); |
885 | assert(tail != this); |
886 | |
887 | head->prev_->next_ = tail->next_; |
888 | tail->next_->prev_ = head->prev_; |
889 | |
890 | head->prev_ = tail; |
891 | tail->next_ = head; |
892 | |
893 | return std::unique_ptr<IOBuf>(head); |
894 | } |
895 | |
896 | /** |
897 | * Return true if at least one of the IOBufs in this chain are shared, |
898 | * or false if all of the IOBufs point to unique buffers. |
899 | * |
900 | * Use isSharedOne() to only check this IOBuf rather than the entire chain. |
901 | */ |
902 | bool isShared() const { |
903 | const IOBuf* current = this; |
904 | while (true) { |
905 | if (current->isSharedOne()) { |
906 | return true; |
907 | } |
908 | current = current->next_; |
909 | if (current == this) { |
910 | return false; |
911 | } |
912 | } |
913 | } |
914 | |
915 | /** |
916 | * |
917 | * Returns the SharedInfo::userData if sharedInfo() |
918 | * is not nullptr, nullptr otherwise |
919 | * |
920 | **/ |
921 | void* getUserData() const { |
922 | SharedInfo* info = sharedInfo(); |
923 | return info ? info->userData : nullptr; |
924 | } |
925 | |
926 | /** |
927 | * |
928 | * Returns free function if sharedInfo() is not nullputr, nullptr otherwise |
929 | * |
930 | **/ |
931 | FreeFunction getFreeFn() const { |
932 | SharedInfo* info = sharedInfo(); |
933 | return info ? info->freeFn : nullptr; |
934 | } |
935 | |
936 | /** |
937 | * Return true if all IOBufs in this chain are managed by the usual |
938 | * refcounting mechanism (and so the lifetime of the underlying memory |
939 | * can be extended by clone()). |
940 | */ |
941 | bool isManaged() const { |
942 | const IOBuf* current = this; |
943 | while (true) { |
944 | if (!current->isManagedOne()) { |
945 | return false; |
946 | } |
947 | current = current->next_; |
948 | if (current == this) { |
949 | return true; |
950 | } |
951 | } |
952 | } |
953 | |
954 | /** |
955 | * Return true if this IOBuf is managed by the usual refcounting mechanism |
956 | * (and so the lifetime of the underlying memory can be extended by |
957 | * cloneOne()). |
958 | */ |
959 | bool isManagedOne() const { |
960 | return sharedInfo(); |
961 | } |
962 | |
963 | /** |
964 | * Return true if other IOBufs are also pointing to the buffer used by this |
965 | * IOBuf, and false otherwise. |
966 | * |
967 | * If this IOBuf points at a buffer owned by another (non-IOBuf) part of the |
968 | * code (i.e., if the IOBuf was created using wrapBuffer(), or was cloned |
969 | * from such an IOBuf), it is always considered shared. |
970 | * |
971 | * This only checks the current IOBuf, and not other IOBufs in the chain. |
972 | */ |
973 | bool isSharedOne() const { |
974 | // If this is a user-owned buffer, it is always considered shared |
975 | if (UNLIKELY(!sharedInfo())) { |
976 | return true; |
977 | } |
978 | |
979 | if (UNLIKELY(sharedInfo()->externallyShared)) { |
980 | return true; |
981 | } |
982 | |
983 | if (LIKELY(!(flags() & kFlagMaybeShared))) { |
984 | return false; |
985 | } |
986 | |
987 | // kFlagMaybeShared is set, so we need to check the reference count. |
988 | // (Checking the reference count requires an atomic operation, which is why |
989 | // we prefer to only check kFlagMaybeShared if possible.) |
990 | bool shared = sharedInfo()->refcount.load(std::memory_order_acquire) > 1; |
991 | if (!shared) { |
992 | // we're the last one left |
993 | clearFlags(kFlagMaybeShared); |
994 | } |
995 | return shared; |
996 | } |
997 | |
998 | /** |
999 | * Ensure that this IOBuf has a unique buffer that is not shared by other |
1000 | * IOBufs. |
1001 | * |
1002 | * unshare() operates on an entire chain of IOBuf objects. If the chain is |
1003 | * shared, it may also coalesce the chain when making it unique. If the |
1004 | * chain is coalesced, subsequent IOBuf objects in the current chain will be |
1005 | * automatically deleted. |
1006 | * |
1007 | * Note that buffers owned by other (non-IOBuf) users are automatically |
1008 | * considered shared. |
1009 | * |
1010 | * Throws std::bad_alloc on error. On error the IOBuf chain will be |
1011 | * unmodified. |
1012 | * |
1013 | * Currently unshare may also throw std::overflow_error if it tries to |
1014 | * coalesce. (TODO: In the future it would be nice if unshare() were smart |
1015 | * enough not to coalesce the entire buffer if the data is too large. |
1016 | * However, in practice this seems unlikely to become an issue.) |
1017 | */ |
1018 | void unshare() { |
1019 | if (isChained()) { |
1020 | unshareChained(); |
1021 | } else { |
1022 | unshareOne(); |
1023 | } |
1024 | } |
1025 | |
1026 | /** |
1027 | * Ensure that this IOBuf has a unique buffer that is not shared by other |
1028 | * IOBufs. |
1029 | * |
1030 | * unshareOne() operates on a single IOBuf object. This IOBuf will have a |
1031 | * unique buffer after unshareOne() returns, but other IOBufs in the chain |
1032 | * may still be shared after unshareOne() returns. |
1033 | * |
1034 | * Throws std::bad_alloc on error. On error the IOBuf will be unmodified. |
1035 | */ |
1036 | void unshareOne() { |
1037 | if (isSharedOne()) { |
1038 | unshareOneSlow(); |
1039 | } |
1040 | } |
1041 | |
1042 | /** |
1043 | * Mark the underlying buffers in this chain as shared with external memory |
1044 | * management mechanism. This will make isShared() always returns true. |
1045 | * |
1046 | * This function is not thread-safe, and only safe to call immediately after |
1047 | * creating an IOBuf, before it has been shared with other threads. |
1048 | */ |
1049 | void markExternallyShared(); |
1050 | |
1051 | /** |
1052 | * Mark the underlying buffer that this IOBuf refers to as shared with |
1053 | * external memory management mechanism. This will make isSharedOne() always |
1054 | * returns true. |
1055 | * |
1056 | * This function is not thread-safe, and only safe to call immediately after |
1057 | * creating an IOBuf, before it has been shared with other threads. |
1058 | */ |
1059 | void markExternallySharedOne() { |
1060 | SharedInfo* info = sharedInfo(); |
1061 | if (info) { |
1062 | info->externallyShared = true; |
1063 | } |
1064 | } |
1065 | |
1066 | /** |
1067 | * Ensure that the memory that IOBufs in this chain refer to will continue to |
1068 | * be allocated for as long as the IOBufs of the chain (or any clone()s |
1069 | * created from this point onwards) is alive. |
1070 | * |
1071 | * This only has an effect for user-owned buffers (created with the |
1072 | * WRAP_BUFFER constructor or wrapBuffer factory function), in which case |
1073 | * those buffers are unshared. |
1074 | */ |
1075 | void makeManaged() { |
1076 | if (isChained()) { |
1077 | makeManagedChained(); |
1078 | } else { |
1079 | makeManagedOne(); |
1080 | } |
1081 | } |
1082 | |
1083 | /** |
1084 | * Ensure that the memory that this IOBuf refers to will continue to be |
1085 | * allocated for as long as this IOBuf (or any clone()s created from this |
1086 | * point onwards) is alive. |
1087 | * |
1088 | * This only has an effect for user-owned buffers (created with the |
1089 | * WRAP_BUFFER constructor or wrapBuffer factory function), in which case |
1090 | * those buffers are unshared. |
1091 | */ |
1092 | void makeManagedOne() { |
1093 | if (!isManagedOne()) { |
1094 | // We can call the internal function directly; unmanaged implies shared. |
1095 | unshareOneSlow(); |
1096 | } |
1097 | } |
1098 | |
1099 | /** |
1100 | * Coalesce this IOBuf chain into a single buffer. |
1101 | * |
1102 | * This method moves all of the data in this IOBuf chain into a single |
1103 | * contiguous buffer, if it is not already in one buffer. After coalesce() |
1104 | * returns, this IOBuf will be a chain of length one. Other IOBufs in the |
1105 | * chain will be automatically deleted. |
1106 | * |
1107 | * After coalescing, the IOBuf will have at least as much headroom as the |
1108 | * first IOBuf in the chain, and at least as much tailroom as the last IOBuf |
1109 | * in the chain. |
1110 | * |
1111 | * Throws std::bad_alloc on error. On error the IOBuf chain will be |
1112 | * unmodified. |
1113 | * |
1114 | * Returns ByteRange that points to the data IOBuf stores. |
1115 | */ |
1116 | ByteRange coalesce() { |
1117 | const std::size_t newHeadroom = headroom(); |
1118 | const std::size_t newTailroom = prev()->tailroom(); |
1119 | return coalesceWithHeadroomTailroom(newHeadroom, newTailroom); |
1120 | } |
1121 | |
1122 | /** |
1123 | * This is similar to the coalesce() method, except this allows to set a |
1124 | * headroom and tailroom after coalescing. |
1125 | * |
1126 | * Returns ByteRange that points to the data IOBuf stores. |
1127 | */ |
1128 | ByteRange coalesceWithHeadroomTailroom( |
1129 | std::size_t newHeadroom, |
1130 | std::size_t newTailroom) { |
1131 | if (isChained()) { |
1132 | coalesceAndReallocate( |
1133 | newHeadroom, computeChainDataLength(), this, newTailroom); |
1134 | } |
1135 | return ByteRange(data_, length_); |
1136 | } |
1137 | |
1138 | /** |
1139 | * Ensure that this chain has at least maxLength bytes available as a |
1140 | * contiguous memory range. |
1141 | * |
1142 | * This method coalesces whole buffers in the chain into this buffer as |
1143 | * necessary until this buffer's length() is at least maxLength. |
1144 | * |
1145 | * After coalescing, the IOBuf will have at least as much headroom as the |
1146 | * first IOBuf in the chain, and at least as much tailroom as the last IOBuf |
1147 | * that was coalesced. |
1148 | * |
1149 | * Throws std::bad_alloc or std::overflow_error on error. On error the IOBuf |
1150 | * chain will be unmodified. Throws std::overflow_error if maxLength is |
1151 | * longer than the total chain length. |
1152 | * |
1153 | * Upon return, either enough of the chain was coalesced into a contiguous |
1154 | * region, or the entire chain was coalesced. That is, |
1155 | * length() >= maxLength || !isChained() is true. |
1156 | */ |
1157 | void gather(std::size_t maxLength) { |
1158 | if (!isChained() || length_ >= maxLength) { |
1159 | return; |
1160 | } |
1161 | coalesceSlow(maxLength); |
1162 | } |
1163 | |
1164 | /** |
1165 | * Return a new IOBuf chain sharing the same data as this chain. |
1166 | * |
1167 | * The new IOBuf chain will normally point to the same underlying data |
1168 | * buffers as the original chain. (The one exception to this is if some of |
1169 | * the IOBufs in this chain contain small internal data buffers which cannot |
1170 | * be shared.) |
1171 | */ |
1172 | std::unique_ptr<IOBuf> clone() const; |
1173 | |
1174 | /** |
1175 | * Similar to clone(). But returns IOBuf by value rather than heap-allocating |
1176 | * it. |
1177 | */ |
1178 | IOBuf cloneAsValue() const; |
1179 | |
1180 | /** |
1181 | * Return a new IOBuf with the same data as this IOBuf. |
1182 | * |
1183 | * The new IOBuf returned will not be part of a chain (even if this IOBuf is |
1184 | * part of a larger chain). |
1185 | */ |
1186 | std::unique_ptr<IOBuf> cloneOne() const; |
1187 | |
1188 | /** |
1189 | * Similar to cloneOne(). But returns IOBuf by value rather than |
1190 | * heap-allocating it. |
1191 | */ |
1192 | IOBuf cloneOneAsValue() const; |
1193 | |
1194 | /** |
1195 | * Return a new unchained IOBuf that may share the same data as this chain. |
1196 | * |
1197 | * If the IOBuf chain is not chained then the new IOBuf will point to the same |
1198 | * underlying data buffer as the original chain. Otherwise, it will clone and |
1199 | * coalesce the IOBuf chain. |
1200 | * |
1201 | * The new IOBuf will have at least as much headroom as the first IOBuf in the |
1202 | * chain, and at least as much tailroom as the last IOBuf in the chain. |
1203 | * |
1204 | * Throws std::bad_alloc on error. |
1205 | */ |
1206 | std::unique_ptr<IOBuf> cloneCoalesced() const; |
1207 | |
1208 | /** |
1209 | * This is similar to the cloneCoalesced() method, except this allows to set a |
1210 | * headroom and tailroom for the new IOBuf. |
1211 | */ |
1212 | std::unique_ptr<IOBuf> cloneCoalescedWithHeadroomTailroom( |
1213 | std::size_t newHeadroom, |
1214 | std::size_t newTailroom) const; |
1215 | |
1216 | /** |
1217 | * Similar to cloneCoalesced(). But returns IOBuf by value rather than |
1218 | * heap-allocating it. |
1219 | */ |
1220 | IOBuf cloneCoalescedAsValue() const; |
1221 | |
1222 | /** |
1223 | * This is similar to the cloneCoalescedAsValue() method, except this allows |
1224 | * to set a headroom and tailroom for the new IOBuf. |
1225 | */ |
1226 | IOBuf cloneCoalescedAsValueWithHeadroomTailroom( |
1227 | std::size_t newHeadroom, |
1228 | std::size_t newTailroom) const; |
1229 | |
1230 | /** |
1231 | * Similar to Clone(). But use other as the head node. Other nodes in the |
1232 | * chain (if any) will be allocted on heap. |
1233 | */ |
1234 | void cloneInto(IOBuf& other) const { |
1235 | other = cloneAsValue(); |
1236 | } |
1237 | |
1238 | /** |
1239 | * Similar to CloneOne(). But to fill an existing IOBuf instead of a new |
1240 | * IOBuf. |
1241 | */ |
1242 | void cloneOneInto(IOBuf& other) const { |
1243 | other = cloneOneAsValue(); |
1244 | } |
1245 | |
1246 | /** |
1247 | * Return an iovector suitable for e.g. writev() |
1248 | * |
1249 | * auto iov = buf->getIov(); |
1250 | * auto xfer = writev(fd, iov.data(), iov.size()); |
1251 | * |
1252 | * Naturally, the returned iovector is invalid if you modify the buffer |
1253 | * chain. |
1254 | */ |
1255 | folly::fbvector<struct iovec> getIov() const; |
1256 | |
1257 | /** |
1258 | * Update an existing iovec array with the IOBuf data. |
1259 | * |
1260 | * New iovecs will be appended to the existing vector; anything already |
1261 | * present in the vector will be left unchanged. |
1262 | * |
1263 | * Naturally, the returned iovec data will be invalid if you modify the |
1264 | * buffer chain. |
1265 | */ |
1266 | void appendToIov(folly::fbvector<struct iovec>* iov) const; |
1267 | |
1268 | struct FillIovResult { |
1269 | // How many iovecs were filled (or 0 on error). |
1270 | size_t numIovecs; |
1271 | // The total length of filled iovecs (or 0 on error). |
1272 | size_t totalLength; |
1273 | }; |
1274 | |
1275 | /** |
1276 | * Fill an iovec array with the IOBuf data. |
1277 | * |
1278 | * Returns a struct with two fields: the number of iovec filled, and total |
1279 | * size of the iovecs filled. If there are more buffer than iovec, returns 0 |
1280 | * in both fields. |
1281 | * This version is suitable to use with stack iovec arrays. |
1282 | * |
1283 | * Naturally, the filled iovec data will be invalid if you modify the |
1284 | * buffer chain. |
1285 | */ |
1286 | FillIovResult fillIov(struct iovec* iov, size_t len) const; |
1287 | |
1288 | /** |
1289 | * A helper that wraps a number of iovecs into an IOBuf chain. If count == 0, |
1290 | * then a zero length buf is returned. This function never returns nullptr. |
1291 | */ |
1292 | static std::unique_ptr<IOBuf> wrapIov(const iovec* vec, size_t count); |
1293 | |
1294 | /** |
1295 | * A helper that takes ownerships a number of iovecs into an IOBuf chain. If |
1296 | * count == 0, then a zero length buf is returned. This function never |
1297 | * returns nullptr. |
1298 | */ |
1299 | static std::unique_ptr<IOBuf> takeOwnershipIov( |
1300 | const iovec* vec, |
1301 | size_t count, |
1302 | FreeFunction freeFn = nullptr, |
1303 | void* userData = nullptr, |
1304 | bool freeOnError = true); |
1305 | |
1306 | /* |
1307 | * Overridden operator new and delete. |
1308 | * These perform specialized memory management to help support |
1309 | * createCombined(), which allocates IOBuf objects together with the buffer |
1310 | * data. |
1311 | */ |
1312 | void* operator new(size_t size); |
1313 | void* operator new(size_t size, void* ptr); |
1314 | void operator delete(void* ptr); |
1315 | void operator delete(void* ptr, void* placement); |
1316 | |
1317 | /** |
1318 | * Destructively convert this IOBuf to a fbstring efficiently. |
1319 | * We rely on fbstring's AcquireMallocatedString constructor to |
1320 | * transfer memory. |
1321 | */ |
1322 | fbstring moveToFbString(); |
1323 | |
1324 | /** |
1325 | * Iteration support: a chain of IOBufs may be iterated through using |
1326 | * STL-style iterators over const ByteRanges. Iterators are only invalidated |
1327 | * if the IOBuf that they currently point to is removed. |
1328 | */ |
1329 | Iterator cbegin() const; |
1330 | Iterator cend() const; |
1331 | Iterator begin() const; |
1332 | Iterator end() const; |
1333 | |
1334 | /** |
1335 | * Allocate a new null buffer. |
1336 | * |
1337 | * This can be used to allocate an empty IOBuf on the stack. It will have no |
1338 | * space allocated for it. This is generally useful only to later use move |
1339 | * assignment to fill out the IOBuf. |
1340 | */ |
1341 | IOBuf() noexcept; |
1342 | |
1343 | /** |
1344 | * Move constructor and assignment operator. |
1345 | * |
1346 | * In general, you should only ever move the head of an IOBuf chain. |
1347 | * Internal nodes in an IOBuf chain are owned by the head of the chain, and |
1348 | * should not be moved from. (Technically, nothing prevents you from moving |
1349 | * a non-head node, but the moved-to node will replace the moved-from node in |
1350 | * the chain. This has implications for ownership, since non-head nodes are |
1351 | * owned by the chain head. You are then responsible for relinquishing |
1352 | * ownership of the moved-to node, and manually deleting the moved-from |
1353 | * node.) |
1354 | * |
1355 | * With the move assignment operator, the destination of the move should be |
1356 | * the head of an IOBuf chain or a solitary IOBuf not part of a chain. If |
1357 | * the move destination is part of a chain, all other IOBufs in the chain |
1358 | * will be deleted. |
1359 | */ |
1360 | IOBuf(IOBuf&& other) noexcept; |
1361 | IOBuf& operator=(IOBuf&& other) noexcept; |
1362 | |
1363 | IOBuf(const IOBuf& other); |
1364 | IOBuf& operator=(const IOBuf& other); |
1365 | |
1366 | private: |
1367 | enum FlagsEnum : uintptr_t { |
1368 | // Adding any more flags would not work on 32-bit architectures, |
1369 | // as these flags are stashed in the least significant 2 bits of a |
1370 | // max-align-aligned pointer. |
1371 | kFlagFreeSharedInfo = 0x1, |
1372 | kFlagMaybeShared = 0x2, |
1373 | kFlagMask = kFlagFreeSharedInfo | kFlagMaybeShared |
1374 | }; |
1375 | |
1376 | struct SharedInfo { |
1377 | SharedInfo(); |
1378 | SharedInfo(FreeFunction fn, void* arg, bool hfs = false); |
1379 | |
1380 | static void releaseStorage(SharedInfo* info); |
1381 | |
1382 | // A pointer to a function to call to free the buffer when the refcount |
1383 | // hits 0. If this is null, free() will be used instead. |
1384 | FreeFunction freeFn; |
1385 | void* userData; |
1386 | std::atomic<uint32_t> refcount; |
1387 | bool externallyShared{false}; |
1388 | bool useHeapFullStorage{false}; |
1389 | }; |
1390 | // Helper structs for use by operator new and delete |
1391 | struct HeapPrefix; |
1392 | struct HeapStorage; |
1393 | struct HeapFullStorage; |
1394 | |
1395 | /** |
1396 | * Create a new IOBuf pointing to an external buffer. |
1397 | * |
1398 | * The caller is responsible for holding a reference count for this new |
1399 | * IOBuf. The IOBuf constructor does not automatically increment the |
1400 | * reference count. |
1401 | */ |
1402 | struct InternalConstructor {}; // avoid conflicts |
1403 | IOBuf( |
1404 | InternalConstructor, |
1405 | uintptr_t flagsAndSharedInfo, |
1406 | uint8_t* buf, |
1407 | std::size_t capacity, |
1408 | uint8_t* data, |
1409 | std::size_t length) noexcept; |
1410 | |
1411 | void unshareOneSlow(); |
1412 | void unshareChained(); |
1413 | void makeManagedChained(); |
1414 | void coalesceSlow(); |
1415 | void coalesceSlow(size_t maxLength); |
1416 | // newLength must be the entire length of the buffers between this and |
1417 | // end (no truncation) |
1418 | void coalesceAndReallocate( |
1419 | size_t newHeadroom, |
1420 | size_t newLength, |
1421 | IOBuf* end, |
1422 | size_t newTailroom); |
1423 | void coalesceAndReallocate(size_t newLength, IOBuf* end) { |
1424 | coalesceAndReallocate(headroom(), newLength, end, end->prev_->tailroom()); |
1425 | } |
1426 | void decrementRefcount(); |
1427 | void reserveSlow(std::size_t minHeadroom, std::size_t minTailroom); |
1428 | void freeExtBuffer(); |
1429 | |
1430 | static size_t goodExtBufferSize(std::size_t minCapacity); |
1431 | static void initExtBuffer( |
1432 | uint8_t* buf, |
1433 | size_t mallocSize, |
1434 | SharedInfo** infoReturn, |
1435 | std::size_t* capacityReturn); |
1436 | static void allocExtBuffer( |
1437 | std::size_t minCapacity, |
1438 | uint8_t** bufReturn, |
1439 | SharedInfo** infoReturn, |
1440 | std::size_t* capacityReturn); |
1441 | static void releaseStorage(HeapStorage* storage, uint16_t freeFlags); |
1442 | static void freeInternalBuf(void* buf, void* userData); |
1443 | |
1444 | /* |
1445 | * Member variables |
1446 | */ |
1447 | |
1448 | /* |
1449 | * Links to the next and the previous IOBuf in this chain. |
1450 | * |
1451 | * The chain is circularly linked (the last element in the chain points back |
1452 | * at the head), and next_ and prev_ can never be null. If this IOBuf is the |
1453 | * only element in the chain, next_ and prev_ will both point to this. |
1454 | */ |
1455 | IOBuf* next_{this}; |
1456 | IOBuf* prev_{this}; |
1457 | |
1458 | /* |
1459 | * A pointer to the start of the data referenced by this IOBuf, and the |
1460 | * length of the data. |
1461 | * |
1462 | * This may refer to any subsection of the actual buffer capacity. |
1463 | */ |
1464 | uint8_t* data_{nullptr}; |
1465 | uint8_t* buf_{nullptr}; |
1466 | std::size_t length_{0}; |
1467 | std::size_t capacity_{0}; |
1468 | |
1469 | // Pack flags in least significant 2 bits, sharedInfo in the rest |
1470 | mutable uintptr_t flagsAndSharedInfo_{0}; |
1471 | |
1472 | static inline uintptr_t packFlagsAndSharedInfo( |
1473 | uintptr_t flags, |
1474 | SharedInfo* info) { |
1475 | uintptr_t uinfo = reinterpret_cast<uintptr_t>(info); |
1476 | DCHECK_EQ(flags & ~kFlagMask, 0u); |
1477 | DCHECK_EQ(uinfo & kFlagMask, 0u); |
1478 | return flags | uinfo; |
1479 | } |
1480 | |
1481 | inline SharedInfo* sharedInfo() const { |
1482 | return reinterpret_cast<SharedInfo*>(flagsAndSharedInfo_ & ~kFlagMask); |
1483 | } |
1484 | |
1485 | inline void setSharedInfo(SharedInfo* info) { |
1486 | uintptr_t uinfo = reinterpret_cast<uintptr_t>(info); |
1487 | DCHECK_EQ(uinfo & kFlagMask, 0u); |
1488 | flagsAndSharedInfo_ = (flagsAndSharedInfo_ & kFlagMask) | uinfo; |
1489 | } |
1490 | |
1491 | inline uintptr_t flags() const { |
1492 | return flagsAndSharedInfo_ & kFlagMask; |
1493 | } |
1494 | |
1495 | // flags_ are changed from const methods |
1496 | inline void setFlags(uintptr_t flags) const { |
1497 | DCHECK_EQ(flags & ~kFlagMask, 0u); |
1498 | flagsAndSharedInfo_ |= flags; |
1499 | } |
1500 | |
1501 | inline void clearFlags(uintptr_t flags) const { |
1502 | DCHECK_EQ(flags & ~kFlagMask, 0u); |
1503 | flagsAndSharedInfo_ &= ~flags; |
1504 | } |
1505 | |
1506 | inline void setFlagsAndSharedInfo(uintptr_t flags, SharedInfo* info) { |
1507 | flagsAndSharedInfo_ = packFlagsAndSharedInfo(flags, info); |
1508 | } |
1509 | |
1510 | struct DeleterBase { |
1511 | virtual ~DeleterBase() {} |
1512 | virtual void dispose(void* p) = 0; |
1513 | }; |
1514 | |
1515 | template <class UniquePtr> |
1516 | struct UniquePtrDeleter : public DeleterBase { |
1517 | typedef typename UniquePtr::pointer Pointer; |
1518 | typedef typename UniquePtr::deleter_type Deleter; |
1519 | |
1520 | explicit UniquePtrDeleter(Deleter deleter) : deleter_(std::move(deleter)) {} |
1521 | void dispose(void* p) override { |
1522 | try { |
1523 | deleter_(static_cast<Pointer>(p)); |
1524 | delete this; |
1525 | } catch (...) { |
1526 | abort(); |
1527 | } |
1528 | } |
1529 | |
1530 | private: |
1531 | Deleter deleter_; |
1532 | }; |
1533 | |
1534 | static void freeUniquePtrBuffer(void* ptr, void* userData) { |
1535 | static_cast<DeleterBase*>(userData)->dispose(ptr); |
1536 | } |
1537 | }; |
1538 | |
1539 | /** |
1540 | * Hasher for IOBuf objects. Hashes the entire chain using SpookyHashV2. |
1541 | */ |
1542 | struct IOBufHash { |
1543 | size_t operator()(const IOBuf& buf) const noexcept; |
1544 | size_t operator()(const std::unique_ptr<IOBuf>& buf) const noexcept { |
1545 | return operator()(buf.get()); |
1546 | } |
1547 | size_t operator()(const IOBuf* buf) const noexcept { |
1548 | return buf ? (*this)(*buf) : 0; |
1549 | } |
1550 | }; |
1551 | |
1552 | /** |
1553 | * Ordering for IOBuf objects. Compares data in the entire chain. |
1554 | */ |
1555 | struct IOBufCompare { |
1556 | ordering operator()(const IOBuf& a, const IOBuf& b) const { |
1557 | return &a == &b ? ordering::eq : impl(a, b); |
1558 | } |
1559 | ordering operator()( |
1560 | const std::unique_ptr<IOBuf>& a, |
1561 | const std::unique_ptr<IOBuf>& b) const { |
1562 | return operator()(a.get(), b.get()); |
1563 | } |
1564 | ordering operator()(const IOBuf* a, const IOBuf* b) const { |
1565 | // clang-format off |
1566 | return |
1567 | !a && !b ? ordering::eq : |
1568 | !a && b ? ordering::lt : |
1569 | a && !b ? ordering::gt : |
1570 | operator()(*a, *b); |
1571 | // clang-format on |
1572 | } |
1573 | |
1574 | private: |
1575 | ordering impl(IOBuf const& a, IOBuf const& b) const noexcept; |
1576 | }; |
1577 | |
1578 | /** |
1579 | * Equality predicate for IOBuf objects. Compares data in the entire chain. |
1580 | */ |
1581 | struct IOBufEqualTo : compare_equal_to<IOBufCompare> {}; |
1582 | |
1583 | /** |
1584 | * Inequality predicate for IOBuf objects. Compares data in the entire chain. |
1585 | */ |
1586 | struct IOBufNotEqualTo : compare_not_equal_to<IOBufCompare> {}; |
1587 | |
1588 | /** |
1589 | * Less predicate for IOBuf objects. Compares data in the entire chain. |
1590 | */ |
1591 | struct IOBufLess : compare_less<IOBufCompare> {}; |
1592 | |
1593 | /** |
1594 | * At-most predicate for IOBuf objects. Compares data in the entire chain. |
1595 | */ |
1596 | struct IOBufLessEqual : compare_less_equal<IOBufCompare> {}; |
1597 | |
1598 | /** |
1599 | * Greater predicate for IOBuf objects. Compares data in the entire chain. |
1600 | */ |
1601 | struct IOBufGreater : compare_greater<IOBufCompare> {}; |
1602 | |
1603 | /** |
1604 | * At-least predicate for IOBuf objects. Compares data in the entire chain. |
1605 | */ |
1606 | struct IOBufGreaterEqual : compare_greater_equal<IOBufCompare> {}; |
1607 | |
1608 | template <class UniquePtr> |
1609 | typename std::enable_if< |
1610 | detail::IsUniquePtrToSL<UniquePtr>::value, |
1611 | std::unique_ptr<IOBuf>>::type |
1612 | IOBuf::takeOwnership(UniquePtr&& buf, size_t count) { |
1613 | size_t size = count * sizeof(typename UniquePtr::element_type); |
1614 | auto deleter = new UniquePtrDeleter<UniquePtr>(buf.get_deleter()); |
1615 | return takeOwnership( |
1616 | buf.release(), size, &IOBuf::freeUniquePtrBuffer, deleter); |
1617 | } |
1618 | |
1619 | inline std::unique_ptr<IOBuf> IOBuf::copyBuffer( |
1620 | const void* data, |
1621 | std::size_t size, |
1622 | std::size_t headroom, |
1623 | std::size_t minTailroom) { |
1624 | std::size_t capacity = headroom + size + minTailroom; |
1625 | std::unique_ptr<IOBuf> buf = create(capacity); |
1626 | buf->advance(headroom); |
1627 | if (size != 0) { |
1628 | memcpy(buf->writableData(), data, size); |
1629 | } |
1630 | buf->append(size); |
1631 | return buf; |
1632 | } |
1633 | |
1634 | inline std::unique_ptr<IOBuf> IOBuf::copyBuffer( |
1635 | const std::string& buf, |
1636 | std::size_t headroom, |
1637 | std::size_t minTailroom) { |
1638 | return copyBuffer(buf.data(), buf.size(), headroom, minTailroom); |
1639 | } |
1640 | |
1641 | inline std::unique_ptr<IOBuf> IOBuf::maybeCopyBuffer( |
1642 | const std::string& buf, |
1643 | std::size_t headroom, |
1644 | std::size_t minTailroom) { |
1645 | if (buf.empty()) { |
1646 | return nullptr; |
1647 | } |
1648 | return copyBuffer(buf.data(), buf.size(), headroom, minTailroom); |
1649 | } |
1650 | |
1651 | class IOBuf::Iterator : public detail::IteratorFacade< |
1652 | IOBuf::Iterator, |
1653 | ByteRange const, |
1654 | std::forward_iterator_tag> { |
1655 | public: |
1656 | // Note that IOBufs are stored as a circular list without a guard node, |
1657 | // so pos == end is ambiguous (it may mean "begin" or "end"). To solve |
1658 | // the ambiguity (at the cost of one extra comparison in the "increment" |
1659 | // code path), we define end iterators as having pos_ == end_ == nullptr |
1660 | // and we only allow forward iteration. |
1661 | explicit Iterator(const IOBuf* pos, const IOBuf* end) : pos_(pos), end_(end) { |
1662 | // Sadly, we must return by const reference, not by value. |
1663 | if (pos_) { |
1664 | setVal(); |
1665 | } |
1666 | } |
1667 | |
1668 | Iterator() {} |
1669 | |
1670 | Iterator(Iterator const& rhs) : Iterator(rhs.pos_, rhs.end_) {} |
1671 | |
1672 | Iterator& operator=(Iterator const& rhs) { |
1673 | pos_ = rhs.pos_; |
1674 | end_ = rhs.end_; |
1675 | if (pos_) { |
1676 | setVal(); |
1677 | } |
1678 | return *this; |
1679 | } |
1680 | |
1681 | const ByteRange& dereference() const { |
1682 | return val_; |
1683 | } |
1684 | |
1685 | bool equal(const Iterator& other) const { |
1686 | // We must compare end_ in addition to pos_, because forward traversal |
1687 | // requires that if two iterators are equal (a == b) and dereferenceable, |
1688 | // then ++a == ++b. |
1689 | return pos_ == other.pos_ && end_ == other.end_; |
1690 | } |
1691 | |
1692 | void increment() { |
1693 | pos_ = pos_->next(); |
1694 | adjustForEnd(); |
1695 | } |
1696 | |
1697 | private: |
1698 | void setVal() { |
1699 | val_ = ByteRange(pos_->data(), pos_->tail()); |
1700 | } |
1701 | |
1702 | void adjustForEnd() { |
1703 | if (pos_ == end_) { |
1704 | pos_ = end_ = nullptr; |
1705 | val_ = ByteRange(); |
1706 | } else { |
1707 | setVal(); |
1708 | } |
1709 | } |
1710 | |
1711 | const IOBuf* pos_{nullptr}; |
1712 | const IOBuf* end_{nullptr}; |
1713 | ByteRange val_; |
1714 | }; |
1715 | |
1716 | inline IOBuf::Iterator IOBuf::begin() const { |
1717 | return cbegin(); |
1718 | } |
1719 | inline IOBuf::Iterator IOBuf::end() const { |
1720 | return cend(); |
1721 | } |
1722 | |
1723 | } // namespace folly |
1724 | |
1725 | FOLLY_POP_WARNING |
1726 | |