1/*
2 * Copyright 2013-present Facebook, Inc.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#pragma once
18
19#include <glog/logging.h>
20#include <atomic>
21#include <cassert>
22#include <cinttypes>
23#include <cstddef>
24#include <cstring>
25#include <iterator>
26#include <limits>
27#include <memory>
28#include <type_traits>
29
30#include <folly/FBString.h>
31#include <folly/FBVector.h>
32#include <folly/Portability.h>
33#include <folly/Range.h>
34#include <folly/detail/Iterators.h>
35#include <folly/lang/Ordering.h>
36#include <folly/portability/SysUio.h>
37
38// Ignore shadowing warnings within this file, so includers can use -Wshadow.
39FOLLY_PUSH_WARNING
40FOLLY_GNU_DISABLE_WARNING("-Wshadow")
41
42namespace folly {
43
44/**
45 * An IOBuf is a pointer to a buffer of data.
46 *
47 * IOBuf objects are intended to be used primarily for networking code, and are
48 * modelled somewhat after FreeBSD's mbuf data structure, and Linux's sk_buff
49 * structure.
50 *
51 * IOBuf objects facilitate zero-copy network programming, by allowing multiple
52 * IOBuf objects to point to the same underlying buffer of data, using a
53 * reference count to track when the buffer is no longer needed and can be
54 * freed.
55 *
56 *
57 * Data Layout
58 * -----------
59 *
60 * The IOBuf itself is a small object containing a pointer to the buffer and
61 * information about which segment of the buffer contains valid data.
62 *
63 * The data layout looks like this:
64 *
65 * +-------+
66 * | IOBuf |
67 * +-------+
68 * /
69 * |
70 * v
71 * +------------+--------------------+-----------+
72 * | headroom | data | tailroom |
73 * +------------+--------------------+-----------+
74 * ^ ^ ^ ^
75 * buffer() data() tail() bufferEnd()
76 *
77 * The length() method returns the length of the valid data; capacity()
78 * returns the entire capacity of the buffer (from buffer() to bufferEnd()).
79 * The headroom() and tailroom() methods return the amount of unused capacity
80 * available before and after the data.
81 *
82 *
83 * Buffer Sharing
84 * --------------
85 *
86 * The buffer itself is reference counted, and multiple IOBuf objects may point
87 * to the same buffer. Each IOBuf may point to a different section of valid
88 * data within the underlying buffer. For example, if multiple protocol
89 * requests are read from the network into a single buffer, a separate IOBuf
90 * may be created for each request, all sharing the same underlying buffer.
91 *
92 * In other words, when multiple IOBufs share the same underlying buffer, the
93 * data() and tail() methods on each IOBuf may point to a different segment of
94 * the data. However, the buffer() and bufferEnd() methods will point to the
95 * same location for all IOBufs sharing the same underlying buffer.
96 *
97 * +-----------+ +---------+
98 * | IOBuf 1 | | IOBuf 2 |
99 * +-----------+ +---------+
100 * | | _____/ |
101 * data | tail |/ data | tail
102 * v v v
103 * +-------------------------------------+
104 * | | | | |
105 * +-------------------------------------+
106 *
107 * If you only read data from an IOBuf, you don't need to worry about other
108 * IOBuf objects possibly sharing the same underlying buffer. However, if you
109 * ever write to the buffer you need to first ensure that no other IOBufs point
110 * to the same buffer. The unshare() method may be used to ensure that you
111 * have an unshared buffer.
112 *
113 *
114 * IOBuf Chains
115 * ------------
116 *
117 * IOBuf objects also contain pointers to next and previous IOBuf objects.
118 * This can be used to represent a single logical piece of data that its stored
119 * in non-contiguous chunks in separate buffers.
120 *
121 * A single IOBuf object can only belong to one chain at a time.
122 *
123 * IOBuf chains are always circular. The "prev" pointer in the head of the
124 * chain points to the tail of the chain. However, it is up to the user to
125 * decide which IOBuf is the head. Internally the IOBuf code does not care
126 * which element is the head.
127 *
128 * The lifetime of all IOBufs in the chain are linked: when one element in the
129 * chain is deleted, all other chained elements are also deleted. Conceptually
130 * it is simplest to treat this as if the head of the chain owns all other
131 * IOBufs in the chain. When you delete the head of the chain, it will delete
132 * the other elements as well. For this reason, prependChain() and
133 * appendChain() take ownership of of the new elements being added to this
134 * chain.
135 *
136 * When the coalesce() method is used to coalesce an entire IOBuf chain into a
137 * single IOBuf, all other IOBufs in the chain are eliminated and automatically
138 * deleted. The unshare() method may coalesce the chain; if it does it will
139 * similarly delete all IOBufs eliminated from the chain.
140 *
141 * As discussed in the following section, it is up to the user to maintain a
142 * lock around the entire IOBuf chain if multiple threads need to access the
143 * chain. IOBuf does not provide any internal locking.
144 *
145 *
146 * Synchronization
147 * ---------------
148 *
149 * When used in multithread programs, a single IOBuf object should only be used
150 * in a single thread at a time. If a caller uses a single IOBuf across
151 * multiple threads the caller is responsible for using an external lock to
152 * synchronize access to the IOBuf.
153 *
154 * Two separate IOBuf objects may be accessed concurrently in separate threads
155 * without locking, even if they point to the same underlying buffer. The
156 * buffer reference count is always accessed atomically, and no other
157 * operations should affect other IOBufs that point to the same data segment.
158 * The caller is responsible for using unshare() to ensure that the data buffer
159 * is not shared by other IOBufs before writing to it, and this ensures that
160 * the data itself is not modified in one thread while also being accessed from
161 * another thread.
162 *
163 * For IOBuf chains, no two IOBufs in the same chain should be accessed
164 * simultaneously in separate threads. The caller must maintain a lock around
165 * the entire chain if the chain, or individual IOBufs in the chain, may be
166 * accessed by multiple threads.
167 *
168 *
169 * IOBuf Object Allocation
170 * -----------------------
171 *
172 * IOBuf objects themselves exist separately from the data buffer they point
173 * to. Therefore one must also consider how to allocate and manage the IOBuf
174 * objects.
175 *
176 * It is more common to allocate IOBuf objects on the heap, using the create(),
177 * takeOwnership(), or wrapBuffer() factory functions. The clone()/cloneOne()
178 * functions also return new heap-allocated IOBufs. The createCombined()
179 * function allocates the IOBuf object and data storage space together, in a
180 * single memory allocation. This can improve performance, particularly if you
181 * know that the data buffer and the IOBuf itself will have similar lifetimes.
182 *
183 * That said, it is also possible to allocate IOBufs on the stack or inline
184 * inside another object as well. This is useful for cases where the IOBuf is
185 * short-lived, or when the overhead of allocating the IOBuf on the heap is
186 * undesirable.
187 *
188 * However, note that stack-allocated IOBufs may only be used as the head of a
189 * chain (or standalone as the only IOBuf in a chain). All non-head members of
190 * an IOBuf chain must be heap allocated. (All functions to add nodes to a
191 * chain require a std::unique_ptr<IOBuf>, which enforces this requrement.)
192 *
193 * Copying IOBufs is only meaningful for the head of a chain. The entire chain
194 * is cloned; the IOBufs will become shared, and the old and new IOBufs will
195 * refer to the same underlying memory.
196 *
197 * IOBuf Sharing
198 * -------------
199 *
200 * The IOBuf class manages sharing of the underlying buffer that it points to,
201 * maintaining a reference count if multiple IOBufs are pointing at the same
202 * buffer.
203 *
204 * However, it is the callers responsibility to manage sharing and ownership of
205 * IOBuf objects themselves. The IOBuf structure does not provide room for an
206 * intrusive refcount on the IOBuf object itself, only the underlying data
207 * buffer is reference counted. If users want to share the same IOBuf object
208 * between multiple parts of the code, they are responsible for managing this
209 * sharing on their own. (For example, by using a shared_ptr. Alternatively,
210 * users always have the option of using clone() to create a second IOBuf that
211 * points to the same underlying buffer.)
212 */
213namespace detail {
214// Is T a unique_ptr<> to a standard-layout type?
215template <typename T>
216struct IsUniquePtrToSL : std::false_type {};
217template <typename T, typename D>
218struct IsUniquePtrToSL<std::unique_ptr<T, D>> : std::is_standard_layout<T> {};
219} // namespace detail
220
221class IOBuf {
222 public:
223 class Iterator;
224
225 enum CreateOp { CREATE };
226 enum WrapBufferOp { WRAP_BUFFER };
227 enum TakeOwnershipOp { TAKE_OWNERSHIP };
228 enum CopyBufferOp { COPY_BUFFER };
229
230 typedef ByteRange value_type;
231 typedef Iterator iterator;
232 typedef Iterator const_iterator;
233
234 typedef void (*FreeFunction)(void* buf, void* userData);
235
236 /**
237 * Allocate a new IOBuf object with the requested capacity.
238 *
239 * Returns a new IOBuf object that must be (eventually) deleted by the
240 * caller. The returned IOBuf may actually have slightly more capacity than
241 * requested.
242 *
243 * The data pointer will initially point to the start of the newly allocated
244 * buffer, and will have a data length of 0.
245 *
246 * Throws std::bad_alloc on error.
247 */
248 static std::unique_ptr<IOBuf> create(std::size_t capacity);
249 IOBuf(CreateOp, std::size_t capacity);
250
251 /**
252 * Create a new IOBuf, using a single memory allocation to allocate space
253 * for both the IOBuf object and the data storage space.
254 *
255 * This saves one memory allocation. However, it can be wasteful if you
256 * later need to grow the buffer using reserve(). If the buffer needs to be
257 * reallocated, the space originally allocated will not be freed() until the
258 * IOBuf object itself is also freed. (It can also be slightly wasteful in
259 * some cases where you clone this IOBuf and then free the original IOBuf.)
260 */
261 static std::unique_ptr<IOBuf> createCombined(std::size_t capacity);
262
263 /**
264 * Create a new IOBuf, using separate memory allocations for the IOBuf object
265 * for the IOBuf and the data storage space.
266 *
267 * This requires two memory allocations, but saves space in the long run
268 * if you know that you will need to reallocate the data buffer later.
269 */
270 static std::unique_ptr<IOBuf> createSeparate(std::size_t capacity);
271
272 /**
273 * Allocate a new IOBuf chain with the requested total capacity, allocating
274 * no more than maxBufCapacity to each buffer.
275 */
276 static std::unique_ptr<IOBuf> createChain(
277 size_t totalCapacity,
278 std::size_t maxBufCapacity);
279
280 /**
281 * Create a new IOBuf pointing to an existing data buffer.
282 *
283 * The new IOBuffer will assume ownership of the buffer, and free it by
284 * calling the specified FreeFunction when the last IOBuf pointing to this
285 * buffer is destroyed. The function will be called with a pointer to the
286 * buffer as the first argument, and the supplied userData value as the
287 * second argument. The free function must never throw exceptions.
288 *
289 * If no FreeFunction is specified, the buffer will be freed using free()
290 * which will result in undefined behavior if the memory was allocated
291 * using 'new'.
292 *
293 * The IOBuf data pointer will initially point to the start of the buffer,
294 *
295 * In the first version of this function, the length of data is unspecified
296 * and is initialized to the capacity of the buffer
297 *
298 * In the second version, the user specifies the valid length of data
299 * in the buffer
300 *
301 * On error, std::bad_alloc will be thrown. If freeOnError is true (the
302 * default) the buffer will be freed before throwing the error.
303 */
304 static std::unique_ptr<IOBuf> takeOwnership(
305 void* buf,
306 std::size_t capacity,
307 FreeFunction freeFn = nullptr,
308 void* userData = nullptr,
309 bool freeOnError = true) {
310 return takeOwnership(
311 buf, capacity, capacity, freeFn, userData, freeOnError);
312 }
313 IOBuf(
314 TakeOwnershipOp op,
315 void* buf,
316 std::size_t capacity,
317 FreeFunction freeFn = nullptr,
318 void* userData = nullptr,
319 bool freeOnError = true)
320 : IOBuf(op, buf, capacity, capacity, freeFn, userData, freeOnError) {}
321
322 static std::unique_ptr<IOBuf> takeOwnership(
323 void* buf,
324 std::size_t capacity,
325 std::size_t length,
326 FreeFunction freeFn = nullptr,
327 void* userData = nullptr,
328 bool freeOnError = true);
329 IOBuf(
330 TakeOwnershipOp,
331 void* buf,
332 std::size_t capacity,
333 std::size_t length,
334 FreeFunction freeFn = nullptr,
335 void* userData = nullptr,
336 bool freeOnError = true);
337
338 /**
339 * Create a new IOBuf pointing to an existing data buffer made up of
340 * count objects of a given standard-layout type.
341 *
342 * This is dangerous -- it is essentially equivalent to doing
343 * reinterpret_cast<unsigned char*> on your data -- but it's often useful
344 * for serialization / deserialization.
345 *
346 * The new IOBuffer will assume ownership of the buffer, and free it
347 * appropriately (by calling the UniquePtr's custom deleter, or by calling
348 * delete or delete[] appropriately if there is no custom deleter)
349 * when the buffer is destroyed. The custom deleter, if any, must never
350 * throw exceptions.
351 *
352 * The IOBuf data pointer will initially point to the start of the buffer,
353 * and the length will be the full capacity of the buffer (count *
354 * sizeof(T)).
355 *
356 * On error, std::bad_alloc will be thrown, and the buffer will be freed
357 * before throwing the error.
358 */
359 template <class UniquePtr>
360 static typename std::enable_if<
361 detail::IsUniquePtrToSL<UniquePtr>::value,
362 std::unique_ptr<IOBuf>>::type
363 takeOwnership(UniquePtr&& buf, size_t count = 1);
364
365 /**
366 * Create a new IOBuf object that points to an existing user-owned buffer.
367 *
368 * This should only be used when the caller knows the lifetime of the IOBuf
369 * object ahead of time and can ensure that all IOBuf objects that will point
370 * to this buffer will be destroyed before the buffer itself is destroyed.
371 *
372 * This buffer will not be freed automatically when the last IOBuf
373 * referencing it is destroyed. It is the caller's responsibility to free
374 * the buffer after the last IOBuf has been destroyed.
375 *
376 * The IOBuf data pointer will initially point to the start of the buffer,
377 * and the length will be the full capacity of the buffer.
378 *
379 * An IOBuf created using wrapBuffer() will always be reported as shared.
380 * unshare() may be used to create a writable copy of the buffer.
381 *
382 * On error, std::bad_alloc will be thrown.
383 */
384 static std::unique_ptr<IOBuf> wrapBuffer(
385 const void* buf,
386 std::size_t capacity);
387 static std::unique_ptr<IOBuf> wrapBuffer(ByteRange br) {
388 return wrapBuffer(br.data(), br.size());
389 }
390
391 /**
392 * Similar to wrapBuffer(), but returns IOBuf by value rather than
393 * heap-allocating it.
394 */
395 static IOBuf wrapBufferAsValue(
396 const void* buf,
397 std::size_t capacity) noexcept;
398 static IOBuf wrapBufferAsValue(ByteRange br) noexcept {
399 return wrapBufferAsValue(br.data(), br.size());
400 }
401
402 IOBuf(WrapBufferOp op, const void* buf, std::size_t capacity) noexcept;
403 IOBuf(WrapBufferOp op, ByteRange br) noexcept;
404
405 /**
406 * Convenience function to create a new IOBuf object that copies data from a
407 * user-supplied buffer, optionally allocating a given amount of
408 * headroom and tailroom.
409 */
410 static std::unique_ptr<IOBuf> copyBuffer(
411 const void* buf,
412 std::size_t size,
413 std::size_t headroom = 0,
414 std::size_t minTailroom = 0);
415 static std::unique_ptr<IOBuf> copyBuffer(
416 ByteRange br,
417 std::size_t headroom = 0,
418 std::size_t minTailroom = 0) {
419 return copyBuffer(br.data(), br.size(), headroom, minTailroom);
420 }
421 IOBuf(
422 CopyBufferOp op,
423 const void* buf,
424 std::size_t size,
425 std::size_t headroom = 0,
426 std::size_t minTailroom = 0);
427 IOBuf(
428 CopyBufferOp op,
429 ByteRange br,
430 std::size_t headroom = 0,
431 std::size_t minTailroom = 0);
432
433 /**
434 * Convenience function to create a new IOBuf object that copies data from a
435 * user-supplied string, optionally allocating a given amount of
436 * headroom and tailroom.
437 *
438 * Beware when attempting to invoke this function with a constant string
439 * literal and a headroom argument: you will likely end up invoking the
440 * version of copyBuffer() above. IOBuf::copyBuffer("hello", 3) will treat
441 * the first argument as a const void*, and will invoke the version of
442 * copyBuffer() above, with the size argument of 3.
443 */
444 static std::unique_ptr<IOBuf> copyBuffer(
445 const std::string& buf,
446 std::size_t headroom = 0,
447 std::size_t minTailroom = 0);
448 IOBuf(
449 CopyBufferOp op,
450 const std::string& buf,
451 std::size_t headroom = 0,
452 std::size_t minTailroom = 0)
453 : IOBuf(op, buf.data(), buf.size(), headroom, minTailroom) {}
454
455 /**
456 * A version of copyBuffer() that returns a null pointer if the input string
457 * is empty.
458 */
459 static std::unique_ptr<IOBuf> maybeCopyBuffer(
460 const std::string& buf,
461 std::size_t headroom = 0,
462 std::size_t minTailroom = 0);
463
464 /**
465 * Convenience function to free a chain of IOBufs held by a unique_ptr.
466 */
467 static void destroy(std::unique_ptr<IOBuf>&& data) {
468 auto destroyer = std::move(data);
469 }
470
471 /**
472 * Destroy this IOBuf.
473 *
474 * Deleting an IOBuf will automatically destroy all IOBufs in the chain.
475 * (See the comments above regarding the ownership model of IOBuf chains.
476 * All subsequent IOBufs in the chain are considered to be owned by the head
477 * of the chain. Users should only explicitly delete the head of a chain.)
478 *
479 * When each individual IOBuf is destroyed, it will release its reference
480 * count on the underlying buffer. If it was the last user of the buffer,
481 * the buffer will be freed.
482 */
483 ~IOBuf();
484
485 /**
486 * Check whether the chain is empty (i.e., whether the IOBufs in the
487 * chain have a total data length of zero).
488 *
489 * This method is semantically equivalent to
490 * i->computeChainDataLength()==0
491 * but may run faster because it can short-circuit as soon as it
492 * encounters a buffer with length()!=0
493 */
494 bool empty() const;
495
496 /**
497 * Get the pointer to the start of the data.
498 */
499 const uint8_t* data() const {
500 return data_;
501 }
502
503 /**
504 * Get a writable pointer to the start of the data.
505 *
506 * The caller is responsible for calling unshare() first to ensure that it is
507 * actually safe to write to the buffer.
508 */
509 uint8_t* writableData() {
510 return data_;
511 }
512
513 /**
514 * Get the pointer to the end of the data.
515 */
516 const uint8_t* tail() const {
517 return data_ + length_;
518 }
519
520 /**
521 * Get a writable pointer to the end of the data.
522 *
523 * The caller is responsible for calling unshare() first to ensure that it is
524 * actually safe to write to the buffer.
525 */
526 uint8_t* writableTail() {
527 return data_ + length_;
528 }
529
530 /**
531 * Get the data length.
532 */
533 std::size_t length() const {
534 return length_;
535 }
536
537 /**
538 * Get the amount of head room.
539 *
540 * Returns the number of bytes in the buffer before the start of the data.
541 */
542 std::size_t headroom() const {
543 return std::size_t(data_ - buffer());
544 }
545
546 /**
547 * Get the amount of tail room.
548 *
549 * Returns the number of bytes in the buffer after the end of the data.
550 */
551 std::size_t tailroom() const {
552 return std::size_t(bufferEnd() - tail());
553 }
554
555 /**
556 * Get the pointer to the start of the buffer.
557 *
558 * Note that this is the pointer to the very beginning of the usable buffer,
559 * not the start of valid data within the buffer. Use the data() method to
560 * get a pointer to the start of the data within the buffer.
561 */
562 const uint8_t* buffer() const {
563 return buf_;
564 }
565
566 /**
567 * Get a writable pointer to the start of the buffer.
568 *
569 * The caller is responsible for calling unshare() first to ensure that it is
570 * actually safe to write to the buffer.
571 */
572 uint8_t* writableBuffer() {
573 return buf_;
574 }
575
576 /**
577 * Get the pointer to the end of the buffer.
578 *
579 * Note that this is the pointer to the very end of the usable buffer,
580 * not the end of valid data within the buffer. Use the tail() method to
581 * get a pointer to the end of the data within the buffer.
582 */
583 const uint8_t* bufferEnd() const {
584 return buf_ + capacity_;
585 }
586
587 /**
588 * Get the total size of the buffer.
589 *
590 * This returns the total usable length of the buffer. Use the length()
591 * method to get the length of the actual valid data in this IOBuf.
592 */
593 std::size_t capacity() const {
594 return capacity_;
595 }
596
597 /**
598 * Get a pointer to the next IOBuf in this chain.
599 */
600 IOBuf* next() {
601 return next_;
602 }
603 const IOBuf* next() const {
604 return next_;
605 }
606
607 /**
608 * Get a pointer to the previous IOBuf in this chain.
609 */
610 IOBuf* prev() {
611 return prev_;
612 }
613 const IOBuf* prev() const {
614 return prev_;
615 }
616
617 /**
618 * Shift the data forwards in the buffer.
619 *
620 * This shifts the data pointer forwards in the buffer to increase the
621 * headroom. This is commonly used to increase the headroom in a newly
622 * allocated buffer.
623 *
624 * The caller is responsible for ensuring that there is sufficient
625 * tailroom in the buffer before calling advance().
626 *
627 * If there is a non-zero data length, advance() will use memmove() to shift
628 * the data forwards in the buffer. In this case, the caller is responsible
629 * for making sure the buffer is unshared, so it will not affect other IOBufs
630 * that may be sharing the same underlying buffer.
631 */
632 void advance(std::size_t amount) {
633 // In debug builds, assert if there is a problem.
634 assert(amount <= tailroom());
635
636 if (length_ > 0) {
637 memmove(data_ + amount, data_, length_);
638 }
639 data_ += amount;
640 }
641
642 /**
643 * Shift the data backwards in the buffer.
644 *
645 * The caller is responsible for ensuring that there is sufficient headroom
646 * in the buffer before calling retreat().
647 *
648 * If there is a non-zero data length, retreat() will use memmove() to shift
649 * the data backwards in the buffer. In this case, the caller is responsible
650 * for making sure the buffer is unshared, so it will not affect other IOBufs
651 * that may be sharing the same underlying buffer.
652 */
653 void retreat(std::size_t amount) {
654 // In debug builds, assert if there is a problem.
655 assert(amount <= headroom());
656
657 if (length_ > 0) {
658 memmove(data_ - amount, data_, length_);
659 }
660 data_ -= amount;
661 }
662
663 /**
664 * Adjust the data pointer to include more valid data at the beginning.
665 *
666 * This moves the data pointer backwards to include more of the available
667 * buffer. The caller is responsible for ensuring that there is sufficient
668 * headroom for the new data. The caller is also responsible for populating
669 * this section with valid data.
670 *
671 * This does not modify any actual data in the buffer.
672 */
673 void prepend(std::size_t amount) {
674 DCHECK_LE(amount, headroom());
675 data_ -= amount;
676 length_ += amount;
677 }
678
679 /**
680 * Adjust the tail pointer to include more valid data at the end.
681 *
682 * This moves the tail pointer forwards to include more of the available
683 * buffer. The caller is responsible for ensuring that there is sufficient
684 * tailroom for the new data. The caller is also responsible for populating
685 * this section with valid data.
686 *
687 * This does not modify any actual data in the buffer.
688 */
689 void append(std::size_t amount) {
690 DCHECK_LE(amount, tailroom());
691 length_ += amount;
692 }
693
694 /**
695 * Adjust the data pointer forwards to include less valid data.
696 *
697 * This moves the data pointer forwards so that the first amount bytes are no
698 * longer considered valid data. The caller is responsible for ensuring that
699 * amount is less than or equal to the actual data length.
700 *
701 * This does not modify any actual data in the buffer.
702 */
703 void trimStart(std::size_t amount) {
704 DCHECK_LE(amount, length_);
705 data_ += amount;
706 length_ -= amount;
707 }
708
709 /**
710 * Adjust the tail pointer backwards to include less valid data.
711 *
712 * This moves the tail pointer backwards so that the last amount bytes are no
713 * longer considered valid data. The caller is responsible for ensuring that
714 * amount is less than or equal to the actual data length.
715 *
716 * This does not modify any actual data in the buffer.
717 */
718 void trimEnd(std::size_t amount) {
719 DCHECK_LE(amount, length_);
720 length_ -= amount;
721 }
722
723 /**
724 * Clear the buffer.
725 *
726 * Postcondition: headroom() == 0, length() == 0, tailroom() == capacity()
727 */
728 void clear() {
729 data_ = writableBuffer();
730 length_ = 0;
731 }
732
733 /**
734 * Ensure that this buffer has at least minHeadroom headroom bytes and at
735 * least minTailroom tailroom bytes. The buffer must be writable
736 * (you must call unshare() before this, if necessary).
737 *
738 * Postcondition: headroom() >= minHeadroom, tailroom() >= minTailroom,
739 * the data (between data() and data() + length()) is preserved.
740 */
741 void reserve(std::size_t minHeadroom, std::size_t minTailroom) {
742 // Maybe we don't need to do anything.
743 if (headroom() >= minHeadroom && tailroom() >= minTailroom) {
744 return;
745 }
746 // If the buffer is empty but we have enough total room (head + tail),
747 // move the data_ pointer around.
748 if (length() == 0 && headroom() + tailroom() >= minHeadroom + minTailroom) {
749 data_ = writableBuffer() + minHeadroom;
750 return;
751 }
752 // Bah, we have to do actual work.
753 reserveSlow(minHeadroom, minTailroom);
754 }
755
756 /**
757 * Return true if this IOBuf is part of a chain of multiple IOBufs, or false
758 * if this is the only IOBuf in its chain.
759 */
760 bool isChained() const {
761 assert((next_ == this) == (prev_ == this));
762 return next_ != this;
763 }
764
765 /**
766 * Get the number of IOBufs in this chain.
767 *
768 * Beware that this method has to walk the entire chain.
769 * Use isChained() if you just want to check if this IOBuf is part of a chain
770 * or not.
771 */
772 size_t countChainElements() const;
773
774 /**
775 * Get the length of all the data in this IOBuf chain.
776 *
777 * Beware that this method has to walk the entire chain.
778 */
779 std::size_t computeChainDataLength() const;
780
781 /**
782 * Insert another IOBuf chain immediately before this IOBuf.
783 *
784 * For example, if there are two IOBuf chains (A, B, C) and (D, E, F),
785 * and B->prependChain(D) is called, the (D, E, F) chain will be subsumed
786 * and become part of the chain starting at A, which will now look like
787 * (A, D, E, F, B, C)
788 *
789 * Note that since IOBuf chains are circular, head->prependChain(other) can
790 * be used to append the other chain at the very end of the chain pointed to
791 * by head. For example, if there are two IOBuf chains (A, B, C) and
792 * (D, E, F), and A->prependChain(D) is called, the chain starting at A will
793 * now consist of (A, B, C, D, E, F)
794 *
795 * The elements in the specified IOBuf chain will become part of this chain,
796 * and will be owned by the head of this chain. When this chain is
797 * destroyed, all elements in the supplied chain will also be destroyed.
798 *
799 * For this reason, appendChain() only accepts an rvalue-reference to a
800 * unique_ptr(), to make it clear that it is taking ownership of the supplied
801 * chain. If you have a raw pointer, you can pass in a new temporary
802 * unique_ptr around the raw pointer. If you have an existing,
803 * non-temporary unique_ptr, you must call std::move(ptr) to make it clear
804 * that you are destroying the original pointer.
805 */
806 void prependChain(std::unique_ptr<IOBuf>&& iobuf);
807
808 /**
809 * Append another IOBuf chain immediately after this IOBuf.
810 *
811 * For example, if there are two IOBuf chains (A, B, C) and (D, E, F),
812 * and B->appendChain(D) is called, the (D, E, F) chain will be subsumed
813 * and become part of the chain starting at A, which will now look like
814 * (A, B, D, E, F, C)
815 *
816 * The elements in the specified IOBuf chain will become part of this chain,
817 * and will be owned by the head of this chain. When this chain is
818 * destroyed, all elements in the supplied chain will also be destroyed.
819 *
820 * For this reason, appendChain() only accepts an rvalue-reference to a
821 * unique_ptr(), to make it clear that it is taking ownership of the supplied
822 * chain. If you have a raw pointer, you can pass in a new temporary
823 * unique_ptr around the raw pointer. If you have an existing,
824 * non-temporary unique_ptr, you must call std::move(ptr) to make it clear
825 * that you are destroying the original pointer.
826 */
827 void appendChain(std::unique_ptr<IOBuf>&& iobuf) {
828 // Just use prependChain() on the next element in our chain
829 next_->prependChain(std::move(iobuf));
830 }
831
832 /**
833 * Remove this IOBuf from its current chain.
834 *
835 * Since ownership of all elements an IOBuf chain is normally maintained by
836 * the head of the chain, unlink() transfers ownership of this IOBuf from the
837 * chain and gives it to the caller. A new unique_ptr to the IOBuf is
838 * returned to the caller. The caller must store the returned unique_ptr (or
839 * call release() on it) to take ownership, otherwise the IOBuf will be
840 * immediately destroyed.
841 *
842 * Since unlink transfers ownership of the IOBuf to the caller, be careful
843 * not to call unlink() on the head of a chain if you already maintain
844 * ownership on the head of the chain via other means. The pop() method
845 * is a better choice for that situation.
846 */
847 std::unique_ptr<IOBuf> unlink() {
848 next_->prev_ = prev_;
849 prev_->next_ = next_;
850 prev_ = this;
851 next_ = this;
852 return std::unique_ptr<IOBuf>(this);
853 }
854
855 /**
856 * Remove this IOBuf from its current chain and return a unique_ptr to
857 * the IOBuf that formerly followed it in the chain.
858 */
859 std::unique_ptr<IOBuf> pop() {
860 IOBuf* next = next_;
861 next_->prev_ = prev_;
862 prev_->next_ = next_;
863 prev_ = this;
864 next_ = this;
865 return std::unique_ptr<IOBuf>((next == this) ? nullptr : next);
866 }
867
868 /**
869 * Remove a subchain from this chain.
870 *
871 * Remove the subchain starting at head and ending at tail from this chain.
872 *
873 * Returns a unique_ptr pointing to head. (In other words, ownership of the
874 * head of the subchain is transferred to the caller.) If the caller ignores
875 * the return value and lets the unique_ptr be destroyed, the subchain will
876 * be immediately destroyed.
877 *
878 * The subchain referenced by the specified head and tail must be part of the
879 * same chain as the current IOBuf, but must not contain the current IOBuf.
880 * However, the specified head and tail may be equal to each other (i.e.,
881 * they may be a subchain of length 1).
882 */
883 std::unique_ptr<IOBuf> separateChain(IOBuf* head, IOBuf* tail) {
884 assert(head != this);
885 assert(tail != this);
886
887 head->prev_->next_ = tail->next_;
888 tail->next_->prev_ = head->prev_;
889
890 head->prev_ = tail;
891 tail->next_ = head;
892
893 return std::unique_ptr<IOBuf>(head);
894 }
895
896 /**
897 * Return true if at least one of the IOBufs in this chain are shared,
898 * or false if all of the IOBufs point to unique buffers.
899 *
900 * Use isSharedOne() to only check this IOBuf rather than the entire chain.
901 */
902 bool isShared() const {
903 const IOBuf* current = this;
904 while (true) {
905 if (current->isSharedOne()) {
906 return true;
907 }
908 current = current->next_;
909 if (current == this) {
910 return false;
911 }
912 }
913 }
914
915 /**
916 *
917 * Returns the SharedInfo::userData if sharedInfo()
918 * is not nullptr, nullptr otherwise
919 *
920 **/
921 void* getUserData() const {
922 SharedInfo* info = sharedInfo();
923 return info ? info->userData : nullptr;
924 }
925
926 /**
927 *
928 * Returns free function if sharedInfo() is not nullputr, nullptr otherwise
929 *
930 **/
931 FreeFunction getFreeFn() const {
932 SharedInfo* info = sharedInfo();
933 return info ? info->freeFn : nullptr;
934 }
935
936 /**
937 * Return true if all IOBufs in this chain are managed by the usual
938 * refcounting mechanism (and so the lifetime of the underlying memory
939 * can be extended by clone()).
940 */
941 bool isManaged() const {
942 const IOBuf* current = this;
943 while (true) {
944 if (!current->isManagedOne()) {
945 return false;
946 }
947 current = current->next_;
948 if (current == this) {
949 return true;
950 }
951 }
952 }
953
954 /**
955 * Return true if this IOBuf is managed by the usual refcounting mechanism
956 * (and so the lifetime of the underlying memory can be extended by
957 * cloneOne()).
958 */
959 bool isManagedOne() const {
960 return sharedInfo();
961 }
962
963 /**
964 * Return true if other IOBufs are also pointing to the buffer used by this
965 * IOBuf, and false otherwise.
966 *
967 * If this IOBuf points at a buffer owned by another (non-IOBuf) part of the
968 * code (i.e., if the IOBuf was created using wrapBuffer(), or was cloned
969 * from such an IOBuf), it is always considered shared.
970 *
971 * This only checks the current IOBuf, and not other IOBufs in the chain.
972 */
973 bool isSharedOne() const {
974 // If this is a user-owned buffer, it is always considered shared
975 if (UNLIKELY(!sharedInfo())) {
976 return true;
977 }
978
979 if (UNLIKELY(sharedInfo()->externallyShared)) {
980 return true;
981 }
982
983 if (LIKELY(!(flags() & kFlagMaybeShared))) {
984 return false;
985 }
986
987 // kFlagMaybeShared is set, so we need to check the reference count.
988 // (Checking the reference count requires an atomic operation, which is why
989 // we prefer to only check kFlagMaybeShared if possible.)
990 bool shared = sharedInfo()->refcount.load(std::memory_order_acquire) > 1;
991 if (!shared) {
992 // we're the last one left
993 clearFlags(kFlagMaybeShared);
994 }
995 return shared;
996 }
997
998 /**
999 * Ensure that this IOBuf has a unique buffer that is not shared by other
1000 * IOBufs.
1001 *
1002 * unshare() operates on an entire chain of IOBuf objects. If the chain is
1003 * shared, it may also coalesce the chain when making it unique. If the
1004 * chain is coalesced, subsequent IOBuf objects in the current chain will be
1005 * automatically deleted.
1006 *
1007 * Note that buffers owned by other (non-IOBuf) users are automatically
1008 * considered shared.
1009 *
1010 * Throws std::bad_alloc on error. On error the IOBuf chain will be
1011 * unmodified.
1012 *
1013 * Currently unshare may also throw std::overflow_error if it tries to
1014 * coalesce. (TODO: In the future it would be nice if unshare() were smart
1015 * enough not to coalesce the entire buffer if the data is too large.
1016 * However, in practice this seems unlikely to become an issue.)
1017 */
1018 void unshare() {
1019 if (isChained()) {
1020 unshareChained();
1021 } else {
1022 unshareOne();
1023 }
1024 }
1025
1026 /**
1027 * Ensure that this IOBuf has a unique buffer that is not shared by other
1028 * IOBufs.
1029 *
1030 * unshareOne() operates on a single IOBuf object. This IOBuf will have a
1031 * unique buffer after unshareOne() returns, but other IOBufs in the chain
1032 * may still be shared after unshareOne() returns.
1033 *
1034 * Throws std::bad_alloc on error. On error the IOBuf will be unmodified.
1035 */
1036 void unshareOne() {
1037 if (isSharedOne()) {
1038 unshareOneSlow();
1039 }
1040 }
1041
1042 /**
1043 * Mark the underlying buffers in this chain as shared with external memory
1044 * management mechanism. This will make isShared() always returns true.
1045 *
1046 * This function is not thread-safe, and only safe to call immediately after
1047 * creating an IOBuf, before it has been shared with other threads.
1048 */
1049 void markExternallyShared();
1050
1051 /**
1052 * Mark the underlying buffer that this IOBuf refers to as shared with
1053 * external memory management mechanism. This will make isSharedOne() always
1054 * returns true.
1055 *
1056 * This function is not thread-safe, and only safe to call immediately after
1057 * creating an IOBuf, before it has been shared with other threads.
1058 */
1059 void markExternallySharedOne() {
1060 SharedInfo* info = sharedInfo();
1061 if (info) {
1062 info->externallyShared = true;
1063 }
1064 }
1065
1066 /**
1067 * Ensure that the memory that IOBufs in this chain refer to will continue to
1068 * be allocated for as long as the IOBufs of the chain (or any clone()s
1069 * created from this point onwards) is alive.
1070 *
1071 * This only has an effect for user-owned buffers (created with the
1072 * WRAP_BUFFER constructor or wrapBuffer factory function), in which case
1073 * those buffers are unshared.
1074 */
1075 void makeManaged() {
1076 if (isChained()) {
1077 makeManagedChained();
1078 } else {
1079 makeManagedOne();
1080 }
1081 }
1082
1083 /**
1084 * Ensure that the memory that this IOBuf refers to will continue to be
1085 * allocated for as long as this IOBuf (or any clone()s created from this
1086 * point onwards) is alive.
1087 *
1088 * This only has an effect for user-owned buffers (created with the
1089 * WRAP_BUFFER constructor or wrapBuffer factory function), in which case
1090 * those buffers are unshared.
1091 */
1092 void makeManagedOne() {
1093 if (!isManagedOne()) {
1094 // We can call the internal function directly; unmanaged implies shared.
1095 unshareOneSlow();
1096 }
1097 }
1098
1099 /**
1100 * Coalesce this IOBuf chain into a single buffer.
1101 *
1102 * This method moves all of the data in this IOBuf chain into a single
1103 * contiguous buffer, if it is not already in one buffer. After coalesce()
1104 * returns, this IOBuf will be a chain of length one. Other IOBufs in the
1105 * chain will be automatically deleted.
1106 *
1107 * After coalescing, the IOBuf will have at least as much headroom as the
1108 * first IOBuf in the chain, and at least as much tailroom as the last IOBuf
1109 * in the chain.
1110 *
1111 * Throws std::bad_alloc on error. On error the IOBuf chain will be
1112 * unmodified.
1113 *
1114 * Returns ByteRange that points to the data IOBuf stores.
1115 */
1116 ByteRange coalesce() {
1117 const std::size_t newHeadroom = headroom();
1118 const std::size_t newTailroom = prev()->tailroom();
1119 return coalesceWithHeadroomTailroom(newHeadroom, newTailroom);
1120 }
1121
1122 /**
1123 * This is similar to the coalesce() method, except this allows to set a
1124 * headroom and tailroom after coalescing.
1125 *
1126 * Returns ByteRange that points to the data IOBuf stores.
1127 */
1128 ByteRange coalesceWithHeadroomTailroom(
1129 std::size_t newHeadroom,
1130 std::size_t newTailroom) {
1131 if (isChained()) {
1132 coalesceAndReallocate(
1133 newHeadroom, computeChainDataLength(), this, newTailroom);
1134 }
1135 return ByteRange(data_, length_);
1136 }
1137
1138 /**
1139 * Ensure that this chain has at least maxLength bytes available as a
1140 * contiguous memory range.
1141 *
1142 * This method coalesces whole buffers in the chain into this buffer as
1143 * necessary until this buffer's length() is at least maxLength.
1144 *
1145 * After coalescing, the IOBuf will have at least as much headroom as the
1146 * first IOBuf in the chain, and at least as much tailroom as the last IOBuf
1147 * that was coalesced.
1148 *
1149 * Throws std::bad_alloc or std::overflow_error on error. On error the IOBuf
1150 * chain will be unmodified. Throws std::overflow_error if maxLength is
1151 * longer than the total chain length.
1152 *
1153 * Upon return, either enough of the chain was coalesced into a contiguous
1154 * region, or the entire chain was coalesced. That is,
1155 * length() >= maxLength || !isChained() is true.
1156 */
1157 void gather(std::size_t maxLength) {
1158 if (!isChained() || length_ >= maxLength) {
1159 return;
1160 }
1161 coalesceSlow(maxLength);
1162 }
1163
1164 /**
1165 * Return a new IOBuf chain sharing the same data as this chain.
1166 *
1167 * The new IOBuf chain will normally point to the same underlying data
1168 * buffers as the original chain. (The one exception to this is if some of
1169 * the IOBufs in this chain contain small internal data buffers which cannot
1170 * be shared.)
1171 */
1172 std::unique_ptr<IOBuf> clone() const;
1173
1174 /**
1175 * Similar to clone(). But returns IOBuf by value rather than heap-allocating
1176 * it.
1177 */
1178 IOBuf cloneAsValue() const;
1179
1180 /**
1181 * Return a new IOBuf with the same data as this IOBuf.
1182 *
1183 * The new IOBuf returned will not be part of a chain (even if this IOBuf is
1184 * part of a larger chain).
1185 */
1186 std::unique_ptr<IOBuf> cloneOne() const;
1187
1188 /**
1189 * Similar to cloneOne(). But returns IOBuf by value rather than
1190 * heap-allocating it.
1191 */
1192 IOBuf cloneOneAsValue() const;
1193
1194 /**
1195 * Return a new unchained IOBuf that may share the same data as this chain.
1196 *
1197 * If the IOBuf chain is not chained then the new IOBuf will point to the same
1198 * underlying data buffer as the original chain. Otherwise, it will clone and
1199 * coalesce the IOBuf chain.
1200 *
1201 * The new IOBuf will have at least as much headroom as the first IOBuf in the
1202 * chain, and at least as much tailroom as the last IOBuf in the chain.
1203 *
1204 * Throws std::bad_alloc on error.
1205 */
1206 std::unique_ptr<IOBuf> cloneCoalesced() const;
1207
1208 /**
1209 * This is similar to the cloneCoalesced() method, except this allows to set a
1210 * headroom and tailroom for the new IOBuf.
1211 */
1212 std::unique_ptr<IOBuf> cloneCoalescedWithHeadroomTailroom(
1213 std::size_t newHeadroom,
1214 std::size_t newTailroom) const;
1215
1216 /**
1217 * Similar to cloneCoalesced(). But returns IOBuf by value rather than
1218 * heap-allocating it.
1219 */
1220 IOBuf cloneCoalescedAsValue() const;
1221
1222 /**
1223 * This is similar to the cloneCoalescedAsValue() method, except this allows
1224 * to set a headroom and tailroom for the new IOBuf.
1225 */
1226 IOBuf cloneCoalescedAsValueWithHeadroomTailroom(
1227 std::size_t newHeadroom,
1228 std::size_t newTailroom) const;
1229
1230 /**
1231 * Similar to Clone(). But use other as the head node. Other nodes in the
1232 * chain (if any) will be allocted on heap.
1233 */
1234 void cloneInto(IOBuf& other) const {
1235 other = cloneAsValue();
1236 }
1237
1238 /**
1239 * Similar to CloneOne(). But to fill an existing IOBuf instead of a new
1240 * IOBuf.
1241 */
1242 void cloneOneInto(IOBuf& other) const {
1243 other = cloneOneAsValue();
1244 }
1245
1246 /**
1247 * Return an iovector suitable for e.g. writev()
1248 *
1249 * auto iov = buf->getIov();
1250 * auto xfer = writev(fd, iov.data(), iov.size());
1251 *
1252 * Naturally, the returned iovector is invalid if you modify the buffer
1253 * chain.
1254 */
1255 folly::fbvector<struct iovec> getIov() const;
1256
1257 /**
1258 * Update an existing iovec array with the IOBuf data.
1259 *
1260 * New iovecs will be appended to the existing vector; anything already
1261 * present in the vector will be left unchanged.
1262 *
1263 * Naturally, the returned iovec data will be invalid if you modify the
1264 * buffer chain.
1265 */
1266 void appendToIov(folly::fbvector<struct iovec>* iov) const;
1267
1268 struct FillIovResult {
1269 // How many iovecs were filled (or 0 on error).
1270 size_t numIovecs;
1271 // The total length of filled iovecs (or 0 on error).
1272 size_t totalLength;
1273 };
1274
1275 /**
1276 * Fill an iovec array with the IOBuf data.
1277 *
1278 * Returns a struct with two fields: the number of iovec filled, and total
1279 * size of the iovecs filled. If there are more buffer than iovec, returns 0
1280 * in both fields.
1281 * This version is suitable to use with stack iovec arrays.
1282 *
1283 * Naturally, the filled iovec data will be invalid if you modify the
1284 * buffer chain.
1285 */
1286 FillIovResult fillIov(struct iovec* iov, size_t len) const;
1287
1288 /**
1289 * A helper that wraps a number of iovecs into an IOBuf chain. If count == 0,
1290 * then a zero length buf is returned. This function never returns nullptr.
1291 */
1292 static std::unique_ptr<IOBuf> wrapIov(const iovec* vec, size_t count);
1293
1294 /**
1295 * A helper that takes ownerships a number of iovecs into an IOBuf chain. If
1296 * count == 0, then a zero length buf is returned. This function never
1297 * returns nullptr.
1298 */
1299 static std::unique_ptr<IOBuf> takeOwnershipIov(
1300 const iovec* vec,
1301 size_t count,
1302 FreeFunction freeFn = nullptr,
1303 void* userData = nullptr,
1304 bool freeOnError = true);
1305
1306 /*
1307 * Overridden operator new and delete.
1308 * These perform specialized memory management to help support
1309 * createCombined(), which allocates IOBuf objects together with the buffer
1310 * data.
1311 */
1312 void* operator new(size_t size);
1313 void* operator new(size_t size, void* ptr);
1314 void operator delete(void* ptr);
1315 void operator delete(void* ptr, void* placement);
1316
1317 /**
1318 * Destructively convert this IOBuf to a fbstring efficiently.
1319 * We rely on fbstring's AcquireMallocatedString constructor to
1320 * transfer memory.
1321 */
1322 fbstring moveToFbString();
1323
1324 /**
1325 * Iteration support: a chain of IOBufs may be iterated through using
1326 * STL-style iterators over const ByteRanges. Iterators are only invalidated
1327 * if the IOBuf that they currently point to is removed.
1328 */
1329 Iterator cbegin() const;
1330 Iterator cend() const;
1331 Iterator begin() const;
1332 Iterator end() const;
1333
1334 /**
1335 * Allocate a new null buffer.
1336 *
1337 * This can be used to allocate an empty IOBuf on the stack. It will have no
1338 * space allocated for it. This is generally useful only to later use move
1339 * assignment to fill out the IOBuf.
1340 */
1341 IOBuf() noexcept;
1342
1343 /**
1344 * Move constructor and assignment operator.
1345 *
1346 * In general, you should only ever move the head of an IOBuf chain.
1347 * Internal nodes in an IOBuf chain are owned by the head of the chain, and
1348 * should not be moved from. (Technically, nothing prevents you from moving
1349 * a non-head node, but the moved-to node will replace the moved-from node in
1350 * the chain. This has implications for ownership, since non-head nodes are
1351 * owned by the chain head. You are then responsible for relinquishing
1352 * ownership of the moved-to node, and manually deleting the moved-from
1353 * node.)
1354 *
1355 * With the move assignment operator, the destination of the move should be
1356 * the head of an IOBuf chain or a solitary IOBuf not part of a chain. If
1357 * the move destination is part of a chain, all other IOBufs in the chain
1358 * will be deleted.
1359 */
1360 IOBuf(IOBuf&& other) noexcept;
1361 IOBuf& operator=(IOBuf&& other) noexcept;
1362
1363 IOBuf(const IOBuf& other);
1364 IOBuf& operator=(const IOBuf& other);
1365
1366 private:
1367 enum FlagsEnum : uintptr_t {
1368 // Adding any more flags would not work on 32-bit architectures,
1369 // as these flags are stashed in the least significant 2 bits of a
1370 // max-align-aligned pointer.
1371 kFlagFreeSharedInfo = 0x1,
1372 kFlagMaybeShared = 0x2,
1373 kFlagMask = kFlagFreeSharedInfo | kFlagMaybeShared
1374 };
1375
1376 struct SharedInfo {
1377 SharedInfo();
1378 SharedInfo(FreeFunction fn, void* arg, bool hfs = false);
1379
1380 static void releaseStorage(SharedInfo* info);
1381
1382 // A pointer to a function to call to free the buffer when the refcount
1383 // hits 0. If this is null, free() will be used instead.
1384 FreeFunction freeFn;
1385 void* userData;
1386 std::atomic<uint32_t> refcount;
1387 bool externallyShared{false};
1388 bool useHeapFullStorage{false};
1389 };
1390 // Helper structs for use by operator new and delete
1391 struct HeapPrefix;
1392 struct HeapStorage;
1393 struct HeapFullStorage;
1394
1395 /**
1396 * Create a new IOBuf pointing to an external buffer.
1397 *
1398 * The caller is responsible for holding a reference count for this new
1399 * IOBuf. The IOBuf constructor does not automatically increment the
1400 * reference count.
1401 */
1402 struct InternalConstructor {}; // avoid conflicts
1403 IOBuf(
1404 InternalConstructor,
1405 uintptr_t flagsAndSharedInfo,
1406 uint8_t* buf,
1407 std::size_t capacity,
1408 uint8_t* data,
1409 std::size_t length) noexcept;
1410
1411 void unshareOneSlow();
1412 void unshareChained();
1413 void makeManagedChained();
1414 void coalesceSlow();
1415 void coalesceSlow(size_t maxLength);
1416 // newLength must be the entire length of the buffers between this and
1417 // end (no truncation)
1418 void coalesceAndReallocate(
1419 size_t newHeadroom,
1420 size_t newLength,
1421 IOBuf* end,
1422 size_t newTailroom);
1423 void coalesceAndReallocate(size_t newLength, IOBuf* end) {
1424 coalesceAndReallocate(headroom(), newLength, end, end->prev_->tailroom());
1425 }
1426 void decrementRefcount();
1427 void reserveSlow(std::size_t minHeadroom, std::size_t minTailroom);
1428 void freeExtBuffer();
1429
1430 static size_t goodExtBufferSize(std::size_t minCapacity);
1431 static void initExtBuffer(
1432 uint8_t* buf,
1433 size_t mallocSize,
1434 SharedInfo** infoReturn,
1435 std::size_t* capacityReturn);
1436 static void allocExtBuffer(
1437 std::size_t minCapacity,
1438 uint8_t** bufReturn,
1439 SharedInfo** infoReturn,
1440 std::size_t* capacityReturn);
1441 static void releaseStorage(HeapStorage* storage, uint16_t freeFlags);
1442 static void freeInternalBuf(void* buf, void* userData);
1443
1444 /*
1445 * Member variables
1446 */
1447
1448 /*
1449 * Links to the next and the previous IOBuf in this chain.
1450 *
1451 * The chain is circularly linked (the last element in the chain points back
1452 * at the head), and next_ and prev_ can never be null. If this IOBuf is the
1453 * only element in the chain, next_ and prev_ will both point to this.
1454 */
1455 IOBuf* next_{this};
1456 IOBuf* prev_{this};
1457
1458 /*
1459 * A pointer to the start of the data referenced by this IOBuf, and the
1460 * length of the data.
1461 *
1462 * This may refer to any subsection of the actual buffer capacity.
1463 */
1464 uint8_t* data_{nullptr};
1465 uint8_t* buf_{nullptr};
1466 std::size_t length_{0};
1467 std::size_t capacity_{0};
1468
1469 // Pack flags in least significant 2 bits, sharedInfo in the rest
1470 mutable uintptr_t flagsAndSharedInfo_{0};
1471
1472 static inline uintptr_t packFlagsAndSharedInfo(
1473 uintptr_t flags,
1474 SharedInfo* info) {
1475 uintptr_t uinfo = reinterpret_cast<uintptr_t>(info);
1476 DCHECK_EQ(flags & ~kFlagMask, 0u);
1477 DCHECK_EQ(uinfo & kFlagMask, 0u);
1478 return flags | uinfo;
1479 }
1480
1481 inline SharedInfo* sharedInfo() const {
1482 return reinterpret_cast<SharedInfo*>(flagsAndSharedInfo_ & ~kFlagMask);
1483 }
1484
1485 inline void setSharedInfo(SharedInfo* info) {
1486 uintptr_t uinfo = reinterpret_cast<uintptr_t>(info);
1487 DCHECK_EQ(uinfo & kFlagMask, 0u);
1488 flagsAndSharedInfo_ = (flagsAndSharedInfo_ & kFlagMask) | uinfo;
1489 }
1490
1491 inline uintptr_t flags() const {
1492 return flagsAndSharedInfo_ & kFlagMask;
1493 }
1494
1495 // flags_ are changed from const methods
1496 inline void setFlags(uintptr_t flags) const {
1497 DCHECK_EQ(flags & ~kFlagMask, 0u);
1498 flagsAndSharedInfo_ |= flags;
1499 }
1500
1501 inline void clearFlags(uintptr_t flags) const {
1502 DCHECK_EQ(flags & ~kFlagMask, 0u);
1503 flagsAndSharedInfo_ &= ~flags;
1504 }
1505
1506 inline void setFlagsAndSharedInfo(uintptr_t flags, SharedInfo* info) {
1507 flagsAndSharedInfo_ = packFlagsAndSharedInfo(flags, info);
1508 }
1509
1510 struct DeleterBase {
1511 virtual ~DeleterBase() {}
1512 virtual void dispose(void* p) = 0;
1513 };
1514
1515 template <class UniquePtr>
1516 struct UniquePtrDeleter : public DeleterBase {
1517 typedef typename UniquePtr::pointer Pointer;
1518 typedef typename UniquePtr::deleter_type Deleter;
1519
1520 explicit UniquePtrDeleter(Deleter deleter) : deleter_(std::move(deleter)) {}
1521 void dispose(void* p) override {
1522 try {
1523 deleter_(static_cast<Pointer>(p));
1524 delete this;
1525 } catch (...) {
1526 abort();
1527 }
1528 }
1529
1530 private:
1531 Deleter deleter_;
1532 };
1533
1534 static void freeUniquePtrBuffer(void* ptr, void* userData) {
1535 static_cast<DeleterBase*>(userData)->dispose(ptr);
1536 }
1537};
1538
1539/**
1540 * Hasher for IOBuf objects. Hashes the entire chain using SpookyHashV2.
1541 */
1542struct IOBufHash {
1543 size_t operator()(const IOBuf& buf) const noexcept;
1544 size_t operator()(const std::unique_ptr<IOBuf>& buf) const noexcept {
1545 return operator()(buf.get());
1546 }
1547 size_t operator()(const IOBuf* buf) const noexcept {
1548 return buf ? (*this)(*buf) : 0;
1549 }
1550};
1551
1552/**
1553 * Ordering for IOBuf objects. Compares data in the entire chain.
1554 */
1555struct IOBufCompare {
1556 ordering operator()(const IOBuf& a, const IOBuf& b) const {
1557 return &a == &b ? ordering::eq : impl(a, b);
1558 }
1559 ordering operator()(
1560 const std::unique_ptr<IOBuf>& a,
1561 const std::unique_ptr<IOBuf>& b) const {
1562 return operator()(a.get(), b.get());
1563 }
1564 ordering operator()(const IOBuf* a, const IOBuf* b) const {
1565 // clang-format off
1566 return
1567 !a && !b ? ordering::eq :
1568 !a && b ? ordering::lt :
1569 a && !b ? ordering::gt :
1570 operator()(*a, *b);
1571 // clang-format on
1572 }
1573
1574 private:
1575 ordering impl(IOBuf const& a, IOBuf const& b) const noexcept;
1576};
1577
1578/**
1579 * Equality predicate for IOBuf objects. Compares data in the entire chain.
1580 */
1581struct IOBufEqualTo : compare_equal_to<IOBufCompare> {};
1582
1583/**
1584 * Inequality predicate for IOBuf objects. Compares data in the entire chain.
1585 */
1586struct IOBufNotEqualTo : compare_not_equal_to<IOBufCompare> {};
1587
1588/**
1589 * Less predicate for IOBuf objects. Compares data in the entire chain.
1590 */
1591struct IOBufLess : compare_less<IOBufCompare> {};
1592
1593/**
1594 * At-most predicate for IOBuf objects. Compares data in the entire chain.
1595 */
1596struct IOBufLessEqual : compare_less_equal<IOBufCompare> {};
1597
1598/**
1599 * Greater predicate for IOBuf objects. Compares data in the entire chain.
1600 */
1601struct IOBufGreater : compare_greater<IOBufCompare> {};
1602
1603/**
1604 * At-least predicate for IOBuf objects. Compares data in the entire chain.
1605 */
1606struct IOBufGreaterEqual : compare_greater_equal<IOBufCompare> {};
1607
1608template <class UniquePtr>
1609typename std::enable_if<
1610 detail::IsUniquePtrToSL<UniquePtr>::value,
1611 std::unique_ptr<IOBuf>>::type
1612IOBuf::takeOwnership(UniquePtr&& buf, size_t count) {
1613 size_t size = count * sizeof(typename UniquePtr::element_type);
1614 auto deleter = new UniquePtrDeleter<UniquePtr>(buf.get_deleter());
1615 return takeOwnership(
1616 buf.release(), size, &IOBuf::freeUniquePtrBuffer, deleter);
1617}
1618
1619inline std::unique_ptr<IOBuf> IOBuf::copyBuffer(
1620 const void* data,
1621 std::size_t size,
1622 std::size_t headroom,
1623 std::size_t minTailroom) {
1624 std::size_t capacity = headroom + size + minTailroom;
1625 std::unique_ptr<IOBuf> buf = create(capacity);
1626 buf->advance(headroom);
1627 if (size != 0) {
1628 memcpy(buf->writableData(), data, size);
1629 }
1630 buf->append(size);
1631 return buf;
1632}
1633
1634inline std::unique_ptr<IOBuf> IOBuf::copyBuffer(
1635 const std::string& buf,
1636 std::size_t headroom,
1637 std::size_t minTailroom) {
1638 return copyBuffer(buf.data(), buf.size(), headroom, minTailroom);
1639}
1640
1641inline std::unique_ptr<IOBuf> IOBuf::maybeCopyBuffer(
1642 const std::string& buf,
1643 std::size_t headroom,
1644 std::size_t minTailroom) {
1645 if (buf.empty()) {
1646 return nullptr;
1647 }
1648 return copyBuffer(buf.data(), buf.size(), headroom, minTailroom);
1649}
1650
1651class IOBuf::Iterator : public detail::IteratorFacade<
1652 IOBuf::Iterator,
1653 ByteRange const,
1654 std::forward_iterator_tag> {
1655 public:
1656 // Note that IOBufs are stored as a circular list without a guard node,
1657 // so pos == end is ambiguous (it may mean "begin" or "end"). To solve
1658 // the ambiguity (at the cost of one extra comparison in the "increment"
1659 // code path), we define end iterators as having pos_ == end_ == nullptr
1660 // and we only allow forward iteration.
1661 explicit Iterator(const IOBuf* pos, const IOBuf* end) : pos_(pos), end_(end) {
1662 // Sadly, we must return by const reference, not by value.
1663 if (pos_) {
1664 setVal();
1665 }
1666 }
1667
1668 Iterator() {}
1669
1670 Iterator(Iterator const& rhs) : Iterator(rhs.pos_, rhs.end_) {}
1671
1672 Iterator& operator=(Iterator const& rhs) {
1673 pos_ = rhs.pos_;
1674 end_ = rhs.end_;
1675 if (pos_) {
1676 setVal();
1677 }
1678 return *this;
1679 }
1680
1681 const ByteRange& dereference() const {
1682 return val_;
1683 }
1684
1685 bool equal(const Iterator& other) const {
1686 // We must compare end_ in addition to pos_, because forward traversal
1687 // requires that if two iterators are equal (a == b) and dereferenceable,
1688 // then ++a == ++b.
1689 return pos_ == other.pos_ && end_ == other.end_;
1690 }
1691
1692 void increment() {
1693 pos_ = pos_->next();
1694 adjustForEnd();
1695 }
1696
1697 private:
1698 void setVal() {
1699 val_ = ByteRange(pos_->data(), pos_->tail());
1700 }
1701
1702 void adjustForEnd() {
1703 if (pos_ == end_) {
1704 pos_ = end_ = nullptr;
1705 val_ = ByteRange();
1706 } else {
1707 setVal();
1708 }
1709 }
1710
1711 const IOBuf* pos_{nullptr};
1712 const IOBuf* end_{nullptr};
1713 ByteRange val_;
1714};
1715
1716inline IOBuf::Iterator IOBuf::begin() const {
1717 return cbegin();
1718}
1719inline IOBuf::Iterator IOBuf::end() const {
1720 return cend();
1721}
1722
1723} // namespace folly
1724
1725FOLLY_POP_WARNING
1726