1//===----------------------------------------------------------------------===//
2// DuckDB
3//
4// duckdb/storage/partial_block_manager.hpp
5//
6//
7//===----------------------------------------------------------------------===//
8
9#pragma once
10
11#include "duckdb/common/common.hpp"
12#include "duckdb/common/map.hpp"
13#include "duckdb/storage/storage_manager.hpp"
14#include "duckdb/storage/meta_block_writer.hpp"
15#include "duckdb/storage/data_pointer.hpp"
16
17namespace duckdb {
18class DatabaseInstance;
19class ClientContext;
20class ColumnSegment;
21class MetaBlockReader;
22class SchemaCatalogEntry;
23class SequenceCatalogEntry;
24class TableCatalogEntry;
25class ViewCatalogEntry;
26class TypeCatalogEntry;
27
28struct PartialBlockState {
29 block_id_t block_id;
30 //! How big is the block we're writing to. (Total bytes to assign).
31 uint32_t block_size;
32 //! How many bytes of the allocation are used. (offset_in_block of next allocation)
33 uint32_t offset_in_block;
34 //! How many times has the block been used?
35 uint32_t block_use_count;
36};
37
38struct PartialBlock {
39 explicit PartialBlock(PartialBlockState state) : state(std::move(state)) {
40 }
41 virtual ~PartialBlock() {
42 }
43
44 PartialBlockState state;
45
46public:
47 virtual void AddUninitializedRegion(idx_t start, idx_t end) = 0;
48 virtual void Flush(idx_t free_space_left) = 0;
49 virtual void Clear() {
50 }
51 virtual void Merge(PartialBlock &other, idx_t offset, idx_t other_size);
52
53public:
54 template <class TARGET>
55 TARGET &Cast() {
56 D_ASSERT(dynamic_cast<TARGET *>(this));
57 return reinterpret_cast<TARGET &>(*this);
58 }
59};
60
61struct PartialBlockAllocation {
62 // BlockManager owning the block_id
63 BlockManager *block_manager {nullptr};
64 //! How many bytes assigned to the caller?
65 uint32_t allocation_size;
66 //! State of assigned block.
67 PartialBlockState state;
68 //! Arbitrary state related to partial block storage.
69 unique_ptr<PartialBlock> partial_block;
70};
71
72enum class CheckpointType { FULL_CHECKPOINT, APPEND_TO_TABLE };
73
74//! Enables sharing blocks across some scope. Scope is whatever we want to share
75//! blocks across. It may be an entire checkpoint or just a single row group.
76//! In any case, they must share a block manager.
77class PartialBlockManager {
78public:
79 // 20% free / 80% utilization
80 static constexpr const idx_t DEFAULT_MAX_PARTIAL_BLOCK_SIZE = Storage::BLOCK_SIZE / 5 * 4;
81 // Max number of shared references to a block. No effective limit by default.
82 static constexpr const idx_t DEFAULT_MAX_USE_COUNT = 1u << 20;
83 // No point letting map size grow unbounded. We'll drop blocks with the
84 // least free space first.
85 static constexpr const idx_t MAX_BLOCK_MAP_SIZE = 1u << 31;
86
87public:
88 PartialBlockManager(BlockManager &block_manager, CheckpointType checkpoint_type,
89 uint32_t max_partial_block_size = DEFAULT_MAX_PARTIAL_BLOCK_SIZE,
90 uint32_t max_use_count = DEFAULT_MAX_USE_COUNT);
91 virtual ~PartialBlockManager();
92
93public:
94 //! Flush any remaining partial blocks to disk
95 void FlushPartialBlocks();
96
97 PartialBlockAllocation GetBlockAllocation(uint32_t segment_size);
98
99 virtual void AllocateBlock(PartialBlockState &state, uint32_t segment_size);
100
101 void Merge(PartialBlockManager &other);
102 //! Register a partially filled block that is filled with "segment_size" entries
103 void RegisterPartialBlock(PartialBlockAllocation &&allocation);
104
105 //! Clear remaining blocks without writing them to disk
106 void ClearBlocks();
107
108 //! Rollback all data written by this partial block manager
109 void Rollback();
110
111protected:
112 BlockManager &block_manager;
113 CheckpointType checkpoint_type;
114 //! A map of (available space -> PartialBlock) for partially filled blocks
115 //! This is a multimap because there might be outstanding partial blocks with
116 //! the same amount of left-over space
117 multimap<idx_t, unique_ptr<PartialBlock>> partially_filled_blocks;
118 //! The set of written blocks
119 unordered_set<block_id_t> written_blocks;
120
121 //! The maximum size (in bytes) at which a partial block will be considered a partial block
122 uint32_t max_partial_block_size;
123 uint32_t max_use_count;
124
125protected:
126 //! Try to obtain a partially filled block that can fit "segment_size" bytes
127 //! If successful, returns true and returns the block_id and offset_in_block to write to
128 //! Otherwise, returns false
129 bool GetPartialBlock(idx_t segment_size, unique_ptr<PartialBlock> &state);
130
131 bool HasBlockAllocation(uint32_t segment_size);
132 void AddWrittenBlock(block_id_t block);
133};
134
135} // namespace duckdb
136