1#include "duckdb/storage/storage_manager.hpp"
2#include "duckdb/storage/checkpoint_manager.hpp"
3#include "duckdb/storage/in_memory_block_manager.hpp"
4#include "duckdb/storage/single_file_block_manager.hpp"
5#include "duckdb/storage/object_cache.hpp"
6
7#include "duckdb/catalog/catalog.hpp"
8#include "duckdb/common/file_system.hpp"
9#include "duckdb/main/database.hpp"
10#include "duckdb/main/client_context.hpp"
11#include "duckdb/function/function.hpp"
12#include "duckdb/transaction/transaction_manager.hpp"
13#include "duckdb/common/serializer/buffered_file_reader.hpp"
14#include "duckdb/main/attached_database.hpp"
15#include "duckdb/main/database_manager.hpp"
16
17namespace duckdb {
18
19StorageManager::StorageManager(AttachedDatabase &db, string path_p, bool read_only)
20 : db(db), path(std::move(path_p)), read_only(read_only) {
21 if (path.empty()) {
22 path = ":memory:";
23 } else {
24 auto &fs = FileSystem::Get(db);
25 this->path = fs.ExpandPath(path);
26 }
27}
28
29StorageManager::~StorageManager() {
30}
31
32StorageManager &StorageManager::Get(AttachedDatabase &db) {
33 return db.GetStorageManager();
34}
35StorageManager &StorageManager::Get(Catalog &catalog) {
36 return StorageManager::Get(db&: catalog.GetAttached());
37}
38
39DatabaseInstance &StorageManager::GetDatabase() {
40 return db.GetDatabase();
41}
42
43BufferManager &BufferManager::GetBufferManager(ClientContext &context) {
44 return BufferManager::GetBufferManager(db&: *context.db);
45}
46
47ObjectCache &ObjectCache::GetObjectCache(ClientContext &context) {
48 return context.db->GetObjectCache();
49}
50
51bool ObjectCache::ObjectCacheEnabled(ClientContext &context) {
52 return context.db->config.options.object_cache_enable;
53}
54
55bool StorageManager::InMemory() {
56 D_ASSERT(!path.empty());
57 return path == ":memory:";
58}
59
60void StorageManager::Initialize() {
61 bool in_memory = InMemory();
62 if (in_memory && read_only) {
63 throw CatalogException("Cannot launch in-memory database in read-only mode!");
64 }
65
66 // create or load the database from disk, if not in-memory mode
67 LoadDatabase();
68}
69
70///////////////////////////////////////////////////////////////////////////
71class SingleFileTableIOManager : public TableIOManager {
72public:
73 explicit SingleFileTableIOManager(BlockManager &block_manager) : block_manager(block_manager) {
74 }
75
76 BlockManager &block_manager;
77
78public:
79 BlockManager &GetIndexBlockManager() override {
80 return block_manager;
81 }
82 BlockManager &GetBlockManagerForRowData() override {
83 return block_manager;
84 }
85};
86
87SingleFileStorageManager::SingleFileStorageManager(AttachedDatabase &db, string path, bool read_only)
88 : StorageManager(db, std::move(path), read_only) {
89}
90
91void SingleFileStorageManager::LoadDatabase() {
92 if (InMemory()) {
93 block_manager = make_uniq<InMemoryBlockManager>(args&: BufferManager::GetBufferManager(db));
94 table_io_manager = make_uniq<SingleFileTableIOManager>(args&: *block_manager);
95 return;
96 }
97
98 string wal_path = path + ".wal";
99 auto &fs = FileSystem::Get(db);
100 auto &config = DBConfig::Get(db);
101 bool truncate_wal = false;
102 if (!config.options.enable_external_access) {
103 if (!db.IsInitialDatabase()) {
104 throw PermissionException("Attaching on-disk databases is disabled through configuration");
105 }
106 }
107
108 StorageManagerOptions options;
109 options.read_only = read_only;
110 options.use_direct_io = config.options.use_direct_io;
111 options.debug_initialize = config.options.debug_initialize;
112 // first check if the database exists
113 if (!fs.FileExists(filename: path)) {
114 if (read_only) {
115 throw CatalogException("Cannot open database \"%s\" in read-only mode: database does not exist", path);
116 }
117 // check if the WAL exists
118 if (fs.FileExists(filename: wal_path)) {
119 // WAL file exists but database file does not
120 // remove the WAL
121 fs.RemoveFile(filename: wal_path);
122 }
123 // initialize the block manager while creating a new db file
124 auto sf_block_manager = make_uniq<SingleFileBlockManager>(args&: db, args&: path, args&: options);
125 sf_block_manager->CreateNewDatabase();
126 block_manager = std::move(sf_block_manager);
127 table_io_manager = make_uniq<SingleFileTableIOManager>(args&: *block_manager);
128 } else {
129 // initialize the block manager while loading the current db file
130 auto sf_block_manager = make_uniq<SingleFileBlockManager>(args&: db, args&: path, args&: options);
131 sf_block_manager->LoadExistingDatabase();
132 block_manager = std::move(sf_block_manager);
133 table_io_manager = make_uniq<SingleFileTableIOManager>(args&: *block_manager);
134
135 //! Load from storage
136 auto checkpointer = SingleFileCheckpointReader(*this);
137 checkpointer.LoadFromStorage();
138 // finish load checkpoint, clear the cached handles of meta blocks
139 block_manager->ClearMetaBlockHandles();
140 // check if the WAL file exists
141 if (fs.FileExists(filename: wal_path)) {
142 // replay the WAL
143 truncate_wal = WriteAheadLog::Replay(database&: db, path&: wal_path);
144 }
145 }
146 // initialize the WAL file
147 if (!read_only) {
148 wal = make_uniq<WriteAheadLog>(args&: db, args&: wal_path);
149 if (truncate_wal) {
150 wal->Truncate(size: 0);
151 }
152 }
153}
154
155///////////////////////////////////////////////////////////////////////////////
156
157class SingleFileStorageCommitState : public StorageCommitState {
158 idx_t initial_wal_size = 0;
159 idx_t initial_written = 0;
160 optional_ptr<WriteAheadLog> log;
161 bool checkpoint;
162
163public:
164 SingleFileStorageCommitState(StorageManager &storage_manager, bool checkpoint);
165 ~SingleFileStorageCommitState() override {
166 // If log is non-null, then commit threw an exception before flushing.
167 if (log) {
168 auto &wal = *log.get();
169 wal.skip_writing = false;
170 if (wal.GetTotalWritten() > initial_written) {
171 // remove any entries written into the WAL by truncating it
172 wal.Truncate(size: initial_wal_size);
173 }
174 }
175 }
176
177 // Make the commit persistent
178 void FlushCommit() override;
179};
180
181SingleFileStorageCommitState::SingleFileStorageCommitState(StorageManager &storage_manager, bool checkpoint)
182 : checkpoint(checkpoint) {
183 log = storage_manager.GetWriteAheadLog();
184 if (log) {
185 auto initial_size = log->GetWALSize();
186 initial_written = log->GetTotalWritten();
187 initial_wal_size = initial_size < 0 ? 0 : idx_t(initial_size);
188
189 if (checkpoint) {
190 // check if we are checkpointing after this commit
191 // if we are checkpointing, we don't need to write anything to the WAL
192 // this saves us a lot of unnecessary writes to disk in the case of large commits
193 log->skip_writing = true;
194 }
195 } else {
196 D_ASSERT(!checkpoint);
197 }
198}
199
200// Make the commit persistent
201void SingleFileStorageCommitState::FlushCommit() {
202 if (log) {
203 // flush the WAL if any changes were made
204 if (log->GetTotalWritten() > initial_written) {
205 (void)checkpoint;
206 D_ASSERT(!checkpoint);
207 D_ASSERT(!log->skip_writing);
208 log->Flush();
209 }
210 log->skip_writing = false;
211 }
212 // Null so that the destructor will not truncate the log.
213 log = nullptr;
214}
215
216unique_ptr<StorageCommitState> SingleFileStorageManager::GenStorageCommitState(Transaction &transaction,
217 bool checkpoint) {
218 return make_uniq<SingleFileStorageCommitState>(args&: *this, args&: checkpoint);
219}
220
221bool SingleFileStorageManager::IsCheckpointClean(block_id_t checkpoint_id) {
222 return block_manager->IsRootBlock(root: checkpoint_id);
223}
224
225void SingleFileStorageManager::CreateCheckpoint(bool delete_wal, bool force_checkpoint) {
226 if (InMemory() || read_only || !wal) {
227 return;
228 }
229 auto &config = DBConfig::Get(db);
230 if (wal->GetWALSize() > 0 || config.options.force_checkpoint || force_checkpoint) {
231 // we only need to checkpoint if there is anything in the WAL
232 SingleFileCheckpointWriter checkpointer(db, *block_manager);
233 checkpointer.CreateCheckpoint();
234 }
235 if (delete_wal) {
236 wal->Delete();
237 wal.reset();
238 }
239}
240
241DatabaseSize SingleFileStorageManager::GetDatabaseSize() {
242 // All members default to zero
243 DatabaseSize ds;
244 if (!InMemory()) {
245 ds.total_blocks = block_manager->TotalBlocks();
246 ds.block_size = Storage::BLOCK_ALLOC_SIZE;
247 ds.free_blocks = block_manager->FreeBlocks();
248 ds.used_blocks = ds.total_blocks - ds.free_blocks;
249 ds.bytes = (ds.total_blocks * ds.block_size);
250 if (auto wal = GetWriteAheadLog()) {
251 ds.wal_size = wal->GetWALSize();
252 }
253 }
254 return ds;
255}
256
257bool SingleFileStorageManager::AutomaticCheckpoint(idx_t estimated_wal_bytes) {
258 auto log = GetWriteAheadLog();
259 if (!log) {
260 return false;
261 }
262
263 auto &config = DBConfig::Get(db);
264 auto initial_size = log->GetWALSize();
265 idx_t expected_wal_size = initial_size + estimated_wal_bytes;
266 return expected_wal_size > config.options.checkpoint_wal_size;
267}
268
269shared_ptr<TableIOManager> SingleFileStorageManager::GetTableIOManager(BoundCreateTableInfo *info /*info*/) {
270 // This is an unmanaged reference. No ref/deref overhead. Lifetime of the
271 // TableIoManager follows lifetime of the StorageManager (this).
272 return shared_ptr<TableIOManager>(shared_ptr<char>(nullptr), table_io_manager.get());
273}
274
275} // namespace duckdb
276