1//===----------------------------------------------------------------------===//
2// DuckDB
3//
4// duckdb/transaction/local_storage.hpp
5//
6//
7//===----------------------------------------------------------------------===//
8
9#pragma once
10
11#include "duckdb/storage/table/row_group_collection.hpp"
12#include "duckdb/storage/table/table_index_list.hpp"
13#include "duckdb/storage/table/table_statistics.hpp"
14#include "duckdb/storage/optimistic_data_writer.hpp"
15
16namespace duckdb {
17class AttachedDatabase;
18class DataTable;
19class Transaction;
20class WriteAheadLog;
21struct LocalAppendState;
22struct TableAppendState;
23
24class LocalTableStorage : public std::enable_shared_from_this<LocalTableStorage> {
25public:
26 // Create a new LocalTableStorage
27 explicit LocalTableStorage(DataTable &table);
28 // Create a LocalTableStorage from an ALTER TYPE
29 LocalTableStorage(ClientContext &context, DataTable &table, LocalTableStorage &parent, idx_t changed_idx,
30 const LogicalType &target_type, const vector<column_t> &bound_columns, Expression &cast_expr);
31 // Create a LocalTableStorage from a DROP COLUMN
32 LocalTableStorage(DataTable &table, LocalTableStorage &parent, idx_t drop_idx);
33 // Create a LocalTableStorage from an ADD COLUMN
34 LocalTableStorage(ClientContext &context, DataTable &table, LocalTableStorage &parent, ColumnDefinition &new_column,
35 optional_ptr<Expression> default_value);
36 ~LocalTableStorage();
37
38 reference<DataTable> table_ref;
39
40 Allocator &allocator;
41 //! The main chunk collection holding the data
42 shared_ptr<RowGroupCollection> row_groups;
43 //! The set of unique indexes
44 TableIndexList indexes;
45 //! The number of deleted rows
46 idx_t deleted_rows;
47 //! The main optimistic data writer
48 OptimisticDataWriter optimistic_writer;
49 //! The set of all optimistic data writers associated with this table
50 vector<unique_ptr<OptimisticDataWriter>> optimistic_writers;
51 //! Whether or not storage was merged
52 bool merged_storage = false;
53
54public:
55 void InitializeScan(CollectionScanState &state, optional_ptr<TableFilterSet> table_filters = nullptr);
56 //! Write a new row group to disk (if possible)
57 void WriteNewRowGroup();
58 void FlushBlocks();
59 void Rollback();
60 idx_t EstimatedSize();
61
62 void AppendToIndexes(DuckTransaction &transaction, TableAppendState &append_state, idx_t append_count,
63 bool append_to_table);
64 PreservedError AppendToIndexes(DuckTransaction &transaction, RowGroupCollection &source, TableIndexList &index_list,
65 const vector<LogicalType> &table_types, row_t &start_row);
66
67 //! Creates an optimistic writer for this table
68 OptimisticDataWriter &CreateOptimisticWriter();
69 void FinalizeOptimisticWriter(OptimisticDataWriter &writer);
70};
71
72class LocalTableManager {
73public:
74 shared_ptr<LocalTableStorage> MoveEntry(DataTable &table);
75 reference_map_t<DataTable, shared_ptr<LocalTableStorage>> MoveEntries();
76 optional_ptr<LocalTableStorage> GetStorage(DataTable &table);
77 LocalTableStorage &GetOrCreateStorage(DataTable &table);
78 idx_t EstimatedSize();
79 bool IsEmpty();
80 void InsertEntry(DataTable &table, shared_ptr<LocalTableStorage> entry);
81
82private:
83 mutex table_storage_lock;
84 reference_map_t<DataTable, shared_ptr<LocalTableStorage>> table_storage;
85};
86
87//! The LocalStorage class holds appends that have not been committed yet
88class LocalStorage {
89public:
90 // Threshold to merge row groups instead of appending
91 static constexpr const idx_t MERGE_THRESHOLD = RowGroup::ROW_GROUP_SIZE;
92
93public:
94 struct CommitState {
95 CommitState();
96 ~CommitState();
97
98 reference_map_t<DataTable, unique_ptr<TableAppendState>> append_states;
99 };
100
101public:
102 explicit LocalStorage(ClientContext &context, DuckTransaction &transaction);
103
104 static LocalStorage &Get(DuckTransaction &transaction);
105 static LocalStorage &Get(ClientContext &context, AttachedDatabase &db);
106 static LocalStorage &Get(ClientContext &context, Catalog &catalog);
107
108 //! Initialize a scan of the local storage
109 void InitializeScan(DataTable &table, CollectionScanState &state, optional_ptr<TableFilterSet> table_filters);
110 //! Scan
111 void Scan(CollectionScanState &state, const vector<storage_t> &column_ids, DataChunk &result);
112
113 void InitializeParallelScan(DataTable &table, ParallelCollectionScanState &state);
114 bool NextParallelScan(ClientContext &context, DataTable &table, ParallelCollectionScanState &state,
115 CollectionScanState &scan_state);
116
117 //! Begin appending to the local storage
118 void InitializeAppend(LocalAppendState &state, DataTable &table);
119 //! Append a chunk to the local storage
120 static void Append(LocalAppendState &state, DataChunk &chunk);
121 //! Finish appending to the local storage
122 static void FinalizeAppend(LocalAppendState &state);
123 //! Merge a row group collection into the transaction-local storage
124 void LocalMerge(DataTable &table, RowGroupCollection &collection);
125 //! Create an optimistic writer for the specified table
126 OptimisticDataWriter &CreateOptimisticWriter(DataTable &table);
127 void FinalizeOptimisticWriter(DataTable &table, OptimisticDataWriter &writer);
128
129 //! Delete a set of rows from the local storage
130 idx_t Delete(DataTable &table, Vector &row_ids, idx_t count);
131 //! Update a set of rows in the local storage
132 void Update(DataTable &table, Vector &row_ids, const vector<PhysicalIndex> &column_ids, DataChunk &data);
133
134 //! Commits the local storage, writing it to the WAL and completing the commit
135 void Commit(LocalStorage::CommitState &commit_state, DuckTransaction &transaction);
136 //! Rollback the local storage
137 void Rollback();
138
139 bool ChangesMade() noexcept;
140 idx_t EstimatedSize();
141
142 bool Find(DataTable &table);
143
144 idx_t AddedRows(DataTable &table);
145
146 void AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinition &new_column,
147 optional_ptr<Expression> default_value);
148 void DropColumn(DataTable &old_dt, DataTable &new_dt, idx_t removed_column);
149 void ChangeType(DataTable &old_dt, DataTable &new_dt, idx_t changed_idx, const LogicalType &target_type,
150 const vector<column_t> &bound_columns, Expression &cast_expr);
151
152 void MoveStorage(DataTable &old_dt, DataTable &new_dt);
153 void FetchChunk(DataTable &table, Vector &row_ids, idx_t count, const vector<column_t> &col_ids, DataChunk &chunk,
154 ColumnFetchState &fetch_state);
155 TableIndexList &GetIndexes(DataTable &table);
156
157 void VerifyNewConstraint(DataTable &parent, const BoundConstraint &constraint);
158
159private:
160 ClientContext &context;
161 DuckTransaction &transaction;
162 LocalTableManager table_manager;
163
164 void Flush(DataTable &table, LocalTableStorage &storage);
165};
166
167} // namespace duckdb
168