1/*-------------------------------------------------------------------------
2 *
3 * tuptoaster.h
4 * POSTGRES definitions for external and compressed storage
5 * of variable size attributes.
6 *
7 * Copyright (c) 2000-2019, PostgreSQL Global Development Group
8 *
9 * src/include/access/tuptoaster.h
10 *
11 *-------------------------------------------------------------------------
12 */
13#ifndef TUPTOASTER_H
14#define TUPTOASTER_H
15
16#include "access/htup_details.h"
17#include "storage/lockdefs.h"
18#include "utils/relcache.h"
19
20/*
21 * This enables de-toasting of index entries. Needed until VACUUM is
22 * smart enough to rebuild indexes from scratch.
23 */
24#define TOAST_INDEX_HACK
25
26
27/*
28 * Find the maximum size of a tuple if there are to be N tuples per page.
29 */
30#define MaximumBytesPerTuple(tuplesPerPage) \
31 MAXALIGN_DOWN((BLCKSZ - \
32 MAXALIGN(SizeOfPageHeaderData + (tuplesPerPage) * sizeof(ItemIdData))) \
33 / (tuplesPerPage))
34
35/*
36 * These symbols control toaster activation. If a tuple is larger than
37 * TOAST_TUPLE_THRESHOLD, we will try to toast it down to no more than
38 * TOAST_TUPLE_TARGET bytes through compressing compressible fields and
39 * moving EXTENDED and EXTERNAL data out-of-line.
40 *
41 * The numbers need not be the same, though they currently are. It doesn't
42 * make sense for TARGET to exceed THRESHOLD, but it could be useful to make
43 * it be smaller.
44 *
45 * Currently we choose both values to match the largest tuple size for which
46 * TOAST_TUPLES_PER_PAGE tuples can fit on a heap page.
47 *
48 * XXX while these can be modified without initdb, some thought needs to be
49 * given to needs_toast_table() in toasting.c before unleashing random
50 * changes. Also see LOBLKSIZE in large_object.h, which can *not* be
51 * changed without initdb.
52 */
53#define TOAST_TUPLES_PER_PAGE 4
54
55#define TOAST_TUPLE_THRESHOLD MaximumBytesPerTuple(TOAST_TUPLES_PER_PAGE)
56
57#define TOAST_TUPLE_TARGET TOAST_TUPLE_THRESHOLD
58
59/*
60 * The code will also consider moving MAIN data out-of-line, but only as a
61 * last resort if the previous steps haven't reached the target tuple size.
62 * In this phase we use a different target size, currently equal to the
63 * largest tuple that will fit on a heap page. This is reasonable since
64 * the user has told us to keep the data in-line if at all possible.
65 */
66#define TOAST_TUPLES_PER_PAGE_MAIN 1
67
68#define TOAST_TUPLE_TARGET_MAIN MaximumBytesPerTuple(TOAST_TUPLES_PER_PAGE_MAIN)
69
70/*
71 * If an index value is larger than TOAST_INDEX_TARGET, we will try to
72 * compress it (we can't move it out-of-line, however). Note that this
73 * number is per-datum, not per-tuple, for simplicity in index_form_tuple().
74 */
75#define TOAST_INDEX_TARGET (MaxHeapTupleSize / 16)
76
77/*
78 * When we store an oversize datum externally, we divide it into chunks
79 * containing at most TOAST_MAX_CHUNK_SIZE data bytes. This number *must*
80 * be small enough that the completed toast-table tuple (including the
81 * ID and sequence fields and all overhead) will fit on a page.
82 * The coding here sets the size on the theory that we want to fit
83 * EXTERN_TUPLES_PER_PAGE tuples of maximum size onto a page.
84 *
85 * NB: Changing TOAST_MAX_CHUNK_SIZE requires an initdb.
86 */
87#define EXTERN_TUPLES_PER_PAGE 4 /* tweak only this */
88
89#define EXTERN_TUPLE_MAX_SIZE MaximumBytesPerTuple(EXTERN_TUPLES_PER_PAGE)
90
91#define TOAST_MAX_CHUNK_SIZE \
92 (EXTERN_TUPLE_MAX_SIZE - \
93 MAXALIGN(SizeofHeapTupleHeader) - \
94 sizeof(Oid) - \
95 sizeof(int32) - \
96 VARHDRSZ)
97
98/* Size of an EXTERNAL datum that contains a standard TOAST pointer */
99#define TOAST_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external))
100
101/* Size of an EXTERNAL datum that contains an indirection pointer */
102#define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect))
103
104/*
105 * Testing whether an externally-stored value is compressed now requires
106 * comparing extsize (the actual length of the external data) to rawsize
107 * (the original uncompressed datum's size). The latter includes VARHDRSZ
108 * overhead, the former doesn't. We never use compression unless it actually
109 * saves space, so we expect either equality or less-than.
110 */
111#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \
112 ((toast_pointer).va_extsize < (toast_pointer).va_rawsize - VARHDRSZ)
113
114/*
115 * Macro to fetch the possibly-unaligned contents of an EXTERNAL datum
116 * into a local "struct varatt_external" toast pointer. This should be
117 * just a memcpy, but some versions of gcc seem to produce broken code
118 * that assumes the datum contents are aligned. Introducing an explicit
119 * intermediate "varattrib_1b_e *" variable seems to fix it.
120 */
121#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr) \
122do { \
123 varattrib_1b_e *attre = (varattrib_1b_e *) (attr); \
124 Assert(VARATT_IS_EXTERNAL(attre)); \
125 Assert(VARSIZE_EXTERNAL(attre) == sizeof(toast_pointer) + VARHDRSZ_EXTERNAL); \
126 memcpy(&(toast_pointer), VARDATA_EXTERNAL(attre), sizeof(toast_pointer)); \
127} while (0)
128
129/* ----------
130 * toast_insert_or_update -
131 *
132 * Called by heap_insert() and heap_update().
133 * ----------
134 */
135extern HeapTuple toast_insert_or_update(Relation rel,
136 HeapTuple newtup, HeapTuple oldtup,
137 int options);
138
139/* ----------
140 * toast_delete -
141 *
142 * Called by heap_delete().
143 * ----------
144 */
145extern void toast_delete(Relation rel, HeapTuple oldtup, bool is_speculative);
146
147/* ----------
148 * heap_tuple_fetch_attr() -
149 *
150 * Fetches an external stored attribute from the toast
151 * relation. Does NOT decompress it, if stored external
152 * in compressed format.
153 * ----------
154 */
155extern struct varlena *heap_tuple_fetch_attr(struct varlena *attr);
156
157/* ----------
158 * heap_tuple_untoast_attr() -
159 *
160 * Fully detoasts one attribute, fetching and/or decompressing
161 * it as needed.
162 * ----------
163 */
164extern struct varlena *heap_tuple_untoast_attr(struct varlena *attr);
165
166/* ----------
167 * heap_tuple_untoast_attr_slice() -
168 *
169 * Fetches only the specified portion of an attribute.
170 * (Handles all cases for attribute storage)
171 * ----------
172 */
173extern struct varlena *heap_tuple_untoast_attr_slice(struct varlena *attr,
174 int32 sliceoffset,
175 int32 slicelength);
176
177/* ----------
178 * toast_flatten_tuple -
179 *
180 * "Flatten" a tuple to contain no out-of-line toasted fields.
181 * (This does not eliminate compressed or short-header datums.)
182 * ----------
183 */
184extern HeapTuple toast_flatten_tuple(HeapTuple tup, TupleDesc tupleDesc);
185
186/* ----------
187 * toast_flatten_tuple_to_datum -
188 *
189 * "Flatten" a tuple containing out-of-line toasted fields into a Datum.
190 * ----------
191 */
192extern Datum toast_flatten_tuple_to_datum(HeapTupleHeader tup,
193 uint32 tup_len,
194 TupleDesc tupleDesc);
195
196/* ----------
197 * toast_build_flattened_tuple -
198 *
199 * Build a tuple containing no out-of-line toasted fields.
200 * (This does not eliminate compressed or short-header datums.)
201 * ----------
202 */
203extern HeapTuple toast_build_flattened_tuple(TupleDesc tupleDesc,
204 Datum *values,
205 bool *isnull);
206
207/* ----------
208 * toast_compress_datum -
209 *
210 * Create a compressed version of a varlena datum, if possible
211 * ----------
212 */
213extern Datum toast_compress_datum(Datum value);
214
215/* ----------
216 * toast_raw_datum_size -
217 *
218 * Return the raw (detoasted) size of a varlena datum
219 * ----------
220 */
221extern Size toast_raw_datum_size(Datum value);
222
223/* ----------
224 * toast_datum_size -
225 *
226 * Return the storage size of a varlena datum
227 * ----------
228 */
229extern Size toast_datum_size(Datum value);
230
231/* ----------
232 * toast_get_valid_index -
233 *
234 * Return OID of valid index associated to a toast relation
235 * ----------
236 */
237extern Oid toast_get_valid_index(Oid toastoid, LOCKMODE lock);
238
239#endif /* TUPTOASTER_H */
240