1//===----------------------------------------------------------------------===//
2// DuckDB
3//
4// duckdb/common/types/row_operations/row_external.cpp
5//
6//
7//===----------------------------------------------------------------------===//
8#include "duckdb/common/row_operations/row_operations.hpp"
9#include "duckdb/common/types/row/row_layout.hpp"
10
11namespace duckdb {
12
13using ValidityBytes = RowLayout::ValidityBytes;
14
15void RowOperations::SwizzleColumns(const RowLayout &layout, const data_ptr_t base_row_ptr, const idx_t count) {
16 const idx_t row_width = layout.GetRowWidth();
17 data_ptr_t heap_row_ptrs[STANDARD_VECTOR_SIZE];
18 idx_t done = 0;
19 while (done != count) {
20 const idx_t next = MinValue<idx_t>(a: count - done, STANDARD_VECTOR_SIZE);
21 const data_ptr_t row_ptr = base_row_ptr + done * row_width;
22 // Load heap row pointers
23 data_ptr_t heap_ptr_ptr = row_ptr + layout.GetHeapOffset();
24 for (idx_t i = 0; i < next; i++) {
25 heap_row_ptrs[i] = Load<data_ptr_t>(ptr: heap_ptr_ptr);
26 heap_ptr_ptr += row_width;
27 }
28 // Loop through the blob columns
29 for (idx_t col_idx = 0; col_idx < layout.ColumnCount(); col_idx++) {
30 auto physical_type = layout.GetTypes()[col_idx].InternalType();
31 if (TypeIsConstantSize(type: physical_type)) {
32 continue;
33 }
34 data_ptr_t col_ptr = row_ptr + layout.GetOffsets()[col_idx];
35 if (physical_type == PhysicalType::VARCHAR) {
36 data_ptr_t string_ptr = col_ptr + string_t::HEADER_SIZE;
37 for (idx_t i = 0; i < next; i++) {
38 if (Load<uint32_t>(ptr: col_ptr) > string_t::INLINE_LENGTH) {
39 // Overwrite the string pointer with the within-row offset (if not inlined)
40 Store<idx_t>(val: Load<data_ptr_t>(ptr: string_ptr) - heap_row_ptrs[i], ptr: string_ptr);
41 }
42 col_ptr += row_width;
43 string_ptr += row_width;
44 }
45 } else {
46 // Non-varchar blob columns
47 for (idx_t i = 0; i < next; i++) {
48 // Overwrite the column data pointer with the within-row offset
49 Store<idx_t>(val: Load<data_ptr_t>(ptr: col_ptr) - heap_row_ptrs[i], ptr: col_ptr);
50 col_ptr += row_width;
51 }
52 }
53 }
54 done += next;
55 }
56}
57
58void RowOperations::SwizzleHeapPointer(const RowLayout &layout, data_ptr_t row_ptr, const data_ptr_t heap_base_ptr,
59 const idx_t count, const idx_t base_offset) {
60 const idx_t row_width = layout.GetRowWidth();
61 row_ptr += layout.GetHeapOffset();
62 idx_t cumulative_offset = 0;
63 for (idx_t i = 0; i < count; i++) {
64 Store<idx_t>(val: base_offset + cumulative_offset, ptr: row_ptr);
65 cumulative_offset += Load<uint32_t>(ptr: heap_base_ptr + cumulative_offset);
66 row_ptr += row_width;
67 }
68}
69
70void RowOperations::CopyHeapAndSwizzle(const RowLayout &layout, data_ptr_t row_ptr, const data_ptr_t heap_base_ptr,
71 data_ptr_t heap_ptr, const idx_t count) {
72 const auto row_width = layout.GetRowWidth();
73 const auto heap_offset = layout.GetHeapOffset();
74 for (idx_t i = 0; i < count; i++) {
75 // Figure out source and size
76 const auto source_heap_ptr = Load<data_ptr_t>(ptr: row_ptr + heap_offset);
77 const auto size = Load<uint32_t>(ptr: source_heap_ptr);
78 D_ASSERT(size >= sizeof(uint32_t));
79
80 // Copy and swizzle
81 memcpy(dest: heap_ptr, src: source_heap_ptr, n: size);
82 Store<idx_t>(val: heap_ptr - heap_base_ptr, ptr: row_ptr + heap_offset);
83
84 // Increment for next iteration
85 row_ptr += row_width;
86 heap_ptr += size;
87 }
88}
89
90void RowOperations::UnswizzleHeapPointer(const RowLayout &layout, const data_ptr_t base_row_ptr,
91 const data_ptr_t base_heap_ptr, const idx_t count) {
92 const auto row_width = layout.GetRowWidth();
93 data_ptr_t heap_ptr_ptr = base_row_ptr + layout.GetHeapOffset();
94 for (idx_t i = 0; i < count; i++) {
95 Store<data_ptr_t>(val: base_heap_ptr + Load<idx_t>(ptr: heap_ptr_ptr), ptr: heap_ptr_ptr);
96 heap_ptr_ptr += row_width;
97 }
98}
99
100static inline void VerifyUnswizzledString(const RowLayout &layout, const idx_t &col_idx, const data_ptr_t &row_ptr) {
101#ifdef DEBUG
102 if (layout.GetTypes()[col_idx] == LogicalTypeId::BLOB) {
103 return;
104 }
105 idx_t entry_idx;
106 idx_t idx_in_entry;
107 ValidityBytes::GetEntryIndex(col_idx, entry_idx, idx_in_entry);
108
109 ValidityBytes row_mask(row_ptr);
110 if (row_mask.RowIsValid(row_mask.GetValidityEntry(entry_idx), idx_in_entry)) {
111 auto str = Load<string_t>(row_ptr + layout.GetOffsets()[col_idx]);
112 str.Verify();
113 }
114#endif
115}
116
117void RowOperations::UnswizzlePointers(const RowLayout &layout, const data_ptr_t base_row_ptr,
118 const data_ptr_t base_heap_ptr, const idx_t count) {
119 const idx_t row_width = layout.GetRowWidth();
120 data_ptr_t heap_row_ptrs[STANDARD_VECTOR_SIZE];
121 idx_t done = 0;
122 while (done != count) {
123 const idx_t next = MinValue<idx_t>(a: count - done, STANDARD_VECTOR_SIZE);
124 const data_ptr_t row_ptr = base_row_ptr + done * row_width;
125 // Restore heap row pointers
126 data_ptr_t heap_ptr_ptr = row_ptr + layout.GetHeapOffset();
127 for (idx_t i = 0; i < next; i++) {
128 heap_row_ptrs[i] = base_heap_ptr + Load<idx_t>(ptr: heap_ptr_ptr);
129 Store<data_ptr_t>(val: heap_row_ptrs[i], ptr: heap_ptr_ptr);
130 heap_ptr_ptr += row_width;
131 }
132 // Loop through the blob columns
133 for (idx_t col_idx = 0; col_idx < layout.ColumnCount(); col_idx++) {
134 auto physical_type = layout.GetTypes()[col_idx].InternalType();
135 if (TypeIsConstantSize(type: physical_type)) {
136 continue;
137 }
138 data_ptr_t col_ptr = row_ptr + layout.GetOffsets()[col_idx];
139 if (physical_type == PhysicalType::VARCHAR) {
140 data_ptr_t string_ptr = col_ptr + string_t::HEADER_SIZE;
141 for (idx_t i = 0; i < next; i++) {
142 if (Load<uint32_t>(ptr: col_ptr) > string_t::INLINE_LENGTH) {
143 // Overwrite the string offset with the pointer (if not inlined)
144 Store<data_ptr_t>(val: heap_row_ptrs[i] + Load<idx_t>(ptr: string_ptr), ptr: string_ptr);
145 VerifyUnswizzledString(layout, col_idx, row_ptr: row_ptr + i * row_width);
146 }
147 col_ptr += row_width;
148 string_ptr += row_width;
149 }
150 } else {
151 // Non-varchar blob columns
152 for (idx_t i = 0; i < next; i++) {
153 // Overwrite the column data offset with the pointer
154 Store<data_ptr_t>(val: heap_row_ptrs[i] + Load<idx_t>(ptr: col_ptr), ptr: col_ptr);
155 col_ptr += row_width;
156 }
157 }
158 }
159 done += next;
160 }
161}
162
163} // namespace duckdb
164