1#include "duckdb/main/extension_helper.hpp"
2#include "duckdb/common/gzip_file_system.hpp"
3#include "duckdb/common/types/uuid.hpp"
4#include "duckdb/common/string_util.hpp"
5
6#ifndef DISABLE_DUCKDB_REMOTE_INSTALL
7#include "httplib.hpp"
8#endif
9#include "duckdb/common/windows_undefs.hpp"
10
11#include <fstream>
12
13namespace duckdb {
14
15//===--------------------------------------------------------------------===//
16// Install Extension
17//===--------------------------------------------------------------------===//
18const string ExtensionHelper::NormalizeVersionTag(const string &version_tag) {
19 if (version_tag.length() > 0 && version_tag[0] != 'v') {
20 return "v" + version_tag;
21 }
22 return version_tag;
23}
24
25bool ExtensionHelper::IsRelease(const string &version_tag) {
26 return !StringUtil::Contains(haystack: version_tag, needle: "-dev");
27}
28
29const string ExtensionHelper::GetVersionDirectoryName() {
30 if (IsRelease(version_tag: DuckDB::LibraryVersion())) {
31 return NormalizeVersionTag(version_tag: DuckDB::LibraryVersion());
32 } else {
33 return DuckDB::SourceID();
34 }
35}
36
37const vector<string> ExtensionHelper::PathComponents() {
38 return vector<string> {".duckdb", "extensions", GetVersionDirectoryName(), DuckDB::Platform()};
39}
40
41string ExtensionHelper::ExtensionDirectory(DBConfig &config, FileSystem &fs) {
42#ifdef WASM_LOADABLE_EXTENSIONS
43 static_assertion(0, "ExtensionDirectory functionality is not supported in duckdb-wasm");
44#endif
45 string extension_directory;
46 if (!config.options.extension_directory.empty()) { // create the extension directory if not present
47 extension_directory = config.options.extension_directory;
48 // TODO this should probably live in the FileSystem
49 // convert random separators to platform-canonic
50 extension_directory = fs.ConvertSeparators(path: extension_directory);
51 // expand ~ in extension directory
52 extension_directory = fs.ExpandPath(path: extension_directory);
53 if (!fs.DirectoryExists(directory: extension_directory)) {
54 auto sep = fs.PathSeparator();
55 auto splits = StringUtil::Split(input: extension_directory, split: sep);
56 D_ASSERT(!splits.empty());
57 string extension_directory_prefix;
58 if (StringUtil::StartsWith(str: extension_directory, prefix: sep)) {
59 extension_directory_prefix = sep; // this is swallowed by Split otherwise
60 }
61 for (auto &split : splits) {
62 extension_directory_prefix = extension_directory_prefix + split + sep;
63 if (!fs.DirectoryExists(directory: extension_directory_prefix)) {
64 fs.CreateDirectory(directory: extension_directory_prefix);
65 }
66 }
67 }
68 } else { // otherwise default to home
69 string home_directory = fs.GetHomeDirectory();
70 // exception if the home directory does not exist, don't create whatever we think is home
71 if (!fs.DirectoryExists(directory: home_directory)) {
72 throw IOException("Can't find the home directory at '%s'\nSpecify a home directory using the SET "
73 "home_directory='/path/to/dir' option.",
74 home_directory);
75 }
76 extension_directory = home_directory;
77 }
78 D_ASSERT(fs.DirectoryExists(extension_directory));
79
80 auto path_components = PathComponents();
81 for (auto &path_ele : path_components) {
82 extension_directory = fs.JoinPath(a: extension_directory, path: path_ele);
83 if (!fs.DirectoryExists(directory: extension_directory)) {
84 fs.CreateDirectory(directory: extension_directory);
85 }
86 }
87 return extension_directory;
88}
89
90string ExtensionHelper::ExtensionDirectory(ClientContext &context) {
91 auto &config = DBConfig::GetConfig(context);
92 auto &fs = FileSystem::GetFileSystem(context);
93 return ExtensionDirectory(config, fs);
94}
95
96bool ExtensionHelper::CreateSuggestions(const string &extension_name, string &message) {
97 vector<string> candidates;
98 for (idx_t ext_count = ExtensionHelper::DefaultExtensionCount(), i = 0; i < ext_count; i++) {
99 candidates.emplace_back(args: ExtensionHelper::GetDefaultExtension(index: i).name);
100 }
101 for (idx_t ext_count = ExtensionHelper::ExtensionAliasCount(), i = 0; i < ext_count; i++) {
102 candidates.emplace_back(args: ExtensionHelper::GetExtensionAlias(index: i).alias);
103 }
104 auto closest_extensions = StringUtil::TopNLevenshtein(strings: candidates, target: extension_name);
105 message = StringUtil::CandidatesMessage(candidates: closest_extensions, candidate: "Candidate extensions");
106 for (auto &closest : closest_extensions) {
107 if (closest == extension_name) {
108 message = "Extension \"" + extension_name + "\" is an existing extension.\n";
109 return true;
110 }
111 }
112 return false;
113}
114
115void ExtensionHelper::InstallExtension(DBConfig &config, FileSystem &fs, const string &extension, bool force_install) {
116#ifdef WASM_LOADABLE_EXTENSIONS
117 // Install is currently a no-op
118 return;
119#endif
120 string local_path = ExtensionDirectory(config, fs);
121 InstallExtensionInternal(config, client_config: nullptr, fs, local_path, extension, force_install);
122}
123
124void ExtensionHelper::InstallExtension(ClientContext &context, const string &extension, bool force_install) {
125#ifdef WASM_LOADABLE_EXTENSIONS
126 // Install is currently a no-op
127 return;
128#endif
129 auto &config = DBConfig::GetConfig(context);
130 auto &fs = FileSystem::GetFileSystem(context);
131 string local_path = ExtensionDirectory(context);
132 auto &client_config = ClientConfig::GetConfig(context);
133 InstallExtensionInternal(config, client_config: &client_config, fs, local_path, extension, force_install);
134}
135
136unsafe_unique_array<data_t> ReadExtensionFileFromDisk(FileSystem &fs, const string &path, idx_t &file_size) {
137 auto source_file = fs.OpenFile(path, flags: FileFlags::FILE_FLAGS_READ);
138 file_size = source_file->GetFileSize();
139 auto in_buffer = make_unsafe_uniq_array<data_t>(n: file_size);
140 source_file->Read(buffer: in_buffer.get(), nr_bytes: file_size);
141 source_file->Close();
142 return in_buffer;
143}
144
145void WriteExtensionFileToDisk(FileSystem &fs, const string &path, void *data, idx_t data_size) {
146 auto target_file = fs.OpenFile(path, flags: FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_APPEND |
147 FileFlags::FILE_FLAGS_FILE_CREATE_NEW);
148 target_file->Write(buffer: data, nr_bytes: data_size);
149 target_file->Close();
150 target_file.reset();
151}
152
153void ExtensionHelper::InstallExtensionInternal(DBConfig &config, ClientConfig *client_config, FileSystem &fs,
154 const string &local_path, const string &extension, bool force_install) {
155 if (!config.options.enable_external_access) {
156 throw PermissionException("Installing extensions is disabled through configuration");
157 }
158 auto extension_name = ApplyExtensionAlias(extension_name: fs.ExtractBaseName(path: extension));
159
160 string local_extension_path = fs.JoinPath(a: local_path, path: extension_name + ".duckdb_extension");
161 if (fs.FileExists(filename: local_extension_path) && !force_install) {
162 return;
163 }
164
165 auto uuid = UUID::ToString(input: UUID::GenerateRandomUUID());
166 string temp_path = local_extension_path + ".tmp-" + uuid;
167 if (fs.FileExists(filename: temp_path)) {
168 fs.RemoveFile(filename: temp_path);
169 }
170 auto is_http_url = StringUtil::Contains(haystack: extension, needle: "http://");
171 if (fs.FileExists(filename: extension)) {
172 idx_t file_size;
173 auto in_buffer = ReadExtensionFileFromDisk(fs, path: extension, file_size);
174 WriteExtensionFileToDisk(fs, path: temp_path, data: in_buffer.get(), data_size: file_size);
175
176 fs.MoveFile(source: temp_path, target: local_extension_path);
177 return;
178 } else if (StringUtil::Contains(haystack: extension, needle: "/") && !is_http_url) {
179 throw IOException("Failed to read extension from \"%s\": no such file", extension);
180 }
181
182#ifdef DISABLE_DUCKDB_REMOTE_INSTALL
183 throw BinderException("Remote extension installation is disabled through configuration");
184#else
185
186 string default_endpoint = "http://extensions.duckdb.org";
187 string versioned_path = "/${REVISION}/${PLATFORM}/${NAME}.duckdb_extension.gz";
188 string custom_endpoint = client_config ? client_config->custom_extension_repo : string();
189 string &endpoint = !custom_endpoint.empty() ? custom_endpoint : default_endpoint;
190 string url_template = endpoint + versioned_path;
191
192 if (is_http_url) {
193 url_template = extension;
194 extension_name = "";
195 }
196
197 auto url = StringUtil::Replace(source: url_template, from: "${REVISION}", to: GetVersionDirectoryName());
198 url = StringUtil::Replace(source: url, from: "${PLATFORM}", to: DuckDB::Platform());
199 url = StringUtil::Replace(source: url, from: "${NAME}", to: extension_name);
200
201 string no_http = StringUtil::Replace(source: url, from: "http://", to: "");
202
203 idx_t next = no_http.find(c: '/', pos: 0);
204 if (next == string::npos) {
205 throw IOException("No slash in URL template");
206 }
207
208 // Push the substring [last, next) on to splits
209 auto hostname_without_http = no_http.substr(pos: 0, n: next);
210 auto url_local_part = no_http.substr(pos: next);
211
212 auto url_base = "http://" + hostname_without_http;
213 duckdb_httplib::Client cli(url_base.c_str());
214
215 duckdb_httplib::Headers headers = {{"User-Agent", StringUtil::Format(fmt_str: "DuckDB %s %s %s", params: DuckDB::LibraryVersion(),
216 params: DuckDB::SourceID(), params: DuckDB::Platform())}};
217
218 auto res = cli.Get(path: url_local_part.c_str(), headers);
219
220 if (!res || res->status != 200) {
221 // create suggestions
222 string message;
223 auto exact_match = ExtensionHelper::CreateSuggestions(extension_name, message);
224 if (exact_match) {
225 message += "\nAre you using a development build? In this case, extensions might not (yet) be uploaded.";
226 }
227 if (res.error() == duckdb_httplib::Error::Success) {
228 throw HTTPException(res.value(), "Failed to download extension \"%s\" at URL \"%s%s\"\n%s", extension_name,
229 url_base, url_local_part, message);
230 } else {
231 throw IOException("Failed to download extension \"%s\" at URL \"%s%s\"\n%s (ERROR %s)", extension_name,
232 url_base, url_local_part, message, to_string(error: res.error()));
233 }
234 }
235 auto decompressed_body = GZipFileSystem::UncompressGZIPString(in: res->body);
236
237 WriteExtensionFileToDisk(fs, path: temp_path, data: (void *)decompressed_body.data(), data_size: decompressed_body.size());
238 fs.MoveFile(source: temp_path, target: local_extension_path);
239#endif
240}
241
242} // namespace duckdb
243