1#pragma once
2
3#include <chrono>
4#include <functional>
5#include <unordered_map>
6#include <Core/Types.h>
7#include <Interpreters/IExternalLoadable.h>
8#include <Interpreters/IExternalLoaderConfigRepository.h>
9#include <common/logger_useful.h>
10#include <ext/scope_guard.h>
11
12
13namespace DB
14{
15/* External configuration structure.
16 *
17 * <external_group>
18 * <external_config>
19 * <external_name>name</external_name>
20 * ....
21 * </external_config>
22 * </external_group>
23 */
24struct ExternalLoaderConfigSettings
25{
26 std::string external_config;
27 std::string external_name;
28 std::string external_database;
29};
30
31/** Interface for manage user-defined objects.
32 * Monitors configuration file and automatically reloads objects in separate threads.
33 * The monitoring thread wakes up every 'check_period_sec' seconds and checks
34 * modification time of objects' configuration file. If said time is greater than
35 * 'config_last_modified', the objects are created from scratch using configuration file,
36 * possibly overriding currently existing objects with the same name (previous versions of
37 * overridden objects will live as long as there are any users retaining them).
38 *
39 * Apart from checking configuration file for modifications, each object
40 * has a lifetime of its own and may be updated if it supportUpdates.
41 * The time of next update is calculated by choosing uniformly a random number
42 * distributed between lifetime.min_sec and lifetime.max_sec.
43 * If either of lifetime.min_sec and lifetime.max_sec is zero, such object is never updated.
44 */
45class ExternalLoader
46{
47public:
48 using LoadablePtr = std::shared_ptr<const IExternalLoadable>;
49 using Loadables = std::vector<LoadablePtr>;
50
51 enum class Status
52 {
53 NOT_LOADED, /// Object hasn't been tried to load. This is an initial state.
54 LOADED, /// Object has been loaded successfully.
55 FAILED, /// Object has been failed to load.
56 LOADING, /// Object is being loaded right now for the first time.
57 FAILED_AND_RELOADING, /// Object was failed to load before and it's being reloaded right now.
58 LOADED_AND_RELOADING, /// Object was loaded successfully before and it's being reloaded right now.
59 NOT_EXIST, /// Object with this name wasn't found in the configuration.
60 };
61
62 static std::vector<std::pair<String, Int8>> getStatusEnumAllPossibleValues();
63
64 using Duration = std::chrono::milliseconds;
65 using TimePoint = std::chrono::system_clock::time_point;
66
67 struct LoadResult
68 {
69 Status status = Status::NOT_EXIST;
70 String name;
71 LoadablePtr object;
72 String origin;
73 TimePoint loading_start_time;
74 Duration loading_duration;
75 std::exception_ptr exception;
76 std::string repository_name;
77 };
78
79 using LoadResults = std::vector<LoadResult>;
80
81 template <typename T>
82 static constexpr bool is_scalar_load_result_type = std::is_same_v<T, LoadResult> || std::is_same_v<T, LoadablePtr>;
83
84 template <typename T>
85 static constexpr bool is_vector_load_result_type = std::is_same_v<T, LoadResults> || std::is_same_v<T, Loadables>;
86
87 ExternalLoader(const String & type_name_, Logger * log);
88 virtual ~ExternalLoader();
89
90 /// Adds a repository which will be used to read configurations from.
91 ext::scope_guard addConfigRepository(std::unique_ptr<IExternalLoaderConfigRepository> config_repository);
92
93 void setConfigSettings(const ExternalLoaderConfigSettings & settings_);
94
95 /// Sets whether all the objects from the configuration should be always loaded (even those which are never used).
96 void enableAlwaysLoadEverything(bool enable);
97
98 /// Sets whether the objects should be loaded asynchronously, each loading in a new thread (from the thread pool).
99 void enableAsyncLoading(bool enable);
100
101 /// Sets settings for periodic updates.
102 void enablePeriodicUpdates(bool enable);
103
104 /// Returns the status of the object.
105 /// If the object has not been loaded yet then the function returns Status::NOT_LOADED.
106 /// If the specified name isn't found in the configuration then the function returns Status::NOT_EXIST.
107 Status getCurrentStatus(const String & name) const;
108
109 /// Returns the result of loading the object.
110 /// The function doesn't load anything, it just returns the current load result as is.
111 template <typename ReturnType = LoadResult, typename = std::enable_if_t<is_scalar_load_result_type<ReturnType>, void>>
112 ReturnType getCurrentLoadResult(const String & name) const;
113
114 using FilterByNameFunction = std::function<bool(const String &)>;
115
116 /// Returns all the load results as a map.
117 /// The function doesn't load anything, it just returns the current load results as is.
118 template <typename ReturnType = LoadResults, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
119 ReturnType getCurrentLoadResults() const { return getCurrentLoadResults<ReturnType>(alwaysTrue); }
120
121 template <typename ReturnType = LoadResults, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
122 ReturnType getCurrentLoadResults(const FilterByNameFunction & filter) const;
123
124 /// Returns all loaded objects as a map.
125 /// The function doesn't load anything, it just returns the current load results as is.
126 Loadables getCurrentlyLoadedObjects() const;
127 Loadables getCurrentlyLoadedObjects(const FilterByNameFunction & filter) const;
128
129 /// Returns true if any object was loaded.
130 bool hasCurrentlyLoadedObjects() const;
131 size_t getNumberOfCurrentlyLoadedObjects() const;
132
133 static constexpr Duration NO_WAIT = Duration::zero();
134 static constexpr Duration WAIT = Duration::max();
135
136 /// Loads a specified object.
137 /// The function does nothing if it's already loaded.
138 /// The function doesn't throw an exception if it's failed to load.
139 template <typename ReturnType = LoadablePtr, typename = std::enable_if_t<is_scalar_load_result_type<ReturnType>, void>>
140 ReturnType tryLoad(const String & name, Duration timeout = WAIT) const;
141
142 /// Loads objects by filter.
143 /// The function does nothing for already loaded objects, it just returns them.
144 /// The function doesn't throw an exception if it's failed to load something.
145 template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
146 ReturnType tryLoad(const FilterByNameFunction & filter, Duration timeout = WAIT) const;
147
148 /// Loads all objects.
149 /// The function does nothing for already loaded objects, it just returns them.
150 /// The function doesn't throw an exception if it's failed to load something.
151 template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
152 ReturnType tryLoadAll(Duration timeout = WAIT) const { return tryLoad<ReturnType>(alwaysTrue, timeout); }
153
154 /// Loads a specified object.
155 /// The function does nothing if it's already loaded.
156 /// The function throws an exception if it's failed to load.
157 template <typename ReturnType = LoadablePtr, typename = std::enable_if_t<is_scalar_load_result_type<ReturnType>, void>>
158 ReturnType load(const String & name) const;
159
160 /// Loads objects by filter.
161 /// The function does nothing for already loaded objects, it just returns them.
162 /// The function throws an exception if it's failed to load something.
163 template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
164 ReturnType load(const FilterByNameFunction & filter) const;
165
166 /// Loads all objects. Not recommended to use.
167 /// The function does nothing for already loaded objects, it just returns them.
168 /// The function throws an exception if it's failed to load something.
169 template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
170 ReturnType loadAll() const { return load<ReturnType>(alwaysTrue); }
171
172 /// Loads or reloads a specified object.
173 /// The function reloads the object if it's already loaded.
174 /// The function throws an exception if it's failed to load or reload.
175 template <typename ReturnType = LoadablePtr, typename = std::enable_if_t<is_scalar_load_result_type<ReturnType>, void>>
176 ReturnType loadOrReload(const String & name) const;
177
178 /// Loads or reloads objects by filter.
179 /// The function reloads the objects which are already loaded.
180 /// The function throws an exception if it's failed to load or reload something.
181 template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
182 ReturnType loadOrReload(const FilterByNameFunction & filter) const;
183
184 /// Load or reloads all objects. Not recommended to use.
185 /// The function throws an exception if it's failed to load or reload something.
186 template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
187 ReturnType loadOrReloadAll() const { return loadOrReload<ReturnType>(alwaysTrue); }
188
189 /// Reloads objects by filter which were tried to load before (successfully or not).
190 /// The function throws an exception if it's failed to load or reload something.
191 template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>>
192 ReturnType reloadAllTriedToLoad() const;
193
194 /// Reloads all config repositories.
195 void reloadConfig() const;
196
197 /// Reloads only a specified config repository.
198 void reloadConfig(const String & repository_name) const;
199
200 /// Reload only a specified path in a specified config repository.
201 void reloadConfig(const String & repository_name, const String & path) const;
202
203protected:
204 virtual LoadablePtr create(const String & name, const Poco::Util::AbstractConfiguration & config, const String & key_in_config, const String & repository_name) const = 0;
205
206private:
207 void checkLoaded(const LoadResult & result, bool check_no_errors) const;
208 void checkLoaded(const LoadResults & results, bool check_no_errors) const;
209
210 static bool alwaysTrue(const String &) { return true; }
211 Strings getAllTriedToLoadNames() const;
212
213 struct ObjectConfig;
214 LoadablePtr createObject(const String & name, const ObjectConfig & config, const LoadablePtr & previous_version) const;
215
216 class LoadablesConfigReader;
217 std::unique_ptr<LoadablesConfigReader> config_files_reader;
218
219 class LoadingDispatcher;
220 std::unique_ptr<LoadingDispatcher> loading_dispatcher;
221
222 class PeriodicUpdater;
223 std::unique_ptr<PeriodicUpdater> periodic_updater;
224
225 const String type_name;
226 Poco::Logger * log;
227};
228
229String toString(ExternalLoader::Status status);
230std::ostream & operator<<(std::ostream & out, ExternalLoader::Status status);
231
232}
233