1 | #pragma once |
2 | |
3 | #include <chrono> |
4 | #include <functional> |
5 | #include <unordered_map> |
6 | #include <Core/Types.h> |
7 | #include <Interpreters/IExternalLoadable.h> |
8 | #include <Interpreters/IExternalLoaderConfigRepository.h> |
9 | #include <common/logger_useful.h> |
10 | #include <ext/scope_guard.h> |
11 | |
12 | |
13 | namespace DB |
14 | { |
15 | /* External configuration structure. |
16 | * |
17 | * <external_group> |
18 | * <external_config> |
19 | * <external_name>name</external_name> |
20 | * .... |
21 | * </external_config> |
22 | * </external_group> |
23 | */ |
24 | struct ExternalLoaderConfigSettings |
25 | { |
26 | std::string external_config; |
27 | std::string external_name; |
28 | std::string external_database; |
29 | }; |
30 | |
31 | /** Interface for manage user-defined objects. |
32 | * Monitors configuration file and automatically reloads objects in separate threads. |
33 | * The monitoring thread wakes up every 'check_period_sec' seconds and checks |
34 | * modification time of objects' configuration file. If said time is greater than |
35 | * 'config_last_modified', the objects are created from scratch using configuration file, |
36 | * possibly overriding currently existing objects with the same name (previous versions of |
37 | * overridden objects will live as long as there are any users retaining them). |
38 | * |
39 | * Apart from checking configuration file for modifications, each object |
40 | * has a lifetime of its own and may be updated if it supportUpdates. |
41 | * The time of next update is calculated by choosing uniformly a random number |
42 | * distributed between lifetime.min_sec and lifetime.max_sec. |
43 | * If either of lifetime.min_sec and lifetime.max_sec is zero, such object is never updated. |
44 | */ |
45 | class ExternalLoader |
46 | { |
47 | public: |
48 | using LoadablePtr = std::shared_ptr<const IExternalLoadable>; |
49 | using Loadables = std::vector<LoadablePtr>; |
50 | |
51 | enum class Status |
52 | { |
53 | NOT_LOADED, /// Object hasn't been tried to load. This is an initial state. |
54 | LOADED, /// Object has been loaded successfully. |
55 | FAILED, /// Object has been failed to load. |
56 | LOADING, /// Object is being loaded right now for the first time. |
57 | FAILED_AND_RELOADING, /// Object was failed to load before and it's being reloaded right now. |
58 | LOADED_AND_RELOADING, /// Object was loaded successfully before and it's being reloaded right now. |
59 | NOT_EXIST, /// Object with this name wasn't found in the configuration. |
60 | }; |
61 | |
62 | static std::vector<std::pair<String, Int8>> getStatusEnumAllPossibleValues(); |
63 | |
64 | using Duration = std::chrono::milliseconds; |
65 | using TimePoint = std::chrono::system_clock::time_point; |
66 | |
67 | struct LoadResult |
68 | { |
69 | Status status = Status::NOT_EXIST; |
70 | String name; |
71 | LoadablePtr object; |
72 | String origin; |
73 | TimePoint loading_start_time; |
74 | Duration loading_duration; |
75 | std::exception_ptr exception; |
76 | std::string repository_name; |
77 | }; |
78 | |
79 | using LoadResults = std::vector<LoadResult>; |
80 | |
81 | template <typename T> |
82 | static constexpr bool is_scalar_load_result_type = std::is_same_v<T, LoadResult> || std::is_same_v<T, LoadablePtr>; |
83 | |
84 | template <typename T> |
85 | static constexpr bool is_vector_load_result_type = std::is_same_v<T, LoadResults> || std::is_same_v<T, Loadables>; |
86 | |
87 | ExternalLoader(const String & type_name_, Logger * log); |
88 | virtual ~ExternalLoader(); |
89 | |
90 | /// Adds a repository which will be used to read configurations from. |
91 | ext::scope_guard addConfigRepository(std::unique_ptr<IExternalLoaderConfigRepository> config_repository); |
92 | |
93 | void setConfigSettings(const ExternalLoaderConfigSettings & settings_); |
94 | |
95 | /// Sets whether all the objects from the configuration should be always loaded (even those which are never used). |
96 | void enableAlwaysLoadEverything(bool enable); |
97 | |
98 | /// Sets whether the objects should be loaded asynchronously, each loading in a new thread (from the thread pool). |
99 | void enableAsyncLoading(bool enable); |
100 | |
101 | /// Sets settings for periodic updates. |
102 | void enablePeriodicUpdates(bool enable); |
103 | |
104 | /// Returns the status of the object. |
105 | /// If the object has not been loaded yet then the function returns Status::NOT_LOADED. |
106 | /// If the specified name isn't found in the configuration then the function returns Status::NOT_EXIST. |
107 | Status getCurrentStatus(const String & name) const; |
108 | |
109 | /// Returns the result of loading the object. |
110 | /// The function doesn't load anything, it just returns the current load result as is. |
111 | template <typename ReturnType = LoadResult, typename = std::enable_if_t<is_scalar_load_result_type<ReturnType>, void>> |
112 | ReturnType getCurrentLoadResult(const String & name) const; |
113 | |
114 | using FilterByNameFunction = std::function<bool(const String &)>; |
115 | |
116 | /// Returns all the load results as a map. |
117 | /// The function doesn't load anything, it just returns the current load results as is. |
118 | template <typename ReturnType = LoadResults, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>> |
119 | ReturnType getCurrentLoadResults() const { return getCurrentLoadResults<ReturnType>(alwaysTrue); } |
120 | |
121 | template <typename ReturnType = LoadResults, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>> |
122 | ReturnType getCurrentLoadResults(const FilterByNameFunction & filter) const; |
123 | |
124 | /// Returns all loaded objects as a map. |
125 | /// The function doesn't load anything, it just returns the current load results as is. |
126 | Loadables getCurrentlyLoadedObjects() const; |
127 | Loadables getCurrentlyLoadedObjects(const FilterByNameFunction & filter) const; |
128 | |
129 | /// Returns true if any object was loaded. |
130 | bool hasCurrentlyLoadedObjects() const; |
131 | size_t getNumberOfCurrentlyLoadedObjects() const; |
132 | |
133 | static constexpr Duration NO_WAIT = Duration::zero(); |
134 | static constexpr Duration WAIT = Duration::max(); |
135 | |
136 | /// Loads a specified object. |
137 | /// The function does nothing if it's already loaded. |
138 | /// The function doesn't throw an exception if it's failed to load. |
139 | template <typename ReturnType = LoadablePtr, typename = std::enable_if_t<is_scalar_load_result_type<ReturnType>, void>> |
140 | ReturnType tryLoad(const String & name, Duration timeout = WAIT) const; |
141 | |
142 | /// Loads objects by filter. |
143 | /// The function does nothing for already loaded objects, it just returns them. |
144 | /// The function doesn't throw an exception if it's failed to load something. |
145 | template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>> |
146 | ReturnType tryLoad(const FilterByNameFunction & filter, Duration timeout = WAIT) const; |
147 | |
148 | /// Loads all objects. |
149 | /// The function does nothing for already loaded objects, it just returns them. |
150 | /// The function doesn't throw an exception if it's failed to load something. |
151 | template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>> |
152 | ReturnType tryLoadAll(Duration timeout = WAIT) const { return tryLoad<ReturnType>(alwaysTrue, timeout); } |
153 | |
154 | /// Loads a specified object. |
155 | /// The function does nothing if it's already loaded. |
156 | /// The function throws an exception if it's failed to load. |
157 | template <typename ReturnType = LoadablePtr, typename = std::enable_if_t<is_scalar_load_result_type<ReturnType>, void>> |
158 | ReturnType load(const String & name) const; |
159 | |
160 | /// Loads objects by filter. |
161 | /// The function does nothing for already loaded objects, it just returns them. |
162 | /// The function throws an exception if it's failed to load something. |
163 | template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>> |
164 | ReturnType load(const FilterByNameFunction & filter) const; |
165 | |
166 | /// Loads all objects. Not recommended to use. |
167 | /// The function does nothing for already loaded objects, it just returns them. |
168 | /// The function throws an exception if it's failed to load something. |
169 | template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>> |
170 | ReturnType loadAll() const { return load<ReturnType>(alwaysTrue); } |
171 | |
172 | /// Loads or reloads a specified object. |
173 | /// The function reloads the object if it's already loaded. |
174 | /// The function throws an exception if it's failed to load or reload. |
175 | template <typename ReturnType = LoadablePtr, typename = std::enable_if_t<is_scalar_load_result_type<ReturnType>, void>> |
176 | ReturnType loadOrReload(const String & name) const; |
177 | |
178 | /// Loads or reloads objects by filter. |
179 | /// The function reloads the objects which are already loaded. |
180 | /// The function throws an exception if it's failed to load or reload something. |
181 | template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>> |
182 | ReturnType loadOrReload(const FilterByNameFunction & filter) const; |
183 | |
184 | /// Load or reloads all objects. Not recommended to use. |
185 | /// The function throws an exception if it's failed to load or reload something. |
186 | template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>> |
187 | ReturnType loadOrReloadAll() const { return loadOrReload<ReturnType>(alwaysTrue); } |
188 | |
189 | /// Reloads objects by filter which were tried to load before (successfully or not). |
190 | /// The function throws an exception if it's failed to load or reload something. |
191 | template <typename ReturnType = Loadables, typename = std::enable_if_t<is_vector_load_result_type<ReturnType>, void>> |
192 | ReturnType reloadAllTriedToLoad() const; |
193 | |
194 | /// Reloads all config repositories. |
195 | void reloadConfig() const; |
196 | |
197 | /// Reloads only a specified config repository. |
198 | void reloadConfig(const String & repository_name) const; |
199 | |
200 | /// Reload only a specified path in a specified config repository. |
201 | void reloadConfig(const String & repository_name, const String & path) const; |
202 | |
203 | protected: |
204 | virtual LoadablePtr create(const String & name, const Poco::Util::AbstractConfiguration & config, const String & key_in_config, const String & repository_name) const = 0; |
205 | |
206 | private: |
207 | void checkLoaded(const LoadResult & result, bool check_no_errors) const; |
208 | void checkLoaded(const LoadResults & results, bool check_no_errors) const; |
209 | |
210 | static bool alwaysTrue(const String &) { return true; } |
211 | Strings getAllTriedToLoadNames() const; |
212 | |
213 | struct ObjectConfig; |
214 | LoadablePtr createObject(const String & name, const ObjectConfig & config, const LoadablePtr & previous_version) const; |
215 | |
216 | class LoadablesConfigReader; |
217 | std::unique_ptr<LoadablesConfigReader> config_files_reader; |
218 | |
219 | class LoadingDispatcher; |
220 | std::unique_ptr<LoadingDispatcher> loading_dispatcher; |
221 | |
222 | class PeriodicUpdater; |
223 | std::unique_ptr<PeriodicUpdater> periodic_updater; |
224 | |
225 | const String type_name; |
226 | Poco::Logger * log; |
227 | }; |
228 | |
229 | String toString(ExternalLoader::Status status); |
230 | std::ostream & operator<<(std::ostream & out, ExternalLoader::Status status); |
231 | |
232 | } |
233 | |