1 | #pragma once |
2 | |
3 | #include <string> |
4 | #include <unordered_set> |
5 | #include <vector> |
6 | #include <memory> |
7 | |
8 | #include <Poco/DOM/Document.h> |
9 | #include <Poco/DOM/DOMParser.h> |
10 | #include <Poco/DOM/DOMWriter.h> |
11 | #include <Poco/DOM/NodeList.h> |
12 | #include <Poco/DOM/NamedNodeMap.h> |
13 | #include <Poco/AutoPtr.h> |
14 | #include <Poco/File.h> |
15 | #include <Poco/Path.h> |
16 | #include <Poco/DirectoryIterator.h> |
17 | #include <Poco/ConsoleChannel.h> |
18 | #include <Poco/Util/AbstractConfiguration.h> |
19 | |
20 | #include <common/logger_useful.h> |
21 | |
22 | |
23 | namespace zkutil |
24 | { |
25 | class ZooKeeperNodeCache; |
26 | using EventPtr = std::shared_ptr<Poco::Event>; |
27 | } |
28 | |
29 | namespace DB |
30 | { |
31 | |
32 | using ConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfiguration>; |
33 | using XMLDocumentPtr = Poco::AutoPtr<Poco::XML::Document>; |
34 | |
35 | class ConfigProcessor |
36 | { |
37 | public: |
38 | using Substitutions = std::vector<std::pair<std::string, std::string>>; |
39 | |
40 | /// Set log_to_console to true if the logging subsystem is not initialized yet. |
41 | explicit ConfigProcessor( |
42 | const std::string & path, |
43 | bool throw_on_bad_incl = false, |
44 | bool log_to_console = false, |
45 | const Substitutions & substitutions = Substitutions()); |
46 | |
47 | ~ConfigProcessor(); |
48 | |
49 | /// Perform config includes and substitutions and return the resulting XML-document. |
50 | /// |
51 | /// Suppose path is "/path/file.xml" |
52 | /// 1) Merge XML trees of /path/file.xml with XML trees of all files from /path/{conf,file}.d/*.{conf,xml} |
53 | /// * If an element has a "replace" attribute, replace the matching element with it. |
54 | /// * If an element has a "remove" attribute, remove the matching element. |
55 | /// * Else, recursively merge child elements. |
56 | /// 2) Determine the includes file from the config: <include_from>/path2/metrika.xml</include_from> |
57 | /// If this path is not configured, use /etc/metrika.xml |
58 | /// 3) Replace elements matching the "<foo incl="bar"/>" pattern with |
59 | /// "<foo>contents of the yandex/bar element in metrika.xml</foo>" |
60 | /// 4) If zk_node_cache is non-NULL, replace elements matching the "<foo from_zk="/bar">" pattern with |
61 | /// "<foo>contents of the /bar ZooKeeper node</foo>". |
62 | /// If has_zk_includes is non-NULL and there are such elements, set has_zk_includes to true. |
63 | /// 5) (Yandex.Metrika-specific) Substitute "<layer/>" with "<layer>layer number from the hostname</layer>". |
64 | XMLDocumentPtr processConfig( |
65 | bool * has_zk_includes = nullptr, |
66 | zkutil::ZooKeeperNodeCache * zk_node_cache = nullptr, |
67 | const zkutil::EventPtr & zk_changed_event = nullptr); |
68 | |
69 | |
70 | /// loadConfig* functions apply processConfig and create Poco::Util::XMLConfiguration. |
71 | /// The resulting XML document is saved into a file with the name |
72 | /// resulting from adding "-preprocessed" suffix to the path file name. |
73 | /// E.g., config.xml -> config-preprocessed.xml |
74 | |
75 | struct LoadedConfig |
76 | { |
77 | ConfigurationPtr configuration; |
78 | bool has_zk_includes; |
79 | bool loaded_from_preprocessed; |
80 | XMLDocumentPtr preprocessed_xml; |
81 | std::string config_path; |
82 | }; |
83 | |
84 | /// If allow_zk_includes is true, expect that the configuration XML can contain from_zk nodes. |
85 | /// If it is the case, set has_zk_includes to true and don't write config-preprocessed.xml, |
86 | /// expecting that config would be reloaded with zookeeper later. |
87 | LoadedConfig loadConfig(bool allow_zk_includes = false); |
88 | |
89 | /// If fallback_to_preprocessed is true, then if KeeperException is thrown during config |
90 | /// processing, load the configuration from the preprocessed file. |
91 | LoadedConfig loadConfigWithZooKeeperIncludes( |
92 | zkutil::ZooKeeperNodeCache & zk_node_cache, |
93 | const zkutil::EventPtr & zk_changed_event, |
94 | bool fallback_to_preprocessed = false); |
95 | |
96 | /// Save preprocessed config to specified directory. |
97 | /// If preprocessed_dir is empty - calculate from loaded_config.path + /preprocessed_configs/ |
98 | void savePreprocessedConfig(const LoadedConfig & loaded_config, std::string preprocessed_dir); |
99 | |
100 | /// Set path of main config.xml . It will be cutted from all configs placed to preprocessed_configs/ |
101 | void setConfigPath(const std::string & config_path); |
102 | |
103 | public: |
104 | using Files = std::vector<std::string>; |
105 | |
106 | static Files getConfigMergeFiles(const std::string & config_path); |
107 | |
108 | /// Is the file named as result of config preprocessing, not as original files. |
109 | static bool isPreprocessedFile(const std::string & config_path); |
110 | |
111 | static inline const auto SUBSTITUTION_ATTRS = {"incl" , "from_zk" , "from_env" }; |
112 | |
113 | private: |
114 | const std::string path; |
115 | std::string preprocessed_path; |
116 | |
117 | bool throw_on_bad_incl; |
118 | |
119 | Logger * log; |
120 | Poco::AutoPtr<Poco::Channel> channel_ptr; |
121 | |
122 | Substitutions substitutions; |
123 | |
124 | Poco::AutoPtr<Poco::XML::NamePool> name_pool; |
125 | Poco::XML::DOMParser dom_parser; |
126 | |
127 | private: |
128 | using NodePtr = Poco::AutoPtr<Poco::XML::Node>; |
129 | |
130 | void mergeRecursive(XMLDocumentPtr config, Poco::XML::Node * config_node, const Poco::XML::Node * with_node); |
131 | |
132 | void merge(XMLDocumentPtr config, XMLDocumentPtr with); |
133 | |
134 | std::string layerFromHost(); |
135 | |
136 | void doIncludesRecursive( |
137 | XMLDocumentPtr config, |
138 | XMLDocumentPtr include_from, |
139 | Poco::XML::Node * node, |
140 | zkutil::ZooKeeperNodeCache * zk_node_cache, |
141 | const zkutil::EventPtr & zk_changed_event, |
142 | std::unordered_set<std::string> & contributing_zk_paths); |
143 | }; |
144 | |
145 | } |
146 | |