1#pragma once
2
3#include <string>
4#include <unordered_set>
5#include <vector>
6#include <memory>
7
8#include <Poco/DOM/Document.h>
9#include <Poco/DOM/DOMParser.h>
10#include <Poco/DOM/DOMWriter.h>
11#include <Poco/DOM/NodeList.h>
12#include <Poco/DOM/NamedNodeMap.h>
13#include <Poco/AutoPtr.h>
14#include <Poco/File.h>
15#include <Poco/Path.h>
16#include <Poco/DirectoryIterator.h>
17#include <Poco/ConsoleChannel.h>
18#include <Poco/Util/AbstractConfiguration.h>
19
20#include <common/logger_useful.h>
21
22
23namespace zkutil
24{
25 class ZooKeeperNodeCache;
26 using EventPtr = std::shared_ptr<Poco::Event>;
27}
28
29namespace DB
30{
31
32using ConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfiguration>;
33using XMLDocumentPtr = Poco::AutoPtr<Poco::XML::Document>;
34
35class ConfigProcessor
36{
37public:
38 using Substitutions = std::vector<std::pair<std::string, std::string>>;
39
40 /// Set log_to_console to true if the logging subsystem is not initialized yet.
41 explicit ConfigProcessor(
42 const std::string & path,
43 bool throw_on_bad_incl = false,
44 bool log_to_console = false,
45 const Substitutions & substitutions = Substitutions());
46
47 ~ConfigProcessor();
48
49 /// Perform config includes and substitutions and return the resulting XML-document.
50 ///
51 /// Suppose path is "/path/file.xml"
52 /// 1) Merge XML trees of /path/file.xml with XML trees of all files from /path/{conf,file}.d/*.{conf,xml}
53 /// * If an element has a "replace" attribute, replace the matching element with it.
54 /// * If an element has a "remove" attribute, remove the matching element.
55 /// * Else, recursively merge child elements.
56 /// 2) Determine the includes file from the config: <include_from>/path2/metrika.xml</include_from>
57 /// If this path is not configured, use /etc/metrika.xml
58 /// 3) Replace elements matching the "<foo incl="bar"/>" pattern with
59 /// "<foo>contents of the yandex/bar element in metrika.xml</foo>"
60 /// 4) If zk_node_cache is non-NULL, replace elements matching the "<foo from_zk="/bar">" pattern with
61 /// "<foo>contents of the /bar ZooKeeper node</foo>".
62 /// If has_zk_includes is non-NULL and there are such elements, set has_zk_includes to true.
63 /// 5) (Yandex.Metrika-specific) Substitute "<layer/>" with "<layer>layer number from the hostname</layer>".
64 XMLDocumentPtr processConfig(
65 bool * has_zk_includes = nullptr,
66 zkutil::ZooKeeperNodeCache * zk_node_cache = nullptr,
67 const zkutil::EventPtr & zk_changed_event = nullptr);
68
69
70 /// loadConfig* functions apply processConfig and create Poco::Util::XMLConfiguration.
71 /// The resulting XML document is saved into a file with the name
72 /// resulting from adding "-preprocessed" suffix to the path file name.
73 /// E.g., config.xml -> config-preprocessed.xml
74
75 struct LoadedConfig
76 {
77 ConfigurationPtr configuration;
78 bool has_zk_includes;
79 bool loaded_from_preprocessed;
80 XMLDocumentPtr preprocessed_xml;
81 std::string config_path;
82 };
83
84 /// If allow_zk_includes is true, expect that the configuration XML can contain from_zk nodes.
85 /// If it is the case, set has_zk_includes to true and don't write config-preprocessed.xml,
86 /// expecting that config would be reloaded with zookeeper later.
87 LoadedConfig loadConfig(bool allow_zk_includes = false);
88
89 /// If fallback_to_preprocessed is true, then if KeeperException is thrown during config
90 /// processing, load the configuration from the preprocessed file.
91 LoadedConfig loadConfigWithZooKeeperIncludes(
92 zkutil::ZooKeeperNodeCache & zk_node_cache,
93 const zkutil::EventPtr & zk_changed_event,
94 bool fallback_to_preprocessed = false);
95
96 /// Save preprocessed config to specified directory.
97 /// If preprocessed_dir is empty - calculate from loaded_config.path + /preprocessed_configs/
98 void savePreprocessedConfig(const LoadedConfig & loaded_config, std::string preprocessed_dir);
99
100 /// Set path of main config.xml . It will be cutted from all configs placed to preprocessed_configs/
101 void setConfigPath(const std::string & config_path);
102
103public:
104 using Files = std::vector<std::string>;
105
106 static Files getConfigMergeFiles(const std::string & config_path);
107
108 /// Is the file named as result of config preprocessing, not as original files.
109 static bool isPreprocessedFile(const std::string & config_path);
110
111 static inline const auto SUBSTITUTION_ATTRS = {"incl", "from_zk", "from_env"};
112
113private:
114 const std::string path;
115 std::string preprocessed_path;
116
117 bool throw_on_bad_incl;
118
119 Logger * log;
120 Poco::AutoPtr<Poco::Channel> channel_ptr;
121
122 Substitutions substitutions;
123
124 Poco::AutoPtr<Poco::XML::NamePool> name_pool;
125 Poco::XML::DOMParser dom_parser;
126
127private:
128 using NodePtr = Poco::AutoPtr<Poco::XML::Node>;
129
130 void mergeRecursive(XMLDocumentPtr config, Poco::XML::Node * config_node, const Poco::XML::Node * with_node);
131
132 void merge(XMLDocumentPtr config, XMLDocumentPtr with);
133
134 std::string layerFromHost();
135
136 void doIncludesRecursive(
137 XMLDocumentPtr config,
138 XMLDocumentPtr include_from,
139 Poco::XML::Node * node,
140 zkutil::ZooKeeperNodeCache * zk_node_cache,
141 const zkutil::EventPtr & zk_changed_event,
142 std::unordered_set<std::string> & contributing_zk_paths);
143};
144
145}
146