1#include <IO/HTTPCommon.h>
2
3#include <Common/config.h>
4#include <Common/DNSResolver.h>
5#include <Common/Exception.h>
6#include <Common/PoolBase.h>
7#include <Common/ProfileEvents.h>
8#include <Common/SipHash.h>
9
10#include <Poco/Version.h>
11
12#if USE_POCO_NETSSL
13#include <Poco/Net/AcceptCertificateHandler.h>
14#include <Poco/Net/Context.h>
15#include <Poco/Net/HTTPSClientSession.h>
16#include <Poco/Net/InvalidCertificateHandler.h>
17#include <Poco/Net/PrivateKeyPassphraseHandler.h>
18#include <Poco/Net/RejectCertificateHandler.h>
19#include <Poco/Net/SSLManager.h>
20#endif
21
22#include <Poco/Net/HTTPServerResponse.h>
23#include <Poco/Util/Application.h>
24
25#include <tuple>
26#include <unordered_map>
27#include <sstream>
28
29
30namespace ProfileEvents
31{
32 extern const Event CreatedHTTPConnections;
33}
34
35namespace DB
36{
37namespace ErrorCodes
38{
39 extern const int RECEIVED_ERROR_FROM_REMOTE_IO_SERVER;
40 extern const int RECEIVED_ERROR_TOO_MANY_REQUESTS;
41 extern const int FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME;
42 extern const int UNSUPPORTED_URI_SCHEME;
43 extern const int TOO_MANY_REDIRECTS;
44}
45
46
47namespace
48{
49 void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts)
50 {
51#if defined(POCO_CLICKHOUSE_PATCH) || POCO_VERSION >= 0x02000000
52 session.setTimeout(timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout);
53#else
54 session.setTimeout(std::max({timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout}));
55#endif
56 session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout);
57 }
58
59 bool isHTTPS(const Poco::URI & uri)
60 {
61 if (uri.getScheme() == "https")
62 return true;
63 else if (uri.getScheme() == "http")
64 return false;
65 else
66 throw Exception("Unsupported scheme in URI '" + uri.toString() + "'", ErrorCodes::UNSUPPORTED_URI_SCHEME);
67 }
68
69 HTTPSessionPtr makeHTTPSessionImpl(const std::string & host, UInt16 port, bool https, bool keep_alive)
70 {
71 HTTPSessionPtr session;
72
73 if (https)
74#if USE_POCO_NETSSL
75 session = std::make_shared<Poco::Net::HTTPSClientSession>();
76#else
77 throw Exception("ClickHouse was built without HTTPS support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME);
78#endif
79 else
80 session = std::make_shared<Poco::Net::HTTPClientSession>();
81
82 ProfileEvents::increment(ProfileEvents::CreatedHTTPConnections);
83
84 session->setHost(DNSResolver::instance().resolveHost(host).toString());
85 session->setPort(port);
86
87 /// doesn't work properly without patch
88#if defined(POCO_CLICKHOUSE_PATCH)
89 session->setKeepAlive(keep_alive);
90#else
91 (void)keep_alive; // Avoid warning: unused parameter
92#endif
93
94 return session;
95 }
96
97 class SingleEndpointHTTPSessionPool : public PoolBase<Poco::Net::HTTPClientSession>
98 {
99 private:
100 const std::string host;
101 const UInt16 port;
102 bool https;
103 using Base = PoolBase<Poco::Net::HTTPClientSession>;
104 ObjectPtr allocObject() override
105 {
106 return makeHTTPSessionImpl(host, port, https, true);
107 }
108
109 public:
110 SingleEndpointHTTPSessionPool(const std::string & host_, UInt16 port_, bool https_, size_t max_pool_size_)
111 : Base(max_pool_size_, &Poco::Logger::get("HTTPSessionPool")), host(host_), port(port_), https(https_)
112 {
113 }
114 };
115
116 class HTTPSessionPool : private boost::noncopyable
117 {
118 private:
119 using Key = std::tuple<std::string, UInt16, bool>;
120 using PoolPtr = std::shared_ptr<SingleEndpointHTTPSessionPool>;
121 using Entry = SingleEndpointHTTPSessionPool::Entry;
122
123 struct Hasher
124 {
125 size_t operator()(const Key & k) const
126 {
127 SipHash s;
128 s.update(std::get<0>(k));
129 s.update(std::get<1>(k));
130 s.update(std::get<2>(k));
131 return s.get64();
132 }
133 };
134
135 std::mutex mutex;
136 std::unordered_map<Key, PoolPtr, Hasher> endpoints_pool;
137
138 protected:
139 HTTPSessionPool() = default;
140
141 public:
142 static auto & instance()
143 {
144 static HTTPSessionPool instance;
145 return instance;
146 }
147
148 Entry getSession(
149 const Poco::URI & uri,
150 const ConnectionTimeouts & timeouts,
151 size_t max_connections_per_endpoint)
152 {
153 std::unique_lock lock(mutex);
154 const std::string & host = uri.getHost();
155 UInt16 port = uri.getPort();
156 bool https = isHTTPS(uri);
157 auto key = std::make_tuple(host, port, https);
158 auto pool_ptr = endpoints_pool.find(key);
159 if (pool_ptr == endpoints_pool.end())
160 std::tie(pool_ptr, std::ignore) = endpoints_pool.emplace(
161 key, std::make_shared<SingleEndpointHTTPSessionPool>(host, port, https, max_connections_per_endpoint));
162
163 auto retry_timeout = timeouts.connection_timeout.totalMicroseconds();
164 auto session = pool_ptr->second->get(retry_timeout);
165
166 /// We store exception messages in session data.
167 /// Poco HTTPSession also stores exception, but it can be removed at any time.
168 const auto & sessionData = session->sessionData();
169 if (!sessionData.empty())
170 {
171 auto msg = Poco::AnyCast<std::string>(sessionData);
172 if (!msg.empty())
173 {
174 LOG_TRACE((&Logger::get("HTTPCommon")), "Failed communicating with " << host << " with error '" << msg << "' will try to reconnect session");
175 /// Host can change IP
176 const auto ip = DNSResolver::instance().resolveHost(host).toString();
177 if (ip != session->getHost())
178 {
179 session->reset();
180 session->setHost(ip);
181 session->attachSessionData({});
182 }
183 }
184 }
185
186 setTimeouts(*session, timeouts);
187
188 return session;
189 }
190 };
191}
192
193void setResponseDefaultHeaders(Poco::Net::HTTPServerResponse & response, unsigned keep_alive_timeout)
194{
195 if (!response.getKeepAlive())
196 return;
197
198 Poco::Timespan timeout(keep_alive_timeout, 0);
199 if (timeout.totalSeconds())
200 response.set("Keep-Alive", "timeout=" + std::to_string(timeout.totalSeconds()));
201}
202
203HTTPSessionPtr makeHTTPSession(const Poco::URI & uri, const ConnectionTimeouts & timeouts)
204{
205 const std::string & host = uri.getHost();
206 UInt16 port = uri.getPort();
207 bool https = isHTTPS(uri);
208
209 auto session = makeHTTPSessionImpl(host, port, https, false);
210 setTimeouts(*session, timeouts);
211 return session;
212}
213
214
215PooledHTTPSessionPtr makePooledHTTPSession(const Poco::URI & uri, const ConnectionTimeouts & timeouts, size_t per_endpoint_pool_size)
216{
217 return HTTPSessionPool::instance().getSession(uri, timeouts, per_endpoint_pool_size);
218}
219
220bool isRedirect(const Poco::Net::HTTPResponse::HTTPStatus status) { return status == Poco::Net::HTTPResponse::HTTP_MOVED_PERMANENTLY || status == Poco::Net::HTTPResponse::HTTP_FOUND || status == Poco::Net::HTTPResponse::HTTP_SEE_OTHER || status == Poco::Net::HTTPResponse::HTTP_TEMPORARY_REDIRECT; }
221
222std::istream * receiveResponse(
223 Poco::Net::HTTPClientSession & session, const Poco::Net::HTTPRequest & request, Poco::Net::HTTPResponse & response, const bool allow_redirects)
224{
225 auto & istr = session.receiveResponse(response);
226 assertResponseIsOk(request, response, istr, allow_redirects);
227 return &istr;
228}
229
230void assertResponseIsOk(const Poco::Net::HTTPRequest & request, Poco::Net::HTTPResponse & response, std::istream & istr, const bool allow_redirects)
231{
232 auto status = response.getStatus();
233
234 if (!(status == Poco::Net::HTTPResponse::HTTP_OK || (isRedirect(status) && allow_redirects)))
235 {
236 std::stringstream error_message;
237 error_message << "Received error from remote server " << request.getURI() << ". HTTP status code: " << status << " "
238 << response.getReason() << ", body: " << istr.rdbuf();
239
240 throw Exception(error_message.str(),
241 status == HTTP_TOO_MANY_REQUESTS ? ErrorCodes::RECEIVED_ERROR_TOO_MANY_REQUESTS
242 : ErrorCodes::RECEIVED_ERROR_FROM_REMOTE_IO_SERVER);
243 }
244}
245
246}
247