1/*
2 * IXHttpClient.cpp
3 * Author: Benjamin Sergeant
4 * Copyright (c) 2019 Machine Zone, Inc. All rights reserved.
5 */
6
7#include "IXHttpClient.h"
8
9#include "IXGzipCodec.h"
10#include "IXSocketFactory.h"
11#include "IXUrlParser.h"
12#include "IXUserAgent.h"
13#include "IXWebSocketHttpHeaders.h"
14#include <assert.h>
15#include <cstring>
16#include <iomanip>
17#include <random>
18#include <sstream>
19#include <vector>
20
21namespace ix
22{
23 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods
24 const std::string HttpClient::kPost = "POST";
25 const std::string HttpClient::kGet = "GET";
26 const std::string HttpClient::kHead = "HEAD";
27 const std::string HttpClient::kDelete = "DELETE";
28 const std::string HttpClient::kPut = "PUT";
29 const std::string HttpClient::kPatch = "PATCH";
30
31 HttpClient::HttpClient(bool async)
32 : _async(async)
33 , _stop(false)
34 , _forceBody(false)
35 {
36 if (!_async) return;
37
38 _thread = std::thread(&HttpClient::run, this);
39 }
40
41 HttpClient::~HttpClient()
42 {
43 if (!_thread.joinable()) return;
44
45 _stop = true;
46 _condition.notify_one();
47 _thread.join();
48 }
49
50 void HttpClient::setTLSOptions(const SocketTLSOptions& tlsOptions)
51 {
52 _tlsOptions = tlsOptions;
53 }
54
55 void HttpClient::setForceBody(bool value)
56 {
57 _forceBody = value;
58 }
59
60 HttpRequestArgsPtr HttpClient::createRequest(const std::string& url, const std::string& verb)
61 {
62 auto request = std::make_shared<HttpRequestArgs>();
63 request->url = url;
64 request->verb = verb;
65 return request;
66 }
67
68 bool HttpClient::performRequest(HttpRequestArgsPtr args,
69 const OnResponseCallback& onResponseCallback)
70 {
71 assert(_async && "HttpClient needs its async parameter set to true "
72 "in order to call performRequest");
73 if (!_async) return false;
74
75 // Enqueue the task
76 {
77 // acquire lock
78 std::unique_lock<std::mutex> lock(_queueMutex);
79
80 // add the task
81 _queue.push(std::make_pair(args, onResponseCallback));
82 } // release lock
83
84 // wake up one thread
85 _condition.notify_one();
86
87 return true;
88 }
89
90 void HttpClient::run()
91 {
92 while (true)
93 {
94 HttpRequestArgsPtr args;
95 OnResponseCallback onResponseCallback;
96
97 {
98 std::unique_lock<std::mutex> lock(_queueMutex);
99
100 while (!_stop && _queue.empty())
101 {
102 _condition.wait(lock);
103 }
104
105 if (_stop) return;
106
107 auto p = _queue.front();
108 _queue.pop();
109
110 args = p.first;
111 onResponseCallback = p.second;
112 }
113
114 if (_stop) return;
115
116 HttpResponsePtr response = request(args->url, args->verb, args->body, args);
117 onResponseCallback(response);
118
119 if (_stop) return;
120 }
121 }
122
123 HttpResponsePtr HttpClient::request(const std::string& url,
124 const std::string& verb,
125 const std::string& body,
126 HttpRequestArgsPtr args,
127 int redirects)
128 {
129 // We only have one socket connection, so we cannot
130 // make multiple requests concurrently.
131 std::lock_guard<std::recursive_mutex> lock(_mutex);
132
133 uint64_t uploadSize = 0;
134 uint64_t downloadSize = 0;
135 int code = 0;
136 WebSocketHttpHeaders headers;
137 std::string payload;
138 std::string description;
139
140 std::string protocol, host, path, query;
141 int port;
142
143 if (!UrlParser::parse(url, protocol, host, path, query, port))
144 {
145 std::stringstream ss;
146 ss << "Cannot parse url: " << url;
147 return std::make_shared<HttpResponse>(code,
148 description,
149 HttpErrorCode::UrlMalformed,
150 headers,
151 payload,
152 ss.str(),
153 uploadSize,
154 downloadSize);
155 }
156
157 bool tls = protocol == "https";
158 std::string errorMsg;
159 _socket = createSocket(tls, -1, errorMsg, _tlsOptions);
160
161 if (!_socket)
162 {
163 return std::make_shared<HttpResponse>(code,
164 description,
165 HttpErrorCode::CannotCreateSocket,
166 headers,
167 payload,
168 errorMsg,
169 uploadSize,
170 downloadSize);
171 }
172
173 // Build request string
174 std::stringstream ss;
175 ss << verb << " " << path << " HTTP/1.1\r\n";
176 ss << "Host: " << host << "\r\n";
177
178#ifdef IXWEBSOCKET_USE_ZLIB
179 if (args->compress && !args->onChunkCallback)
180 {
181 ss << "Accept-Encoding: gzip"
182 << "\r\n";
183 }
184#endif
185
186 // Append extra headers
187 for (auto&& it : args->extraHeaders)
188 {
189 ss << it.first << ": " << it.second << "\r\n";
190 }
191
192 // Set a default Accept header if none is present
193 if (args->extraHeaders.find("Accept") == args->extraHeaders.end())
194 {
195 ss << "Accept: */*"
196 << "\r\n";
197 }
198
199 // Set a default User agent if none is present
200 if (args->extraHeaders.find("User-Agent") == args->extraHeaders.end())
201 {
202 ss << "User-Agent: " << userAgent() << "\r\n";
203 }
204
205 if (verb == kPost || verb == kPut || verb == kPatch || _forceBody)
206 {
207 // Set request compression header
208#ifdef IXWEBSOCKET_USE_ZLIB
209 if (args->compressRequest)
210 {
211 ss << "Content-Encoding: gzip"
212 << "\r\n";
213 }
214#endif
215
216 ss << "Content-Length: " << body.size() << "\r\n";
217
218 // Set default Content-Type if unspecified
219 if (args->extraHeaders.find("Content-Type") == args->extraHeaders.end())
220 {
221 if (args->multipartBoundary.empty())
222 {
223 ss << "Content-Type: application/x-www-form-urlencoded"
224 << "\r\n";
225 }
226 else
227 {
228 ss << "Content-Type: multipart/form-data; boundary=" << args->multipartBoundary
229 << "\r\n";
230 }
231 }
232 ss << "\r\n";
233 ss << body;
234 }
235 else
236 {
237 ss << "\r\n";
238 }
239
240 std::string req(ss.str());
241 std::string errMsg;
242
243 // Make a cancellation object dealing with connection timeout
244 auto cancelled = makeCancellationRequestWithTimeout(args->connectTimeout, args->cancel);
245
246 auto isCancellationRequested = [&]() {
247 return cancelled() || _stop;
248 };
249
250 bool success = _socket->connect(host, port, errMsg, isCancellationRequested);
251 if (!success)
252 {
253 auto errorCode = args->cancel ? HttpErrorCode::Cancelled : HttpErrorCode::CannotConnect;
254 std::stringstream ss;
255 ss << "Cannot connect to url: " << url << " / error : " << errMsg;
256 return std::make_shared<HttpResponse>(code,
257 description,
258 errorCode,
259 headers,
260 payload,
261 ss.str(),
262 uploadSize,
263 downloadSize);
264 }
265
266 // Make a new cancellation object dealing with transfer timeout
267 cancelled = makeCancellationRequestWithTimeout(args->transferTimeout, args->cancel);
268
269 if (args->verbose)
270 {
271 std::stringstream ss;
272 ss << "Sending " << verb << " request "
273 << "to " << host << ":" << port << std::endl
274 << "request size: " << req.size() << " bytes" << std::endl
275 << "=============" << std::endl
276 << req << "=============" << std::endl
277 << std::endl;
278
279 log(ss.str(), args);
280 }
281
282 if (!_socket->writeBytes(req, isCancellationRequested))
283 {
284 auto errorCode = args->cancel ? HttpErrorCode::Cancelled : HttpErrorCode::SendError;
285 std::string errorMsg("Cannot send request");
286 return std::make_shared<HttpResponse>(code,
287 description,
288 errorCode,
289 headers,
290 payload,
291 errorMsg,
292 uploadSize,
293 downloadSize);
294 }
295
296 uploadSize = req.size();
297
298 auto lineResult = _socket->readLine(isCancellationRequested);
299 auto lineValid = lineResult.first;
300 auto line = lineResult.second;
301
302 if (!lineValid)
303 {
304 auto errorCode = args->cancel ? HttpErrorCode::Cancelled : HttpErrorCode::CannotReadStatusLine;
305 std::string errorMsg("Cannot retrieve status line");
306 return std::make_shared<HttpResponse>(code,
307 description,
308 errorCode,
309 headers,
310 payload,
311 errorMsg,
312 uploadSize,
313 downloadSize);
314 }
315
316 if (args->verbose)
317 {
318 std::stringstream ss;
319 ss << "Status line " << line;
320 log(ss.str(), args);
321 }
322
323 if (sscanf(line.c_str(), "HTTP/1.1 %d", &code) != 1)
324 {
325 std::string errorMsg("Cannot parse response code from status line");
326 return std::make_shared<HttpResponse>(code,
327 description,
328 HttpErrorCode::MissingStatus,
329 headers,
330 payload,
331 errorMsg,
332 uploadSize,
333 downloadSize);
334 }
335
336 auto result = parseHttpHeaders(_socket, isCancellationRequested);
337 auto headersValid = result.first;
338 headers = result.second;
339
340 if (!headersValid)
341 {
342 auto errorCode = args->cancel ? HttpErrorCode::Cancelled : HttpErrorCode::HeaderParsingError;
343 std::string errorMsg("Cannot parse http headers");
344 return std::make_shared<HttpResponse>(code,
345 description,
346 errorCode,
347 headers,
348 payload,
349 errorMsg,
350 uploadSize,
351 downloadSize);
352 }
353
354 // Redirect ?
355 if ((code >= 301 && code <= 308) && args->followRedirects)
356 {
357 if (headers.find("Location") == headers.end())
358 {
359 std::string errorMsg("Missing location header for redirect");
360 return std::make_shared<HttpResponse>(code,
361 description,
362 HttpErrorCode::MissingLocation,
363 headers,
364 payload,
365 errorMsg,
366 uploadSize,
367 downloadSize);
368 }
369
370 if (redirects >= args->maxRedirects)
371 {
372 std::stringstream ss;
373 ss << "Too many redirects: " << redirects;
374 return std::make_shared<HttpResponse>(code,
375 description,
376 HttpErrorCode::TooManyRedirects,
377 headers,
378 payload,
379 ss.str(),
380 uploadSize,
381 downloadSize);
382 }
383
384 // Recurse
385 std::string location = headers["Location"];
386 return request(location, verb, body, args, redirects + 1);
387 }
388
389 if (verb == "HEAD")
390 {
391 return std::make_shared<HttpResponse>(code,
392 description,
393 HttpErrorCode::Ok,
394 headers,
395 payload,
396 std::string(),
397 uploadSize,
398 downloadSize);
399 }
400
401 // Parse response:
402 if (headers.find("Content-Length") != headers.end())
403 {
404 ssize_t contentLength = -1;
405 ss.str("");
406 ss << headers["Content-Length"];
407 ss >> contentLength;
408
409 auto chunkResult = _socket->readBytes(contentLength,
410 args->onProgressCallback,
411 args->onChunkCallback,
412 isCancellationRequested);
413 if (!chunkResult.first)
414 {
415 auto errorCode = args->cancel ? HttpErrorCode::Cancelled : HttpErrorCode::ChunkReadError;
416 errorMsg = "Cannot read chunk";
417 return std::make_shared<HttpResponse>(code,
418 description,
419 errorCode,
420 headers,
421 payload,
422 errorMsg,
423 uploadSize,
424 downloadSize);
425 }
426
427 if (!args->onChunkCallback)
428 {
429 payload.reserve(contentLength);
430 payload += chunkResult.second;
431 }
432 }
433 else if (headers.find("Transfer-Encoding") != headers.end() &&
434 headers["Transfer-Encoding"] == "chunked")
435 {
436 std::stringstream ss;
437
438 while (true)
439 {
440 auto errorCode = args->cancel ? HttpErrorCode::Cancelled : HttpErrorCode::ChunkReadError;
441 lineResult = _socket->readLine(isCancellationRequested);
442 line = lineResult.second;
443
444 if (!lineResult.first)
445 {
446 return std::make_shared<HttpResponse>(code,
447 description,
448 errorCode,
449 headers,
450 payload,
451 errorMsg,
452 uploadSize,
453 downloadSize);
454 }
455
456 uint64_t chunkSize;
457 ss.str("");
458 ss << std::hex << line;
459 ss >> chunkSize;
460
461 if (args->verbose)
462 {
463 std::stringstream oss;
464 oss << "Reading " << chunkSize << " bytes" << std::endl;
465 log(oss.str(), args);
466 }
467
468 // Read a chunk
469 auto chunkResult = _socket->readBytes((size_t) chunkSize,
470 args->onProgressCallback,
471 args->onChunkCallback,
472 isCancellationRequested);
473 if (!chunkResult.first)
474 {
475 auto errorCode = args->cancel ? HttpErrorCode::Cancelled : HttpErrorCode::ChunkReadError;
476 errorMsg = "Cannot read chunk";
477 return std::make_shared<HttpResponse>(code,
478 description,
479 errorCode,
480 headers,
481 payload,
482 errorMsg,
483 uploadSize,
484 downloadSize);
485 }
486
487 if (!args->onChunkCallback)
488 {
489 payload.reserve(payload.size() + (size_t) chunkSize);
490 payload += chunkResult.second;
491 }
492
493 // Read the line that terminates the chunk (\r\n)
494 lineResult = _socket->readLine(isCancellationRequested);
495
496 if (!lineResult.first)
497 {
498 auto errorCode = args->cancel ? HttpErrorCode::Cancelled : HttpErrorCode::ChunkReadError;
499 return std::make_shared<HttpResponse>(code,
500 description,
501 errorCode,
502 headers,
503 payload,
504 errorMsg,
505 uploadSize,
506 downloadSize);
507 }
508
509 if (chunkSize == 0) break;
510 }
511 }
512 else if (code == 204)
513 {
514 ; // 204 is NoContent response code
515 }
516 else
517 {
518 std::string errorMsg("Cannot read http body");
519 return std::make_shared<HttpResponse>(code,
520 description,
521 HttpErrorCode::CannotReadBody,
522 headers,
523 payload,
524 errorMsg,
525 uploadSize,
526 downloadSize);
527 }
528
529 downloadSize = payload.size();
530
531 // If the content was compressed with gzip, decode it
532 if (headers["Content-Encoding"] == "gzip")
533 {
534#ifdef IXWEBSOCKET_USE_ZLIB
535 std::string decompressedPayload;
536 if (!gzipDecompress(payload, decompressedPayload))
537 {
538 std::string errorMsg("Error decompressing payload");
539 return std::make_shared<HttpResponse>(code,
540 description,
541 HttpErrorCode::Gzip,
542 headers,
543 payload,
544 errorMsg,
545 uploadSize,
546 downloadSize);
547 }
548 payload = decompressedPayload;
549#else
550 std::string errorMsg("ixwebsocket was not compiled with gzip support on");
551 return std::make_shared<HttpResponse>(code,
552 description,
553 HttpErrorCode::Gzip,
554 headers,
555 payload,
556 errorMsg,
557 uploadSize,
558 downloadSize);
559#endif
560 }
561
562 return std::make_shared<HttpResponse>(code,
563 description,
564 HttpErrorCode::Ok,
565 headers,
566 payload,
567 std::string(),
568 uploadSize,
569 downloadSize);
570 }
571
572 HttpResponsePtr HttpClient::get(const std::string& url, HttpRequestArgsPtr args)
573 {
574 return request(url, kGet, std::string(), args);
575 }
576
577 HttpResponsePtr HttpClient::head(const std::string& url, HttpRequestArgsPtr args)
578 {
579 return request(url, kHead, std::string(), args);
580 }
581
582 HttpResponsePtr HttpClient::Delete(const std::string& url, HttpRequestArgsPtr args)
583 {
584 return request(url, kDelete, std::string(), args);
585 }
586
587 HttpResponsePtr HttpClient::request(const std::string& url,
588 const std::string& verb,
589 const HttpParameters& httpParameters,
590 const HttpFormDataParameters& httpFormDataParameters,
591 HttpRequestArgsPtr args)
592 {
593 std::string body;
594
595 if (httpFormDataParameters.empty())
596 {
597 body = serializeHttpParameters(httpParameters);
598 }
599 else
600 {
601 std::string multipartBoundary = generateMultipartBoundary();
602 args->multipartBoundary = multipartBoundary;
603 body = serializeHttpFormDataParameters(
604 multipartBoundary, httpFormDataParameters, httpParameters);
605 }
606
607#ifdef IXWEBSOCKET_USE_ZLIB
608 if (args->compressRequest)
609 {
610 body = gzipCompress(body);
611 }
612#endif
613
614 return request(url, verb, body, args);
615 }
616
617 HttpResponsePtr HttpClient::post(const std::string& url,
618 const HttpParameters& httpParameters,
619 const HttpFormDataParameters& httpFormDataParameters,
620 HttpRequestArgsPtr args)
621 {
622 return request(url, kPost, httpParameters, httpFormDataParameters, args);
623 }
624
625 HttpResponsePtr HttpClient::post(const std::string& url,
626 const std::string& body,
627 HttpRequestArgsPtr args)
628 {
629 return request(url, kPost, body, args);
630 }
631
632 HttpResponsePtr HttpClient::put(const std::string& url,
633 const HttpParameters& httpParameters,
634 const HttpFormDataParameters& httpFormDataParameters,
635 HttpRequestArgsPtr args)
636 {
637 return request(url, kPut, httpParameters, httpFormDataParameters, args);
638 }
639
640 HttpResponsePtr HttpClient::put(const std::string& url,
641 const std::string& body,
642 const HttpRequestArgsPtr args)
643 {
644 return request(url, kPut, body, args);
645 }
646
647 HttpResponsePtr HttpClient::patch(const std::string& url,
648 const HttpParameters& httpParameters,
649 const HttpFormDataParameters& httpFormDataParameters,
650 HttpRequestArgsPtr args)
651 {
652 return request(url, kPatch, httpParameters, httpFormDataParameters, args);
653 }
654
655 HttpResponsePtr HttpClient::patch(const std::string& url,
656 const std::string& body,
657 const HttpRequestArgsPtr args)
658 {
659 return request(url, kPatch, body, args);
660 }
661
662 std::string HttpClient::urlEncode(const std::string& value)
663 {
664 std::ostringstream escaped;
665 escaped.fill('0');
666 escaped << std::hex;
667
668 for (std::string::const_iterator i = value.begin(), n = value.end(); i != n; ++i)
669 {
670 std::string::value_type c = (*i);
671
672 // Keep alphanumeric and other accepted characters intact
673 if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~')
674 {
675 escaped << c;
676 continue;
677 }
678
679 // Any other characters are percent-encoded
680 escaped << std::uppercase;
681 escaped << '%' << std::setw(2) << int((unsigned char) c);
682 escaped << std::nouppercase;
683 }
684
685 return escaped.str();
686 }
687
688 std::string HttpClient::serializeHttpParameters(const HttpParameters& httpParameters)
689 {
690 std::stringstream ss;
691 size_t count = httpParameters.size();
692 size_t i = 0;
693
694 for (auto&& it : httpParameters)
695 {
696 ss << urlEncode(it.first) << "=" << urlEncode(it.second);
697
698 if (i++ < (count - 1))
699 {
700 ss << "&";
701 }
702 }
703 return ss.str();
704 }
705
706 std::string HttpClient::serializeHttpFormDataParameters(
707 const std::string& multipartBoundary,
708 const HttpFormDataParameters& httpFormDataParameters,
709 const HttpParameters& httpParameters)
710 {
711 //
712 // --AaB03x
713 // Content-Disposition: form-data; name="submit-name"
714
715 // Larry
716 // --AaB03x
717 // Content-Disposition: form-data; name="foo.txt"; filename="file1.txt"
718 // Content-Type: text/plain
719
720 // ... contents of file1.txt ...
721 // --AaB03x--
722 //
723 std::stringstream ss;
724
725 for (auto&& it : httpFormDataParameters)
726 {
727 ss << "--" << multipartBoundary << "\r\n"
728 << "Content-Disposition:"
729 << " form-data; name=\"" << it.first << "\";"
730 << " filename=\"" << it.first << "\""
731 << "\r\n"
732 << "Content-Type: application/octet-stream"
733 << "\r\n"
734 << "\r\n"
735 << it.second << "\r\n";
736 }
737
738 for (auto&& it : httpParameters)
739 {
740 ss << "--" << multipartBoundary << "\r\n"
741 << "Content-Disposition:"
742 << " form-data; name=\"" << it.first << "\";"
743 << "\r\n"
744 << "\r\n"
745 << it.second << "\r\n";
746 }
747
748 ss << "--" << multipartBoundary << "--\r\n";
749
750 return ss.str();
751 }
752
753 void HttpClient::log(const std::string& msg, HttpRequestArgsPtr args)
754 {
755 if (args->logger)
756 {
757 args->logger(msg);
758 }
759 }
760
761 std::string HttpClient::generateMultipartBoundary()
762 {
763 std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");
764
765 static std::random_device rd;
766 static std::mt19937 generator(rd());
767
768 std::shuffle(str.begin(), str.end(), generator);
769
770 return str;
771 }
772} // namespace ix
773