1/***************************************************************************
2 * _ _ ____ _
3 * Project ___| | | | _ \| |
4 * / __| | | | |_) | |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
7 *
8 * Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
9 *
10 * This software is licensed as described in the file COPYING, which
11 * you should have received as part of this distribution. The terms
12 * are also available at https://curl.se/docs/copyright.html.
13 *
14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15 * copies of the Software, and permit persons to whom the Software is
16 * furnished to do so, under the terms of the COPYING file.
17 *
18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19 * KIND, either express or implied.
20 *
21 ***************************************************************************/
22
23#include "curl_setup.h"
24#include "strtoofft.h"
25
26#ifdef HAVE_NETINET_IN_H
27#include <netinet/in.h>
28#endif
29#ifdef HAVE_NETDB_H
30#include <netdb.h>
31#endif
32#ifdef HAVE_ARPA_INET_H
33#include <arpa/inet.h>
34#endif
35#ifdef HAVE_NET_IF_H
36#include <net/if.h>
37#endif
38#ifdef HAVE_SYS_IOCTL_H
39#include <sys/ioctl.h>
40#endif
41#ifdef HAVE_SIGNAL_H
42#include <signal.h>
43#endif
44
45#ifdef HAVE_SYS_PARAM_H
46#include <sys/param.h>
47#endif
48
49#ifdef HAVE_SYS_SELECT_H
50#include <sys/select.h>
51#elif defined(HAVE_UNISTD_H)
52#include <unistd.h>
53#endif
54
55#ifndef HAVE_SOCKET
56#error "We can't compile without socket() support!"
57#endif
58
59#include "urldata.h"
60#include <curl/curl.h>
61#include "netrc.h"
62
63#include "content_encoding.h"
64#include "hostip.h"
65#include "transfer.h"
66#include "sendf.h"
67#include "speedcheck.h"
68#include "progress.h"
69#include "http.h"
70#include "url.h"
71#include "getinfo.h"
72#include "vtls/vtls.h"
73#include "select.h"
74#include "multiif.h"
75#include "connect.h"
76#include "non-ascii.h"
77#include "http2.h"
78#include "mime.h"
79#include "strcase.h"
80#include "urlapi-int.h"
81#include "hsts.h"
82#include "setopt.h"
83
84/* The last 3 #include files should be in this order */
85#include "curl_printf.h"
86#include "curl_memory.h"
87#include "memdebug.h"
88
89#if !defined(CURL_DISABLE_HTTP) || !defined(CURL_DISABLE_SMTP) || \
90 !defined(CURL_DISABLE_IMAP)
91/*
92 * checkheaders() checks the linked list of custom headers for a
93 * particular header (prefix). Provide the prefix without colon!
94 *
95 * Returns a pointer to the first matching header or NULL if none matched.
96 */
97char *Curl_checkheaders(const struct Curl_easy *data,
98 const char *thisheader)
99{
100 struct curl_slist *head;
101 size_t thislen = strlen(thisheader);
102 DEBUGASSERT(thislen);
103 DEBUGASSERT(thisheader[thislen-1] != ':');
104
105 for(head = data->set.headers; head; head = head->next) {
106 if(strncasecompare(head->data, thisheader, thislen) &&
107 Curl_headersep(head->data[thislen]) )
108 return head->data;
109 }
110
111 return NULL;
112}
113#endif
114
115CURLcode Curl_get_upload_buffer(struct Curl_easy *data)
116{
117 if(!data->state.ulbuf) {
118 data->state.ulbuf = malloc(data->set.upload_buffer_size);
119 if(!data->state.ulbuf)
120 return CURLE_OUT_OF_MEMORY;
121 }
122 return CURLE_OK;
123}
124
125#ifndef CURL_DISABLE_HTTP
126/*
127 * This function will be called to loop through the trailers buffer
128 * until no more data is available for sending.
129 */
130static size_t trailers_read(char *buffer, size_t size, size_t nitems,
131 void *raw)
132{
133 struct Curl_easy *data = (struct Curl_easy *)raw;
134 struct dynbuf *trailers_buf = &data->state.trailers_buf;
135 size_t bytes_left = Curl_dyn_len(trailers_buf) -
136 data->state.trailers_bytes_sent;
137 size_t to_copy = (size*nitems < bytes_left) ? size*nitems : bytes_left;
138 if(to_copy) {
139 memcpy(buffer,
140 Curl_dyn_ptr(trailers_buf) + data->state.trailers_bytes_sent,
141 to_copy);
142 data->state.trailers_bytes_sent += to_copy;
143 }
144 return to_copy;
145}
146
147static size_t trailers_left(void *raw)
148{
149 struct Curl_easy *data = (struct Curl_easy *)raw;
150 struct dynbuf *trailers_buf = &data->state.trailers_buf;
151 return Curl_dyn_len(trailers_buf) - data->state.trailers_bytes_sent;
152}
153#endif
154
155/*
156 * This function will call the read callback to fill our buffer with data
157 * to upload.
158 */
159CURLcode Curl_fillreadbuffer(struct Curl_easy *data, size_t bytes,
160 size_t *nreadp)
161{
162 size_t buffersize = bytes;
163 size_t nread;
164
165 curl_read_callback readfunc = NULL;
166 void *extra_data = NULL;
167
168#ifdef CURL_DOES_CONVERSIONS
169 bool sending_http_headers = FALSE;
170 struct connectdata *conn = data->conn;
171
172 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) {
173 const struct HTTP *http = data->req.p.http;
174
175 if(http->sending == HTTPSEND_REQUEST)
176 /* We're sending the HTTP request headers, not the data.
177 Remember that so we don't re-translate them into garbage. */
178 sending_http_headers = TRUE;
179 }
180#endif
181
182#ifndef CURL_DISABLE_HTTP
183 if(data->state.trailers_state == TRAILERS_INITIALIZED) {
184 struct curl_slist *trailers = NULL;
185 CURLcode result;
186 int trailers_ret_code;
187
188 /* at this point we already verified that the callback exists
189 so we compile and store the trailers buffer, then proceed */
190 infof(data,
191 "Moving trailers state machine from initialized to sending.");
192 data->state.trailers_state = TRAILERS_SENDING;
193 Curl_dyn_init(&data->state.trailers_buf, DYN_TRAILERS);
194
195 data->state.trailers_bytes_sent = 0;
196 Curl_set_in_callback(data, true);
197 trailers_ret_code = data->set.trailer_callback(&trailers,
198 data->set.trailer_data);
199 Curl_set_in_callback(data, false);
200 if(trailers_ret_code == CURL_TRAILERFUNC_OK) {
201 result = Curl_http_compile_trailers(trailers, &data->state.trailers_buf,
202 data);
203 }
204 else {
205 failf(data, "operation aborted by trailing headers callback");
206 *nreadp = 0;
207 result = CURLE_ABORTED_BY_CALLBACK;
208 }
209 if(result) {
210 Curl_dyn_free(&data->state.trailers_buf);
211 curl_slist_free_all(trailers);
212 return result;
213 }
214 infof(data, "Successfully compiled trailers.");
215 curl_slist_free_all(trailers);
216 }
217#endif
218
219 /* if we are transmitting trailing data, we don't need to write
220 a chunk size so we skip this */
221 if(data->req.upload_chunky &&
222 data->state.trailers_state == TRAILERS_NONE) {
223 /* if chunked Transfer-Encoding */
224 buffersize -= (8 + 2 + 2); /* 32bit hex + CRLF + CRLF */
225 data->req.upload_fromhere += (8 + 2); /* 32bit hex + CRLF */
226 }
227
228#ifndef CURL_DISABLE_HTTP
229 if(data->state.trailers_state == TRAILERS_SENDING) {
230 /* if we're here then that means that we already sent the last empty chunk
231 but we didn't send a final CR LF, so we sent 0 CR LF. We then start
232 pulling trailing data until we have no more at which point we
233 simply return to the previous point in the state machine as if
234 nothing happened.
235 */
236 readfunc = trailers_read;
237 extra_data = (void *)data;
238 }
239 else
240#endif
241 {
242 readfunc = data->state.fread_func;
243 extra_data = data->state.in;
244 }
245
246 Curl_set_in_callback(data, true);
247 nread = readfunc(data->req.upload_fromhere, 1,
248 buffersize, extra_data);
249 Curl_set_in_callback(data, false);
250
251 if(nread == CURL_READFUNC_ABORT) {
252 failf(data, "operation aborted by callback");
253 *nreadp = 0;
254 return CURLE_ABORTED_BY_CALLBACK;
255 }
256 if(nread == CURL_READFUNC_PAUSE) {
257 struct SingleRequest *k = &data->req;
258
259 if(data->conn->handler->flags & PROTOPT_NONETWORK) {
260 /* protocols that work without network cannot be paused. This is
261 actually only FILE:// just now, and it can't pause since the transfer
262 isn't done using the "normal" procedure. */
263 failf(data, "Read callback asked for PAUSE when not supported!");
264 return CURLE_READ_ERROR;
265 }
266
267 /* CURL_READFUNC_PAUSE pauses read callbacks that feed socket writes */
268 k->keepon |= KEEP_SEND_PAUSE; /* mark socket send as paused */
269 if(data->req.upload_chunky) {
270 /* Back out the preallocation done above */
271 data->req.upload_fromhere -= (8 + 2);
272 }
273 *nreadp = 0;
274
275 return CURLE_OK; /* nothing was read */
276 }
277 else if(nread > buffersize) {
278 /* the read function returned a too large value */
279 *nreadp = 0;
280 failf(data, "read function returned funny value");
281 return CURLE_READ_ERROR;
282 }
283
284 if(!data->req.forbidchunk && data->req.upload_chunky) {
285 /* if chunked Transfer-Encoding
286 * build chunk:
287 *
288 * <HEX SIZE> CRLF
289 * <DATA> CRLF
290 */
291 /* On non-ASCII platforms the <DATA> may or may not be
292 translated based on state.prefer_ascii while the protocol
293 portion must always be translated to the network encoding.
294 To further complicate matters, line end conversion might be
295 done later on, so we need to prevent CRLFs from becoming
296 CRCRLFs if that's the case. To do this we use bare LFs
297 here, knowing they'll become CRLFs later on.
298 */
299
300 bool added_crlf = FALSE;
301 int hexlen = 0;
302 const char *endofline_native;
303 const char *endofline_network;
304
305 if(
306#ifdef CURL_DO_LINEEND_CONV
307 (data->state.prefer_ascii) ||
308#endif
309 (data->set.crlf)) {
310 /* \n will become \r\n later on */
311 endofline_native = "\n";
312 endofline_network = "\x0a";
313 }
314 else {
315 endofline_native = "\r\n";
316 endofline_network = "\x0d\x0a";
317 }
318
319 /* if we're not handling trailing data, proceed as usual */
320 if(data->state.trailers_state != TRAILERS_SENDING) {
321 char hexbuffer[11] = "";
322 hexlen = msnprintf(hexbuffer, sizeof(hexbuffer),
323 "%zx%s", nread, endofline_native);
324
325 /* move buffer pointer */
326 data->req.upload_fromhere -= hexlen;
327 nread += hexlen;
328
329 /* copy the prefix to the buffer, leaving out the NUL */
330 memcpy(data->req.upload_fromhere, hexbuffer, hexlen);
331
332 /* always append ASCII CRLF to the data unless
333 we have a valid trailer callback */
334#ifndef CURL_DISABLE_HTTP
335 if((nread-hexlen) == 0 &&
336 data->set.trailer_callback != NULL &&
337 data->state.trailers_state == TRAILERS_NONE) {
338 data->state.trailers_state = TRAILERS_INITIALIZED;
339 }
340 else
341#endif
342 {
343 memcpy(data->req.upload_fromhere + nread,
344 endofline_network,
345 strlen(endofline_network));
346 added_crlf = TRUE;
347 }
348 }
349
350#ifdef CURL_DOES_CONVERSIONS
351 {
352 CURLcode result;
353 size_t length;
354 if(data->state.prefer_ascii)
355 /* translate the protocol and data */
356 length = nread;
357 else
358 /* just translate the protocol portion */
359 length = hexlen;
360 if(length) {
361 result = Curl_convert_to_network(data, data->req.upload_fromhere,
362 length);
363 /* Curl_convert_to_network calls failf if unsuccessful */
364 if(result)
365 return result;
366 }
367 }
368#endif /* CURL_DOES_CONVERSIONS */
369
370#ifndef CURL_DISABLE_HTTP
371 if(data->state.trailers_state == TRAILERS_SENDING &&
372 !trailers_left(data)) {
373 Curl_dyn_free(&data->state.trailers_buf);
374 data->state.trailers_state = TRAILERS_DONE;
375 data->set.trailer_data = NULL;
376 data->set.trailer_callback = NULL;
377 /* mark the transfer as done */
378 data->req.upload_done = TRUE;
379 infof(data, "Signaling end of chunked upload after trailers.");
380 }
381 else
382#endif
383 if((nread - hexlen) == 0 &&
384 data->state.trailers_state != TRAILERS_INITIALIZED) {
385 /* mark this as done once this chunk is transferred */
386 data->req.upload_done = TRUE;
387 infof(data,
388 "Signaling end of chunked upload via terminating chunk.");
389 }
390
391 if(added_crlf)
392 nread += strlen(endofline_network); /* for the added end of line */
393 }
394#ifdef CURL_DOES_CONVERSIONS
395 else if((data->state.prefer_ascii) && (!sending_http_headers)) {
396 CURLcode result;
397 result = Curl_convert_to_network(data, data->req.upload_fromhere, nread);
398 /* Curl_convert_to_network calls failf if unsuccessful */
399 if(result)
400 return result;
401 }
402#endif /* CURL_DOES_CONVERSIONS */
403
404 *nreadp = nread;
405
406 return CURLE_OK;
407}
408
409
410/*
411 * Curl_readrewind() rewinds the read stream. This is typically used for HTTP
412 * POST/PUT with multi-pass authentication when a sending was denied and a
413 * resend is necessary.
414 */
415CURLcode Curl_readrewind(struct Curl_easy *data)
416{
417 struct connectdata *conn = data->conn;
418 curl_mimepart *mimepart = &data->set.mimepost;
419
420 conn->bits.rewindaftersend = FALSE; /* we rewind now */
421
422 /* explicitly switch off sending data on this connection now since we are
423 about to restart a new transfer and thus we want to avoid inadvertently
424 sending more data on the existing connection until the next transfer
425 starts */
426 data->req.keepon &= ~KEEP_SEND;
427
428 /* We have sent away data. If not using CURLOPT_POSTFIELDS or
429 CURLOPT_HTTPPOST, call app to rewind
430 */
431 if(conn->handler->protocol & PROTO_FAMILY_HTTP) {
432 struct HTTP *http = data->req.p.http;
433
434 if(http->sendit)
435 mimepart = http->sendit;
436 }
437 if(data->set.postfields)
438 ; /* do nothing */
439 else if(data->state.httpreq == HTTPREQ_POST_MIME ||
440 data->state.httpreq == HTTPREQ_POST_FORM) {
441 CURLcode result = Curl_mime_rewind(mimepart);
442 if(result) {
443 failf(data, "Cannot rewind mime/post data");
444 return result;
445 }
446 }
447 else {
448 if(data->set.seek_func) {
449 int err;
450
451 Curl_set_in_callback(data, true);
452 err = (data->set.seek_func)(data->set.seek_client, 0, SEEK_SET);
453 Curl_set_in_callback(data, false);
454 if(err) {
455 failf(data, "seek callback returned error %d", (int)err);
456 return CURLE_SEND_FAIL_REWIND;
457 }
458 }
459 else if(data->set.ioctl_func) {
460 curlioerr err;
461
462 Curl_set_in_callback(data, true);
463 err = (data->set.ioctl_func)(data, CURLIOCMD_RESTARTREAD,
464 data->set.ioctl_client);
465 Curl_set_in_callback(data, false);
466 infof(data, "the ioctl callback returned %d", (int)err);
467
468 if(err) {
469 failf(data, "ioctl callback returned error %d", (int)err);
470 return CURLE_SEND_FAIL_REWIND;
471 }
472 }
473 else {
474 /* If no CURLOPT_READFUNCTION is used, we know that we operate on a
475 given FILE * stream and we can actually attempt to rewind that
476 ourselves with fseek() */
477 if(data->state.fread_func == (curl_read_callback)fread) {
478 if(-1 != fseek(data->state.in, 0, SEEK_SET))
479 /* successful rewind */
480 return CURLE_OK;
481 }
482
483 /* no callback set or failure above, makes us fail at once */
484 failf(data, "necessary data rewind wasn't possible");
485 return CURLE_SEND_FAIL_REWIND;
486 }
487 }
488 return CURLE_OK;
489}
490
491static int data_pending(const struct Curl_easy *data)
492{
493 struct connectdata *conn = data->conn;
494
495#ifdef ENABLE_QUIC
496 if(conn->transport == TRNSPRT_QUIC)
497 return Curl_quic_data_pending(data);
498#endif
499
500 if(conn->handler->protocol&PROTO_FAMILY_FTP)
501 return Curl_ssl_data_pending(conn, SECONDARYSOCKET);
502
503 /* in the case of libssh2, we can never be really sure that we have emptied
504 its internal buffers so we MUST always try until we get EAGAIN back */
505 return conn->handler->protocol&(CURLPROTO_SCP|CURLPROTO_SFTP) ||
506#if defined(USE_NGHTTP2)
507 /* For HTTP/2, we may read up everything including response body
508 with header fields in Curl_http_readwrite_headers. If no
509 content-length is provided, curl waits for the connection
510 close, which we emulate it using conn->proto.httpc.closed =
511 TRUE. The thing is if we read everything, then http2_recv won't
512 be called and we cannot signal the HTTP/2 stream has closed. As
513 a workaround, we return nonzero here to call http2_recv. */
514 ((conn->handler->protocol&PROTO_FAMILY_HTTP) && conn->httpversion >= 20) ||
515#endif
516 Curl_ssl_data_pending(conn, FIRSTSOCKET);
517}
518
519/*
520 * Check to see if CURLOPT_TIMECONDITION was met by comparing the time of the
521 * remote document with the time provided by CURLOPT_TIMEVAL
522 */
523bool Curl_meets_timecondition(struct Curl_easy *data, time_t timeofdoc)
524{
525 if((timeofdoc == 0) || (data->set.timevalue == 0))
526 return TRUE;
527
528 switch(data->set.timecondition) {
529 case CURL_TIMECOND_IFMODSINCE:
530 default:
531 if(timeofdoc <= data->set.timevalue) {
532 infof(data,
533 "The requested document is not new enough");
534 data->info.timecond = TRUE;
535 return FALSE;
536 }
537 break;
538 case CURL_TIMECOND_IFUNMODSINCE:
539 if(timeofdoc >= data->set.timevalue) {
540 infof(data,
541 "The requested document is not old enough");
542 data->info.timecond = TRUE;
543 return FALSE;
544 }
545 break;
546 }
547
548 return TRUE;
549}
550
551/*
552 * Go ahead and do a read if we have a readable socket or if
553 * the stream was rewound (in which case we have data in a
554 * buffer)
555 *
556 * return '*comeback' TRUE if we didn't properly drain the socket so this
557 * function should get called again without select() or similar in between!
558 */
559static CURLcode readwrite_data(struct Curl_easy *data,
560 struct connectdata *conn,
561 struct SingleRequest *k,
562 int *didwhat, bool *done,
563 bool *comeback)
564{
565 CURLcode result = CURLE_OK;
566 ssize_t nread; /* number of bytes read */
567 size_t excess = 0; /* excess bytes read */
568 bool readmore = FALSE; /* used by RTP to signal for more data */
569 int maxloops = 100;
570 char *buf = data->state.buffer;
571 DEBUGASSERT(buf);
572
573 *done = FALSE;
574 *comeback = FALSE;
575
576 /* This is where we loop until we have read everything there is to
577 read or we get a CURLE_AGAIN */
578 do {
579 bool is_empty_data = FALSE;
580 size_t buffersize = data->set.buffer_size;
581 size_t bytestoread = buffersize;
582#ifdef USE_NGHTTP2
583 bool is_http2 = ((conn->handler->protocol & PROTO_FAMILY_HTTP) &&
584 (conn->httpversion == 20));
585#endif
586
587 if(
588#ifdef USE_NGHTTP2
589 /* For HTTP/2, read data without caring about the content
590 length. This is safe because body in HTTP/2 is always
591 segmented thanks to its framing layer. Meanwhile, we have to
592 call Curl_read to ensure that http2_handle_stream_close is
593 called when we read all incoming bytes for a particular
594 stream. */
595 !is_http2 &&
596#endif
597 k->size != -1 && !k->header) {
598 /* make sure we don't read too much */
599 curl_off_t totalleft = k->size - k->bytecount;
600 if(totalleft < (curl_off_t)bytestoread)
601 bytestoread = (size_t)totalleft;
602 }
603
604 if(bytestoread) {
605 /* receive data from the network! */
606 result = Curl_read(data, conn->sockfd, buf, bytestoread, &nread);
607
608 /* read would've blocked */
609 if(CURLE_AGAIN == result)
610 break; /* get out of loop */
611
612 if(result>0)
613 return result;
614 }
615 else {
616 /* read nothing but since we wanted nothing we consider this an OK
617 situation to proceed from */
618 DEBUGF(infof(data, "readwrite_data: we're done!"));
619 nread = 0;
620 }
621
622 if(!k->bytecount) {
623 Curl_pgrsTime(data, TIMER_STARTTRANSFER);
624 if(k->exp100 > EXP100_SEND_DATA)
625 /* set time stamp to compare with when waiting for the 100 */
626 k->start100 = Curl_now();
627 }
628
629 *didwhat |= KEEP_RECV;
630 /* indicates data of zero size, i.e. empty file */
631 is_empty_data = ((nread == 0) && (k->bodywrites == 0)) ? TRUE : FALSE;
632
633 if(0 < nread || is_empty_data) {
634 buf[nread] = 0;
635 }
636 else {
637 /* if we receive 0 or less here, either the http2 stream is closed or the
638 server closed the connection and we bail out from this! */
639#ifdef USE_NGHTTP2
640 if(is_http2 && !nread)
641 DEBUGF(infof(data, "nread == 0, stream closed, bailing"));
642 else
643#endif
644 DEBUGF(infof(data, "nread <= 0, server closed connection, bailing"));
645 k->keepon &= ~KEEP_RECV;
646 break;
647 }
648
649 /* Default buffer to use when we write the buffer, it may be changed
650 in the flow below before the actual storing is done. */
651 k->str = buf;
652
653 if(conn->handler->readwrite) {
654 result = conn->handler->readwrite(data, conn, &nread, &readmore);
655 if(result)
656 return result;
657 if(readmore)
658 break;
659 }
660
661#ifndef CURL_DISABLE_HTTP
662 /* Since this is a two-state thing, we check if we are parsing
663 headers at the moment or not. */
664 if(k->header) {
665 /* we are in parse-the-header-mode */
666 bool stop_reading = FALSE;
667 result = Curl_http_readwrite_headers(data, conn, &nread, &stop_reading);
668 if(result)
669 return result;
670
671 if(conn->handler->readwrite &&
672 (k->maxdownload <= 0 && nread > 0)) {
673 result = conn->handler->readwrite(data, conn, &nread, &readmore);
674 if(result)
675 return result;
676 if(readmore)
677 break;
678 }
679
680 if(stop_reading) {
681 /* We've stopped dealing with input, get out of the do-while loop */
682
683 if(nread > 0) {
684 infof(data,
685 "Excess found:"
686 " excess = %zd"
687 " url = %s (zero-length body)",
688 nread, data->state.up.path);
689 }
690
691 break;
692 }
693 }
694#endif /* CURL_DISABLE_HTTP */
695
696
697 /* This is not an 'else if' since it may be a rest from the header
698 parsing, where the beginning of the buffer is headers and the end
699 is non-headers. */
700 if(!k->header && (nread > 0 || is_empty_data)) {
701
702 if(data->set.opt_no_body) {
703 /* data arrives although we want none, bail out */
704 streamclose(conn, "ignoring body");
705 *done = TRUE;
706 return CURLE_WEIRD_SERVER_REPLY;
707 }
708
709#ifndef CURL_DISABLE_HTTP
710 if(0 == k->bodywrites && !is_empty_data) {
711 /* These checks are only made the first time we are about to
712 write a piece of the body */
713 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) {
714 /* HTTP-only checks */
715 result = Curl_http_firstwrite(data, conn, done);
716 if(result || *done)
717 return result;
718 }
719 } /* this is the first time we write a body part */
720#endif /* CURL_DISABLE_HTTP */
721
722 k->bodywrites++;
723
724 /* pass data to the debug function before it gets "dechunked" */
725 if(data->set.verbose) {
726 if(k->badheader) {
727 Curl_debug(data, CURLINFO_DATA_IN,
728 Curl_dyn_ptr(&data->state.headerb),
729 Curl_dyn_len(&data->state.headerb));
730 if(k->badheader == HEADER_PARTHEADER)
731 Curl_debug(data, CURLINFO_DATA_IN,
732 k->str, (size_t)nread);
733 }
734 else
735 Curl_debug(data, CURLINFO_DATA_IN,
736 k->str, (size_t)nread);
737 }
738
739#ifndef CURL_DISABLE_HTTP
740 if(k->chunk) {
741 /*
742 * Here comes a chunked transfer flying and we need to decode this
743 * properly. While the name says read, this function both reads
744 * and writes away the data. The returned 'nread' holds the number
745 * of actual data it wrote to the client.
746 */
747 CURLcode extra;
748 CHUNKcode res =
749 Curl_httpchunk_read(data, k->str, nread, &nread, &extra);
750
751 if(CHUNKE_OK < res) {
752 if(CHUNKE_PASSTHRU_ERROR == res) {
753 failf(data, "Failed reading the chunked-encoded stream");
754 return extra;
755 }
756 failf(data, "%s in chunked-encoding", Curl_chunked_strerror(res));
757 return CURLE_RECV_ERROR;
758 }
759 if(CHUNKE_STOP == res) {
760 /* we're done reading chunks! */
761 k->keepon &= ~KEEP_RECV; /* read no more */
762
763 /* N number of bytes at the end of the str buffer that weren't
764 written to the client. */
765 if(conn->chunk.datasize) {
766 infof(data, "Leftovers after chunking: % "
767 CURL_FORMAT_CURL_OFF_T "u bytes",
768 conn->chunk.datasize);
769 }
770 }
771 /* If it returned OK, we just keep going */
772 }
773#endif /* CURL_DISABLE_HTTP */
774
775 /* Account for body content stored in the header buffer */
776 if((k->badheader == HEADER_PARTHEADER) && !k->ignorebody) {
777 size_t headlen = Curl_dyn_len(&data->state.headerb);
778 DEBUGF(infof(data, "Increasing bytecount by %zu", headlen));
779 k->bytecount += headlen;
780 }
781
782 if((-1 != k->maxdownload) &&
783 (k->bytecount + nread >= k->maxdownload)) {
784
785 excess = (size_t)(k->bytecount + nread - k->maxdownload);
786 if(excess > 0 && !k->ignorebody) {
787 infof(data,
788 "Excess found in a read:"
789 " excess = %zu"
790 ", size = %" CURL_FORMAT_CURL_OFF_T
791 ", maxdownload = %" CURL_FORMAT_CURL_OFF_T
792 ", bytecount = %" CURL_FORMAT_CURL_OFF_T,
793 excess, k->size, k->maxdownload, k->bytecount);
794 connclose(conn, "excess found in a read");
795 }
796
797 nread = (ssize_t) (k->maxdownload - k->bytecount);
798 if(nread < 0) /* this should be unusual */
799 nread = 0;
800
801 k->keepon &= ~KEEP_RECV; /* we're done reading */
802 }
803
804 k->bytecount += nread;
805
806 Curl_pgrsSetDownloadCounter(data, k->bytecount);
807
808 if(!k->chunk && (nread || k->badheader || is_empty_data)) {
809 /* If this is chunky transfer, it was already written */
810
811 if(k->badheader && !k->ignorebody) {
812 /* we parsed a piece of data wrongly assuming it was a header
813 and now we output it as body instead */
814 size_t headlen = Curl_dyn_len(&data->state.headerb);
815
816 /* Don't let excess data pollute body writes */
817 if(k->maxdownload == -1 || (curl_off_t)headlen <= k->maxdownload)
818 result = Curl_client_write(data, CLIENTWRITE_BODY,
819 Curl_dyn_ptr(&data->state.headerb),
820 headlen);
821 else
822 result = Curl_client_write(data, CLIENTWRITE_BODY,
823 Curl_dyn_ptr(&data->state.headerb),
824 (size_t)k->maxdownload);
825
826 if(result)
827 return result;
828 }
829 if(k->badheader < HEADER_ALLBAD) {
830 /* This switch handles various content encodings. If there's an
831 error here, be sure to check over the almost identical code
832 in http_chunks.c.
833 Make sure that ALL_CONTENT_ENCODINGS contains all the
834 encodings handled here. */
835 if(data->set.http_ce_skip || !k->writer_stack) {
836 if(!k->ignorebody && nread) {
837#ifndef CURL_DISABLE_POP3
838 if(conn->handler->protocol & PROTO_FAMILY_POP3)
839 result = Curl_pop3_write(data, k->str, nread);
840 else
841#endif /* CURL_DISABLE_POP3 */
842 result = Curl_client_write(data, CLIENTWRITE_BODY, k->str,
843 nread);
844 }
845 }
846 else if(!k->ignorebody && nread)
847 result = Curl_unencode_write(data, k->writer_stack, k->str, nread);
848 }
849 k->badheader = HEADER_NORMAL; /* taken care of now */
850
851 if(result)
852 return result;
853 }
854
855 } /* if(!header and data to read) */
856
857 if(conn->handler->readwrite && excess) {
858 /* Parse the excess data */
859 k->str += nread;
860
861 if(&k->str[excess] > &buf[data->set.buffer_size]) {
862 /* the excess amount was too excessive(!), make sure
863 it doesn't read out of buffer */
864 excess = &buf[data->set.buffer_size] - k->str;
865 }
866 nread = (ssize_t)excess;
867
868 result = conn->handler->readwrite(data, conn, &nread, &readmore);
869 if(result)
870 return result;
871
872 if(readmore)
873 k->keepon |= KEEP_RECV; /* we're not done reading */
874 break;
875 }
876
877 if(is_empty_data) {
878 /* if we received nothing, the server closed the connection and we
879 are done */
880 k->keepon &= ~KEEP_RECV;
881 }
882
883 if(k->keepon & KEEP_RECV_PAUSE) {
884 /* this is a paused transfer */
885 break;
886 }
887
888 } while(data_pending(data) && maxloops--);
889
890 if(maxloops <= 0) {
891 /* we mark it as read-again-please */
892 conn->cselect_bits = CURL_CSELECT_IN;
893 *comeback = TRUE;
894 }
895
896 if(((k->keepon & (KEEP_RECV|KEEP_SEND)) == KEEP_SEND) &&
897 conn->bits.close) {
898 /* When we've read the entire thing and the close bit is set, the server
899 may now close the connection. If there's now any kind of sending going
900 on from our side, we need to stop that immediately. */
901 infof(data, "we are done reading and this is set to close, stop send");
902 k->keepon &= ~KEEP_SEND; /* no writing anymore either */
903 }
904
905 return CURLE_OK;
906}
907
908CURLcode Curl_done_sending(struct Curl_easy *data,
909 struct SingleRequest *k)
910{
911 struct connectdata *conn = data->conn;
912 k->keepon &= ~KEEP_SEND; /* we're done writing */
913
914 /* These functions should be moved into the handler struct! */
915 Curl_http2_done_sending(data, conn);
916 Curl_quic_done_sending(data);
917
918 if(conn->bits.rewindaftersend) {
919 CURLcode result = Curl_readrewind(data);
920 if(result)
921 return result;
922 }
923 return CURLE_OK;
924}
925
926#if defined(WIN32) && defined(USE_WINSOCK)
927#ifndef SIO_IDEAL_SEND_BACKLOG_QUERY
928#define SIO_IDEAL_SEND_BACKLOG_QUERY 0x4004747B
929#endif
930
931static void win_update_buffer_size(curl_socket_t sockfd)
932{
933 int result;
934 ULONG ideal;
935 DWORD ideallen;
936 result = WSAIoctl(sockfd, SIO_IDEAL_SEND_BACKLOG_QUERY, 0, 0,
937 &ideal, sizeof(ideal), &ideallen, 0, 0);
938 if(result == 0) {
939 setsockopt(sockfd, SOL_SOCKET, SO_SNDBUF,
940 (const char *)&ideal, sizeof(ideal));
941 }
942}
943#else
944#define win_update_buffer_size(x)
945#endif
946
947/*
948 * Send data to upload to the server, when the socket is writable.
949 */
950static CURLcode readwrite_upload(struct Curl_easy *data,
951 struct connectdata *conn,
952 int *didwhat)
953{
954 ssize_t i, si;
955 ssize_t bytes_written;
956 CURLcode result;
957 ssize_t nread; /* number of bytes read */
958 bool sending_http_headers = FALSE;
959 struct SingleRequest *k = &data->req;
960
961 if((k->bytecount == 0) && (k->writebytecount == 0))
962 Curl_pgrsTime(data, TIMER_STARTTRANSFER);
963
964 *didwhat |= KEEP_SEND;
965
966 do {
967 curl_off_t nbody;
968
969 /* only read more data if there's no upload data already
970 present in the upload buffer */
971 if(0 == k->upload_present) {
972 result = Curl_get_upload_buffer(data);
973 if(result)
974 return result;
975 /* init the "upload from here" pointer */
976 k->upload_fromhere = data->state.ulbuf;
977
978 if(!k->upload_done) {
979 /* HTTP pollution, this should be written nicer to become more
980 protocol agnostic. */
981 size_t fillcount;
982 struct HTTP *http = k->p.http;
983
984 if((k->exp100 == EXP100_SENDING_REQUEST) &&
985 (http->sending == HTTPSEND_BODY)) {
986 /* If this call is to send body data, we must take some action:
987 We have sent off the full HTTP 1.1 request, and we shall now
988 go into the Expect: 100 state and await such a header */
989 k->exp100 = EXP100_AWAITING_CONTINUE; /* wait for the header */
990 k->keepon &= ~KEEP_SEND; /* disable writing */
991 k->start100 = Curl_now(); /* timeout count starts now */
992 *didwhat &= ~KEEP_SEND; /* we didn't write anything actually */
993 /* set a timeout for the multi interface */
994 Curl_expire(data, data->set.expect_100_timeout, EXPIRE_100_TIMEOUT);
995 break;
996 }
997
998 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) {
999 if(http->sending == HTTPSEND_REQUEST)
1000 /* We're sending the HTTP request headers, not the data.
1001 Remember that so we don't change the line endings. */
1002 sending_http_headers = TRUE;
1003 else
1004 sending_http_headers = FALSE;
1005 }
1006
1007 result = Curl_fillreadbuffer(data, data->set.upload_buffer_size,
1008 &fillcount);
1009 if(result)
1010 return result;
1011
1012 nread = fillcount;
1013 }
1014 else
1015 nread = 0; /* we're done uploading/reading */
1016
1017 if(!nread && (k->keepon & KEEP_SEND_PAUSE)) {
1018 /* this is a paused transfer */
1019 break;
1020 }
1021 if(nread <= 0) {
1022 result = Curl_done_sending(data, k);
1023 if(result)
1024 return result;
1025 break;
1026 }
1027
1028 /* store number of bytes available for upload */
1029 k->upload_present = nread;
1030
1031 /* convert LF to CRLF if so asked */
1032 if((!sending_http_headers) && (
1033#ifdef CURL_DO_LINEEND_CONV
1034 /* always convert if we're FTPing in ASCII mode */
1035 (data->state.prefer_ascii) ||
1036#endif
1037 (data->set.crlf))) {
1038 /* Do we need to allocate a scratch buffer? */
1039 if(!data->state.scratch) {
1040 data->state.scratch = malloc(2 * data->set.upload_buffer_size);
1041 if(!data->state.scratch) {
1042 failf(data, "Failed to alloc scratch buffer!");
1043
1044 return CURLE_OUT_OF_MEMORY;
1045 }
1046 }
1047
1048 /*
1049 * ASCII/EBCDIC Note: This is presumably a text (not binary)
1050 * transfer so the data should already be in ASCII.
1051 * That means the hex values for ASCII CR (0x0d) & LF (0x0a)
1052 * must be used instead of the escape sequences \r & \n.
1053 */
1054 for(i = 0, si = 0; i < nread; i++, si++) {
1055 if(k->upload_fromhere[i] == 0x0a) {
1056 data->state.scratch[si++] = 0x0d;
1057 data->state.scratch[si] = 0x0a;
1058 if(!data->set.crlf) {
1059 /* we're here only because FTP is in ASCII mode...
1060 bump infilesize for the LF we just added */
1061 if(data->state.infilesize != -1)
1062 data->state.infilesize++;
1063 }
1064 }
1065 else
1066 data->state.scratch[si] = k->upload_fromhere[i];
1067 }
1068
1069 if(si != nread) {
1070 /* only perform the special operation if we really did replace
1071 anything */
1072 nread = si;
1073
1074 /* upload from the new (replaced) buffer instead */
1075 k->upload_fromhere = data->state.scratch;
1076
1077 /* set the new amount too */
1078 k->upload_present = nread;
1079 }
1080 }
1081
1082#ifndef CURL_DISABLE_SMTP
1083 if(conn->handler->protocol & PROTO_FAMILY_SMTP) {
1084 result = Curl_smtp_escape_eob(data, nread);
1085 if(result)
1086 return result;
1087 }
1088#endif /* CURL_DISABLE_SMTP */
1089 } /* if 0 == k->upload_present */
1090 else {
1091 /* We have a partial buffer left from a previous "round". Use
1092 that instead of reading more data */
1093 }
1094
1095 /* write to socket (send away data) */
1096 result = Curl_write(data,
1097 conn->writesockfd, /* socket to send to */
1098 k->upload_fromhere, /* buffer pointer */
1099 k->upload_present, /* buffer size */
1100 &bytes_written); /* actually sent */
1101 if(result)
1102 return result;
1103
1104 win_update_buffer_size(conn->writesockfd);
1105
1106 if(k->pendingheader) {
1107 /* parts of what was sent was header */
1108 curl_off_t n = CURLMIN(k->pendingheader, bytes_written);
1109 /* show the data before we change the pointer upload_fromhere */
1110 Curl_debug(data, CURLINFO_HEADER_OUT, k->upload_fromhere, (size_t)n);
1111 k->pendingheader -= n;
1112 nbody = bytes_written - n; /* size of the written body part */
1113 }
1114 else
1115 nbody = bytes_written;
1116
1117 if(nbody) {
1118 /* show the data before we change the pointer upload_fromhere */
1119 Curl_debug(data, CURLINFO_DATA_OUT,
1120 &k->upload_fromhere[bytes_written - nbody],
1121 (size_t)nbody);
1122
1123 k->writebytecount += nbody;
1124 Curl_pgrsSetUploadCounter(data, k->writebytecount);
1125 }
1126
1127 if((!k->upload_chunky || k->forbidchunk) &&
1128 (k->writebytecount == data->state.infilesize)) {
1129 /* we have sent all data we were supposed to */
1130 k->upload_done = TRUE;
1131 infof(data, "We are completely uploaded and fine");
1132 }
1133
1134 if(k->upload_present != bytes_written) {
1135 /* we only wrote a part of the buffer (if anything), deal with it! */
1136
1137 /* store the amount of bytes left in the buffer to write */
1138 k->upload_present -= bytes_written;
1139
1140 /* advance the pointer where to find the buffer when the next send
1141 is to happen */
1142 k->upload_fromhere += bytes_written;
1143 }
1144 else {
1145 /* we've uploaded that buffer now */
1146 result = Curl_get_upload_buffer(data);
1147 if(result)
1148 return result;
1149 k->upload_fromhere = data->state.ulbuf;
1150 k->upload_present = 0; /* no more bytes left */
1151
1152 if(k->upload_done) {
1153 result = Curl_done_sending(data, k);
1154 if(result)
1155 return result;
1156 }
1157 }
1158
1159
1160 } while(0); /* just to break out from! */
1161
1162 return CURLE_OK;
1163}
1164
1165/*
1166 * Curl_readwrite() is the low-level function to be called when data is to
1167 * be read and written to/from the connection.
1168 *
1169 * return '*comeback' TRUE if we didn't properly drain the socket so this
1170 * function should get called again without select() or similar in between!
1171 */
1172CURLcode Curl_readwrite(struct connectdata *conn,
1173 struct Curl_easy *data,
1174 bool *done,
1175 bool *comeback)
1176{
1177 struct SingleRequest *k = &data->req;
1178 CURLcode result;
1179 int didwhat = 0;
1180
1181 curl_socket_t fd_read;
1182 curl_socket_t fd_write;
1183 int select_res = conn->cselect_bits;
1184
1185 conn->cselect_bits = 0;
1186
1187 /* only use the proper socket if the *_HOLD bit is not set simultaneously as
1188 then we are in rate limiting state in that transfer direction */
1189
1190 if((k->keepon & KEEP_RECVBITS) == KEEP_RECV)
1191 fd_read = conn->sockfd;
1192 else
1193 fd_read = CURL_SOCKET_BAD;
1194
1195 if((k->keepon & KEEP_SENDBITS) == KEEP_SEND)
1196 fd_write = conn->writesockfd;
1197 else
1198 fd_write = CURL_SOCKET_BAD;
1199
1200 if(data->state.drain) {
1201 select_res |= CURL_CSELECT_IN;
1202 DEBUGF(infof(data, "Curl_readwrite: forcibly told to drain data"));
1203 }
1204
1205 if(!select_res) /* Call for select()/poll() only, if read/write/error
1206 status is not known. */
1207 select_res = Curl_socket_check(fd_read, CURL_SOCKET_BAD, fd_write, 0);
1208
1209 if(select_res == CURL_CSELECT_ERR) {
1210 failf(data, "select/poll returned error");
1211 return CURLE_SEND_ERROR;
1212 }
1213
1214#ifdef USE_HYPER
1215 if(conn->datastream) {
1216 result = conn->datastream(data, conn, &didwhat, done, select_res);
1217 if(result || *done)
1218 return result;
1219 }
1220 else {
1221#endif
1222 /* We go ahead and do a read if we have a readable socket or if
1223 the stream was rewound (in which case we have data in a
1224 buffer) */
1225 if((k->keepon & KEEP_RECV) && (select_res & CURL_CSELECT_IN)) {
1226 result = readwrite_data(data, conn, k, &didwhat, done, comeback);
1227 if(result || *done)
1228 return result;
1229 }
1230
1231 /* If we still have writing to do, we check if we have a writable socket. */
1232 if((k->keepon & KEEP_SEND) && (select_res & CURL_CSELECT_OUT)) {
1233 /* write */
1234
1235 result = readwrite_upload(data, conn, &didwhat);
1236 if(result)
1237 return result;
1238 }
1239#ifdef USE_HYPER
1240 }
1241#endif
1242
1243 k->now = Curl_now();
1244 if(!didwhat) {
1245 /* no read no write, this is a timeout? */
1246 if(k->exp100 == EXP100_AWAITING_CONTINUE) {
1247 /* This should allow some time for the header to arrive, but only a
1248 very short time as otherwise it'll be too much wasted time too
1249 often. */
1250
1251 /* Quoting RFC2616, section "8.2.3 Use of the 100 (Continue) Status":
1252
1253 Therefore, when a client sends this header field to an origin server
1254 (possibly via a proxy) from which it has never seen a 100 (Continue)
1255 status, the client SHOULD NOT wait for an indefinite period before
1256 sending the request body.
1257
1258 */
1259
1260 timediff_t ms = Curl_timediff(k->now, k->start100);
1261 if(ms >= data->set.expect_100_timeout) {
1262 /* we've waited long enough, continue anyway */
1263 k->exp100 = EXP100_SEND_DATA;
1264 k->keepon |= KEEP_SEND;
1265 Curl_expire_done(data, EXPIRE_100_TIMEOUT);
1266 infof(data, "Done waiting for 100-continue");
1267 }
1268 }
1269 }
1270
1271 if(Curl_pgrsUpdate(data))
1272 result = CURLE_ABORTED_BY_CALLBACK;
1273 else
1274 result = Curl_speedcheck(data, k->now);
1275 if(result)
1276 return result;
1277
1278 if(k->keepon) {
1279 if(0 > Curl_timeleft(data, &k->now, FALSE)) {
1280 if(k->size != -1) {
1281 failf(data, "Operation timed out after %" CURL_FORMAT_TIMEDIFF_T
1282 " milliseconds with %" CURL_FORMAT_CURL_OFF_T " out of %"
1283 CURL_FORMAT_CURL_OFF_T " bytes received",
1284 Curl_timediff(k->now, data->progress.t_startsingle),
1285 k->bytecount, k->size);
1286 }
1287 else {
1288 failf(data, "Operation timed out after %" CURL_FORMAT_TIMEDIFF_T
1289 " milliseconds with %" CURL_FORMAT_CURL_OFF_T " bytes received",
1290 Curl_timediff(k->now, data->progress.t_startsingle),
1291 k->bytecount);
1292 }
1293 return CURLE_OPERATION_TIMEDOUT;
1294 }
1295 }
1296 else {
1297 /*
1298 * The transfer has been performed. Just make some general checks before
1299 * returning.
1300 */
1301
1302 if(!(data->set.opt_no_body) && (k->size != -1) &&
1303 (k->bytecount != k->size) &&
1304#ifdef CURL_DO_LINEEND_CONV
1305 /* Most FTP servers don't adjust their file SIZE response for CRLFs,
1306 so we'll check to see if the discrepancy can be explained
1307 by the number of CRLFs we've changed to LFs.
1308 */
1309 (k->bytecount != (k->size + data->state.crlf_conversions)) &&
1310#endif /* CURL_DO_LINEEND_CONV */
1311 !k->newurl) {
1312 failf(data, "transfer closed with %" CURL_FORMAT_CURL_OFF_T
1313 " bytes remaining to read", k->size - k->bytecount);
1314 return CURLE_PARTIAL_FILE;
1315 }
1316 if(!(data->set.opt_no_body) && k->chunk &&
1317 (conn->chunk.state != CHUNK_STOP)) {
1318 /*
1319 * In chunked mode, return an error if the connection is closed prior to
1320 * the empty (terminating) chunk is read.
1321 *
1322 * The condition above used to check for
1323 * conn->proto.http->chunk.datasize != 0 which is true after reading
1324 * *any* chunk, not just the empty chunk.
1325 *
1326 */
1327 failf(data, "transfer closed with outstanding read data remaining");
1328 return CURLE_PARTIAL_FILE;
1329 }
1330 if(Curl_pgrsUpdate(data))
1331 return CURLE_ABORTED_BY_CALLBACK;
1332 }
1333
1334 /* Now update the "done" boolean we return */
1335 *done = (0 == (k->keepon&(KEEP_RECV|KEEP_SEND|
1336 KEEP_RECV_PAUSE|KEEP_SEND_PAUSE))) ? TRUE : FALSE;
1337
1338 return CURLE_OK;
1339}
1340
1341/*
1342 * Curl_single_getsock() gets called by the multi interface code when the app
1343 * has requested to get the sockets for the current connection. This function
1344 * will then be called once for every connection that the multi interface
1345 * keeps track of. This function will only be called for connections that are
1346 * in the proper state to have this information available.
1347 */
1348int Curl_single_getsock(struct Curl_easy *data,
1349 struct connectdata *conn,
1350 curl_socket_t *sock)
1351{
1352 int bitmap = GETSOCK_BLANK;
1353 unsigned sockindex = 0;
1354
1355 if(conn->handler->perform_getsock)
1356 return conn->handler->perform_getsock(data, conn, sock);
1357
1358 /* don't include HOLD and PAUSE connections */
1359 if((data->req.keepon & KEEP_RECVBITS) == KEEP_RECV) {
1360
1361 DEBUGASSERT(conn->sockfd != CURL_SOCKET_BAD);
1362
1363 bitmap |= GETSOCK_READSOCK(sockindex);
1364 sock[sockindex] = conn->sockfd;
1365 }
1366
1367 /* don't include HOLD and PAUSE connections */
1368 if((data->req.keepon & KEEP_SENDBITS) == KEEP_SEND) {
1369
1370 if((conn->sockfd != conn->writesockfd) ||
1371 bitmap == GETSOCK_BLANK) {
1372 /* only if they are not the same socket and we have a readable
1373 one, we increase index */
1374 if(bitmap != GETSOCK_BLANK)
1375 sockindex++; /* increase index if we need two entries */
1376
1377 DEBUGASSERT(conn->writesockfd != CURL_SOCKET_BAD);
1378
1379 sock[sockindex] = conn->writesockfd;
1380 }
1381
1382 bitmap |= GETSOCK_WRITESOCK(sockindex);
1383 }
1384
1385 return bitmap;
1386}
1387
1388/* Curl_init_CONNECT() gets called each time the handle switches to CONNECT
1389 which means this gets called once for each subsequent redirect etc */
1390void Curl_init_CONNECT(struct Curl_easy *data)
1391{
1392 data->state.fread_func = data->set.fread_func_set;
1393 data->state.in = data->set.in_set;
1394}
1395
1396/*
1397 * Curl_pretransfer() is called immediately before a transfer starts, and only
1398 * once for one transfer no matter if it has redirects or do multi-pass
1399 * authentication etc.
1400 */
1401CURLcode Curl_pretransfer(struct Curl_easy *data)
1402{
1403 CURLcode result;
1404
1405 if(!data->state.url && !data->set.uh) {
1406 /* we can't do anything without URL */
1407 failf(data, "No URL set!");
1408 return CURLE_URL_MALFORMAT;
1409 }
1410
1411 /* since the URL may have been redirected in a previous use of this handle */
1412 if(data->state.url_alloc) {
1413 /* the already set URL is allocated, free it first! */
1414 Curl_safefree(data->state.url);
1415 data->state.url_alloc = FALSE;
1416 }
1417
1418 if(!data->state.url && data->set.uh) {
1419 CURLUcode uc;
1420 free(data->set.str[STRING_SET_URL]);
1421 uc = curl_url_get(data->set.uh,
1422 CURLUPART_URL, &data->set.str[STRING_SET_URL], 0);
1423 if(uc) {
1424 failf(data, "No URL set!");
1425 return CURLE_URL_MALFORMAT;
1426 }
1427 }
1428
1429 data->state.prefer_ascii = data->set.prefer_ascii;
1430 data->state.list_only = data->set.list_only;
1431 data->state.httpreq = data->set.method;
1432 data->state.url = data->set.str[STRING_SET_URL];
1433
1434 /* Init the SSL session ID cache here. We do it here since we want to do it
1435 after the *_setopt() calls (that could specify the size of the cache) but
1436 before any transfer takes place. */
1437 result = Curl_ssl_initsessions(data, data->set.general_ssl.max_ssl_sessions);
1438 if(result)
1439 return result;
1440
1441 data->state.wildcardmatch = data->set.wildcard_enabled;
1442 data->state.followlocation = 0; /* reset the location-follow counter */
1443 data->state.this_is_a_follow = FALSE; /* reset this */
1444 data->state.errorbuf = FALSE; /* no error has occurred */
1445 data->state.httpwant = data->set.httpwant;
1446 data->state.httpversion = 0;
1447 data->state.authproblem = FALSE;
1448 data->state.authhost.want = data->set.httpauth;
1449 data->state.authproxy.want = data->set.proxyauth;
1450 Curl_safefree(data->info.wouldredirect);
1451
1452 if(data->state.httpreq == HTTPREQ_PUT)
1453 data->state.infilesize = data->set.filesize;
1454 else if((data->state.httpreq != HTTPREQ_GET) &&
1455 (data->state.httpreq != HTTPREQ_HEAD)) {
1456 data->state.infilesize = data->set.postfieldsize;
1457 if(data->set.postfields && (data->state.infilesize == -1))
1458 data->state.infilesize = (curl_off_t)strlen(data->set.postfields);
1459 }
1460 else
1461 data->state.infilesize = 0;
1462
1463 /* If there is a list of cookie files to read, do it now! */
1464 if(data->state.cookielist)
1465 Curl_cookie_loadfiles(data);
1466
1467 /* If there is a list of host pairs to deal with */
1468 if(data->state.resolve)
1469 result = Curl_loadhostpairs(data);
1470
1471 if(!result) {
1472 /* Allow data->set.use_port to set which port to use. This needs to be
1473 * disabled for example when we follow Location: headers to URLs using
1474 * different ports! */
1475 data->state.allow_port = TRUE;
1476
1477#if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL)
1478 /*************************************************************
1479 * Tell signal handler to ignore SIGPIPE
1480 *************************************************************/
1481 if(!data->set.no_signal)
1482 data->state.prev_signal = signal(SIGPIPE, SIG_IGN);
1483#endif
1484
1485 Curl_initinfo(data); /* reset session-specific information "variables" */
1486 Curl_pgrsResetTransferSizes(data);
1487 Curl_pgrsStartNow(data);
1488
1489 /* In case the handle is re-used and an authentication method was picked
1490 in the session we need to make sure we only use the one(s) we now
1491 consider to be fine */
1492 data->state.authhost.picked &= data->state.authhost.want;
1493 data->state.authproxy.picked &= data->state.authproxy.want;
1494
1495#ifndef CURL_DISABLE_FTP
1496 if(data->state.wildcardmatch) {
1497 struct WildcardData *wc = &data->wildcard;
1498 if(wc->state < CURLWC_INIT) {
1499 result = Curl_wildcard_init(wc); /* init wildcard structures */
1500 if(result)
1501 return CURLE_OUT_OF_MEMORY;
1502 }
1503 }
1504#endif
1505 Curl_http2_init_state(&data->state);
1506 result = Curl_hsts_loadcb(data, data->hsts);
1507 }
1508
1509 /*
1510 * Set user-agent. Used for HTTP, but since we can attempt to tunnel
1511 * basically anything through a http proxy we can't limit this based on
1512 * protocol.
1513 */
1514 if(data->set.str[STRING_USERAGENT]) {
1515 Curl_safefree(data->state.aptr.uagent);
1516 data->state.aptr.uagent =
1517 aprintf("User-Agent: %s\r\n", data->set.str[STRING_USERAGENT]);
1518 if(!data->state.aptr.uagent)
1519 return CURLE_OUT_OF_MEMORY;
1520 }
1521
1522 if(!result)
1523 result = Curl_setstropt(&data->state.aptr.user,
1524 data->set.str[STRING_USERNAME]);
1525 if(!result)
1526 result = Curl_setstropt(&data->state.aptr.passwd,
1527 data->set.str[STRING_PASSWORD]);
1528 if(!result)
1529 result = Curl_setstropt(&data->state.aptr.proxyuser,
1530 data->set.str[STRING_PROXYUSERNAME]);
1531 if(!result)
1532 result = Curl_setstropt(&data->state.aptr.proxypasswd,
1533 data->set.str[STRING_PROXYPASSWORD]);
1534
1535 data->req.headerbytecount = 0;
1536 return result;
1537}
1538
1539/*
1540 * Curl_posttransfer() is called immediately after a transfer ends
1541 */
1542CURLcode Curl_posttransfer(struct Curl_easy *data)
1543{
1544#if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL)
1545 /* restore the signal handler for SIGPIPE before we get back */
1546 if(!data->set.no_signal)
1547 signal(SIGPIPE, data->state.prev_signal);
1548#else
1549 (void)data; /* unused parameter */
1550#endif
1551
1552 return CURLE_OK;
1553}
1554
1555/*
1556 * Curl_follow() handles the URL redirect magic. Pass in the 'newurl' string
1557 * as given by the remote server and set up the new URL to request.
1558 *
1559 * This function DOES NOT FREE the given url.
1560 */
1561CURLcode Curl_follow(struct Curl_easy *data,
1562 char *newurl, /* the Location: string */
1563 followtype type) /* see transfer.h */
1564{
1565#ifdef CURL_DISABLE_HTTP
1566 (void)data;
1567 (void)newurl;
1568 (void)type;
1569 /* Location: following will not happen when HTTP is disabled */
1570 return CURLE_TOO_MANY_REDIRECTS;
1571#else
1572
1573 /* Location: redirect */
1574 bool disallowport = FALSE;
1575 bool reachedmax = FALSE;
1576 CURLUcode uc;
1577
1578 DEBUGASSERT(type != FOLLOW_NONE);
1579
1580 if(type == FOLLOW_REDIR) {
1581 if((data->set.maxredirs != -1) &&
1582 (data->state.followlocation >= data->set.maxredirs)) {
1583 reachedmax = TRUE;
1584 type = FOLLOW_FAKE; /* switch to fake to store the would-be-redirected
1585 to URL */
1586 }
1587 else {
1588 /* mark the next request as a followed location: */
1589 data->state.this_is_a_follow = TRUE;
1590
1591 data->state.followlocation++; /* count location-followers */
1592
1593 if(data->set.http_auto_referer) {
1594 CURLU *u;
1595 char *referer = NULL;
1596
1597 /* We are asked to automatically set the previous URL as the referer
1598 when we get the next URL. We pick the ->url field, which may or may
1599 not be 100% correct */
1600
1601 if(data->state.referer_alloc) {
1602 Curl_safefree(data->state.referer);
1603 data->state.referer_alloc = FALSE;
1604 }
1605
1606 /* Make a copy of the URL without crenditals and fragment */
1607 u = curl_url();
1608 if(!u)
1609 return CURLE_OUT_OF_MEMORY;
1610
1611 uc = curl_url_set(u, CURLUPART_URL, data->state.url, 0);
1612 if(!uc)
1613 uc = curl_url_set(u, CURLUPART_FRAGMENT, NULL, 0);
1614 if(!uc)
1615 uc = curl_url_set(u, CURLUPART_USER, NULL, 0);
1616 if(!uc)
1617 uc = curl_url_set(u, CURLUPART_PASSWORD, NULL, 0);
1618 if(!uc)
1619 uc = curl_url_get(u, CURLUPART_URL, &referer, 0);
1620
1621 curl_url_cleanup(u);
1622
1623 if(uc || !referer)
1624 return CURLE_OUT_OF_MEMORY;
1625
1626 data->state.referer = referer;
1627 data->state.referer_alloc = TRUE; /* yes, free this later */
1628 }
1629 }
1630 }
1631
1632 if((type != FOLLOW_RETRY) &&
1633 (data->req.httpcode != 401) && (data->req.httpcode != 407) &&
1634 Curl_is_absolute_url(newurl, NULL, MAX_SCHEME_LEN))
1635 /* If this is not redirect due to a 401 or 407 response and an absolute
1636 URL: don't allow a custom port number */
1637 disallowport = TRUE;
1638
1639 DEBUGASSERT(data->state.uh);
1640 uc = curl_url_set(data->state.uh, CURLUPART_URL, newurl,
1641 (type == FOLLOW_FAKE) ? CURLU_NON_SUPPORT_SCHEME :
1642 ((type == FOLLOW_REDIR) ? CURLU_URLENCODE : 0) |
1643 CURLU_ALLOW_SPACE);
1644 if(uc) {
1645 if(type != FOLLOW_FAKE)
1646 return Curl_uc_to_curlcode(uc);
1647
1648 /* the URL could not be parsed for some reason, but since this is FAKE
1649 mode, just duplicate the field as-is */
1650 newurl = strdup(newurl);
1651 if(!newurl)
1652 return CURLE_OUT_OF_MEMORY;
1653 }
1654 else {
1655
1656 uc = curl_url_get(data->state.uh, CURLUPART_URL, &newurl, 0);
1657 if(uc)
1658 return Curl_uc_to_curlcode(uc);
1659 }
1660
1661 if(type == FOLLOW_FAKE) {
1662 /* we're only figuring out the new url if we would've followed locations
1663 but now we're done so we can get out! */
1664 data->info.wouldredirect = newurl;
1665
1666 if(reachedmax) {
1667 failf(data, "Maximum (%ld) redirects followed", data->set.maxredirs);
1668 return CURLE_TOO_MANY_REDIRECTS;
1669 }
1670 return CURLE_OK;
1671 }
1672
1673 if(disallowport)
1674 data->state.allow_port = FALSE;
1675
1676 if(data->state.url_alloc)
1677 Curl_safefree(data->state.url);
1678
1679 data->state.url = newurl;
1680 data->state.url_alloc = TRUE;
1681
1682 infof(data, "Issue another request to this URL: '%s'", data->state.url);
1683
1684 /*
1685 * We get here when the HTTP code is 300-399 (and 401). We need to perform
1686 * differently based on exactly what return code there was.
1687 *
1688 * News from 7.10.6: we can also get here on a 401 or 407, in case we act on
1689 * a HTTP (proxy-) authentication scheme other than Basic.
1690 */
1691 switch(data->info.httpcode) {
1692 /* 401 - Act on a WWW-Authenticate, we keep on moving and do the
1693 Authorization: XXXX header in the HTTP request code snippet */
1694 /* 407 - Act on a Proxy-Authenticate, we keep on moving and do the
1695 Proxy-Authorization: XXXX header in the HTTP request code snippet */
1696 /* 300 - Multiple Choices */
1697 /* 306 - Not used */
1698 /* 307 - Temporary Redirect */
1699 default: /* for all above (and the unknown ones) */
1700 /* Some codes are explicitly mentioned since I've checked RFC2616 and they
1701 * seem to be OK to POST to.
1702 */
1703 break;
1704 case 301: /* Moved Permanently */
1705 /* (quote from RFC7231, section 6.4.2)
1706 *
1707 * Note: For historical reasons, a user agent MAY change the request
1708 * method from POST to GET for the subsequent request. If this
1709 * behavior is undesired, the 307 (Temporary Redirect) status code
1710 * can be used instead.
1711 *
1712 * ----
1713 *
1714 * Many webservers expect this, so these servers often answers to a POST
1715 * request with an error page. To be sure that libcurl gets the page that
1716 * most user agents would get, libcurl has to force GET.
1717 *
1718 * This behavior is forbidden by RFC1945 and the obsolete RFC2616, and
1719 * can be overridden with CURLOPT_POSTREDIR.
1720 */
1721 if((data->state.httpreq == HTTPREQ_POST
1722 || data->state.httpreq == HTTPREQ_POST_FORM
1723 || data->state.httpreq == HTTPREQ_POST_MIME)
1724 && !(data->set.keep_post & CURL_REDIR_POST_301)) {
1725 infof(data, "Switch from POST to GET");
1726 data->state.httpreq = HTTPREQ_GET;
1727 }
1728 break;
1729 case 302: /* Found */
1730 /* (quote from RFC7231, section 6.4.3)
1731 *
1732 * Note: For historical reasons, a user agent MAY change the request
1733 * method from POST to GET for the subsequent request. If this
1734 * behavior is undesired, the 307 (Temporary Redirect) status code
1735 * can be used instead.
1736 *
1737 * ----
1738 *
1739 * Many webservers expect this, so these servers often answers to a POST
1740 * request with an error page. To be sure that libcurl gets the page that
1741 * most user agents would get, libcurl has to force GET.
1742 *
1743 * This behavior is forbidden by RFC1945 and the obsolete RFC2616, and
1744 * can be overridden with CURLOPT_POSTREDIR.
1745 */
1746 if((data->state.httpreq == HTTPREQ_POST
1747 || data->state.httpreq == HTTPREQ_POST_FORM
1748 || data->state.httpreq == HTTPREQ_POST_MIME)
1749 && !(data->set.keep_post & CURL_REDIR_POST_302)) {
1750 infof(data, "Switch from POST to GET");
1751 data->state.httpreq = HTTPREQ_GET;
1752 }
1753 break;
1754
1755 case 303: /* See Other */
1756 /* 'See Other' location is not the resource but a substitute for the
1757 * resource. In this case we switch the method to GET/HEAD, unless the
1758 * method is POST and the user specified to keep it as POST.
1759 * https://github.com/curl/curl/issues/5237#issuecomment-614641049
1760 */
1761 if(data->state.httpreq != HTTPREQ_GET &&
1762 ((data->state.httpreq != HTTPREQ_POST &&
1763 data->state.httpreq != HTTPREQ_POST_FORM &&
1764 data->state.httpreq != HTTPREQ_POST_MIME) ||
1765 !(data->set.keep_post & CURL_REDIR_POST_303))) {
1766 data->state.httpreq = HTTPREQ_GET;
1767 data->set.upload = false;
1768 infof(data, "Switch to %s",
1769 data->set.opt_no_body?"HEAD":"GET");
1770 }
1771 break;
1772 case 304: /* Not Modified */
1773 /* 304 means we did a conditional request and it was "Not modified".
1774 * We shouldn't get any Location: header in this response!
1775 */
1776 break;
1777 case 305: /* Use Proxy */
1778 /* (quote from RFC2616, section 10.3.6):
1779 * "The requested resource MUST be accessed through the proxy given
1780 * by the Location field. The Location field gives the URI of the
1781 * proxy. The recipient is expected to repeat this single request
1782 * via the proxy. 305 responses MUST only be generated by origin
1783 * servers."
1784 */
1785 break;
1786 }
1787 Curl_pgrsTime(data, TIMER_REDIRECT);
1788 Curl_pgrsResetTransferSizes(data);
1789
1790 return CURLE_OK;
1791#endif /* CURL_DISABLE_HTTP */
1792}
1793
1794/* Returns CURLE_OK *and* sets '*url' if a request retry is wanted.
1795
1796 NOTE: that the *url is malloc()ed. */
1797CURLcode Curl_retry_request(struct Curl_easy *data, char **url)
1798{
1799 struct connectdata *conn = data->conn;
1800 bool retry = FALSE;
1801 *url = NULL;
1802
1803 /* if we're talking upload, we can't do the checks below, unless the protocol
1804 is HTTP as when uploading over HTTP we will still get a response */
1805 if(data->set.upload &&
1806 !(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)))
1807 return CURLE_OK;
1808
1809 if((data->req.bytecount + data->req.headerbytecount == 0) &&
1810 conn->bits.reuse &&
1811 (!data->set.opt_no_body
1812 || (conn->handler->protocol & PROTO_FAMILY_HTTP)) &&
1813 (data->set.rtspreq != RTSPREQ_RECEIVE))
1814 /* We got no data, we attempted to re-use a connection. For HTTP this
1815 can be a retry so we try again regardless if we expected a body.
1816 For other protocols we only try again only if we expected a body.
1817
1818 This might happen if the connection was left alive when we were
1819 done using it before, but that was closed when we wanted to read from
1820 it again. Bad luck. Retry the same request on a fresh connect! */
1821 retry = TRUE;
1822 else if(data->state.refused_stream &&
1823 (data->req.bytecount + data->req.headerbytecount == 0) ) {
1824 /* This was sent on a refused stream, safe to rerun. A refused stream
1825 error can typically only happen on HTTP/2 level if the stream is safe
1826 to issue again, but the nghttp2 API can deliver the message to other
1827 streams as well, which is why this adds the check the data counters
1828 too. */
1829 infof(data, "REFUSED_STREAM, retrying a fresh connect");
1830 data->state.refused_stream = FALSE; /* clear again */
1831 retry = TRUE;
1832 }
1833 if(retry) {
1834#define CONN_MAX_RETRIES 5
1835 if(data->state.retrycount++ >= CONN_MAX_RETRIES) {
1836 failf(data, "Connection died, tried %d times before giving up",
1837 CONN_MAX_RETRIES);
1838 data->state.retrycount = 0;
1839 return CURLE_SEND_ERROR;
1840 }
1841 infof(data, "Connection died, retrying a fresh connect (retry count: %d)",
1842 data->state.retrycount);
1843 *url = strdup(data->state.url);
1844 if(!*url)
1845 return CURLE_OUT_OF_MEMORY;
1846
1847 connclose(conn, "retry"); /* close this connection */
1848 conn->bits.retry = TRUE; /* mark this as a connection we're about
1849 to retry. Marking it this way should
1850 prevent i.e HTTP transfers to return
1851 error just because nothing has been
1852 transferred! */
1853
1854
1855 if(conn->handler->protocol&PROTO_FAMILY_HTTP) {
1856 if(data->req.writebytecount) {
1857 CURLcode result = Curl_readrewind(data);
1858 if(result) {
1859 Curl_safefree(*url);
1860 return result;
1861 }
1862 }
1863 }
1864 }
1865 return CURLE_OK;
1866}
1867
1868/*
1869 * Curl_setup_transfer() is called to setup some basic properties for the
1870 * upcoming transfer.
1871 */
1872void
1873Curl_setup_transfer(
1874 struct Curl_easy *data, /* transfer */
1875 int sockindex, /* socket index to read from or -1 */
1876 curl_off_t size, /* -1 if unknown at this point */
1877 bool getheader, /* TRUE if header parsing is wanted */
1878 int writesockindex /* socket index to write to, it may very well be
1879 the same we read from. -1 disables */
1880 )
1881{
1882 struct SingleRequest *k = &data->req;
1883 struct connectdata *conn = data->conn;
1884 struct HTTP *http = data->req.p.http;
1885 bool httpsending = ((conn->handler->protocol&PROTO_FAMILY_HTTP) &&
1886 (http->sending == HTTPSEND_REQUEST));
1887 DEBUGASSERT(conn != NULL);
1888 DEBUGASSERT((sockindex <= 1) && (sockindex >= -1));
1889
1890 if(conn->bits.multiplex || conn->httpversion == 20 || httpsending) {
1891 /* when multiplexing, the read/write sockets need to be the same! */
1892 conn->sockfd = sockindex == -1 ?
1893 ((writesockindex == -1 ? CURL_SOCKET_BAD : conn->sock[writesockindex])) :
1894 conn->sock[sockindex];
1895 conn->writesockfd = conn->sockfd;
1896 if(httpsending)
1897 /* special and very HTTP-specific */
1898 writesockindex = FIRSTSOCKET;
1899 }
1900 else {
1901 conn->sockfd = sockindex == -1 ?
1902 CURL_SOCKET_BAD : conn->sock[sockindex];
1903 conn->writesockfd = writesockindex == -1 ?
1904 CURL_SOCKET_BAD:conn->sock[writesockindex];
1905 }
1906 k->getheader = getheader;
1907
1908 k->size = size;
1909
1910 /* The code sequence below is placed in this function just because all
1911 necessary input is not always known in do_complete() as this function may
1912 be called after that */
1913
1914 if(!k->getheader) {
1915 k->header = FALSE;
1916 if(size > 0)
1917 Curl_pgrsSetDownloadSize(data, size);
1918 }
1919 /* we want header and/or body, if neither then don't do this! */
1920 if(k->getheader || !data->set.opt_no_body) {
1921
1922 if(sockindex != -1)
1923 k->keepon |= KEEP_RECV;
1924
1925 if(writesockindex != -1) {
1926 /* HTTP 1.1 magic:
1927
1928 Even if we require a 100-return code before uploading data, we might
1929 need to write data before that since the REQUEST may not have been
1930 finished sent off just yet.
1931
1932 Thus, we must check if the request has been sent before we set the
1933 state info where we wait for the 100-return code
1934 */
1935 if((data->state.expect100header) &&
1936 (conn->handler->protocol&PROTO_FAMILY_HTTP) &&
1937 (http->sending == HTTPSEND_BODY)) {
1938 /* wait with write until we either got 100-continue or a timeout */
1939 k->exp100 = EXP100_AWAITING_CONTINUE;
1940 k->start100 = Curl_now();
1941
1942 /* Set a timeout for the multi interface. Add the inaccuracy margin so
1943 that we don't fire slightly too early and get denied to run. */
1944 Curl_expire(data, data->set.expect_100_timeout, EXPIRE_100_TIMEOUT);
1945 }
1946 else {
1947 if(data->state.expect100header)
1948 /* when we've sent off the rest of the headers, we must await a
1949 100-continue but first finish sending the request */
1950 k->exp100 = EXP100_SENDING_REQUEST;
1951
1952 /* enable the write bit when we're not waiting for continue */
1953 k->keepon |= KEEP_SEND;
1954 }
1955 } /* if(writesockindex != -1) */
1956 } /* if(k->getheader || !data->set.opt_no_body) */
1957
1958}
1959