1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright 2005 Colin Percival
5 * All rights reserved
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted providing that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD: stable/11/usr.sbin/portsnap/phttpget/phttpget.c 330449 2018-03-05 07:26:05Z eadler $");
31
32#include <sys/types.h>
33#include <sys/time.h>
34#include <sys/socket.h>
35
36#include <ctype.h>
37#include <err.h>
38#include <errno.h>
39#include <fcntl.h>
40#include <limits.h>
41#include <netdb.h>
42#include <stdint.h>
43#include <stdio.h>
44#include <stdlib.h>
45#include <string.h>
46#include <sysexits.h>
47#include <unistd.h>
48
49static const char *	env_HTTP_PROXY;
50static char *		env_HTTP_PROXY_AUTH;
51static const char *	env_HTTP_USER_AGENT;
52static char *		env_HTTP_TIMEOUT;
53static const char *	proxyport;
54static char *		proxyauth;
55
56static struct timeval	timo = { 15, 0};
57
58static void
59usage(void)
60{
61
62	fprintf(stderr, "usage: phttpget server [file ...]\n");
63	exit(EX_USAGE);
64}
65
66/*
67 * Base64 encode a string; the string returned, if non-NULL, is
68 * allocated using malloc() and must be freed by the caller.
69 */
70static char *
71b64enc(const char *ptext)
72{
73	static const char base64[] =
74	    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
75	    "abcdefghijklmnopqrstuvwxyz"
76	    "0123456789+/";
77	const char *pt;
78	char *ctext, *pc;
79	size_t ptlen, ctlen;
80	uint32_t t;
81	unsigned int j;
82
83	/*
84	 * Encoded length is 4 characters per 3-byte block or partial
85	 * block of plaintext, plus one byte for the terminating NUL
86	 */
87	ptlen = strlen(ptext);
88	if (ptlen > ((SIZE_MAX - 1) / 4) * 3 - 2)
89		return NULL;	/* Possible integer overflow */
90	ctlen = 4 * ((ptlen + 2) / 3) + 1;
91	if ((ctext = malloc(ctlen)) == NULL)
92		return NULL;
93	ctext[ctlen - 1] = 0;
94
95	/*
96	 * Scan through ptext, reading up to 3 bytes from ptext and
97	 * writing 4 bytes to ctext, until we run out of input.
98	 */
99	for (pt = ptext, pc = ctext; ptlen; ptlen -= 3, pc += 4) {
100		/* Read 3 bytes */
101		for (t = j = 0; j < 3; j++) {
102			t <<= 8;
103			if (j < ptlen)
104				t += *pt++;
105		}
106
107		/* Write 4 bytes */
108		for (j = 0; j < 4; j++) {
109			if (j <= ptlen + 1)
110				pc[j] = base64[(t >> 18) & 0x3f];
111			else
112				pc[j] = '=';
113			t <<= 6;
114		}
115
116		/* If we're done, exit the loop */
117		if (ptlen <= 3)
118			break;
119	}
120
121	return (ctext);
122}
123
124static void
125readenv(void)
126{
127	char *proxy_auth_userpass, *proxy_auth_userpass64, *p;
128	char *proxy_auth_user = NULL;
129	char *proxy_auth_pass = NULL;
130	long http_timeout;
131
132	env_HTTP_PROXY = getenv("HTTP_PROXY");
133	if (env_HTTP_PROXY == NULL)
134		env_HTTP_PROXY = getenv("http_proxy");
135	if (env_HTTP_PROXY != NULL) {
136		if (strncmp(env_HTTP_PROXY, "http://", 7) == 0)
137			env_HTTP_PROXY += 7;
138		p = strchr(env_HTTP_PROXY, '/');
139		if (p != NULL)
140			*p = 0;
141		p = strchr(env_HTTP_PROXY, ':');
142		if (p != NULL) {
143			*p = 0;
144			proxyport = p + 1;
145		} else
146			proxyport = "3128";
147	}
148
149	env_HTTP_PROXY_AUTH = getenv("HTTP_PROXY_AUTH");
150	if ((env_HTTP_PROXY != NULL) &&
151	    (env_HTTP_PROXY_AUTH != NULL) &&
152	    (strncasecmp(env_HTTP_PROXY_AUTH, "basic:" , 6) == 0)) {
153		/* Ignore authentication scheme */
154		(void) strsep(&env_HTTP_PROXY_AUTH, ":");
155
156		/* Ignore realm */
157		(void) strsep(&env_HTTP_PROXY_AUTH, ":");
158
159		/* Obtain username and password */
160		proxy_auth_user = strsep(&env_HTTP_PROXY_AUTH, ":");
161		proxy_auth_pass = env_HTTP_PROXY_AUTH;
162	}
163
164	if ((proxy_auth_user != NULL) && (proxy_auth_pass != NULL)) {
165		asprintf(&proxy_auth_userpass, "%s:%s",
166		    proxy_auth_user, proxy_auth_pass);
167		if (proxy_auth_userpass == NULL)
168			err(1, "asprintf");
169
170		proxy_auth_userpass64 = b64enc(proxy_auth_userpass);
171		if (proxy_auth_userpass64 == NULL)
172			err(1, "malloc");
173
174		asprintf(&proxyauth, "Proxy-Authorization: Basic %s\r\n",
175		    proxy_auth_userpass64);
176		if (proxyauth == NULL)
177			err(1, "asprintf");
178
179		free(proxy_auth_userpass);
180		free(proxy_auth_userpass64);
181	} else
182		proxyauth = NULL;
183
184	env_HTTP_USER_AGENT = getenv("HTTP_USER_AGENT");
185	if (env_HTTP_USER_AGENT == NULL)
186		env_HTTP_USER_AGENT = "phttpget/0.1";
187
188	env_HTTP_TIMEOUT = getenv("HTTP_TIMEOUT");
189	if (env_HTTP_TIMEOUT != NULL) {
190		http_timeout = strtol(env_HTTP_TIMEOUT, &p, 10);
191		if ((*env_HTTP_TIMEOUT == '\0') || (*p != '\0') ||
192		    (http_timeout < 0))
193			warnx("HTTP_TIMEOUT (%s) is not a positive integer",
194			    env_HTTP_TIMEOUT);
195		else
196			timo.tv_sec = http_timeout;
197	}
198}
199
200static int
201makerequest(char ** buf, char * path, char * server, int connclose)
202{
203	int buflen;
204
205	buflen = asprintf(buf,
206	    "GET %s%s/%s HTTP/1.1\r\n"
207	    "Host: %s\r\n"
208	    "User-Agent: %s\r\n"
209	    "%s"
210	    "%s"
211	    "\r\n",
212	    env_HTTP_PROXY ? "http://" : "",
213	    env_HTTP_PROXY ? server : "",
214	    path, server, env_HTTP_USER_AGENT,
215	    proxyauth ? proxyauth : "",
216	    connclose ? "Connection: Close\r\n" : "Connection: Keep-Alive\r\n");
217	if (buflen == -1)
218		err(1, "asprintf");
219	return(buflen);
220}
221
222static int
223readln(int sd, char * resbuf, int * resbuflen, int * resbufpos)
224{
225	ssize_t len;
226
227	while (strnstr(resbuf + *resbufpos, "\r\n",
228	    *resbuflen - *resbufpos) == NULL) {
229		/* Move buffered data to the start of the buffer */
230		if (*resbufpos != 0) {
231			memmove(resbuf, resbuf + *resbufpos,
232			    *resbuflen - *resbufpos);
233			*resbuflen -= *resbufpos;
234			*resbufpos = 0;
235		}
236
237		/* If the buffer is full, complain */
238		if (*resbuflen == BUFSIZ)
239			return -1;
240
241		/* Read more data into the buffer */
242		len = recv(sd, resbuf + *resbuflen, BUFSIZ - *resbuflen, 0);
243		if ((len == 0) ||
244		    ((len == -1) && (errno != EINTR)))
245			return -1;
246
247		if (len != -1)
248			*resbuflen += len;
249	}
250
251	return 0;
252}
253
254static int
255copybytes(int sd, int fd, off_t copylen, char * resbuf, int * resbuflen,
256    int * resbufpos)
257{
258	ssize_t len;
259
260	while (copylen) {
261		/* Write data from resbuf to fd */
262		len = *resbuflen - *resbufpos;
263		if (copylen < len)
264			len = copylen;
265		if (len > 0) {
266			if (fd != -1)
267				len = write(fd, resbuf + *resbufpos, len);
268			if (len == -1)
269				err(1, "write");
270			*resbufpos += len;
271			copylen -= len;
272			continue;
273		}
274
275		/* Read more data into buffer */
276		len = recv(sd, resbuf, BUFSIZ, 0);
277		if (len == -1) {
278			if (errno == EINTR)
279				continue;
280			return -1;
281		} else if (len == 0) {
282			return -2;
283		} else {
284			*resbuflen = len;
285			*resbufpos = 0;
286		}
287	}
288
289	return 0;
290}
291
292int
293main(int argc, char *argv[])
294{
295	struct addrinfo hints;	/* Hints to getaddrinfo */
296	struct addrinfo *res;	/* Pointer to server address being used */
297	struct addrinfo *res0;	/* Pointer to server addresses */
298	char * resbuf = NULL;	/* Response buffer */
299	int resbufpos = 0;	/* Response buffer position */
300	int resbuflen = 0;	/* Response buffer length */
301	char * eolp;		/* Pointer to "\r\n" within resbuf */
302	char * hln;		/* Pointer within header line */
303	char * servername;	/* Name of server */
304	char * fname = NULL;	/* Name of downloaded file */
305	char * reqbuf = NULL;	/* Request buffer */
306	int reqbufpos = 0;	/* Request buffer position */
307	int reqbuflen = 0;	/* Request buffer length */
308	ssize_t len;		/* Length sent or received */
309	int nreq = 0;		/* Number of next request to send */
310	int nres = 0;		/* Number of next reply to receive */
311	int pipelined = 0;	/* != 0 if connection in pipelined mode. */
312	int keepalive;		/* != 0 if HTTP/1.0 keep-alive rcvd. */
313	int sd = -1;		/* Socket descriptor */
314	int sdflags = 0;	/* Flags on the socket sd */
315	int fd = -1;		/* Descriptor for download target file */
316	int error;		/* Error code */
317	int statuscode;		/* HTTP Status code */
318	off_t contentlength;	/* Value from Content-Length header */
319	int chunked;		/* != if transfer-encoding is chunked */
320	off_t clen;		/* Chunk length */
321	int firstreq = 0;	/* # of first request for this connection */
322	int val;		/* Value used for setsockopt call */
323
324	/* Check that the arguments are sensible */
325	if (argc < 2)
326		usage();
327
328	/* Read important environment variables */
329	readenv();
330
331	/* Get server name and adjust arg[cv] to point at file names */
332	servername = argv[1];
333	argv += 2;
334	argc -= 2;
335
336	/* Allocate response buffer */
337	resbuf = malloc(BUFSIZ);
338	if (resbuf == NULL)
339		err(1, "malloc");
340
341	/* Look up server */
342	memset(&hints, 0, sizeof(hints));
343	hints.ai_family = PF_UNSPEC;
344	hints.ai_socktype = SOCK_STREAM;
345	error = getaddrinfo(env_HTTP_PROXY ? env_HTTP_PROXY : servername,
346	    env_HTTP_PROXY ? proxyport : "http", &hints, &res0);
347	if (error)
348		errx(1, "host = %s, port = %s: %s",
349		    env_HTTP_PROXY ? env_HTTP_PROXY : servername,
350		    env_HTTP_PROXY ? proxyport : "http",
351		    gai_strerror(error));
352	if (res0 == NULL)
353		errx(1, "could not look up %s", servername);
354	res = res0;
355
356	/* Do the fetching */
357	while (nres < argc) {
358		/* Make sure we have a connected socket */
359		for (; sd == -1; res = res->ai_next) {
360			/* No addresses left to try :-( */
361			if (res == NULL)
362				errx(1, "Could not connect to %s", servername);
363
364			/* Create a socket... */
365			sd = socket(res->ai_family, res->ai_socktype,
366			    res->ai_protocol);
367			if (sd == -1)
368				continue;
369
370			/* ... set 15-second timeouts ... */
371			setsockopt(sd, SOL_SOCKET, SO_SNDTIMEO,
372			    (void *)&timo, (socklen_t)sizeof(timo));
373			setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO,
374			    (void *)&timo, (socklen_t)sizeof(timo));
375
376			/* ... disable SIGPIPE generation ... */
377			val = 1;
378			setsockopt(sd, SOL_SOCKET, SO_NOSIGPIPE,
379			    (void *)&val, sizeof(int));
380
381			/* ... and connect to the server. */
382			if(connect(sd, res->ai_addr, res->ai_addrlen)) {
383				close(sd);
384				sd = -1;
385				continue;
386			}
387
388			firstreq = nres;
389		}
390
391		/*
392		 * If in pipelined HTTP mode, put socket into non-blocking
393		 * mode, since we're probably going to want to try to send
394		 * several HTTP requests.
395		 */
396		if (pipelined) {
397			sdflags = fcntl(sd, F_GETFL);
398			if (fcntl(sd, F_SETFL, sdflags | O_NONBLOCK) == -1)
399				err(1, "fcntl");
400		}
401
402		/* Construct requests and/or send them without blocking */
403		while ((nreq < argc) && ((reqbuf == NULL) || pipelined)) {
404			/* If not in the middle of a request, make one */
405			if (reqbuf == NULL) {
406				reqbuflen = makerequest(&reqbuf, argv[nreq],
407				    servername, (nreq == argc - 1));
408				reqbufpos = 0;
409			}
410
411			/* If in pipelined mode, try to send the request */
412			if (pipelined) {
413				while (reqbufpos < reqbuflen) {
414					len = send(sd, reqbuf + reqbufpos,
415					    reqbuflen - reqbufpos, 0);
416					if (len == -1)
417						break;
418					reqbufpos += len;
419				}
420				if (reqbufpos < reqbuflen) {
421					if (errno != EAGAIN)
422						goto conndied;
423					break;
424				} else {
425					free(reqbuf);
426					reqbuf = NULL;
427					nreq++;
428				}
429			}
430		}
431
432		/* Put connection back into blocking mode */
433		if (pipelined) {
434			if (fcntl(sd, F_SETFL, sdflags) == -1)
435				err(1, "fcntl");
436		}
437
438		/* Do we need to blocking-send a request? */
439		if (nres == nreq) {
440			while (reqbufpos < reqbuflen) {
441				len = send(sd, reqbuf + reqbufpos,
442				    reqbuflen - reqbufpos, 0);
443				if (len == -1)
444					goto conndied;
445				reqbufpos += len;
446			}
447			free(reqbuf);
448			reqbuf = NULL;
449			nreq++;
450		}
451
452		/* Scan through the response processing headers. */
453		statuscode = 0;
454		contentlength = -1;
455		chunked = 0;
456		keepalive = 0;
457		do {
458			/* Get a header line */
459			error = readln(sd, resbuf, &resbuflen, &resbufpos);
460			if (error)
461				goto conndied;
462			hln = resbuf + resbufpos;
463			eolp = strnstr(hln, "\r\n", resbuflen - resbufpos);
464			resbufpos = (eolp - resbuf) + 2;
465			*eolp = '\0';
466
467			/* Make sure it doesn't contain a NUL character */
468			if (strchr(hln, '\0') != eolp)
469				goto conndied;
470
471			if (statuscode == 0) {
472				/* The first line MUST be HTTP/1.x xxx ... */
473				if ((strncmp(hln, "HTTP/1.", 7) != 0) ||
474				    ! isdigit(hln[7]))
475					goto conndied;
476
477				/*
478				 * If the minor version number isn't zero,
479				 * then we can assume that pipelining our
480				 * requests is OK -- as long as we don't
481				 * see a "Connection: close" line later
482				 * and we either have a Content-Length or
483				 * Transfer-Encoding: chunked header to
484				 * tell us the length.
485				 */
486				if (hln[7] != '0')
487					pipelined = 1;
488
489				/* Skip over the minor version number */
490				hln = strchr(hln + 7, ' ');
491				if (hln == NULL)
492					goto conndied;
493				else
494					hln++;
495
496				/* Read the status code */
497				while (isdigit(*hln)) {
498					statuscode = statuscode * 10 +
499					    *hln - '0';
500					hln++;
501				}
502
503				if (statuscode < 100 || statuscode > 599)
504					goto conndied;
505
506				/* Ignore the rest of the line */
507				continue;
508			}
509
510			/*
511			 * Check for "Connection: close" or
512			 * "Connection: Keep-Alive" header
513			 */
514			if (strncasecmp(hln, "Connection:", 11) == 0) {
515				hln += 11;
516				if (strcasestr(hln, "close") != NULL)
517					pipelined = 0;
518				if (strcasestr(hln, "Keep-Alive") != NULL)
519					keepalive = 1;
520
521				/* Next header... */
522				continue;
523			}
524
525			/* Check for "Content-Length:" header */
526			if (strncasecmp(hln, "Content-Length:", 15) == 0) {
527				hln += 15;
528				contentlength = 0;
529
530				/* Find the start of the length */
531				while (!isdigit(*hln) && (*hln != '\0'))
532					hln++;
533
534				/* Compute the length */
535				while (isdigit(*hln)) {
536					if (contentlength >= OFF_MAX / 10) {
537						/* Nasty people... */
538						goto conndied;
539					}
540					contentlength = contentlength * 10 +
541					    *hln - '0';
542					hln++;
543				}
544
545				/* Next header... */
546				continue;
547			}
548
549			/* Check for "Transfer-Encoding: chunked" header */
550			if (strncasecmp(hln, "Transfer-Encoding:", 18) == 0) {
551				hln += 18;
552				if (strcasestr(hln, "chunked") != NULL)
553					chunked = 1;
554
555				/* Next header... */
556				continue;
557			}
558
559			/* We blithely ignore any other header lines */
560
561			/* No more header lines */
562			if (strlen(hln) == 0) {
563				/*
564				 * If the status code was 1xx, then there will
565				 * be a real header later.  Servers may emit
566				 * 1xx header blocks at will, but since we
567				 * don't expect one, we should just ignore it.
568				 */
569				if (100 <= statuscode && statuscode <= 199) {
570					statuscode = 0;
571					continue;
572				}
573
574				/* End of header; message body follows */
575				break;
576			}
577		} while (1);
578
579		/* No message body for 204 or 304 */
580		if (statuscode == 204 || statuscode == 304) {
581			nres++;
582			continue;
583		}
584
585		/*
586		 * There should be a message body coming, but we only want
587		 * to send it to a file if the status code is 200
588		 */
589		if (statuscode == 200) {
590			/* Generate a file name for the download */
591			fname = strrchr(argv[nres], '/');
592			if (fname == NULL)
593				fname = argv[nres];
594			else
595				fname++;
596			if (strlen(fname) == 0)
597				errx(1, "Cannot obtain file name from %s\n",
598				    argv[nres]);
599
600			fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644);
601			if (fd == -1)
602				errx(1, "open(%s)", fname);
603		}
604
605		/* Read the message and send data to fd if appropriate */
606		if (chunked) {
607			/* Handle a chunked-encoded entity */
608
609			/* Read chunks */
610			do {
611				error = readln(sd, resbuf, &resbuflen,
612				    &resbufpos);
613				if (error)
614					goto conndied;
615				hln = resbuf + resbufpos;
616				eolp = strstr(hln, "\r\n");
617				resbufpos = (eolp - resbuf) + 2;
618
619				clen = 0;
620				while (isxdigit(*hln)) {
621					if (clen >= OFF_MAX / 16) {
622						/* Nasty people... */
623						goto conndied;
624					}
625					if (isdigit(*hln))
626						clen = clen * 16 + *hln - '0';
627					else
628						clen = clen * 16 + 10 +
629						    tolower(*hln) - 'a';
630					hln++;
631				}
632
633				error = copybytes(sd, fd, clen, resbuf,
634				    &resbuflen, &resbufpos);
635				if (error) {
636					goto conndied;
637				}
638			} while (clen != 0);
639
640			/* Read trailer and final CRLF */
641			do {
642				error = readln(sd, resbuf, &resbuflen,
643				    &resbufpos);
644				if (error)
645					goto conndied;
646				hln = resbuf + resbufpos;
647				eolp = strstr(hln, "\r\n");
648				resbufpos = (eolp - resbuf) + 2;
649			} while (hln != eolp);
650		} else if (contentlength != -1) {
651			error = copybytes(sd, fd, contentlength, resbuf,
652			    &resbuflen, &resbufpos);
653			if (error)
654				goto conndied;
655		} else {
656			/*
657			 * Not chunked, and no content length header.
658			 * Read everything until the server closes the
659			 * socket.
660			 */
661			error = copybytes(sd, fd, OFF_MAX, resbuf,
662			    &resbuflen, &resbufpos);
663			if (error == -1)
664				goto conndied;
665			pipelined = 0;
666		}
667
668		if (fd != -1) {
669			close(fd);
670			fd = -1;
671		}
672
673		fprintf(stderr, "http://%s/%s: %d ", servername, argv[nres],
674		    statuscode);
675		if (statuscode == 200)
676			fprintf(stderr, "OK\n");
677		else if (statuscode < 300)
678			fprintf(stderr, "Successful (ignored)\n");
679		else if (statuscode < 400)
680			fprintf(stderr, "Redirection (ignored)\n");
681		else
682			fprintf(stderr, "Error (ignored)\n");
683
684		/* We've finished this file! */
685		nres++;
686
687		/*
688		 * If necessary, clean up this connection so that we
689		 * can start a new one.
690		 */
691		if (pipelined == 0 && keepalive == 0)
692			goto cleanupconn;
693		continue;
694
695conndied:
696		/*
697		 * Something went wrong -- our connection died, the server
698		 * sent us garbage, etc.  If this happened on the first
699		 * request we sent over this connection, give up.  Otherwise,
700		 * close this connection, open a new one, and reissue the
701		 * request.
702		 */
703		if (nres == firstreq)
704			errx(1, "Connection failure");
705
706cleanupconn:
707		/*
708		 * Clean up our connection and keep on going
709		 */
710		shutdown(sd, SHUT_RDWR);
711		close(sd);
712		sd = -1;
713		if (fd != -1) {
714			close(fd);
715			fd = -1;
716		}
717		if (reqbuf != NULL) {
718			free(reqbuf);
719			reqbuf = NULL;
720		}
721		nreq = nres;
722		res = res0;
723		pipelined = 0;
724		resbufpos = resbuflen = 0;
725		continue;
726	}
727
728	free(resbuf);
729	freeaddrinfo(res0);
730
731	return 0;
732}
733