phttpget.c revision 171120 
1/*-
2 * Copyright 2005 Colin Percival
3 * All rights reserved
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted providing that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
16 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
18 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
22 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
23 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
24 * POSSIBILITY OF SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/usr.sbin/portsnap/phttpget/phttpget.c 171120 2007-06-30 19:48:28Z cperciva $");
29
30#include <sys/types.h>
31#include <sys/time.h>
32#include <sys/socket.h>
33
34#include <ctype.h>
35#include <err.h>
36#include <errno.h>
37#include <fcntl.h>
38#include <limits.h>
39#include <netdb.h>
40#include <stdint.h>
41#include <stdio.h>
42#include <stdlib.h>
43#include <string.h>
44#include <sysexits.h>
45#include <unistd.h>
46
47static const char *	env_HTTP_PROXY;
48static char *		env_HTTP_PROXY_AUTH;
49static const char *	env_HTTP_USER_AGENT;
50static char *		env_HTTP_TIMEOUT;
51static const char *	proxyport;
52static char *		proxyauth;
53
54static struct timeval	timo = { 15, 0};
55
56static void
57usage(void)
58{
59
60	fprintf(stderr, "usage: phttpget server [file ...]\n");
61	exit(EX_USAGE);
62}
63
64/*
65 * Base64 encode a string; the string returned, if non-NULL, is
66 * allocated using malloc() and must be freed by the caller.
67 */
68static char *
69b64enc(const char *ptext)
70{
71	static const char base64[] =
72	    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
73	    "abcdefghijklmnopqrstuvwxyz"
74	    "0123456789+/";
75	const char *pt;
76	char *ctext, *pc;
77	size_t ptlen, ctlen;
78	uint32_t t;
79	unsigned int j;
80
81	/*
82	 * Encoded length is 4 characters per 3-byte block or partial
83	 * block of plaintext, plus one byte for the terminating NUL
84	 */
85	ptlen = strlen(ptext);
86	if (ptlen > ((SIZE_MAX - 1) / 4) * 3 - 2)
87		return NULL;	/* Possible integer overflow */
88	ctlen = 4 * ((ptlen + 2) / 3) + 1;
89	if ((ctext = malloc(ctlen)) == NULL)
90		return NULL;
91	ctext[ctlen - 1] = 0;
92
93	/*
94	 * Scan through ptext, reading up to 3 bytes from ptext and
95	 * writing 4 bytes to ctext, until we run out of input.
96	 */
97	for (pt = ptext, pc = ctext; ptlen; ptlen -= 3, pc += 4) {
98		/* Read 3 bytes */
99		for (t = j = 0; j < 3; j++) {
100			t <<= 8;
101			if (j < ptlen)
102				t += *pt++;
103		}
104
105		/* Write 4 bytes */
106		for (j = 0; j < 4; j++) {
107			if (j <= ptlen + 1)
108				pc[j] = base64[(t >> 18) & 0x3f];
109			else
110				pc[j] = '=';
111			t <<= 6;
112		}
113
114		/* If we're done, exit the loop */
115		if (ptlen <= 3)
116			break;
117	}
118
119	return (ctext);
120}
121
122static void
123readenv(void)
124{
125	char *proxy_auth_userpass, *proxy_auth_userpass64, *p;
126	char *proxy_auth_user = NULL;
127	char *proxy_auth_pass = NULL;
128	long http_timeout;
129
130	env_HTTP_PROXY = getenv("HTTP_PROXY");
131	if (env_HTTP_PROXY == NULL)
132		env_HTTP_PROXY = getenv("http_proxy");
133	if (env_HTTP_PROXY != NULL) {
134		if (strncmp(env_HTTP_PROXY, "http://", 7) == 0)
135			env_HTTP_PROXY += 7;
136		p = strchr(env_HTTP_PROXY, '/');
137		if (p != NULL)
138			*p = 0;
139		p = strchr(env_HTTP_PROXY, ':');
140		if (p != NULL) {
141			*p = 0;
142			proxyport = p + 1;
143		} else
144			proxyport = "3128";
145	}
146
147	env_HTTP_PROXY_AUTH = getenv("HTTP_PROXY_AUTH");
148	if ((env_HTTP_PROXY != NULL) &&
149	    (env_HTTP_PROXY_AUTH != NULL) &&
150	    (strncasecmp(env_HTTP_PROXY_AUTH, "basic:" , 6) == 0)) {
151		/* Ignore authentication scheme */
152		(void) strsep(&env_HTTP_PROXY_AUTH, ":");
153
154		/* Ignore realm */
155		(void) strsep(&env_HTTP_PROXY_AUTH, ":");
156
157		/* Obtain username and password */
158		proxy_auth_user = strsep(&env_HTTP_PROXY_AUTH, ":");
159		proxy_auth_pass = env_HTTP_PROXY_AUTH;
160	}
161
162	if ((proxy_auth_user != NULL) && (proxy_auth_pass != NULL)) {
163		asprintf(&proxy_auth_userpass, "%s:%s",
164		    proxy_auth_user, proxy_auth_pass);
165		if (proxy_auth_userpass == NULL)
166			err(1, "asprintf");
167
168		proxy_auth_userpass64 = b64enc(proxy_auth_userpass);
169		if (proxy_auth_userpass64 == NULL)
170			err(1, "malloc");
171
172		asprintf(&proxyauth, "Proxy-Authorization: Basic %s\r\n",
173		    proxy_auth_userpass64);
174		if (proxyauth == NULL)
175			err(1, "asprintf");
176
177		free(proxy_auth_userpass);
178		free(proxy_auth_userpass64);
179	} else
180		proxyauth = NULL;
181
182	env_HTTP_USER_AGENT = getenv("HTTP_USER_AGENT");
183	if (env_HTTP_USER_AGENT == NULL)
184		env_HTTP_USER_AGENT = "phttpget/0.1";
185
186	env_HTTP_TIMEOUT = getenv("HTTP_TIMEOUT");
187	if (env_HTTP_TIMEOUT != NULL) {
188		http_timeout = strtol(env_HTTP_TIMEOUT, &p, 10);
189		if ((*env_HTTP_TIMEOUT == '\0') || (*p != '\0') ||
190		    (http_timeout < 0))
191			warnx("HTTP_TIMEOUT (%s) is not a positive integer",
192			    env_HTTP_TIMEOUT);
193		else
194			timo.tv_sec = http_timeout;
195	}
196}
197
198static int
199makerequest(char ** buf, char * path, char * server, int connclose)
200{
201	int buflen;
202
203	buflen = asprintf(buf,
204	    "GET %s%s/%s HTTP/1.1\r\n"
205	    "Host: %s\r\n"
206	    "User-Agent: %s\r\n"
207	    "%s"
208	    "%s"
209	    "\r\n",
210	    env_HTTP_PROXY ? "http://" : "",
211	    env_HTTP_PROXY ? server : "",
212	    path, server, env_HTTP_USER_AGENT,
213	    proxyauth ? proxyauth : "",
214	    connclose ? "Connection: Close\r\n" : "Connection: Keep-Alive\r\n");
215	if (buflen == -1)
216		err(1, "asprintf");
217	return(buflen);
218}
219
220static int
221readln(int sd, char * resbuf, int * resbuflen, int * resbufpos)
222{
223	ssize_t len;
224
225	while (strnstr(resbuf + *resbufpos, "\r\n",
226	    *resbuflen - *resbufpos) == NULL) {
227		/* Move buffered data to the start of the buffer */
228		if (*resbufpos != 0) {
229			memmove(resbuf, resbuf + *resbufpos,
230			    *resbuflen - *resbufpos);
231			*resbuflen -= *resbufpos;
232			*resbufpos = 0;
233		}
234
235		/* If the buffer is full, complain */
236		if (*resbuflen == BUFSIZ)
237			return -1;
238
239		/* Read more data into the buffer */
240		len = recv(sd, resbuf + *resbuflen, BUFSIZ - *resbuflen, 0);
241		if ((len == 0) ||
242		    ((len == -1) && (errno != EINTR)))
243			return -1;
244
245		if (len != -1)
246			*resbuflen += len;
247	}
248
249	return 0;
250}
251
252static int
253copybytes(int sd, int fd, off_t copylen, char * resbuf, int * resbuflen,
254    int * resbufpos)
255{
256	ssize_t len;
257
258	while (copylen) {
259		/* Write data from resbuf to fd */
260		len = *resbuflen - *resbufpos;
261		if (copylen < len)
262			len = copylen;
263		if (len > 0) {
264			if (fd != -1)
265				len = write(fd, resbuf + *resbufpos, len);
266			if (len == -1)
267				err(1, "write");
268			*resbufpos += len;
269			copylen -= len;
270			continue;
271		}
272
273		/* Read more data into buffer */
274		len = recv(sd, resbuf, BUFSIZ, 0);
275		if (len == -1) {
276			if (errno == EINTR)
277				continue;
278			return -1;
279		} else if (len == 0) {
280			return -2;
281		} else {
282			*resbuflen = len;
283			*resbufpos = 0;
284		}
285	}
286
287	return 0;
288}
289
290int
291main(int argc, char *argv[])
292{
293	struct addrinfo hints;	/* Hints to getaddrinfo */
294	struct addrinfo *res;	/* Pointer to server address being used */
295	struct addrinfo *res0;	/* Pointer to server addresses */
296	char * resbuf = NULL;	/* Response buffer */
297	int resbufpos = 0;	/* Response buffer position */
298	int resbuflen = 0;	/* Response buffer length */
299	char * eolp;		/* Pointer to "\r\n" within resbuf */
300	char * hln;		/* Pointer within header line */
301	char * servername;	/* Name of server */
302	char * fname = NULL;	/* Name of downloaded file */
303	char * reqbuf = NULL;	/* Request buffer */
304	int reqbufpos = 0;	/* Request buffer position */
305	int reqbuflen = 0;	/* Request buffer length */
306	ssize_t len;		/* Length sent or received */
307	int nreq = 0;		/* Number of next request to send */
308	int nres = 0;		/* Number of next reply to receive */
309	int pipelined = 0;	/* != 0 if connection in pipelined mode. */
310	int keepalive;		/* != 0 if HTTP/1.0 keep-alive rcvd. */
311	int sd = -1;		/* Socket descriptor */
312	int sdflags = 0;	/* Flags on the socket sd */
313	int fd = -1;		/* Descriptor for download target file */
314	int error;		/* Error code */
315	int statuscode;		/* HTTP Status code */
316	off_t contentlength;	/* Value from Content-Length header */
317	int chunked;		/* != if transfer-encoding is chunked */
318	off_t clen;		/* Chunk length */
319	int firstreq = 0;	/* # of first request for this connection */
320
321	/* Check that the arguments are sensible */
322	if (argc < 2)
323		usage();
324
325	/* Read important environment variables */
326	readenv();
327
328	/* Get server name and adjust arg[cv] to point at file names */
329	servername = argv[1];
330	argv += 2;
331	argc -= 2;
332
333	/* Allocate response buffer */
334	resbuf = malloc(BUFSIZ);
335	if (resbuf == NULL)
336		err(1, "malloc");
337
338	/* Look up server */
339	memset(&hints, 0, sizeof(hints));
340	hints.ai_family = PF_UNSPEC;
341	hints.ai_socktype = SOCK_STREAM;
342	error = getaddrinfo(env_HTTP_PROXY ? env_HTTP_PROXY : servername,
343	    env_HTTP_PROXY ? proxyport : "http", &hints, &res0);
344	if (error)
345		errx(1, "host = %s, port = %s: %s",
346		    env_HTTP_PROXY ? env_HTTP_PROXY : servername,
347		    env_HTTP_PROXY ? proxyport : "http",
348		    gai_strerror(error));
349	if (res0 == NULL)
350		errx(1, "could not look up %s", servername);
351	res = res0;
352
353	/* Do the fetching */
354	while (nres < argc) {
355		/* Make sure we have a connected socket */
356		for (; sd == -1; res = res->ai_next) {
357			/* No addresses left to try :-( */
358			if (res == NULL)
359				errx(1, "Could not connect to %s", servername);
360
361			/* Create a socket... */
362			sd = socket(res->ai_family, res->ai_socktype,
363			    res->ai_protocol);
364			if (sd == -1)
365				continue;
366
367			/* ... set 15-second timeouts ... */
368			setsockopt(sd, SOL_SOCKET, SO_SNDTIMEO,
369			    (void *)&timo, (socklen_t)sizeof(timo));
370			setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO,
371			    (void *)&timo, (socklen_t)sizeof(timo));
372
373			/* ... and connect to the server. */
374			if(connect(sd, res->ai_addr, res->ai_addrlen)) {
375				close(sd);
376				sd = -1;
377				continue;
378			}
379
380			firstreq = nres;
381		}
382
383		/*
384		 * If in pipelined HTTP mode, put socket into non-blocking
385		 * mode, since we're probably going to want to try to send
386		 * several HTTP requests.
387		 */
388		if (pipelined) {
389			sdflags = fcntl(sd, F_GETFL);
390			if (fcntl(sd, F_SETFL, sdflags | O_NONBLOCK) == -1)
391				err(1, "fcntl");
392		}
393
394		/* Construct requests and/or send them without blocking */
395		while ((nreq < argc) && ((reqbuf == NULL) || pipelined)) {
396			/* If not in the middle of a request, make one */
397			if (reqbuf == NULL) {
398				reqbuflen = makerequest(&reqbuf, argv[nreq],
399				    servername, (nreq == argc - 1));
400				reqbufpos = 0;
401			}
402
403			/* If in pipelined mode, try to send the request */
404			if (pipelined) {
405				while (reqbufpos < reqbuflen) {
406					len = send(sd, reqbuf + reqbufpos,
407					    reqbuflen - reqbufpos, 0);
408					if (len == -1)
409						break;
410					reqbufpos += len;
411				}
412				if (reqbufpos < reqbuflen) {
413					if (errno != EAGAIN)
414						goto conndied;
415					break;
416				} else {
417					free(reqbuf);
418					reqbuf = NULL;
419					nreq++;
420				}
421			}
422		}
423
424		/* Put connection back into blocking mode */
425		if (pipelined) {
426			if (fcntl(sd, F_SETFL, sdflags) == -1)
427				err(1, "fcntl");
428		}
429
430		/* Do we need to blocking-send a request? */
431		if (nres == nreq) {
432			while (reqbufpos < reqbuflen) {
433				len = send(sd, reqbuf + reqbufpos,
434				    reqbuflen - reqbufpos, 0);
435				if (len == -1)
436					goto conndied;
437				reqbufpos += len;
438			}
439			free(reqbuf);
440			reqbuf = NULL;
441			nreq++;
442		}
443
444		/* Scan through the response processing headers. */
445		statuscode = 0;
446		contentlength = -1;
447		chunked = 0;
448		keepalive = 0;
449		do {
450			/* Get a header line */
451			error = readln(sd, resbuf, &resbuflen, &resbufpos);
452			if (error)
453				goto conndied;
454			hln = resbuf + resbufpos;
455			eolp = strnstr(hln, "\r\n", resbuflen - resbufpos);
456			resbufpos = (eolp - resbuf) + 2;
457			*eolp = '\0';
458
459			/* Make sure it doesn't contain a NUL character */
460			if (strchr(hln, '\0') != eolp)
461				goto conndied;
462
463			if (statuscode == 0) {
464				/* The first line MUST be HTTP/1.x xxx ... */
465				if ((strncmp(hln, "HTTP/1.", 7) != 0) ||
466				    ! isdigit(hln[7]))
467					goto conndied;
468
469				/*
470				 * If the minor version number isn't zero,
471				 * then we can assume that pipelining our
472				 * requests is OK -- as long as we don't
473				 * see a "Connection: close" line later
474				 * and we either have a Content-Length or
475				 * Transfer-Encoding: chunked header to
476				 * tell us the length.
477				 */
478				if (hln[7] != '0')
479					pipelined = 1;
480
481				/* Skip over the minor version number */
482				hln = strchr(hln + 7, ' ');
483				if (hln == NULL)
484					goto conndied;
485				else
486					hln++;
487
488				/* Read the status code */
489				while (isdigit(*hln)) {
490					statuscode = statuscode * 10 +
491					    *hln - '0';
492					hln++;
493				}
494
495				if (statuscode < 100 || statuscode > 599)
496					goto conndied;
497
498				/* Ignore the rest of the line */
499				continue;
500			}
501
502			/*
503			 * Check for "Connection: close" or
504			 * "Connection: Keep-Alive" header
505			 */
506			if (strncmp(hln, "Connection:", 11) == 0) {
507				hln += 11;
508				if (strstr(hln, "close") != NULL)
509					pipelined = 0;
510				if (strstr(hln, "Keep-Alive") != NULL)
511					keepalive = 1;
512
513				/* Next header... */
514				continue;
515			}
516
517			/* Check for "Content-Length:" header */
518			if (strncmp(hln, "Content-Length:", 15) == 0) {
519				hln += 15;
520				contentlength = 0;
521
522				/* Find the start of the length */
523				while (!isdigit(*hln) && (*hln != '\0'))
524					hln++;
525
526				/* Compute the length */
527				while (isdigit(*hln)) {
528					if (contentlength >= OFF_MAX / 10) {
529						/* Nasty people... */
530						goto conndied;
531					}
532					contentlength = contentlength * 10 +
533					    *hln - '0';
534					hln++;
535				}
536
537				/* Next header... */
538				continue;
539			}
540
541			/* Check for "Transfer-Encoding: chunked" header */
542			if (strncmp(hln, "Transfer-Encoding:", 18) == 0) {
543				hln += 18;
544				if (strstr(hln, "chunked") != NULL)
545					chunked = 1;
546
547				/* Next header... */
548				continue;
549			}
550
551			/* We blithely ignore any other header lines */
552
553			/* No more header lines */
554			if (strlen(hln) == 0) {
555				/*
556				 * If the status code was 1xx, then there will
557				 * be a real header later.  Servers may emit
558				 * 1xx header blocks at will, but since we
559				 * don't expect one, we should just ignore it.
560				 */
561				if (100 <= statuscode && statuscode <= 199) {
562					statuscode = 0;
563					continue;
564				}
565
566				/* End of header; message body follows */
567				break;
568			}
569		} while (1);
570
571		/* No message body for 204 or 304 */
572		if (statuscode == 204 || statuscode == 304) {
573			nres++;
574			continue;
575		}
576
577		/*
578		 * There should be a message body coming, but we only want
579		 * to send it to a file if the status code is 200
580		 */
581		if (statuscode == 200) {
582			/* Generate a file name for the download */
583			fname = strrchr(argv[nres], '/');
584			if (fname == NULL)
585				fname = argv[nres];
586			else
587				fname++;
588			if (strlen(fname) == 0)
589				errx(1, "Cannot obtain file name from %s\n",
590				    argv[nres]);
591
592			fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644);
593			if (fd == -1)
594				errx(1, "open(%s)", fname);
595		};
596
597		/* Read the message and send data to fd if appropriate */
598		if (chunked) {
599			/* Handle a chunked-encoded entity */
600
601			/* Read chunks */
602			do {
603				error = readln(sd, resbuf, &resbuflen,
604				    &resbufpos);
605				if (error)
606					goto conndied;
607				hln = resbuf + resbufpos;
608				eolp = strstr(hln, "\r\n");
609				resbufpos = (eolp - resbuf) + 2;
610
611				clen = 0;
612				while (isxdigit(*hln)) {
613					if (clen >= OFF_MAX / 16) {
614						/* Nasty people... */
615						goto conndied;
616					}
617					if (isdigit(*hln))
618						clen = clen * 16 + *hln - '0';
619					else
620						clen = clen * 16 + 10 +
621						    tolower(*hln) - 'a';
622					hln++;
623				}
624
625				error = copybytes(sd, fd, clen, resbuf,
626				    &resbuflen, &resbufpos);
627				if (error) {
628					goto conndied;
629				}
630			} while (clen != 0);
631
632			/* Read trailer and final CRLF */
633			do {
634				error = readln(sd, resbuf, &resbuflen,
635				    &resbufpos);
636				if (error)
637					goto conndied;
638				hln = resbuf + resbufpos;
639				eolp = strstr(hln, "\r\n");
640				resbufpos = (eolp - resbuf) + 2;
641			} while (hln != eolp);
642		} else if (contentlength != -1) {
643			error = copybytes(sd, fd, contentlength, resbuf,
644			    &resbuflen, &resbufpos);
645			if (error)
646				goto conndied;
647		} else {
648			/*
649			 * Not chunked, and no content length header.
650			 * Read everything until the server closes the
651			 * socket.
652			 */
653			error = copybytes(sd, fd, OFF_MAX, resbuf,
654			    &resbuflen, &resbufpos);
655			if (error == -1)
656				goto conndied;
657			pipelined = 0;
658		}
659
660		if (fd != -1) {
661			close(fd);
662			fd = -1;
663		}
664
665		fprintf(stderr, "http://%s/%s: %d ", servername, argv[nres],
666		    statuscode);
667		if (statuscode == 200)
668			fprintf(stderr, "OK\n");
669		else if (statuscode < 300)
670			fprintf(stderr, "Successful (ignored)\n");
671		else if (statuscode < 400)
672			fprintf(stderr, "Redirection (ignored)\n");
673		else
674			fprintf(stderr, "Error (ignored)\n");
675
676		/* We've finished this file! */
677		nres++;
678
679		/*
680		 * If necessary, clean up this connection so that we
681		 * can start a new one.
682		 */
683		if (pipelined == 0 && keepalive == 0)
684			goto cleanupconn;
685		continue;
686
687conndied:
688		/*
689		 * Something went wrong -- our connection died, the server
690		 * sent us garbage, etc.  If this happened on the first
691		 * request we sent over this connection, give up.  Otherwise,
692		 * close this connection, open a new one, and reissue the
693		 * request.
694		 */
695		if (nres == firstreq)
696			errx(1, "Connection failure");
697
698cleanupconn:
699		/*
700		 * Clean up our connection and keep on going
701		 */
702		shutdown(sd, SHUT_RDWR);
703		close(sd);
704		sd = -1;
705		if (fd != -1) {
706			close(fd);
707			fd = -1;
708		}
709		if (reqbuf != NULL) {
710			free(reqbuf);
711			reqbuf = NULL;
712		}
713		nreq = nres;
714		res = res0;
715		pipelined = 0;
716		resbufpos = resbuflen = 0;
717		continue;
718	}
719
720	free(resbuf);
721	freeaddrinfo(res0);
722
723	return 0;
724}
725