1/*	$NetBSD: fetch.c,v 1.158 2005/05/14 15:26:43 lukem Exp $	*/
2
3/*-
4 * Copyright (c) 1997-2004 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Luke Mewburn.
9 *
10 * This code is derived from software contributed to The NetBSD Foundation
11 * by Scott Aaron Bamford.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 *    notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 *    notice, this list of conditions and the following disclaimer in the
20 *    documentation and/or other materials provided with the distribution.
21 * 3. All advertising materials mentioning features or use of this software
22 *    must display the following acknowledgement:
23 *	This product includes software developed by the NetBSD
24 *	Foundation, Inc. and its contributors.
25 * 4. Neither the name of The NetBSD Foundation nor the names of its
26 *    contributors may be used to endorse or promote products derived
27 *    from this software without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
30 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
31 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
32 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
33 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
34 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
35 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
36 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
37 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
38 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
39 * POSSIBILITY OF SUCH DAMAGE.
40 */
41
42#include <sys/cdefs.h>
43
44/*
45 * FTP User Program -- Command line file retrieval
46 */
47
48#include <sys/types.h>
49#include <sys/param.h>
50#include <sys/socket.h>
51#include <sys/stat.h>
52#include <sys/time.h>
53
54#include <netinet/in.h>
55
56#include <arpa/ftp.h>
57#include <arpa/inet.h>
58
59#include <ctype.h>
60#include <err.h>
61#include <errno.h>
62#include <netdb.h>
63#include <fcntl.h>
64#include <stdio.h>
65#include <stdlib.h>
66#include <string.h>
67#include <unistd.h>
68#include <time.h>
69#include <libutil.h>
70
71#include "ftp_var.h"
72#include "version.h"
73
74typedef enum {
75	UNKNOWN_URL_T=-1,
76	HTTP_URL_T,
77	FTP_URL_T,
78	FILE_URL_T,
79	CLASSIC_URL_T
80} url_t;
81
82void		aborthttp(int);
83#ifndef NO_AUTH
84static int	auth_url(const char *, char **, const char *, const char *);
85static void	base64_encode(const unsigned char *, size_t, unsigned char *);
86#endif
87static int	go_fetch(const char *);
88static int	fetch_ftp(const char *);
89static int	fetch_url(const char *, const char *, char *, char *);
90static const char *match_token(const char **, const char *);
91static int	parse_url(const char *, const char *, url_t *, char **,
92			    char **, char **, char **, in_port_t *, char **);
93static void	url_decode(char *);
94
95static int	redirect_loop;
96
97
98#define	STRNEQUAL(a,b)	(strncasecmp((a), (b), sizeof((b))-1) == 0)
99#define	ISLWS(x)	((x)=='\r' || (x)=='\n' || (x)==' ' || (x)=='\t')
100#define	SKIPLWS(x)	do { while (ISLWS((*x))) x++; } while (0)
101
102
103#define	ABOUT_URL	"about:"	/* propaganda */
104#define	FILE_URL	"file://"	/* file URL prefix */
105#define	FTP_URL		"ftp://"	/* ftp URL prefix */
106#define	HTTP_URL	"http://"	/* http URL prefix */
107
108
109/*
110 * Determine if token is the next word in buf (case insensitive).
111 * If so, advance buf past the token and any trailing LWS, and
112 * return a pointer to the token (in buf).  Otherwise, return NULL.
113 * token may be preceeded by LWS.
114 * token must be followed by LWS or NUL.  (I.e, don't partial match).
115 */
116static const char *
117match_token(const char **buf, const char *token)
118{
119	const char	*p, *orig;
120	size_t		tlen;
121
122	tlen = strlen(token);
123	p = *buf;
124	SKIPLWS(p);
125	orig = p;
126	if (strncasecmp(p, token, tlen) != 0)
127		return NULL;
128	p += tlen;
129	if (*p != '\0' && !ISLWS(*p))
130		return NULL;
131	SKIPLWS(p);
132	orig = *buf;
133	*buf = p;
134	return orig;
135}
136
137#ifndef NO_AUTH
138/*
139 * Generate authorization response based on given authentication challenge.
140 * Returns -1 if an error occurred, otherwise 0.
141 * Sets response to a malloc(3)ed string; caller should free.
142 */
143static int
144auth_url(const char *challenge, char **response, const char *guser,
145	const char *gpass)
146{
147	const char	*cp, *scheme;
148	char		*ep, *clear, *realm;
149	char		 user[BUFSIZ], *pass;
150	int		 rval;
151	size_t		 len, clen, rlen;
152
153	*response = NULL;
154	clear = realm = NULL;
155	rval = -1;
156	cp = challenge;
157	scheme = "Basic";	/* only support Basic authentication */
158
159	if (debug)
160		fprintf(ttyout, "auth_url: challenge `%s'\n", challenge);
161
162	if (! match_token(&cp, scheme)) {
163		warnx("Unsupported authentication challenge - `%s'",
164		    challenge);
165		goto cleanup_auth_url;
166	}
167
168#define	REALM "realm=\""
169	if (STRNEQUAL(cp, REALM))
170		cp += sizeof(REALM) - 1;
171	else {
172		warnx("Unsupported authentication challenge - `%s'",
173		    challenge);
174		goto cleanup_auth_url;
175	}
176/* XXX: need to improve quoted-string parsing to support \ quoting, etc. */
177	if ((ep = strchr(cp, '\"')) != NULL) {
178		size_t len = ep - cp;
179
180		realm = (char *)xmalloc(len + 1);
181		(void)strlcpy(realm, cp, len + 1);
182	} else {
183		warnx("Unsupported authentication challenge - `%s'",
184		    challenge);
185		goto cleanup_auth_url;
186	}
187
188	fprintf(ttyout, "Username for `%s': ", realm);
189	if (guser != NULL) {
190		(void)strlcpy(user, guser, sizeof(user));
191		fprintf(ttyout, "%s\n", user);
192	} else {
193		(void)fflush(ttyout);
194		if (fgets(user, sizeof(user) - 1, stdin) == NULL) {
195			clearerr(stdin);
196			goto cleanup_auth_url;
197		}
198		user[strlen(user) - 1] = '\0';
199	}
200	if (gpass != NULL)
201		pass = (char *)gpass;
202	else
203		pass = getpass("Password: ");
204
205	clen = strlen(user) + strlen(pass) + 2;	/* user + ":" + pass + "\0" */
206	clear = (char *)xmalloc(clen);
207	(void)strlcpy(clear, user, clen);
208	(void)strlcat(clear, ":", clen);
209	(void)strlcat(clear, pass, clen);
210	if (gpass == NULL)
211		memset(pass, 0, strlen(pass));
212
213						/* scheme + " " + enc + "\0" */
214	rlen = strlen(scheme) + 1 + (clen + 2) * 4 / 3 + 1;
215	*response = (char *)xmalloc(rlen);
216	(void)strlcpy(*response, scheme, rlen);
217	len = strlcat(*response, " ", rlen);
218			/* use  `clen - 1'  to not encode the trailing NUL */
219	base64_encode((unsigned char *)clear, clen - 1,
220	    (unsigned char *)*response + len);
221	memset(clear, 0, clen);
222	rval = 0;
223
224 cleanup_auth_url:
225	FREEPTR(clear);
226	FREEPTR(realm);
227	return (rval);
228}
229
230/*
231 * Encode len bytes starting at clear using base64 encoding into encoded,
232 * which should be at least ((len + 2) * 4 / 3 + 1) in size.
233 */
234static void
235base64_encode(const unsigned char *clear, size_t len, unsigned char *encoded)
236{
237	static const unsigned char enc[] =
238	    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
239	unsigned char	*cp;
240	int	 i;
241
242	cp = encoded;
243	for (i = 0; i < len; i += 3) {
244		*(cp++) = enc[((clear[i + 0] >> 2))];
245		*(cp++) = enc[((clear[i + 0] << 4) & 0x30)
246			    | ((clear[i + 1] >> 4) & 0x0f)];
247		*(cp++) = enc[((clear[i + 1] << 2) & 0x3c)
248			    | ((clear[i + 2] >> 6) & 0x03)];
249		*(cp++) = enc[((clear[i + 2]     ) & 0x3f)];
250	}
251	*cp = '\0';
252	while (i-- > len)
253		*(--cp) = '=';
254}
255#endif
256
257/*
258 * Decode %xx escapes in given string, `in-place'.
259 */
260static void
261url_decode(char *url)
262{
263	unsigned char *p, *q;
264
265	if (EMPTYSTRING(url))
266		return;
267	p = q = (unsigned char *)url;
268
269#define	HEXTOINT(x) (x - (isdigit(x) ? '0' : (islower(x) ? 'a' : 'A') - 10))
270	while (*p) {
271		if (p[0] == '%'
272		    && p[1] && isxdigit((unsigned char)p[1])
273		    && p[2] && isxdigit((unsigned char)p[2])) {
274			*q++ = HEXTOINT(p[1]) * 16 + HEXTOINT(p[2]);
275			p+=3;
276		} else
277			*q++ = *p++;
278	}
279	*q = '\0';
280}
281
282
283/*
284 * Parse URL of form:
285 *	<type>://[<user>[:<password>]@]<host>[:<port>][/<path>]
286 * Returns -1 if a parse error occurred, otherwise 0.
287 * It's the caller's responsibility to url_decode() the returned
288 * user, pass and path.
289 *
290 * Sets type to url_t, each of the given char ** pointers to a
291 * malloc(3)ed strings of the relevant section, and port to
292 * the number given, or ftpport if ftp://, or httpport if http://.
293 *
294 * If <host> is surrounded by `[' and ']', it's parsed as an
295 * IPv6 address (as per RFC 2732).
296 *
297 * XXX: this is not totally RFC 1738 compliant; <path> will have the
298 * leading `/' unless it's an ftp:// URL, as this makes things easier
299 * for file:// and http:// URLs. ftp:// URLs have the `/' between the
300 * host and the URL-path removed, but any additional leading slashes
301 * in the URL-path are retained (because they imply that we should
302 * later do "CWD" with a null argument).
303 *
304 * Examples:
305 *	 input URL			 output path
306 *	 ---------			 -----------
307 *	"ftp://host"			NULL
308 *	"http://host/"			NULL
309 *	"file://host/dir/file"		"dir/file"
310 *	"ftp://host/"			""
311 *	"ftp://host//"			NULL
312 *	"ftp://host//dir/file"		"/dir/file"
313 */
314static int
315parse_url(const char *url, const char *desc, url_t *type,
316		char **user, char **pass, char **host, char **port,
317		in_port_t *portnum, char **path)
318{
319	const char	*origurl;
320	char		*cp, *ep, *thost, *tport;
321	size_t		 len;
322
323	if (url == NULL || desc == NULL || type == NULL || user == NULL
324	    || pass == NULL || host == NULL || port == NULL || portnum == NULL
325	    || path == NULL)
326		errx(1, "parse_url: invoked with NULL argument!");
327
328	origurl = url;
329	*type = UNKNOWN_URL_T;
330	*user = *pass = *host = *port = *path = NULL;
331	*portnum = 0;
332	tport = NULL;
333
334	if (STRNEQUAL(url, HTTP_URL)) {
335		url += sizeof(HTTP_URL) - 1;
336		*type = HTTP_URL_T;
337		*portnum = HTTP_PORT;
338		tport = httpport;
339	} else if (STRNEQUAL(url, FTP_URL)) {
340		url += sizeof(FTP_URL) - 1;
341		*type = FTP_URL_T;
342		*portnum = FTP_PORT;
343		tport = ftpport;
344	} else if (STRNEQUAL(url, FILE_URL)) {
345		url += sizeof(FILE_URL) - 1;
346		*type = FILE_URL_T;
347	} else {
348		warnx("Invalid %s `%s'", desc, url);
349 cleanup_parse_url:
350		FREEPTR(*user);
351		FREEPTR(*pass);
352		FREEPTR(*host);
353		FREEPTR(*port);
354		FREEPTR(*path);
355		return (-1);
356	}
357
358	if (*url == '\0')
359		return (0);
360
361			/* find [user[:pass]@]host[:port] */
362	ep = strchr(url, '/');
363	if (ep == NULL)
364		thost = xstrdup(url);
365	else {
366		len = ep - url;
367		thost = (char *)xmalloc(len + 1);
368		(void)strlcpy(thost, url, len + 1);
369		if (*type == FTP_URL_T)	/* skip first / for ftp URLs */
370			ep++;
371		*path = xstrdup(ep);
372	}
373
374	cp = strchr(thost, '@');	/* look for user[:pass]@ in URLs */
375	if (cp != NULL) {
376		if (*type == FTP_URL_T)
377			anonftp = 0;	/* disable anonftp */
378		*user = thost;
379		*cp = '\0';
380		thost = xstrdup(cp + 1);
381		cp = strchr(*user, ':');
382		if (cp != NULL) {
383			*cp = '\0';
384			*pass = xstrdup(cp + 1);
385		}
386		url_decode(*user);
387		if (*pass)
388			url_decode(*pass);
389	}
390
391#ifdef INET6
392			/*
393			 * Check if thost is an encoded IPv6 address, as per
394			 * RFC 2732:
395			 *	`[' ipv6-address ']'
396			 */
397	if (*thost == '[') {
398		cp = thost + 1;
399		if ((ep = strchr(cp, ']')) == NULL ||
400		    (ep[1] != '\0' && ep[1] != ':')) {
401			warnx("Invalid address `%s' in %s `%s'",
402			    thost, desc, origurl);
403			goto cleanup_parse_url;
404		}
405		len = ep - cp;		/* change `[xyz]' -> `xyz' */
406		memmove(thost, thost + 1, len);
407		thost[len] = '\0';
408		if (! isipv6addr(thost)) {
409			warnx("Invalid IPv6 address `%s' in %s `%s'",
410			    thost, desc, origurl);
411			goto cleanup_parse_url;
412		}
413		cp = ep + 1;
414		if (*cp == ':')
415			cp++;
416		else
417			cp = NULL;
418	} else
419#endif /* INET6 */
420	    if ((cp = strchr(thost, ':')) != NULL)
421		*cp++ =  '\0';
422	*host = thost;
423
424			/* look for [:port] */
425	if (cp != NULL) {
426		long	nport;
427
428		nport = parseport(cp, -1);
429		if (nport == -1) {
430			warnx("Unknown port `%s' in %s `%s'",
431			    cp, desc, origurl);
432			goto cleanup_parse_url;
433		}
434		*portnum = nport;
435		tport = cp;
436	}
437
438	if (tport != NULL)
439		*port = xstrdup(tport);
440	if (*path == NULL)
441		*path = xstrdup("/");
442
443	if (debug)
444		fprintf(ttyout,
445		    "parse_url: user `%s' pass `%s' host %s port %s(%d) "
446		    "path `%s'\n",
447		    *user ? *user : "<null>", *pass ? *pass : "<null>",
448		    *host ? *host : "<null>", *port ? *port : "<null>",
449		    *portnum ? *portnum : -1, *path ? *path : "<null>");
450
451	return (0);
452}
453
454sigjmp_buf	httpabort;
455
456/*
457 * Retrieve URL, via a proxy if necessary, using HTTP.
458 * If proxyenv is set, use that for the proxy, otherwise try ftp_proxy or
459 * http_proxy as appropriate.
460 * Supports HTTP redirects.
461 * Returns 1 on failure, 0 on completed xfer, -1 if ftp connection
462 * is still open (e.g, ftp xfer with trailing /)
463 */
464static int
465fetch_url(const char *url, const char *proxyenv, char *proxyauth, char *wwwauth)
466{
467	struct addrinfo		hints, *res, *res0 = NULL;
468	int			error;
469	char			hbuf[NI_MAXHOST];
470	volatile sigfunc	oldintr, oldintp;
471	volatile int		s;
472	struct stat		sb;
473	int			ischunked, isproxy, rval, hcode;
474	size_t			len;
475	static size_t		bufsize;
476	static char		*xferbuf;
477	const char		*cp, *token;
478	char			*ep, *buf, *savefile;
479	char			*auth, *location, *message;
480	char			*user, *pass, *host, *port, *path, *decodedpath;
481	char			*puser, *ppass, *useragent;
482	off_t			hashbytes, rangestart, rangeend, entitylen;
483	int			 (*closefunc)(FILE *);
484	FILE			*fin, *fout;
485	time_t			mtime;
486	url_t			urltype;
487	in_port_t		portnum;
488
489	oldintr = oldintp = NULL;
490	closefunc = NULL;
491	fin = fout = NULL;
492	s = -1;
493	buf = savefile = NULL;
494	auth = location = message = NULL;
495	ischunked = isproxy = hcode = 0;
496	rval = 1;
497	user = pass = host = path = decodedpath = puser = ppass = NULL;
498
499#ifdef __GNUC__			/* shut up gcc warnings */
500	(void)&closefunc;
501	(void)&fin;
502	(void)&fout;
503	(void)&buf;
504	(void)&savefile;
505	(void)&rval;
506	(void)&isproxy;
507	(void)&hcode;
508	(void)&ischunked;
509	(void)&message;
510	(void)&location;
511	(void)&auth;
512	(void)&decodedpath;
513#endif
514
515	if (parse_url(url, "URL", &urltype, &user, &pass, &host, &port,
516	    &portnum, &path) == -1)
517		goto cleanup_fetch_url;
518
519	if (urltype == FILE_URL_T && ! EMPTYSTRING(host)
520	    && strcasecmp(host, "localhost") != 0) {
521		warnx("No support for non local file URL `%s'", url);
522		goto cleanup_fetch_url;
523	}
524
525	if (EMPTYSTRING(path)) {
526		if (urltype == FTP_URL_T) {
527			rval = fetch_ftp(url);
528			goto cleanup_fetch_url;
529		}
530		if (urltype != HTTP_URL_T || outfile == NULL)  {
531			warnx("Invalid URL (no file after host) `%s'", url);
532			goto cleanup_fetch_url;
533		}
534	}
535
536	decodedpath = xstrdup(path);
537	url_decode(decodedpath);
538
539	if (outfile)
540		savefile = xstrdup(outfile);
541	else {
542		cp = strrchr(decodedpath, '/');		/* find savefile */
543		if (cp != NULL)
544			savefile = xstrdup(cp + 1);
545		else
546			savefile = xstrdup(decodedpath);
547	}
548	if (EMPTYSTRING(savefile)) {
549		if (urltype == FTP_URL_T) {
550			rval = fetch_ftp(url);
551			goto cleanup_fetch_url;
552		}
553		warnx("no file after directory (you must specify an "
554		    "output file) `%s'", url);
555		goto cleanup_fetch_url;
556	} else {
557		if (debug)
558			fprintf(ttyout, "savefile `%s'\n", savefile);
559	}
560
561	restart_point = 0;
562	filesize = -1;
563	rangestart = rangeend = entitylen = -1;
564	mtime = -1;
565	if (restartautofetch) {
566		if (strcmp(savefile, "-") != 0 && *savefile != '|' &&
567		    stat(savefile, &sb) == 0)
568			restart_point = sb.st_size;
569	}
570	if (urltype == FILE_URL_T) {		/* file:// URLs */
571		direction = "copied";
572		fin = fopen(decodedpath, "r");
573		if (fin == NULL) {
574			warn("Cannot open file `%s'", decodedpath);
575			goto cleanup_fetch_url;
576		}
577		if (fstat(fileno(fin), &sb) == 0) {
578			mtime = sb.st_mtime;
579			filesize = sb.st_size;
580		}
581		if (restart_point) {
582			if (lseek(fileno(fin), restart_point, SEEK_SET) < 0) {
583				warn("Can't lseek to restart `%s'",
584				    decodedpath);
585				goto cleanup_fetch_url;
586			}
587		}
588		if (verbose) {
589			fprintf(ttyout, "Copying %s", decodedpath);
590			if (restart_point)
591				fprintf(ttyout, " (restarting at " LLF ")",
592				    (LLT)restart_point);
593			fputs("\n", ttyout);
594		}
595	} else {				/* ftp:// or http:// URLs */
596		char *leading;
597		int hasleading;
598
599		if (proxyenv == NULL) {
600			if (urltype == HTTP_URL_T)
601				proxyenv = getoptionvalue("http_proxy");
602			else if (urltype == FTP_URL_T)
603				proxyenv = getoptionvalue("ftp_proxy");
604		}
605		direction = "retrieved";
606		if (! EMPTYSTRING(proxyenv)) {			/* use proxy */
607			url_t purltype;
608			char *phost, *ppath;
609			char *pport, *no_proxy;
610
611			isproxy = 1;
612
613				/* check URL against list of no_proxied sites */
614			no_proxy = getoptionvalue("no_proxy");
615			if (! EMPTYSTRING(no_proxy)) {
616				char *np, *np_copy;
617				long np_port;
618				size_t hlen, plen;
619
620				np_copy = xstrdup(no_proxy);
621				hlen = strlen(host);
622				while ((cp = strsep(&np_copy, " ,")) != NULL) {
623					if (*cp == '\0')
624						continue;
625					if ((np = strrchr(cp, ':')) != NULL) {
626						*np = '\0';
627						np_port =
628						    strtol(np + 1, &ep, 10);
629						if (*ep != '\0')
630							continue;
631						if (np_port != portnum)
632							continue;
633					}
634					plen = strlen(cp);
635					if (hlen < plen)
636						continue;
637					if (strncasecmp(host + hlen - plen,
638					    cp, plen) == 0) {
639						isproxy = 0;
640						break;
641					}
642				}
643				FREEPTR(np_copy);
644				if (isproxy == 0 && urltype == FTP_URL_T) {
645					rval = fetch_ftp(url);
646					goto cleanup_fetch_url;
647				}
648			}
649
650			if (isproxy) {
651				if (parse_url(proxyenv, "proxy URL", &purltype,
652				    &puser, &ppass, &phost, &pport, &portnum,
653				    &ppath) == -1)
654					goto cleanup_fetch_url;
655
656				if ((purltype != HTTP_URL_T
657				     && purltype != FTP_URL_T) ||
658				    EMPTYSTRING(phost) ||
659				    (! EMPTYSTRING(ppath)
660				     && strcmp(ppath, "/") != 0)) {
661					warnx("Malformed proxy URL `%s'",
662					    proxyenv);
663					FREEPTR(phost);
664					FREEPTR(pport);
665					FREEPTR(ppath);
666					goto cleanup_fetch_url;
667				}
668				if (isipv6addr(host) &&
669				    strchr(host, '%') != NULL) {
670					warnx(
671"Scoped address notation `%s' disallowed via web proxy",
672					    host);
673					FREEPTR(phost);
674					FREEPTR(pport);
675					FREEPTR(ppath);
676					goto cleanup_fetch_url;
677				}
678
679				FREEPTR(host);
680				host = phost;
681				FREEPTR(port);
682				port = pport;
683				FREEPTR(path);
684				path = xstrdup(url);
685				FREEPTR(ppath);
686			}
687		} /* ! EMPTYSTRING(proxyenv) */
688
689		memset(&hints, 0, sizeof(hints));
690		hints.ai_flags = 0;
691		hints.ai_family = family;
692		hints.ai_socktype = SOCK_STREAM;
693		hints.ai_protocol = 0;
694		error = getaddrinfo(host, NULL, &hints, &res0);
695		if (error) {
696			warnx("%s", gai_strerror(error));
697			goto cleanup_fetch_url;
698		}
699		if (res0->ai_canonname)
700			host = res0->ai_canonname;
701
702		s = -1;
703		for (res = res0; res; res = res->ai_next) {
704			/*
705			 * see comment in hookup()
706			 */
707			ai_unmapped(res);
708			if (getnameinfo(res->ai_addr, res->ai_addrlen,
709			    hbuf, sizeof(hbuf), NULL, 0, NI_NUMERICHOST) != 0)
710				strlcpy(hbuf, "invalid", sizeof(hbuf));
711
712			if (verbose && res != res0)
713				fprintf(ttyout, "Trying %s...\n", hbuf);
714
715			((struct sockaddr_in *)res->ai_addr)->sin_port =
716			    htons(portnum);
717			s = socket(res->ai_family, SOCK_STREAM,
718			    res->ai_protocol);
719			if (s < 0) {
720				warn("Can't create socket");
721				continue;
722			}
723
724			if (xconnect(s, res->ai_addr, res->ai_addrlen) < 0) {
725				warn("Connect to address `%s'", hbuf);
726				close(s);
727				s = -1;
728				continue;
729			}
730
731			/* success */
732			break;
733		}
734
735		if (s < 0) {
736			warn("Can't connect to %s", host);
737			goto cleanup_fetch_url;
738		}
739
740		fin = fdopen(s, "r+");
741		/*
742		 * Construct and send the request.
743		 */
744		if (verbose)
745			fprintf(ttyout, "Requesting %s\n", url);
746		leading = "  (";
747		hasleading = 0;
748		if (isproxy) {
749			if (verbose) {
750				fprintf(ttyout, "%svia %s:%s", leading,
751				    host, port);
752				leading = ", ";
753				hasleading++;
754			}
755			fprintf(fin, "GET %s HTTP/1.0\r\n", path);
756			if (flushcache)
757				fprintf(fin, "Pragma: no-cache\r\n");
758		} else {
759			fprintf(fin, "GET %s HTTP/1.1\r\n", path);
760			if (strchr(host, ':')) {
761				char *h, *p;
762
763				/*
764				 * strip off IPv6 scope identifier, since it is
765				 * local to the node
766				 */
767				h = xstrdup(host);
768				if (isipv6addr(h) &&
769				    (p = strchr(h, '%')) != NULL) {
770					*p = '\0';
771				}
772				fprintf(fin, "Host: [%s]", h);
773				free(h);
774			} else
775				fprintf(fin, "Host: %s", host);
776			if (portnum != HTTP_PORT)
777				fprintf(fin, ":%u", portnum);
778			fprintf(fin, "\r\n");
779			fprintf(fin, "Accept: */*\r\n");
780			fprintf(fin, "Connection: close\r\n");
781			if (restart_point) {
782				fputs(leading, ttyout);
783				fprintf(fin, "Range: bytes=" LLF "-\r\n",
784				    (LLT)restart_point);
785				fprintf(ttyout, "restarting at " LLF,
786				    (LLT)restart_point);
787				leading = ", ";
788				hasleading++;
789			}
790			if (flushcache)
791				fprintf(fin, "Cache-Control: no-cache\r\n");
792		}
793		if ((useragent=getenv("FTPUSERAGENT")) != NULL) {
794			fprintf(fin, "User-Agent: %s\r\n", useragent);
795		} else {
796			fprintf(fin, "User-Agent: %s/%s\r\n",
797			    FTP_PRODUCT, FTP_VERSION);
798		}
799		if (wwwauth) {
800			if (verbose) {
801				fprintf(ttyout, "%swith authorization",
802				    leading);
803				leading = ", ";
804				hasleading++;
805			}
806			fprintf(fin, "Authorization: %s\r\n", wwwauth);
807		}
808		if (proxyauth) {
809			if (verbose) {
810				fprintf(ttyout,
811				    "%swith proxy authorization", leading);
812				leading = ", ";
813				hasleading++;
814			}
815			fprintf(fin, "Proxy-Authorization: %s\r\n", proxyauth);
816		}
817		if (verbose && hasleading)
818			fputs(")\n", ttyout);
819		fprintf(fin, "\r\n");
820		if (fflush(fin) == EOF) {
821			warn("Writing HTTP request");
822			goto cleanup_fetch_url;
823		}
824
825				/* Read the response */
826		if ((buf = fparseln(fin, &len, NULL, "\0\0\0", 0)) == NULL) {
827			warn("Receiving HTTP reply");
828			goto cleanup_fetch_url;
829		}
830		while (len > 0 && (ISLWS(buf[len-1])))
831			buf[--len] = '\0';
832		if (debug)
833			fprintf(ttyout, "received `%s'\n", buf);
834
835				/* Determine HTTP response code */
836		cp = strchr(buf, ' ');
837		if (cp == NULL)
838			goto improper;
839		else
840			cp++;
841		hcode = strtol(cp, &ep, 10);
842		if (*ep != '\0' && !isspace((unsigned char)*ep))
843			goto improper;
844		message = xstrdup(cp);
845
846				/* Read the rest of the header. */
847		while (1) {
848			FREEPTR(buf);
849			if ((buf = fparseln(fin, &len, NULL, "\0\0\0", 0))
850			    == NULL) {
851				warn("Receiving HTTP reply");
852				goto cleanup_fetch_url;
853			}
854			while (len > 0 && (ISLWS(buf[len-1])))
855				buf[--len] = '\0';
856			if (len == 0)
857				break;
858			if (debug)
859				fprintf(ttyout, "received `%s'\n", buf);
860
861		/*
862		 * Look for some headers
863		 */
864
865			cp = buf;
866
867			if (match_token(&cp, "Content-Length:")) {
868				filesize = STRTOLL(cp, &ep, 10);
869				if (filesize < 0 || *ep != '\0')
870					goto improper;
871				if (debug)
872					fprintf(ttyout,
873					    "parsed len as: " LLF "\n",
874					    (LLT)filesize);
875
876			} else if (match_token(&cp, "Content-Range:")) {
877				if (! match_token(&cp, "bytes"))
878					goto improper;
879
880				if (*cp == '*')
881					cp++;
882				else {
883					rangestart = STRTOLL(cp, &ep, 10);
884					if (rangestart < 0 || *ep != '-')
885						goto improper;
886					cp = ep + 1;
887					rangeend = STRTOLL(cp, &ep, 10);
888					if (rangeend < 0 || rangeend < rangestart)
889						goto improper;
890					cp = ep;
891				}
892				if (*cp != '/')
893					goto improper;
894				cp++;
895				if (*cp == '*')
896					cp++;
897				else {
898					entitylen = STRTOLL(cp, &ep, 10);
899					if (entitylen < 0)
900						goto improper;
901					cp = ep;
902				}
903				if (*cp != '\0')
904					goto improper;
905
906				if (debug) {
907					fprintf(ttyout, "parsed range as: ");
908					if (rangestart == -1)
909						fprintf(ttyout, "*");
910					else
911						fprintf(ttyout, LLF "-" LLF,
912						    (LLT)rangestart,
913						    (LLT)rangeend);
914					fprintf(ttyout, "/" LLF "\n", (LLT)entitylen);
915				}
916				if (! restart_point) {
917					warnx(
918				    "Received unexpected Content-Range header");
919					goto cleanup_fetch_url;
920				}
921
922			} else if (match_token(&cp, "Last-Modified:")) {
923				struct tm parsed;
924				char *t;
925
926							/* RFC 1123 */
927				if ((t = strptime(cp,
928						"%a, %d %b %Y %H:%M:%S GMT",
929						&parsed))
930							/* RFC 850 */
931				    || (t = strptime(cp,
932						"%a, %d-%b-%y %H:%M:%S GMT",
933						&parsed))
934							/* asctime */
935				    || (t = strptime(cp,
936						"%a, %b %d %H:%M:%S %Y",
937						&parsed))) {
938					parsed.tm_isdst = -1;
939					if (*t == '\0')
940						mtime = timegm(&parsed);
941					if (debug && mtime != -1) {
942						fprintf(ttyout,
943						    "parsed date as: %s",
944						    ctime(&mtime));
945					}
946				}
947
948			} else if (match_token(&cp, "Location:")) {
949				location = xstrdup(cp);
950				if (debug)
951					fprintf(ttyout,
952					    "parsed location as `%s'\n", cp);
953
954			} else if (match_token(&cp, "Transfer-Encoding:")) {
955				if (match_token(&cp, "binary")) {
956					warnx(
957			"Bogus transfer encoding - `binary' (fetching anyway)");
958					continue;
959				}
960				if (! (token = match_token(&cp, "chunked"))) {
961					warnx(
962				    "Unsupported transfer encoding - `%s'",
963					    token);
964					goto cleanup_fetch_url;
965				}
966				ischunked++;
967				if (debug)
968					fprintf(ttyout,
969					    "using chunked encoding\n");
970
971			} else if (match_token(&cp, "Proxy-Authenticate:")
972				|| match_token(&cp, "WWW-Authenticate:")) {
973				if (! (token = match_token(&cp, "Basic"))) {
974					if (debug)
975						fprintf(ttyout,
976				"skipping unknown auth scheme `%s'\n",
977						    token);
978					continue;
979				}
980				FREEPTR(auth);
981				auth = xstrdup(token);
982				if (debug)
983					fprintf(ttyout,
984					    "parsed auth as `%s'\n", cp);
985			}
986
987		}
988				/* finished parsing header */
989		FREEPTR(buf);
990
991		switch (hcode) {
992		case 200:
993			break;
994		case 206:
995			if (! restart_point) {
996				warnx("Not expecting partial content header");
997				goto cleanup_fetch_url;
998			}
999			break;
1000		case 300:
1001		case 301:
1002		case 302:
1003		case 303:
1004		case 305:
1005			if (EMPTYSTRING(location)) {
1006				warnx(
1007				"No redirection Location provided by server");
1008				goto cleanup_fetch_url;
1009			}
1010			if (redirect_loop++ > 5) {
1011				warnx("Too many redirections requested");
1012				goto cleanup_fetch_url;
1013			}
1014			if (hcode == 305) {
1015				if (verbose)
1016					fprintf(ttyout, "Redirected via %s\n",
1017					    location);
1018				rval = fetch_url(url, location,
1019				    proxyauth, wwwauth);
1020			} else {
1021				if (verbose)
1022					fprintf(ttyout, "Redirected to %s\n",
1023					    location);
1024				rval = go_fetch(location);
1025			}
1026			goto cleanup_fetch_url;
1027#ifndef NO_AUTH
1028		case 401:
1029		case 407:
1030		    {
1031			char **authp;
1032			char *auser, *apass;
1033
1034			if (hcode == 401) {
1035				authp = &wwwauth;
1036				auser = user;
1037				apass = pass;
1038			} else {
1039				authp = &proxyauth;
1040				auser = puser;
1041				apass = ppass;
1042			}
1043			if (verbose || *authp == NULL ||
1044			    auser == NULL || apass == NULL)
1045				fprintf(ttyout, "%s\n", message);
1046			if (EMPTYSTRING(auth)) {
1047				warnx(
1048			    "No authentication challenge provided by server");
1049				goto cleanup_fetch_url;
1050			}
1051			if (*authp != NULL) {
1052				char reply[10];
1053
1054				fprintf(ttyout,
1055				    "Authorization failed. Retry (y/n)? ");
1056				if (fgets(reply, sizeof(reply), stdin)
1057				    == NULL) {
1058					clearerr(stdin);
1059					goto cleanup_fetch_url;
1060				}
1061				if (tolower((unsigned char)reply[0]) != 'y')
1062					goto cleanup_fetch_url;
1063				auser = NULL;
1064				apass = NULL;
1065			}
1066			if (auth_url(auth, authp, auser, apass) == 0) {
1067				rval = fetch_url(url, proxyenv,
1068				    proxyauth, wwwauth);
1069				memset(*authp, 0, strlen(*authp));
1070				FREEPTR(*authp);
1071			}
1072			goto cleanup_fetch_url;
1073		    }
1074#endif
1075		default:
1076			if (message)
1077				warnx("Error retrieving file - `%s'", message);
1078			else
1079				warnx("Unknown error retrieving file");
1080			goto cleanup_fetch_url;
1081		}
1082	}		/* end of ftp:// or http:// specific setup */
1083
1084			/* Open the output file. */
1085	if (strcmp(savefile, "-") == 0) {
1086		fout = stdout;
1087	} else if (*savefile == '|') {
1088		oldintp = xsignal(SIGPIPE, SIG_IGN);
1089		fout = popen(savefile + 1, "w");
1090		if (fout == NULL) {
1091			warn("Can't run `%s'", savefile + 1);
1092			goto cleanup_fetch_url;
1093		}
1094		closefunc = pclose;
1095	} else {
1096		if ((rangeend != -1 && rangeend <= restart_point) ||
1097		    (rangestart == -1 && filesize != -1 && filesize <= restart_point)) {
1098			/* already done */
1099			if (verbose)
1100				fprintf(ttyout, "already done\n");
1101			rval = 0;
1102			goto cleanup_fetch_url;
1103		}
1104		if (restart_point && rangestart != -1) {
1105			if (entitylen != -1)
1106				filesize = entitylen;
1107			if (rangestart != restart_point) {
1108				warnx(
1109				    "Size of `%s' differs from save file `%s'",
1110				    url, savefile);
1111				goto cleanup_fetch_url;
1112			}
1113			fout = fopen(savefile, "a");
1114		} else
1115			fout = fopen(savefile, "w");
1116		if (fout == NULL) {
1117			warn("Can't open `%s'", savefile);
1118			goto cleanup_fetch_url;
1119		}
1120		closefunc = fclose;
1121	}
1122
1123			/* Trap signals */
1124	if (sigsetjmp(httpabort, 1))
1125		goto cleanup_fetch_url;
1126	(void)xsignal(SIGQUIT, psummary);
1127	oldintr = xsignal(SIGINT, aborthttp);
1128
1129	if (rcvbuf_size > bufsize) {
1130		if (xferbuf)
1131			(void)free(xferbuf);
1132		bufsize = rcvbuf_size;
1133		xferbuf = xmalloc(bufsize);
1134	}
1135
1136	bytes = 0;
1137	hashbytes = mark;
1138	progressmeter(-1);
1139
1140			/* Finally, suck down the file. */
1141	do {
1142		long chunksize;
1143
1144		chunksize = 0;
1145					/* read chunksize */
1146		if (ischunked) {
1147			if (fgets(xferbuf, bufsize, fin) == NULL) {
1148				warnx("Unexpected EOF reading chunksize");
1149				goto cleanup_fetch_url;
1150			}
1151			chunksize = strtol(xferbuf, &ep, 16);
1152
1153				/*
1154				 * XXX:	Work around bug in Apache 1.3.9 and
1155				 *	1.3.11, which incorrectly put trailing
1156				 *	space after the chunksize.
1157				 */
1158			while (*ep == ' ')
1159				ep++;
1160
1161			if (strcmp(ep, "\r\n") != 0) {
1162				warnx("Unexpected data following chunksize");
1163				goto cleanup_fetch_url;
1164			}
1165			if (debug)
1166				fprintf(ttyout, "got chunksize of " LLF "\n",
1167				    (LLT)chunksize);
1168			if (chunksize == 0)
1169				break;
1170		}
1171					/* transfer file or chunk */
1172		while (1) {
1173			struct timeval then, now, td;
1174			off_t bufrem;
1175
1176			if (rate_get)
1177				(void)gettimeofday(&then, NULL);
1178			bufrem = rate_get ? rate_get : bufsize;
1179			if (ischunked)
1180				bufrem = MIN(chunksize, bufrem);
1181			while (bufrem > 0) {
1182				len = fread(xferbuf, sizeof(char),
1183				    MIN(bufsize, bufrem), fin);
1184				if (len <= 0)
1185					goto chunkdone;
1186				bytes += len;
1187				bufrem -= len;
1188				if (fwrite(xferbuf, sizeof(char), len, fout)
1189				    != len) {
1190					warn("Writing `%s'", savefile);
1191					goto cleanup_fetch_url;
1192				}
1193				if (hash && !progress) {
1194					while (bytes >= hashbytes) {
1195						(void)putc('#', ttyout);
1196						hashbytes += mark;
1197					}
1198					(void)fflush(ttyout);
1199				}
1200				if (ischunked) {
1201					chunksize -= len;
1202					if (chunksize <= 0)
1203						break;
1204				}
1205			}
1206			if (rate_get) {
1207				while (1) {
1208					(void)gettimeofday(&now, NULL);
1209					timersub(&now, &then, &td);
1210					if (td.tv_sec > 0)
1211						break;
1212					usleep(1000000 - td.tv_usec);
1213				}
1214			}
1215			if (ischunked && chunksize <= 0)
1216				break;
1217		}
1218					/* read CRLF after chunk*/
1219 chunkdone:
1220		if (ischunked) {
1221			if (fgets(xferbuf, bufsize, fin) == NULL)
1222				break;
1223			if (strcmp(xferbuf, "\r\n") != 0) {
1224				warnx("Unexpected data following chunk");
1225				goto cleanup_fetch_url;
1226			}
1227		}
1228	} while (ischunked);
1229	if (hash && !progress && bytes > 0) {
1230		if (bytes < mark)
1231			(void)putc('#', ttyout);
1232		(void)putc('\n', ttyout);
1233	}
1234	if (ferror(fin)) {
1235		warn("Reading file");
1236		goto cleanup_fetch_url;
1237	}
1238	progressmeter(1);
1239	(void)fflush(fout);
1240	if (closefunc == fclose && mtime != -1) {
1241		struct timeval tval[2];
1242
1243		(void)gettimeofday(&tval[0], NULL);
1244		tval[1].tv_sec = mtime;
1245		tval[1].tv_usec = 0;
1246		(*closefunc)(fout);
1247		fout = NULL;
1248
1249		if (utimes(savefile, tval) == -1) {
1250			fprintf(ttyout,
1251			    "Can't change modification time to %s",
1252			    asctime(localtime(&mtime)));
1253		}
1254	}
1255	if (bytes > 0)
1256		ptransfer(0);
1257	bytes = 0;
1258
1259	rval = 0;
1260	goto cleanup_fetch_url;
1261
1262 improper:
1263	warnx("Improper response from `%s'", host);
1264
1265 cleanup_fetch_url:
1266	if (oldintr)
1267		(void)xsignal(SIGINT, oldintr);
1268	if (oldintp)
1269		(void)xsignal(SIGPIPE, oldintp);
1270	if (fin != NULL)
1271		fclose(fin);
1272	else if (s != -1)
1273		close(s);
1274	if (closefunc != NULL && fout != NULL)
1275		(*closefunc)(fout);
1276	if (res0)
1277		freeaddrinfo(res0);
1278	FREEPTR(savefile);
1279	FREEPTR(user);
1280	FREEPTR(pass);
1281	FREEPTR(host);
1282	FREEPTR(port);
1283	FREEPTR(path);
1284	FREEPTR(decodedpath);
1285	FREEPTR(puser);
1286	FREEPTR(ppass);
1287	FREEPTR(buf);
1288	FREEPTR(auth);
1289	FREEPTR(location);
1290	FREEPTR(message);
1291	return (rval);
1292}
1293
1294/*
1295 * Abort a HTTP retrieval
1296 */
1297void
1298aborthttp(int notused)
1299{
1300	char msgbuf[100];
1301	int len;
1302
1303	sigint_raised = 1;
1304	alarmtimer(0);
1305	len = strlcpy(msgbuf, "\nHTTP fetch aborted.\n", sizeof(msgbuf));
1306	write(fileno(ttyout), msgbuf, len);
1307	siglongjmp(httpabort, 1);
1308}
1309
1310/*
1311 * Retrieve ftp URL or classic ftp argument using FTP.
1312 * Returns 1 on failure, 0 on completed xfer, -1 if ftp connection
1313 * is still open (e.g, ftp xfer with trailing /)
1314 */
1315static int
1316fetch_ftp(const char *url)
1317{
1318	char		*cp, *xargv[5], rempath[MAXPATHLEN];
1319	char		*host, *path, *dir, *file, *user, *pass;
1320	char		*port;
1321	int		 dirhasglob, filehasglob, oautologin, rval, type, xargc;
1322	in_port_t	 portnum;
1323	url_t		 urltype;
1324
1325	host = path = dir = file = user = pass = NULL;
1326	port = NULL;
1327	rval = 1;
1328	type = TYPE_I;
1329
1330	if (STRNEQUAL(url, FTP_URL)) {
1331		if ((parse_url(url, "URL", &urltype, &user, &pass,
1332		    &host, &port, &portnum, &path) == -1) ||
1333		    (user != NULL && *user == '\0') ||
1334		    EMPTYSTRING(host)) {
1335			warnx("Invalid URL `%s'", url);
1336			goto cleanup_fetch_ftp;
1337		}
1338		/*
1339		 * Note: Don't url_decode(path) here.  We need to keep the
1340		 * distinction between "/" and "%2F" until later.
1341		 */
1342
1343					/* check for trailing ';type=[aid]' */
1344		if (! EMPTYSTRING(path) && (cp = strrchr(path, ';')) != NULL) {
1345			if (strcasecmp(cp, ";type=a") == 0)
1346				type = TYPE_A;
1347			else if (strcasecmp(cp, ";type=i") == 0)
1348				type = TYPE_I;
1349			else if (strcasecmp(cp, ";type=d") == 0) {
1350				warnx(
1351			    "Directory listing via a URL is not supported");
1352				goto cleanup_fetch_ftp;
1353			} else {
1354				warnx("Invalid suffix `%s' in URL `%s'", cp,
1355				    url);
1356				goto cleanup_fetch_ftp;
1357			}
1358			*cp = 0;
1359		}
1360	} else {			/* classic style `[user@]host:[file]' */
1361		urltype = CLASSIC_URL_T;
1362		host = xstrdup(url);
1363		cp = strchr(host, '@');
1364		if (cp != NULL) {
1365			*cp = '\0';
1366			user = host;
1367			anonftp = 0;	/* disable anonftp */
1368			host = xstrdup(cp + 1);
1369		}
1370		cp = strchr(host, ':');
1371		if (cp != NULL) {
1372			*cp = '\0';
1373			path = xstrdup(cp + 1);
1374		}
1375	}
1376	if (EMPTYSTRING(host))
1377		goto cleanup_fetch_ftp;
1378
1379			/* Extract the file and (if present) directory name. */
1380	dir = path;
1381	if (! EMPTYSTRING(dir)) {
1382		/*
1383		 * If we are dealing with classic `[user@]host:[path]' syntax,
1384		 * then a path of the form `/file' (resulting from input of the
1385		 * form `host:/file') means that we should do "CWD /" before
1386		 * retrieving the file.  So we set dir="/" and file="file".
1387		 *
1388		 * But if we are dealing with URLs like `ftp://host/path' then
1389		 * a path of the form `/file' (resulting from a URL of the form
1390		 * `ftp://host//file') means that we should do `CWD ' (with an
1391		 * empty argument) before retrieving the file.  So we set
1392		 * dir="" and file="file".
1393		 *
1394		 * If the path does not contain / at all, we set dir=NULL.
1395		 * (We get a path without any slashes if we are dealing with
1396		 * classic `[user@]host:[file]' or URL `ftp://host/file'.)
1397		 *
1398		 * In all other cases, we set dir to a string that does not
1399		 * include the final '/' that separates the dir part from the
1400		 * file part of the path.  (This will be the empty string if
1401		 * and only if we are dealing with a path of the form `/file'
1402		 * resulting from an URL of the form `ftp://host//file'.)
1403		 */
1404		cp = strrchr(dir, '/');
1405		if (cp == dir && urltype == CLASSIC_URL_T) {
1406			file = cp + 1;
1407			dir = "/";
1408		} else if (cp != NULL) {
1409			*cp++ = '\0';
1410			file = cp;
1411		} else {
1412			file = dir;
1413			dir = NULL;
1414		}
1415	} else
1416		dir = NULL;
1417	if (urltype == FTP_URL_T && file != NULL) {
1418		url_decode(file);
1419		/* but still don't url_decode(dir) */
1420	}
1421	if (debug)
1422		fprintf(ttyout,
1423		    "fetch_ftp: user `%s' pass `%s' host %s port %s "
1424		    "path `%s' dir `%s' file `%s'\n",
1425		    user ? user : "<null>", pass ? pass : "<null>",
1426		    host ? host : "<null>", port ? port : "<null>",
1427		    path ? path : "<null>",
1428		    dir ? dir : "<null>", file ? file : "<null>");
1429
1430	dirhasglob = filehasglob = 0;
1431	if (doglob && urltype == CLASSIC_URL_T) {
1432		if (! EMPTYSTRING(dir) && strpbrk(dir, "*?[]{}") != NULL)
1433			dirhasglob = 1;
1434		if (! EMPTYSTRING(file) && strpbrk(file, "*?[]{}") != NULL)
1435			filehasglob = 1;
1436	}
1437
1438			/* Set up the connection */
1439	if (connected)
1440		disconnect(0, NULL);
1441	xargv[0] = (char *)getprogname();	/* XXX discards const */
1442	xargv[1] = host;
1443	xargv[2] = NULL;
1444	xargc = 2;
1445	if (port) {
1446		xargv[2] = port;
1447		xargv[3] = NULL;
1448		xargc = 3;
1449	}
1450	oautologin = autologin;
1451		/* don't autologin in setpeer(), use ftp_login() below */
1452	autologin = 0;
1453	setpeer(xargc, xargv);
1454	autologin = oautologin;
1455	if ((connected == 0) ||
1456	    (connected == 1 && !ftp_login(host, user, pass))) {
1457		warnx("Can't connect or login to host `%s'", host);
1458		goto cleanup_fetch_ftp;
1459	}
1460
1461	switch (type) {
1462	case TYPE_A:
1463		setascii(1, xargv);
1464		break;
1465	case TYPE_I:
1466		setbinary(1, xargv);
1467		break;
1468	default:
1469		errx(1, "fetch_ftp: unknown transfer type %d", type);
1470	}
1471
1472		/*
1473		 * Change directories, if necessary.
1474		 *
1475		 * Note: don't use EMPTYSTRING(dir) below, because
1476		 * dir=="" means something different from dir==NULL.
1477		 */
1478	if (dir != NULL && !dirhasglob) {
1479		char *nextpart;
1480
1481		/*
1482		 * If we are dealing with a classic `[user@]host:[path]'
1483		 * (urltype is CLASSIC_URL_T) then we have a raw directory
1484		 * name (not encoded in any way) and we can change
1485		 * directories in one step.
1486		 *
1487		 * If we are dealing with an `ftp://host/path' URL
1488		 * (urltype is FTP_URL_T), then RFC 1738 says we need to
1489		 * send a separate CWD command for each unescaped "/"
1490		 * in the path, and we have to interpret %hex escaping
1491		 * *after* we find the slashes.  It's possible to get
1492		 * empty components here, (from multiple adjacent
1493		 * slashes in the path) and RFC 1738 says that we should
1494		 * still do `CWD ' (with a null argument) in such cases.
1495		 *
1496		 * Many ftp servers don't support `CWD ', so if there's an
1497		 * error performing that command, bail out with a descriptive
1498		 * message.
1499		 *
1500		 * Examples:
1501		 *
1502		 * host:			dir="", urltype=CLASSIC_URL_T
1503		 *		logged in (to default directory)
1504		 * host:file			dir=NULL, urltype=CLASSIC_URL_T
1505		 *		"RETR file"
1506		 * host:dir/			dir="dir", urltype=CLASSIC_URL_T
1507		 *		"CWD dir", logged in
1508		 * ftp://host/			dir="", urltype=FTP_URL_T
1509		 *		logged in (to default directory)
1510		 * ftp://host/dir/		dir="dir", urltype=FTP_URL_T
1511		 *		"CWD dir", logged in
1512		 * ftp://host/file		dir=NULL, urltype=FTP_URL_T
1513		 *		"RETR file"
1514		 * ftp://host//file		dir="", urltype=FTP_URL_T
1515		 *		"CWD ", "RETR file"
1516		 * host:/file			dir="/", urltype=CLASSIC_URL_T
1517		 *		"CWD /", "RETR file"
1518		 * ftp://host///file		dir="/", urltype=FTP_URL_T
1519		 *		"CWD ", "CWD ", "RETR file"
1520		 * ftp://host/%2F/file		dir="%2F", urltype=FTP_URL_T
1521		 *		"CWD /", "RETR file"
1522		 * ftp://host/foo/file		dir="foo", urltype=FTP_URL_T
1523		 *		"CWD foo", "RETR file"
1524		 * ftp://host/foo/bar/file	dir="foo/bar"
1525		 *		"CWD foo", "CWD bar", "RETR file"
1526		 * ftp://host//foo/bar/file	dir="/foo/bar"
1527		 *		"CWD ", "CWD foo", "CWD bar", "RETR file"
1528		 * ftp://host/foo//bar/file	dir="foo//bar"
1529		 *		"CWD foo", "CWD ", "CWD bar", "RETR file"
1530		 * ftp://host/%2F/foo/bar/file	dir="%2F/foo/bar"
1531		 *		"CWD /", "CWD foo", "CWD bar", "RETR file"
1532		 * ftp://host/%2Ffoo/bar/file	dir="%2Ffoo/bar"
1533		 *		"CWD /foo", "CWD bar", "RETR file"
1534		 * ftp://host/%2Ffoo%2Fbar/file	dir="%2Ffoo%2Fbar"
1535		 *		"CWD /foo/bar", "RETR file"
1536		 * ftp://host/%2Ffoo%2Fbar%2Ffile	dir=NULL
1537		 *		"RETR /foo/bar/file"
1538		 *
1539		 * Note that we don't need `dir' after this point.
1540		 */
1541		do {
1542			if (urltype == FTP_URL_T) {
1543				nextpart = strchr(dir, '/');
1544				if (nextpart) {
1545					*nextpart = '\0';
1546					nextpart++;
1547				}
1548				url_decode(dir);
1549			} else
1550				nextpart = NULL;
1551			if (debug)
1552				fprintf(ttyout, "dir `%s', nextpart `%s'\n",
1553				    dir ? dir : "<null>",
1554				    nextpart ? nextpart : "<null>");
1555			if (urltype == FTP_URL_T || *dir != '\0') {
1556				xargv[0] = "cd";
1557				xargv[1] = dir;
1558				xargv[2] = NULL;
1559				dirchange = 0;
1560				cd(2, xargv);
1561				if (! dirchange) {
1562					if (*dir == '\0' && code == 500)
1563						fprintf(stderr,
1564"\n"
1565"ftp: The `CWD ' command (without a directory), which is required by\n"
1566"     RFC 1738 to support the empty directory in the URL pathname (`//'),\n"
1567"     conflicts with the server's conformance to RFC 959.\n"
1568"     Try the same URL without the `//' in the URL pathname.\n"
1569"\n");
1570					goto cleanup_fetch_ftp;
1571				}
1572			}
1573			dir = nextpart;
1574		} while (dir != NULL);
1575	}
1576
1577	if (EMPTYSTRING(file)) {
1578		rval = -1;
1579		goto cleanup_fetch_ftp;
1580	}
1581
1582	if (dirhasglob) {
1583		(void)strlcpy(rempath, dir,	sizeof(rempath));
1584		(void)strlcat(rempath, "/",	sizeof(rempath));
1585		(void)strlcat(rempath, file,	sizeof(rempath));
1586		file = rempath;
1587	}
1588
1589			/* Fetch the file(s). */
1590	xargc = 2;
1591	xargv[0] = "get";
1592	xargv[1] = file;
1593	xargv[2] = NULL;
1594	if (dirhasglob || filehasglob) {
1595		int ointeractive;
1596
1597		ointeractive = interactive;
1598		interactive = 0;
1599		if (restartautofetch)
1600			xargv[0] = "mreget";
1601		else
1602			xargv[0] = "mget";
1603		mget(xargc, xargv);
1604		interactive = ointeractive;
1605	} else {
1606		if (outfile == NULL) {
1607			cp = strrchr(file, '/');	/* find savefile */
1608			if (cp != NULL)
1609				outfile = cp + 1;
1610			else
1611				outfile = file;
1612		}
1613		xargv[2] = (char *)outfile;
1614		xargv[3] = NULL;
1615		xargc++;
1616		if (restartautofetch)
1617			reget(xargc, xargv);
1618		else
1619			get(xargc, xargv);
1620	}
1621
1622	if ((code / 100) == COMPLETE)
1623		rval = 0;
1624
1625 cleanup_fetch_ftp:
1626	FREEPTR(host);
1627	FREEPTR(path);
1628	FREEPTR(user);
1629	FREEPTR(pass);
1630	return (rval);
1631}
1632
1633/*
1634 * Retrieve the given file to outfile.
1635 * Supports arguments of the form:
1636 *	"host:path", "ftp://host/path"	if $ftpproxy, call fetch_url() else
1637 *					call fetch_ftp()
1638 *	"http://host/path"		call fetch_url() to use HTTP
1639 *	"file:///path"			call fetch_url() to copy
1640 *	"about:..."			print a message
1641 *
1642 * Returns 1 on failure, 0 on completed xfer, -1 if ftp connection
1643 * is still open (e.g, ftp xfer with trailing /)
1644 */
1645static int
1646go_fetch(const char *url)
1647{
1648	char *proxy;
1649
1650#ifndef NO_ABOUT
1651	/*
1652	 * Check for about:*
1653	 */
1654	if (STRNEQUAL(url, ABOUT_URL)) {
1655		url += sizeof(ABOUT_URL) -1;
1656		if (strcasecmp(url, "ftp") == 0 ||
1657		    strcasecmp(url, "tnftp") == 0) {
1658			fputs(
1659"This version of ftp has been enhanced by Luke Mewburn <lukem@NetBSD.org>\n"
1660"for the NetBSD project.  Execute `man ftp' for more details.\n", ttyout);
1661		} else if (strcasecmp(url, "lukem") == 0) {
1662			fputs(
1663"Luke Mewburn is the author of most of the enhancements in this ftp client.\n"
1664"Please email feedback to <lukem@NetBSD.org>.\n", ttyout);
1665		} else if (strcasecmp(url, "netbsd") == 0) {
1666			fputs(
1667"NetBSD is a freely available and redistributable UNIX-like operating system.\n"
1668"For more information, see http://www.NetBSD.org/\n", ttyout);
1669		} else if (strcasecmp(url, "version") == 0) {
1670			fprintf(ttyout, "Version: %s %s%s\n",
1671			    FTP_PRODUCT, FTP_VERSION,
1672#ifdef INET6
1673			    ""
1674#else
1675			    " (-IPv6)"
1676#endif
1677			);
1678		} else {
1679			fprintf(ttyout, "`%s' is an interesting topic.\n", url);
1680		}
1681		fputs("\n", ttyout);
1682		return (0);
1683	}
1684#endif
1685
1686	/*
1687	 * Check for file:// and http:// URLs.
1688	 */
1689	if (STRNEQUAL(url, HTTP_URL) || STRNEQUAL(url, FILE_URL))
1690		return (fetch_url(url, NULL, NULL, NULL));
1691
1692	/*
1693	 * Try FTP URL-style and host:file arguments next.
1694	 * If ftpproxy is set with an FTP URL, use fetch_url()
1695	 * Othewise, use fetch_ftp().
1696	 */
1697	proxy = getoptionvalue("ftp_proxy");
1698	if (!EMPTYSTRING(proxy) && STRNEQUAL(url, FTP_URL))
1699		return (fetch_url(url, NULL, NULL, NULL));
1700
1701	return (fetch_ftp(url));
1702}
1703
1704/*
1705 * Retrieve multiple files from the command line,
1706 * calling go_fetch() for each file.
1707 *
1708 * If an ftp path has a trailing "/", the path will be cd-ed into and
1709 * the connection remains open, and the function will return -1
1710 * (to indicate the connection is alive).
1711 * If an error occurs the return value will be the offset+1 in
1712 * argv[] of the file that caused a problem (i.e, argv[x]
1713 * returns x+1)
1714 * Otherwise, 0 is returned if all files retrieved successfully.
1715 */
1716int
1717auto_fetch(int argc, char *argv[])
1718{
1719	volatile int	argpos;
1720	int		rval;
1721
1722	argpos = 0;
1723
1724	if (sigsetjmp(toplevel, 1)) {
1725		if (connected)
1726			disconnect(0, NULL);
1727		if (rval > 0)
1728			rval = argpos + 1;
1729		return (rval);
1730	}
1731	(void)xsignal(SIGINT, intr);
1732	(void)xsignal(SIGPIPE, lostpeer);
1733
1734	/*
1735	 * Loop through as long as there's files to fetch.
1736	 */
1737	for (rval = 0; (rval == 0) && (argpos < argc); argpos++) {
1738		if (strchr(argv[argpos], ':') == NULL)
1739			break;
1740		redirect_loop = 0;
1741		if (!anonftp)
1742			anonftp = 2;	/* Handle "automatic" transfers. */
1743		rval = go_fetch(argv[argpos]);
1744		if (outfile != NULL && strcmp(outfile, "-") != 0
1745		    && outfile[0] != '|')
1746			outfile = NULL;
1747		if (rval > 0)
1748			rval = argpos + 1;
1749	}
1750
1751	if (connected && rval != -1)
1752		disconnect(0, NULL);
1753	return (rval);
1754}
1755
1756
1757int
1758auto_put(int argc, char **argv, const char *uploadserver)
1759{
1760	char	*uargv[4], *path, *pathsep;
1761	int	 uargc, rval, len;
1762
1763	uargc = 0;
1764	uargv[uargc++] = "mput";
1765	uargv[uargc++] = argv[0];
1766	uargv[2] = uargv[3] = NULL;
1767	pathsep = NULL;
1768	rval = 1;
1769
1770	if (debug)
1771		fprintf(ttyout, "auto_put: target `%s'\n", uploadserver);
1772
1773	path = xstrdup(uploadserver);
1774	len = strlen(path);
1775	if (path[len - 1] != '/' && path[len - 1] != ':') {
1776			/*
1777			 * make sure we always pass a directory to auto_fetch
1778			 */
1779		if (argc > 1) {		/* more than one file to upload */
1780			int len;
1781
1782			len = strlen(uploadserver) + 2;	/* path + "/" + "\0" */
1783			free(path);
1784			path = (char *)xmalloc(len);
1785			(void)strlcpy(path, uploadserver, len);
1786			(void)strlcat(path, "/", len);
1787		} else {		/* single file to upload */
1788			uargv[0] = "put";
1789			pathsep = strrchr(path, '/');
1790			if (pathsep == NULL) {
1791				pathsep = strrchr(path, ':');
1792				if (pathsep == NULL) {
1793					warnx("Invalid URL `%s'", path);
1794					goto cleanup_auto_put;
1795				}
1796				pathsep++;
1797				uargv[2] = xstrdup(pathsep);
1798				pathsep[0] = '/';
1799			} else
1800				uargv[2] = xstrdup(pathsep + 1);
1801			pathsep[1] = '\0';
1802			uargc++;
1803		}
1804	}
1805	if (debug)
1806		fprintf(ttyout, "auto_put: URL `%s' argv[2] `%s'\n",
1807		    path, uargv[2] ? uargv[2] : "<null>");
1808
1809			/* connect and cwd */
1810	rval = auto_fetch(1, &path);
1811	free(path);
1812	if(rval >= 0)
1813		goto cleanup_auto_put;
1814
1815			/* XXX : is this the best way? */
1816	if (uargc == 3) {
1817		uargv[1] = argv[0];
1818		put(uargc, uargv);
1819		goto cleanup_auto_put;
1820	}
1821
1822	for(; argv[0] != NULL; argv++) {
1823		uargv[1] = argv[0];
1824		mput(uargc, uargv);
1825	}
1826	rval = 0;
1827
1828 cleanup_auto_put:
1829	FREEPTR(uargv[2]);
1830	return (rval);
1831}
1832