http.c revision 63842
122347Spst/*- 222347Spst * Copyright (c) 2000 Dag-Erling Co�dan Sm�rgrav 329967Sache * All rights reserved. 492914Smarkm * 522347Spst * Redistribution and use in source and binary forms, with or without 622347Spst * modification, are permitted provided that the following conditions 722347Spst * are met: 822347Spst * 1. Redistributions of source code must retain the above copyright 922347Spst * notice, this list of conditions and the following disclaimer 1022347Spst * in this position and unchanged. 1122347Spst * 2. Redistributions in binary form must reproduce the above copyright 1222347Spst * notice, this list of conditions and the following disclaimer in the 1322347Spst * documentation and/or other materials provided with the distribution. 1422347Spst * 3. The name of the author may not be used to endorse or promote products 1522347Spst * derived from this software without specific prior written permission. 1622347Spst * 1722347Spst * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1822347Spst * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1922347Spst * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 2022347Spst * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 2122347Spst * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2222347Spst * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2322347Spst * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2422347Spst * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2522347Spst * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2622347Spst * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2722347Spst * 2859121Skris * $FreeBSD: head/lib/libfetch/http.c 63842 2000-07-25 11:45:38Z des $ 2922347Spst */ 3022347Spst 3122347Spst/* 3222347Spst * The following copyright applies to the base64 code: 3322347Spst * 3422347Spst *- 3522347Spst * Copyright 1997 Massachusetts Institute of Technology 3622347Spst * 3722347Spst * Permission to use, copy, modify, and distribute this software and 3822347Spst * its documentation for any purpose and without fee is hereby 3922347Spst * granted, provided that both the above copyright notice and this 4022347Spst * permission notice appear in all copies, that both the above 4122347Spst * copyright notice and this permission notice appear in all 4222347Spst * supporting documentation, and that the name of M.I.T. not be used 4322347Spst * in advertising or publicity pertaining to distribution of the 4422347Spst * software without specific, written prior permission. M.I.T. makes 4522347Spst * no representations about the suitability of this software for any 4622347Spst * purpose. It is provided "as is" without express or implied 4722347Spst * warranty. 4822347Spst * 4922347Spst * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS 5022347Spst * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE, 5122347Spst * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 5222347Spst * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT 5322347Spst * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 5422347Spst * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 5522347Spst * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF 5622347Spst * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 5722347Spst * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 5822347Spst * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 5922347Spst * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 6022347Spst * SUCH DAMAGE. 6122347Spst */ 6222347Spst 6322347Spst#include <sys/param.h> 6422347Spst#include <sys/socket.h> 6522347Spst 6622347Spst#include <ctype.h> 6722347Spst#include <err.h> 6822347Spst#include <errno.h> 6922347Spst#include <locale.h> 7022347Spst#include <netdb.h> 7122347Spst#include <stdarg.h> 7222347Spst#include <stdio.h> 7322347Spst#include <stdlib.h> 7422347Spst#include <string.h> 7522347Spst#include <time.h> 7622347Spst#include <unistd.h> 7722347Spst 7822347Spst#include "fetch.h" 7922347Spst#include "common.h" 8022347Spst#include "httperr.h" 8122347Spst 8222347Spstextern char *__progname; /* XXX not portable */ 8322347Spst 8422347Spst/* Maximum number of redirects to follow */ 8522347Spst#define MAX_REDIRECT 5 8622347Spst 8722347Spst/* Symbolic names for reply codes we care about */ 8822347Spst#define HTTP_OK 200 8922347Spst#define HTTP_PARTIAL 206 9022347Spst#define HTTP_MOVED_PERM 301 9122347Spst#define HTTP_MOVED_TEMP 302 9222347Spst#define HTTP_SEE_OTHER 303 9322347Spst#define HTTP_NEED_AUTH 401 9479634Sru#define HTTP_NEED_PROXY_AUTH 403 9522347Spst#define HTTP_PROTOCOL_ERROR 999 9622347Spst 9722347Spst#define HTTP_REDIRECT(xyz) ((xyz) == HTTP_MOVED_PERM \ 9822347Spst || (xyz) == HTTP_MOVED_TEMP \ 9922347Spst || (xyz) == HTTP_SEE_OTHER) 10022347Spst 10122347Spst 10222347Spst 10322347Spst/***************************************************************************** 10422347Spst * I/O functions for decoding chunked streams 10522347Spst */ 10622347Spst 10722347Spststruct cookie 10822347Spst{ 10922347Spst int fd; 11022347Spst char *buf; 11122347Spst size_t b_size; 11222347Spst size_t b_len; 11322347Spst int b_pos; 11422347Spst int eof; 11522347Spst int error; 11622347Spst long chunksize; 11722347Spst#ifndef NDEBUG 11822347Spst long total; 11922347Spst#endif 12022347Spst}; 12122347Spst 12222347Spst/* 12322347Spst * Get next chunk header 12422347Spst */ 12522347Spststatic int 12622347Spst_http_new_chunk(struct cookie *c) 12722347Spst{ 12822347Spst char *p; 12922347Spst 13022347Spst if (_fetch_getln(c->fd, &c->buf, &c->b_size, &c->b_len) == -1) 13122347Spst return -1; 13222347Spst 13322347Spst if (c->b_len < 2 || !ishexnumber(*c->buf)) 13422347Spst return -1; 13522347Spst 13622347Spst for (p = c->buf; !isspace(*p) && *p != ';' && p < c->buf + c->b_len; ++p) 13722347Spst if (!ishexnumber(*p)) 13822347Spst return -1; 13922347Spst else if (isdigit(*p)) 14022347Spst c->chunksize = c->chunksize * 16 + *p - '0'; 14122347Spst else 14222347Spst c->chunksize = c->chunksize * 16 + 10 + tolower(*p) - 'a'; 14322347Spst 14422347Spst#ifndef NDEBUG 14522347Spst c->total += c->chunksize; 14622347Spst if (c->chunksize == 0) 14722347Spst fprintf(stderr, "\033[1m_http_fillbuf(): " 14822347Spst "end of last chunk\033[m\n"); 14922347Spst else 15022347Spst fprintf(stderr, "\033[1m_http_fillbuf(): " 15122347Spst "new chunk: %ld (%ld)\033[m\n", c->chunksize, c->total); 15222347Spst#endif 15322372Spst 15422372Spst return c->chunksize; 15522347Spst} 15622347Spst 15722347Spst/* 15822347Spst * Fill the input buffer, do chunk decoding on the fly 15922347Spst */ 160187920Sgaborstatic int 16122347Spst_http_fillbuf(struct cookie *c) 16222347Spst{ 16322347Spst if (c->error) 16422347Spst return -1; 16522347Spst if (c->eof) 16622347Spst return 0; 16722347Spst 16822347Spst if (c->chunksize == 0) { 16922347Spst switch (_http_new_chunk(c)) { 17022347Spst case -1: 17122347Spst c->error = 1; 17222347Spst return -1; 17322347Spst case 0: 174 c->eof = 1; 175 return 0; 176 } 177 } 178 179 if (c->b_size < c->chunksize) { 180 char *tmp; 181 182 if ((tmp = realloc(c->buf, c->chunksize)) == NULL) 183 return -1; 184 c->buf = tmp; 185 c->b_size = c->chunksize; 186 } 187 188 if ((c->b_len = read(c->fd, c->buf, c->chunksize)) == -1) 189 return -1; 190 c->chunksize -= c->b_len; 191 192 if (c->chunksize == 0) { 193 char endl[2]; 194 read(c->fd, endl, 2); 195 } 196 197 c->b_pos = 0; 198 199 return c->b_len; 200} 201 202/* 203 * Read function 204 */ 205static int 206_http_readfn(void *v, char *buf, int len) 207{ 208 struct cookie *c = (struct cookie *)v; 209 int l, pos; 210 211 if (c->error) 212 return -1; 213 if (c->eof) 214 return 0; 215 216 for (pos = 0; len > 0; pos += l, len -= l) { 217 /* empty buffer */ 218 if (!c->buf || c->b_pos == c->b_len) 219 if (_http_fillbuf(c) < 1) 220 break; 221 l = c->b_len - c->b_pos; 222 if (len < l) 223 l = len; 224 bcopy(c->buf + c->b_pos, buf + pos, l); 225 c->b_pos += l; 226 } 227 228 if (!pos && c->error) 229 return -1; 230 return pos; 231} 232 233/* 234 * Write function 235 */ 236static int 237_http_writefn(void *v, const char *buf, int len) 238{ 239 struct cookie *c = (struct cookie *)v; 240 241 return write(c->fd, buf, len); 242} 243 244/* 245 * Close function 246 */ 247static int 248_http_closefn(void *v) 249{ 250 struct cookie *c = (struct cookie *)v; 251 int r; 252 253 r = close(c->fd); 254 if (c->buf) 255 free(c->buf); 256 free(c); 257 return r; 258} 259 260/* 261 * Wrap a file descriptor up 262 */ 263static FILE * 264_http_funopen(int fd) 265{ 266 struct cookie *c; 267 FILE *f; 268 269 if ((c = calloc(1, sizeof *c)) == NULL) { 270 _fetch_syserr(); 271 return NULL; 272 } 273 c->fd = fd; 274 if (!(f = funopen(c, _http_readfn, _http_writefn, NULL, _http_closefn))) { 275 _fetch_syserr(); 276 free(c); 277 return NULL; 278 } 279 return f; 280} 281 282 283/***************************************************************************** 284 * Helper functions for talking to the server and parsing its replies 285 */ 286 287/* Header types */ 288typedef enum { 289 hdr_syserror = -2, 290 hdr_error = -1, 291 hdr_end = 0, 292 hdr_unknown = 1, 293 hdr_content_length, 294 hdr_content_range, 295 hdr_last_modified, 296 hdr_location, 297 hdr_transfer_encoding 298} hdr; 299 300/* Names of interesting headers */ 301static struct { 302 hdr num; 303 char *name; 304} hdr_names[] = { 305 { hdr_content_length, "Content-Length" }, 306 { hdr_content_range, "Content-Range" }, 307 { hdr_last_modified, "Last-Modified" }, 308 { hdr_location, "Location" }, 309 { hdr_transfer_encoding, "Transfer-Encoding" }, 310 { hdr_unknown, NULL }, 311}; 312 313static char *reply_buf; 314static size_t reply_size; 315static size_t reply_length; 316 317/* 318 * Send a formatted line; optionally echo to terminal 319 */ 320static int 321_http_cmd(int fd, char *fmt, ...) 322{ 323 va_list ap; 324 size_t len; 325 char *msg; 326 int r; 327 328 va_start(ap, fmt); 329 len = vasprintf(&msg, fmt, ap); 330 va_end(ap); 331 332 if (msg == NULL) { 333 errno = ENOMEM; 334 _fetch_syserr(); 335 return -1; 336 } 337 338 r = _fetch_putln(fd, msg, len); 339 free(msg); 340 341 if (r == -1) { 342 _fetch_syserr(); 343 return -1; 344 } 345 346 return 0; 347} 348 349/* 350 * Get and parse status line 351 */ 352static int 353_http_get_reply(int fd) 354{ 355 if (_fetch_getln(fd, &reply_buf, &reply_size, &reply_length) == -1) 356 return -1; 357 /* 358 * A valid status line looks like "HTTP/m.n xyz reason" where m 359 * and n are the major and minor protocol version numbers and xyz 360 * is the reply code. 361 * We grok HTTP 1.0 and 1.1, so m must be 1 and n must be 0 or 1. 362 * We don't care about the reason phrase. 363 */ 364 if (strncmp(reply_buf, "HTTP/1.", 7) != 0 365 || (reply_buf[7] != '0' && reply_buf[7] != '1') || reply_buf[8] != ' ' 366 || !isdigit(reply_buf[9]) 367 || !isdigit(reply_buf[10]) 368 || !isdigit(reply_buf[11])) 369 return HTTP_PROTOCOL_ERROR; 370 371 return ((reply_buf[9] - '0') * 100 372 + (reply_buf[10] - '0') * 10 373 + (reply_buf[11] - '0')); 374} 375 376/* 377 * Check a header; if the type matches the given string, return a 378 * pointer to the beginning of the value. 379 */ 380static char * 381_http_match(char *str, char *hdr) 382{ 383 while (*str && *hdr && tolower(*str++) == tolower(*hdr++)) 384 /* nothing */; 385 if (*str || *hdr != ':') 386 return NULL; 387 while (*hdr && isspace(*++hdr)) 388 /* nothing */; 389 return hdr; 390} 391 392/* 393 * Get the next header and return the appropriate symbolic code. 394 */ 395static hdr 396_http_next_header(int fd, char **p) 397{ 398 int i; 399 400 if (_fetch_getln(fd, &reply_buf, &reply_size, &reply_length) == -1) 401 return hdr_syserror; 402 while (reply_length && isspace(reply_buf[reply_length-1])) 403 reply_length--; 404 reply_buf[reply_length] = 0; 405 if (reply_length == 0) 406 return hdr_end; 407 /* 408 * We could check for malformed headers but we don't really care. 409 * A valid header starts with a token immediately followed by a 410 * colon; a token is any sequence of non-control, non-whitespace 411 * characters except "()<>@,;:\\\"{}". 412 */ 413 for (i = 0; hdr_names[i].num != hdr_unknown; i++) 414 if ((*p = _http_match(hdr_names[i].name, reply_buf)) != NULL) 415 return hdr_names[i].num; 416 return hdr_unknown; 417} 418 419/* 420 * Parse a last-modified header 421 */ 422static int 423_http_parse_mtime(char *p, time_t *mtime) 424{ 425 char locale[64], *r; 426 struct tm tm; 427 428 strncpy(locale, setlocale(LC_TIME, NULL), sizeof locale); 429 setlocale(LC_TIME, "C"); 430 r = strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm); 431 /* XXX should add support for date-2 and date-3 */ 432 setlocale(LC_TIME, locale); 433 if (r == NULL) 434 return -1; 435 DEBUG(fprintf(stderr, "last modified: [\033[1m%04d-%02d-%02d " 436 "%02d:%02d:%02d\033[m]\n", 437 tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, 438 tm.tm_hour, tm.tm_min, tm.tm_sec)); 439 *mtime = timegm(&tm); 440 return 0; 441} 442 443/* 444 * Parse a content-length header 445 */ 446static int 447_http_parse_length(char *p, size_t *length) 448{ 449 size_t len; 450 451 for (len = 0; *p && isdigit(*p); ++p) 452 len = len * 10 + (*p - '0'); 453 DEBUG(fprintf(stderr, "content length: [\033[1m%d\033[m]\n", len)); 454 *length = len; 455 return 0; 456} 457 458/* 459 * Parse a content-range header 460 */ 461static int 462_http_parse_range(char *p, off_t *offset, size_t *length, size_t *size) 463{ 464 int first, last, len; 465 466 if (strncasecmp(p, "bytes ", 6) != 0) 467 return -1; 468 for (first = 0, p += 6; *p && isdigit(*p); ++p) 469 first = first * 10 + *p - '0'; 470 if (*p != '-') 471 return -1; 472 for (last = 0, ++p; *p && isdigit(*p); ++p) 473 last = last * 10 + *p - '0'; 474 if (first > last || *p != '/') 475 return -1; 476 for (len = 0, ++p; *p && isdigit(*p); ++p) 477 len = len * 10 + *p - '0'; 478 if (len < last - first + 1) 479 return -1; 480 DEBUG(fprintf(stderr, "content range: [\033[1m%d-%d/%d\033[m]\n", 481 first, last, len)); 482 *offset = first; 483 *length = last - first + 1; 484 *size = len; 485 return 0; 486} 487 488 489/***************************************************************************** 490 * Helper functions for authorization 491 */ 492 493/* 494 * Base64 encoding 495 */ 496static char * 497_http_base64(char *src) 498{ 499 static const char base64[] = 500 "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 501 "abcdefghijklmnopqrstuvwxyz" 502 "0123456789+/"; 503 char *str, *dst; 504 size_t l; 505 int t, r; 506 507 l = strlen(src); 508 if ((str = malloc(((l + 2) / 3) * 4)) == NULL) 509 return NULL; 510 dst = str; 511 r = 0; 512 513 while (l >= 3) { 514 t = (src[0] << 16) | (src[1] << 8) | src[2]; 515 dst[0] = base64[(t >> 18) & 0x3f]; 516 dst[1] = base64[(t >> 12) & 0x3f]; 517 dst[2] = base64[(t >> 6) & 0x3f]; 518 dst[3] = base64[(t >> 0) & 0x3f]; 519 src += 3; l -= 3; 520 dst += 4; r += 4; 521 } 522 523 switch (l) { 524 case 2: 525 t = (src[0] << 16) | (src[1] << 8); 526 dst[0] = base64[(t >> 18) & 0x3f]; 527 dst[1] = base64[(t >> 12) & 0x3f]; 528 dst[2] = base64[(t >> 6) & 0x3f]; 529 dst[3] = '='; 530 dst += 4; 531 r += 4; 532 break; 533 case 1: 534 t = src[0] << 16; 535 dst[0] = base64[(t >> 18) & 0x3f]; 536 dst[1] = base64[(t >> 12) & 0x3f]; 537 dst[2] = dst[3] = '='; 538 dst += 4; 539 r += 4; 540 break; 541 case 0: 542 break; 543 } 544 545 *dst = 0; 546 return str; 547} 548 549/* 550 * Encode username and password 551 */ 552static int 553_http_basic_auth(int fd, char *hdr, char *usr, char *pwd) 554{ 555 char *upw, *auth; 556 int r; 557 558 if (asprintf(&upw, "%s:%s", usr, pwd) == -1) 559 return -1; 560 auth = _http_base64(upw); 561 free(upw); 562 if (auth == NULL) 563 return -1; 564 r = _http_cmd(fd, "%s: Basic %s", hdr, auth); 565 free(auth); 566 return r; 567} 568 569/* 570 * Send an authorization header 571 */ 572static int 573_http_authorize(int fd, char *hdr, char *p) 574{ 575 /* basic authorization */ 576 if (strncasecmp(p, "basic:", 6) == 0) { 577 char *user, *pwd, *str; 578 int r; 579 580 /* skip realm */ 581 for (p += 6; *p && *p != ':'; ++p) 582 /* nothing */ ; 583 if (!*p || strchr(++p, ':') == NULL) 584 return -1; 585 if ((str = strdup(p)) == NULL) 586 return -1; /* XXX */ 587 user = str; 588 pwd = strchr(str, ':'); 589 *pwd++ = '\0'; 590 r = _http_basic_auth(fd, hdr, user, pwd); 591 free(str); 592 return r; 593 } 594 return -1; 595} 596 597 598/***************************************************************************** 599 * Helper functions for connecting to a server or proxy 600 */ 601 602/* 603 * Return the default port for this scheme 604 */ 605static int 606_http_default_port(char *scheme) 607{ 608 struct servent *se; 609 610 if ((se = getservbyname(scheme, "tcp")) != NULL) 611 return ntohs(se->s_port); 612 if (strcasecmp(scheme, "ftp") == 0) 613 return FTP_DEFAULT_PORT; 614 if (strcasecmp(scheme, "http") == 0) 615 return HTTP_DEFAULT_PORT; 616 return 0; 617} 618 619/* 620 * Connect to the specified HTTP proxy server. 621 */ 622static int 623_http_proxy_connect(char *proxy, int af, int verbose) 624{ 625 char *hostname, *p; 626 int fd, port; 627 628 /* get hostname */ 629 hostname = NULL; 630#ifdef INET6 631 /* host part can be an IPv6 address enclosed in square brackets */ 632 if (*proxy == '[') { 633 if ((p = strchr(proxy, ']')) == NULL) { 634 /* no terminating bracket */ 635 /* XXX should set an error code */ 636 goto ouch; 637 } 638 if (p[1] != '\0' && p[1] != ':') { 639 /* garbage after address */ 640 /* XXX should set an error code */ 641 goto ouch; 642 } 643 if ((hostname = malloc(p - proxy)) == NULL) { 644 errno = ENOMEM; 645 _fetch_syserr(); 646 goto ouch; 647 } 648 strncpy(hostname, proxy + 1, p - proxy - 1); 649 hostname[p - proxy - 1] = '\0'; 650 ++p; 651 } else { 652#endif /* INET6 */ 653 if ((p = strchr(proxy, ':')) == NULL) 654 p = strchr(proxy, '\0'); 655 if ((hostname = malloc(p - proxy + 1)) == NULL) { 656 errno = ENOMEM; 657 _fetch_syserr(); 658 goto ouch; 659 } 660 strncpy(hostname, proxy, p - proxy); 661 hostname[p - proxy] = '\0'; 662#ifdef INET6 663 } 664#endif /* INET6 */ 665 DEBUG(fprintf(stderr, "proxy name: [%s]\n", hostname)); 666 667 /* get port number */ 668 port = 0; 669 if (*p == ':') { 670 ++p; 671 if (strspn(p, "0123456789") != strlen(p) || strlen(p) > 5) { 672 /* port number is non-numeric or too long */ 673 /* XXX should set an error code */ 674 goto ouch; 675 } 676 port = atoi(p); 677 if (port < 1 || port > 65535) { 678 /* port number is out of range */ 679 /* XXX should set an error code */ 680 goto ouch; 681 } 682 } 683 684 if (!port) { 685#if 0 686 /* 687 * commented out, since there is currently no service name 688 * for HTTP proxies 689 */ 690 struct servent *se; 691 692 if ((se = getservbyname("xxxx", "tcp")) != NULL) 693 port = ntohs(se->s_port); 694 else 695#endif 696 port = 3128; 697 } 698 DEBUG(fprintf(stderr, "proxy port: %d\n", port)); 699 700 /* connect */ 701 if ((fd = _fetch_connect(hostname, port, af, verbose)) == -1) 702 _fetch_syserr(); 703 return fd; 704 705 ouch: 706 if (hostname) 707 free(hostname); 708 return -1; 709} 710 711/* 712 * Connect to the correct HTTP server or proxy. 713 */ 714static int 715_http_connect(struct url *URL, int *proxy, char *flags) 716{ 717 int direct, verbose; 718 int af, fd; 719 char *p; 720 721#ifdef INET6 722 af = AF_UNSPEC; 723#else 724 af = AF_INET; 725#endif 726 727 direct = (flags && strchr(flags, 'd')); 728 verbose = (flags && strchr(flags, 'v')); 729 if (flags && strchr(flags, '4')) 730 af = AF_INET; 731 else if (flags && strchr(flags, '6')) 732 af = AF_INET6; 733 734 /* check port */ 735 if (!URL->port) 736 URL->port = _http_default_port(URL->scheme); 737 738 if (!direct && (p = getenv("HTTP_PROXY")) != NULL && *p != '\0') { 739 /* attempt to connect to proxy server */ 740 if ((fd = _http_proxy_connect(p, af, verbose)) == -1) 741 return -1; 742 *proxy = 1; 743 } else { 744 /* if no proxy is configured, try direct */ 745 if (strcasecmp(URL->scheme, "ftp") == 0) { 746 /* can't talk http to an ftp server */ 747 /* XXX should set an error code */ 748 return -1; 749 } 750 if ((fd = _fetch_connect(URL->host, URL->port, af, verbose)) == -1) 751 /* _fetch_connect() has already set an error code */ 752 return -1; 753 *proxy = 0; 754 } 755 756 return fd; 757} 758 759 760/***************************************************************************** 761 * Core 762 */ 763 764/* 765 * Send a request and process the reply 766 */ 767static FILE * 768_http_request(struct url *URL, char *op, struct url_stat *us, char *flags) 769{ 770 struct url *url, *new; 771 int chunked, need_auth, noredirect, proxy, verbose; 772 int code, fd, i, n; 773 off_t offset; 774 size_t clength, length, size; 775 time_t mtime; 776 char *p; 777 FILE *f; 778 hdr h; 779 char *host; 780#ifdef INET6 781 char hbuf[MAXHOSTNAMELEN + 1]; 782#endif 783 784 noredirect = (flags && strchr(flags, 'A')); 785 verbose = (flags && strchr(flags, 'v')); 786 787 /* try the provided URL first */ 788 url = URL; 789 790 /* if the A flag is set, we only get one try */ 791 n = noredirect ? 1 : MAX_REDIRECT; 792 i = 0; 793 794 do { 795 new = NULL; 796 chunked = 0; 797 need_auth = 0; 798 offset = 0; 799 clength = -1; 800 length = -1; 801 size = -1; 802 mtime = 0; 803 retry: 804 /* connect to server or proxy */ 805 if ((fd = _http_connect(url, &proxy, flags)) == -1) 806 goto ouch; 807 808 host = url->host; 809#ifdef INET6 810 if (strchr(url->host, ':')) { 811 snprintf(hbuf, sizeof(hbuf), "[%s]", url->host); 812 host = hbuf; 813 } 814#endif 815 816 /* send request */ 817 if (verbose) 818 _fetch_info("requesting %s://%s:%d%s", 819 url->scheme, host, url->port, url->doc); 820 if (proxy) { 821 _http_cmd(fd, "%s %s://%s:%d%s HTTP/1.1", 822 op, url->scheme, host, url->port, url->doc); 823 } else { 824 _http_cmd(fd, "%s %s HTTP/1.1", 825 op, url->doc); 826 } 827 828 /* proxy authorization */ 829 if (proxy && (p = getenv("HTTP_PROXY_AUTH")) != NULL && *p != '\0') 830 _http_authorize(fd, "Proxy-Authorization", p); 831 832 /* server authorization */ 833 if (need_auth) { 834 if (*url->user || *url->pwd) 835 _http_basic_auth(fd, "Authorization", 836 url->user ? url->user : "", 837 url->pwd ? url->pwd : ""); 838 else if ((p = getenv("HTTP_AUTH")) != NULL && *p != '\0') 839 _http_authorize(fd, "Authorization", p); 840 else { 841 _http_seterr(HTTP_NEED_AUTH); 842 goto ouch; 843 } 844 } 845 846 /* other headers */ 847 if (url->port == _http_default_port(url->scheme)) 848 _http_cmd(fd, "Host: %s", host); 849 else 850 _http_cmd(fd, "Host: %s:%d", host, url->port); 851 _http_cmd(fd, "User-Agent: %s " _LIBFETCH_VER, __progname); 852 if (url->offset) 853 _http_cmd(fd, "Range: bytes=%lld-", url->offset); 854 _http_cmd(fd, "Connection: close"); 855 _http_cmd(fd, ""); 856 857 /* get reply */ 858 switch ((code = _http_get_reply(fd))) { 859 case HTTP_OK: 860 case HTTP_PARTIAL: 861 /* fine */ 862 break; 863 case HTTP_MOVED_PERM: 864 case HTTP_MOVED_TEMP: 865 /* 866 * Not so fine, but we still have to read the headers to 867 * get the new location. 868 */ 869 break; 870 case HTTP_NEED_AUTH: 871 if (need_auth) { 872 /* 873 * We already sent out authorization code, so there's 874 * nothing more we can do. 875 */ 876 _http_seterr(code); 877 goto ouch; 878 } 879 /* try again, but send the password this time */ 880 if (verbose) 881 _fetch_info("server requires authorization"); 882 need_auth = 1; 883 close(fd); 884 goto retry; 885 case HTTP_NEED_PROXY_AUTH: 886 /* 887 * If we're talking to a proxy, we already sent our proxy 888 * authorization code, so there's nothing more we can do. 889 */ 890 _http_seterr(code); 891 goto ouch; 892 case HTTP_PROTOCOL_ERROR: 893 /* fall through */ 894 case -1: 895 _fetch_syserr(); 896 goto ouch; 897 default: 898 _http_seterr(code); 899 goto ouch; 900 } 901 902 /* get headers */ 903 do { 904 switch ((h = _http_next_header(fd, &p))) { 905 case hdr_syserror: 906 _fetch_syserr(); 907 goto ouch; 908 case hdr_error: 909 _http_seterr(HTTP_PROTOCOL_ERROR); 910 goto ouch; 911 case hdr_content_length: 912 _http_parse_length(p, &clength); 913 break; 914 case hdr_content_range: 915 _http_parse_range(p, &offset, &length, &size); 916 break; 917 case hdr_last_modified: 918 _http_parse_mtime(p, &mtime); 919 break; 920 case hdr_location: 921 if (!HTTP_REDIRECT(code)) 922 break; 923 if (new) 924 free(new); 925 if (verbose) 926 _fetch_info("%d redirect to %s", code, p); 927 if (*p == '/') 928 /* absolute path */ 929 new = fetchMakeURL(url->scheme, url->host, url->port, p, 930 url->user, url->pwd); 931 else 932 new = fetchParseURL(p); 933 if (new == NULL) { 934 /* XXX should set an error code */ 935 DEBUG(fprintf(stderr, "failed to parse new URL\n")); 936 goto ouch; 937 } 938 if (!*new->user && !*new->pwd) { 939 strcpy(new->user, url->user); 940 strcpy(new->pwd, url->pwd); 941 } 942 new->offset = url->offset; 943 new->length = url->length; 944 break; 945 case hdr_transfer_encoding: 946 /* XXX weak test*/ 947 chunked = (strcasecmp(p, "chunked") == 0); 948 break; 949 case hdr_end: 950 /* fall through */ 951 case hdr_unknown: 952 /* ignore */ 953 break; 954 } 955 } while (h > hdr_end); 956 957 /* we either have a hit, or a redirect with no Location: header */ 958 if (code == HTTP_OK || code == HTTP_PARTIAL || !new) 959 break; 960 961 /* we have a redirect */ 962 close(fd); 963 fd = -1; 964 if (url != URL) 965 fetchFreeURL(url); 966 url = new; 967 } while (++i < n); 968 969 /* no success */ 970 if (fd == -1) { 971 _http_seterr(code); 972 goto ouch; 973 } 974 975 DEBUG(fprintf(stderr, "offset: %lld, length: %d, size: %d, clength: %d\n", 976 offset, length, size, clength)); 977 978 /* check for inconsistencies */ 979 if (clength != -1 && length != -1 && clength != length) { 980 _http_seterr(HTTP_PROTOCOL_ERROR); 981 goto ouch; 982 } 983 if (clength == -1) 984 clength = length; 985 if (clength != -1) 986 length = offset + clength; 987 if (length != -1 && size != -1 && length != size) { 988 _http_seterr(HTTP_PROTOCOL_ERROR); 989 goto ouch; 990 } 991 if (size == -1) 992 size = length; 993 994 /* fill in stats */ 995 if (us) { 996 us->size = size; 997 us->atime = us->mtime = mtime; 998 } 999 1000 /* too far? */ 1001 if (offset > URL->offset) { 1002 _http_seterr(HTTP_PROTOCOL_ERROR); 1003 goto ouch; 1004 } 1005 1006 /* report back real offset and size */ 1007 URL->offset = offset; 1008 URL->length = clength; 1009 1010 /* wrap it up in a FILE */ 1011 if ((f = chunked ? _http_funopen(fd) : fdopen(fd, "r")) == NULL) { 1012 _fetch_syserr(); 1013 goto ouch; 1014 } 1015 1016 if (url != URL) 1017 fetchFreeURL(url); 1018 1019 return f; 1020 1021 ouch: 1022 if (url != URL) 1023 fetchFreeURL(url); 1024 if (fd != -1) 1025 close(fd); 1026 return NULL; 1027} 1028 1029 1030/***************************************************************************** 1031 * Entry points 1032 */ 1033 1034/* 1035 * Retrieve and stat a file by HTTP 1036 */ 1037FILE * 1038fetchXGetHTTP(struct url *URL, struct url_stat *us, char *flags) 1039{ 1040 return _http_request(URL, "GET", us, flags); 1041} 1042 1043/* 1044 * Retrieve a file by HTTP 1045 */ 1046FILE * 1047fetchGetHTTP(struct url *URL, char *flags) 1048{ 1049 return fetchXGetHTTP(URL, NULL, flags); 1050} 1051 1052/* 1053 * Store a file by HTTP 1054 */ 1055FILE * 1056fetchPutHTTP(struct url *URL, char *flags) 1057{ 1058 warnx("fetchPutHTTP(): not implemented"); 1059 return NULL; 1060} 1061 1062/* 1063 * Get an HTTP document's metadata 1064 */ 1065int 1066fetchStatHTTP(struct url *URL, struct url_stat *us, char *flags) 1067{ 1068 FILE *f; 1069 1070 if ((f = _http_request(URL, "HEAD", us, flags)) == NULL) 1071 return -1; 1072 fclose(f); 1073 return 0; 1074} 1075 1076/* 1077 * List a directory 1078 */ 1079struct url_ent * 1080fetchListHTTP(struct url *url, char *flags) 1081{ 1082 warnx("fetchListHTTP(): not implemented"); 1083 return NULL; 1084} 1085