http.c revision 63716
195584Sanholt/*- 295584Sanholt * Copyright (c) 2000 Dag-Erling Co�dan Sm�rgrav 395584Sanholt * All rights reserved. 495584Sanholt * 595584Sanholt * Redistribution and use in source and binary forms, with or without 695584Sanholt * modification, are permitted provided that the following conditions 795584Sanholt * are met: 895584Sanholt * 1. Redistributions of source code must retain the above copyright 995584Sanholt * notice, this list of conditions and the following disclaimer 1095584Sanholt * in this position and unchanged. 1195584Sanholt * 2. Redistributions in binary form must reproduce the above copyright 1295584Sanholt * notice, this list of conditions and the following disclaimer in the 1395584Sanholt * documentation and/or other materials provided with the distribution. 1495584Sanholt * 3. The name of the author may not be used to endorse or promote products 1595584Sanholt * derived from this software without specific prior written permission. 1695584Sanholt * 1795584Sanholt * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1895584Sanholt * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1995584Sanholt * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 2095584Sanholt * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 2195584Sanholt * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2295584Sanholt * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2395584Sanholt * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2495584Sanholt * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2595584Sanholt * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2695584Sanholt * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2795584Sanholt * 2895584Sanholt * $FreeBSD: head/lib/libfetch/http.c 63716 2000-07-21 11:02:43Z des $ 29112015Sanholt */ 3095584Sanholt 3195584Sanholt/* 3295584Sanholt * The following copyright applies to the base64 code: 3395584Sanholt * 3495584Sanholt *- 3595584Sanholt * Copyright 1997 Massachusetts Institute of Technology 3695584Sanholt * 3795584Sanholt * Permission to use, copy, modify, and distribute this software and 3895584Sanholt * its documentation for any purpose and without fee is hereby 3995584Sanholt * granted, provided that both the above copyright notice and this 4095584Sanholt * permission notice appear in all copies, that both the above 4195584Sanholt * copyright notice and this permission notice appear in all 4295584Sanholt * supporting documentation, and that the name of M.I.T. not be used 43112015Sanholt * in advertising or publicity pertaining to distribution of the 4495584Sanholt * software without specific, written prior permission. M.I.T. makes 4595584Sanholt * no representations about the suitability of this software for any 4695584Sanholt * purpose. It is provided "as is" without express or implied 4795584Sanholt * warranty. 4895584Sanholt * 4995584Sanholt * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS 5095584Sanholt * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE, 5195584Sanholt * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 5295584Sanholt * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT 5395584Sanholt * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 5495584Sanholt * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 5595584Sanholt * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF 5695584Sanholt * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 5795584Sanholt * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 5895584Sanholt * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 5995584Sanholt * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 6095584Sanholt * SUCH DAMAGE. 6195584Sanholt */ 6295584Sanholt 6395584Sanholt#include <sys/param.h> 6495584Sanholt#include <sys/socket.h> 6595584Sanholt 6695584Sanholt#include <ctype.h> 6795584Sanholt#include <err.h> 6895584Sanholt#include <errno.h> 6995584Sanholt#include <locale.h> 7095584Sanholt#include <netdb.h> 7195584Sanholt#include <stdarg.h> 7295584Sanholt#include <stdio.h> 7395584Sanholt#include <stdlib.h> 7495584Sanholt#include <string.h> 7595584Sanholt#include <time.h> 7695584Sanholt#include <unistd.h> 7795584Sanholt 7895584Sanholt#include "fetch.h" 7995584Sanholt#include "common.h" 8095584Sanholt#include "httperr.h" 8195584Sanholt 8295584Sanholtextern char *__progname; /* XXX not portable */ 8395584Sanholt 8495584Sanholt/* Maximum number of redirects to follow */ 8595584Sanholt#define MAX_REDIRECT 5 8695584Sanholt 8795584Sanholt/* Symbolic names for reply codes we care about */ 8895584Sanholt#define HTTP_OK 200 8995584Sanholt#define HTTP_PARTIAL 206 9095584Sanholt#define HTTP_MOVED_PERM 301 9195584Sanholt#define HTTP_MOVED_TEMP 302 9295584Sanholt#define HTTP_SEE_OTHER 303 9395584Sanholt#define HTTP_NEED_AUTH 401 9495584Sanholt#define HTTP_NEED_PROXY_AUTH 403 9595584Sanholt#define HTTP_PROTOCOL_ERROR 999 9695584Sanholt 9795584Sanholt#define HTTP_REDIRECT(xyz) ((xyz) == HTTP_MOVED_PERM \ 9895584Sanholt || (xyz) == HTTP_MOVED_TEMP \ 9995584Sanholt || (xyz) == HTTP_SEE_OTHER) 10095584Sanholt 10195584Sanholt 10295584Sanholt 10395584Sanholt/***************************************************************************** 10495584Sanholt * I/O functions for decoding chunked streams 10595584Sanholt */ 10695584Sanholt 10795584Sanholtstruct cookie 10895584Sanholt{ 10995584Sanholt int fd; 11095584Sanholt char *buf; 11195584Sanholt size_t b_size; 11295584Sanholt size_t b_len; 11395584Sanholt int b_pos; 11495584Sanholt int eof; 11595584Sanholt int error; 11695584Sanholt long chunksize; 11795584Sanholt#ifndef NDEBUG 11895584Sanholt long total; 11995584Sanholt#endif 12095584Sanholt}; 12195584Sanholt 122130331Sanholt/* 123130331Sanholt * Get next chunk header 12495584Sanholt */ 12595584Sanholtstatic int 12695584Sanholt_http_new_chunk(struct cookie *c) 12795584Sanholt{ 12895584Sanholt char *p; 12995584Sanholt 13095584Sanholt if (_fetch_getln(c->fd, &c->buf, &c->b_size, &c->b_len) == -1) 13195584Sanholt return -1; 13295584Sanholt 13395584Sanholt if (c->b_len < 2 || !ishexnumber(*c->buf)) 13495584Sanholt return -1; 13595584Sanholt 13695584Sanholt for (p = c->buf; !isspace(*p) && *p != ';' && p < c->buf + c->b_len; ++p) 13795584Sanholt if (!ishexnumber(*p)) 13895584Sanholt return -1; 13995584Sanholt else if (isdigit(*p)) 14095584Sanholt c->chunksize = c->chunksize * 16 + *p - '0'; 14195584Sanholt else 14295584Sanholt c->chunksize = c->chunksize * 16 + 10 + tolower(*p) - 'a'; 14395584Sanholt 14495584Sanholt#ifndef NDEBUG 14595584Sanholt c->total += c->chunksize; 14695584Sanholt if (c->chunksize == 0) 14795584Sanholt fprintf(stderr, "\033[1m_http_fillbuf(): " 14895584Sanholt "end of last chunk\033[m\n"); 14995584Sanholt else 15095584Sanholt fprintf(stderr, "\033[1m_http_fillbuf(): " 15195584Sanholt "new chunk: %ld (%ld)\033[m\n", c->chunksize, c->total); 15295584Sanholt#endif 15395584Sanholt 15495584Sanholt return c->chunksize; 15595584Sanholt} 15695584Sanholt 15795584Sanholt/* 15895584Sanholt * Fill the input buffer, do chunk decoding on the fly 15995584Sanholt */ 16095584Sanholtstatic int 16195584Sanholt_http_fillbuf(struct cookie *c) 16295584Sanholt{ 16395584Sanholt if (c->error) 16495584Sanholt return -1; 16595584Sanholt if (c->eof) 16695584Sanholt return 0; 16795584Sanholt 16895584Sanholt if (c->chunksize == 0) { 16995584Sanholt switch (_http_new_chunk(c)) { 17095584Sanholt case -1: 17195584Sanholt c->error = 1; 17295584Sanholt return -1; 17395584Sanholt case 0: 17495584Sanholt c->eof = 1; 17595584Sanholt return 0; 17695584Sanholt } 17795584Sanholt } 17895584Sanholt 17995584Sanholt if (c->b_size < c->chunksize) { 18095584Sanholt char *tmp; 18195584Sanholt 18295584Sanholt if ((tmp = realloc(c->buf, c->chunksize)) == NULL) 18395584Sanholt return -1; 18495584Sanholt c->buf = tmp; 18595584Sanholt c->b_size = c->chunksize; 18695584Sanholt } 18795584Sanholt 18895584Sanholt if ((c->b_len = read(c->fd, c->buf, c->chunksize)) == -1) 18995584Sanholt return -1; 19095584Sanholt c->chunksize -= c->b_len; 19195584Sanholt 19295584Sanholt if (c->chunksize == 0) { 19395584Sanholt char endl[2]; 19495584Sanholt read(c->fd, endl, 2); 19595584Sanholt } 19695584Sanholt 19795584Sanholt c->b_pos = 0; 19895584Sanholt 19995584Sanholt return c->b_len; 20095584Sanholt} 20195584Sanholt 20295584Sanholt/* 20395584Sanholt * Read function 20495584Sanholt */ 20595584Sanholtstatic int 20695584Sanholt_http_readfn(void *v, char *buf, int len) 20795584Sanholt{ 20895584Sanholt struct cookie *c = (struct cookie *)v; 20995584Sanholt int l, pos; 21095584Sanholt 21195584Sanholt if (c->error) 21295584Sanholt return -1; 21395584Sanholt if (c->eof) 21495584Sanholt return 0; 21595584Sanholt 21695584Sanholt for (pos = 0; len > 0; pos += l, len -= l) { 21795584Sanholt /* empty buffer */ 21895584Sanholt if (!c->buf || c->b_pos == c->b_len) 21995584Sanholt if (_http_fillbuf(c) < 1) 22095584Sanholt break; 22195584Sanholt l = c->b_len - c->b_pos; 22295584Sanholt if (len < l) 22395584Sanholt l = len; 22495584Sanholt bcopy(c->buf + c->b_pos, buf + pos, l); 22595584Sanholt c->b_pos += l; 22695584Sanholt } 22795584Sanholt 22895584Sanholt if (!pos && c->error) 229112015Sanholt return -1; 230112015Sanholt return pos; 231112015Sanholt} 232112015Sanholt 233112015Sanholt/* 234112015Sanholt * Write function 235112015Sanholt */ 236112015Sanholtstatic int 237130331Sanholt_http_writefn(void *v, const char *buf, int len) 238130331Sanholt{ 239130331Sanholt struct cookie *c = (struct cookie *)v; 240130331Sanholt 241130331Sanholt return write(c->fd, buf, len); 242130331Sanholt} 243130331Sanholt 244130331Sanholt/* 245130331Sanholt * Close function 246130331Sanholt */ 24795584Sanholtstatic int 248130331Sanholt_http_closefn(void *v) 249130331Sanholt{ 250130331Sanholt struct cookie *c = (struct cookie *)v; 251130331Sanholt int r; 252130331Sanholt 253130331Sanholt r = close(c->fd); 254130331Sanholt if (c->buf) 255130331Sanholt free(c->buf); 256130331Sanholt free(c); 257130331Sanholt return r; 258130331Sanholt} 25995584Sanholt 26095584Sanholt/* 26195584Sanholt * Wrap a file descriptor up 26295584Sanholt */ 26395584Sanholtstatic FILE * 26495584Sanholt_http_funopen(int fd) 26595584Sanholt{ 26695584Sanholt struct cookie *c; 26795584Sanholt FILE *f; 26895584Sanholt 26995584Sanholt if ((c = calloc(1, sizeof *c)) == NULL) { 27095584Sanholt _fetch_syserr(); 27195584Sanholt return NULL; 27295584Sanholt } 27395584Sanholt c->fd = fd; 27495584Sanholt if (!(f = funopen(c, _http_readfn, _http_writefn, NULL, _http_closefn))) { 27595584Sanholt _fetch_syserr(); 27695584Sanholt free(c); 27795584Sanholt return NULL; 27895584Sanholt } 27995584Sanholt return f; 28095584Sanholt} 28195584Sanholt 28295584Sanholt 28395584Sanholt/***************************************************************************** 28495584Sanholt * Helper functions for talking to the server and parsing its replies 28595584Sanholt */ 28695584Sanholt 28795584Sanholt/* Header types */ 28895584Sanholttypedef enum { 28995584Sanholt hdr_syserror = -2, 29095584Sanholt hdr_error = -1, 29195584Sanholt hdr_end = 0, 29295584Sanholt hdr_unknown = 1, 29395584Sanholt hdr_content_length, 29495584Sanholt hdr_content_range, 29595584Sanholt hdr_last_modified, 29695584Sanholt hdr_location, 29795584Sanholt hdr_transfer_encoding 29895584Sanholt} hdr; 29995584Sanholt 30095584Sanholt/* Names of interesting headers */ 30195584Sanholtstatic struct { 30295584Sanholt hdr num; 30395584Sanholt char *name; 30495584Sanholt} hdr_names[] = { 30595584Sanholt { hdr_content_length, "Content-Length" }, 30695584Sanholt { hdr_content_range, "Content-Range" }, 30795584Sanholt { hdr_last_modified, "Last-Modified" }, 30895584Sanholt { hdr_location, "Location" }, 30995584Sanholt { hdr_transfer_encoding, "Transfer-Encoding" }, 31095584Sanholt { hdr_unknown, NULL }, 31195584Sanholt}; 31295584Sanholt 31395584Sanholtstatic char *reply_buf; 31495584Sanholtstatic size_t reply_size; 31595584Sanholtstatic size_t reply_length; 31695584Sanholt 31795584Sanholt/* 31895584Sanholt * Send a formatted line; optionally echo to terminal 31995584Sanholt */ 32095584Sanholtstatic int 32195584Sanholt_http_cmd(int fd, char *fmt, ...) 32295584Sanholt{ 32395584Sanholt va_list ap; 32495584Sanholt size_t len; 32595584Sanholt char *msg; 32695584Sanholt int r; 32795584Sanholt 32895584Sanholt va_start(ap, fmt); 32995584Sanholt len = vasprintf(&msg, fmt, ap); 33095584Sanholt va_end(ap); 33195584Sanholt 33295584Sanholt if (msg == NULL) { 33395584Sanholt errno = ENOMEM; 33495584Sanholt _fetch_syserr(); 33595584Sanholt return -1; 33695584Sanholt } 33795584Sanholt 33895584Sanholt r = _fetch_putln(fd, msg, len); 33995584Sanholt free(msg); 34095584Sanholt 341112015Sanholt if (r == -1) { 342112015Sanholt _fetch_syserr(); 343112015Sanholt return -1; 344112015Sanholt } 345112015Sanholt 346112015Sanholt return 0; 347112015Sanholt} 348130331Sanholt 349112015Sanholt/* 350112015Sanholt * Get and parse status line 35195584Sanholt */ 352static int 353_http_get_reply(int fd) 354{ 355 if (_fetch_getln(fd, &reply_buf, &reply_size, &reply_length) == -1) 356 return -1; 357 /* 358 * A valid status line looks like "HTTP/m.n xyz reason" where m 359 * and n are the major and minor protocol version numbers and xyz 360 * is the reply code. 361 * We grok HTTP 1.0 and 1.1, so m must be 1 and n must be 0 or 1. 362 * We don't care about the reason phrase. 363 */ 364 if (strncmp(reply_buf, "HTTP/1.", 7) != 0 365 || (reply_buf[7] != '0' && reply_buf[7] != '1') || reply_buf[8] != ' ' 366 || !isdigit(reply_buf[9]) 367 || !isdigit(reply_buf[10]) 368 || !isdigit(reply_buf[11])) 369 return HTTP_PROTOCOL_ERROR; 370 371 return ((reply_buf[9] - '0') * 100 372 + (reply_buf[10] - '0') * 10 373 + (reply_buf[11] - '0')); 374} 375 376/* 377 * Check a header; if the type matches the given string, return a 378 * pointer to the beginning of the value. 379 */ 380static char * 381_http_match(char *str, char *hdr) 382{ 383 while (*str && *hdr && tolower(*str++) == tolower(*hdr++)) 384 /* nothing */; 385 if (*str || *hdr != ':') 386 return NULL; 387 while (*hdr && isspace(*++hdr)) 388 /* nothing */; 389 return hdr; 390} 391 392/* 393 * Get the next header and return the appropriate symbolic code. 394 */ 395static hdr 396_http_next_header(int fd, char **p) 397{ 398 int i; 399 400 if (_fetch_getln(fd, &reply_buf, &reply_size, &reply_length) == -1) 401 return hdr_syserror; 402 while (reply_length && isspace(reply_buf[reply_length-1])) 403 reply_length--; 404 reply_buf[reply_length] = 0; 405 if (reply_length == 0) 406 return hdr_end; 407 /* 408 * We could check for malformed headers but we don't really care. 409 * A valid header starts with a token immediately followed by a 410 * colon; a token is any sequence of non-control, non-whitespace 411 * characters except "()<>@,;:\\\"{}". 412 */ 413 for (i = 0; hdr_names[i].num != hdr_unknown; i++) 414 if ((*p = _http_match(hdr_names[i].name, reply_buf)) != NULL) 415 return hdr_names[i].num; 416 return hdr_unknown; 417} 418 419/* 420 * Parse a last-modified header 421 */ 422static int 423_http_parse_mtime(char *p, time_t *mtime) 424{ 425 char locale[64], *r; 426 struct tm tm; 427 428 strncpy(locale, setlocale(LC_TIME, NULL), sizeof locale); 429 setlocale(LC_TIME, "C"); 430 r = strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm); 431 /* XXX should add support for date-2 and date-3 */ 432 setlocale(LC_TIME, locale); 433 if (r == NULL) 434 return -1; 435 DEBUG(fprintf(stderr, "last modified: [\033[1m%04d-%02d-%02d " 436 "%02d:%02d:%02d\033[m]\n", 437 tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, 438 tm.tm_hour, tm.tm_min, tm.tm_sec)); 439 *mtime = timegm(&tm); 440 return 0; 441} 442 443/* 444 * Parse a content-length header 445 */ 446static int 447_http_parse_length(char *p, size_t *length) 448{ 449 size_t len; 450 451 for (len = 0; *p && isdigit(*p); ++p) 452 len = len * 10 + (*p - '0'); 453 DEBUG(fprintf(stderr, "content length: [\033[1m%d\033[m]\n", len)); 454 *length = len; 455 return 0; 456} 457 458/* 459 * Parse a content-range header 460 */ 461static int 462_http_parse_range(char *p, off_t *offset, size_t *length, size_t *size) 463{ 464 int first, last, len; 465 466 if (strncasecmp(p, "bytes ", 6) != 0) 467 return -1; 468 for (first = 0, p += 6; *p && isdigit(*p); ++p) 469 first = first * 10 + *p - '0'; 470 if (*p != '-') 471 return -1; 472 for (last = 0, ++p; *p && isdigit(*p); ++p) 473 last = last * 10 + *p - '0'; 474 if (first > last || *p != '/') 475 return -1; 476 for (len = 0, ++p; *p && isdigit(*p); ++p) 477 len = len * 10 + *p - '0'; 478 if (len < last - first + 1) 479 return -1; 480 DEBUG(fprintf(stderr, "content range: [\033[1m%d-%d/%d\033[m]\n", 481 first, last, len)); 482 *offset = first; 483 *length = last - first + 1; 484 *size = len; 485 return 0; 486} 487 488 489/***************************************************************************** 490 * Helper functions for authorization 491 */ 492 493/* 494 * Base64 encoding 495 */ 496static char * 497_http_base64(char *src) 498{ 499 static const char base64[] = 500 "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 501 "abcdefghijklmnopqrstuvwxyz" 502 "0123456789+/"; 503 char *str, *dst; 504 size_t l; 505 int t, r; 506 507 l = strlen(src); 508 if ((str = malloc(((l + 2) / 3) * 4)) == NULL) 509 return NULL; 510 dst = str; 511 r = 0; 512 513 while (l >= 3) { 514 t = (src[0] << 16) | (src[1] << 8) | src[2]; 515 dst[0] = base64[(t >> 18) & 0x3f]; 516 dst[1] = base64[(t >> 12) & 0x3f]; 517 dst[2] = base64[(t >> 6) & 0x3f]; 518 dst[3] = base64[(t >> 0) & 0x3f]; 519 src += 3; l -= 3; 520 dst += 4; r += 4; 521 } 522 523 switch (l) { 524 case 2: 525 t = (src[0] << 16) | (src[1] << 8); 526 dst[0] = base64[(t >> 18) & 0x3f]; 527 dst[1] = base64[(t >> 12) & 0x3f]; 528 dst[2] = base64[(t >> 6) & 0x3f]; 529 dst[3] = '='; 530 dst += 4; 531 r += 4; 532 break; 533 case 1: 534 t = src[0] << 16; 535 dst[0] = base64[(t >> 18) & 0x3f]; 536 dst[1] = base64[(t >> 12) & 0x3f]; 537 dst[2] = dst[3] = '='; 538 dst += 4; 539 r += 4; 540 break; 541 case 0: 542 break; 543 } 544 545 *dst = 0; 546 return str; 547} 548 549/* 550 * Encode username and password 551 */ 552static int 553_http_basic_auth(int fd, char *hdr, char *usr, char *pwd) 554{ 555 char *upw, *auth; 556 int r; 557 558 if (asprintf(&upw, "%s:%s", usr, pwd) == -1) 559 return -1; 560 auth = _http_base64(upw); 561 free(upw); 562 if (auth == NULL) 563 return -1; 564 r = _http_cmd(fd, "%s: Basic %s", hdr, auth); 565 free(auth); 566 return r; 567} 568 569/* 570 * Send an authorization header 571 */ 572static int 573_http_authorize(int fd, char *hdr, char *p) 574{ 575 /* basic authorization */ 576 if (strncasecmp(p, "basic:", 6) == 0) { 577 char *user, *pwd, *str; 578 int r; 579 580 /* skip realm */ 581 for (p += 6; *p && *p != ':'; ++p) 582 /* nothing */ ; 583 if (!*p || strchr(++p, ':') == NULL) 584 return -1; 585 if ((str = strdup(p)) == NULL) 586 return -1; /* XXX */ 587 user = str; 588 pwd = strchr(str, ':'); 589 *pwd++ = '\0'; 590 r = _http_basic_auth(fd, hdr, user, pwd); 591 free(str); 592 return r; 593 } 594 return -1; 595} 596 597 598/***************************************************************************** 599 * Helper functions for connecting to a server or proxy 600 */ 601 602/* 603 * Connect to the specified HTTP proxy server. 604 */ 605static int 606_http_proxy_connect(char *proxy, int af, int verbose) 607{ 608 char *hostname, *p; 609 int fd, port; 610 611 /* get hostname */ 612 hostname = NULL; 613#ifdef INET6 614 /* host part can be an IPv6 address enclosed in square brackets */ 615 if (*proxy == '[') { 616 if ((p = strchr(proxy, ']')) == NULL) { 617 /* no terminating bracket */ 618 /* XXX should set an error code */ 619 goto ouch; 620 } 621 if (p[1] != '\0' && p[1] != ':') { 622 /* garbage after address */ 623 /* XXX should set an error code */ 624 goto ouch; 625 } 626 if ((hostname = malloc(p - proxy)) == NULL) { 627 errno = ENOMEM; 628 _fetch_syserr(); 629 goto ouch; 630 } 631 strncpy(hostname, proxy + 1, p - proxy - 1); 632 hostname[p - proxy - 1] = '\0'; 633 ++p; 634 } else { 635#endif /* INET6 */ 636 if ((p = strchr(proxy, ':')) == NULL) 637 p = strchr(proxy, '\0'); 638 if ((hostname = malloc(p - proxy + 1)) == NULL) { 639 errno = ENOMEM; 640 _fetch_syserr(); 641 goto ouch; 642 } 643 strncpy(hostname, proxy, p - proxy); 644 hostname[p - proxy] = '\0'; 645#ifdef INET6 646 } 647#endif /* INET6 */ 648 DEBUG(fprintf(stderr, "proxy name: [%s]\n", hostname)); 649 650 /* get port number */ 651 port = 0; 652 if (*p == ':') { 653 ++p; 654 if (strspn(p, "0123456789") != strlen(p) || strlen(p) > 5) { 655 /* port number is non-numeric or too long */ 656 /* XXX should set an error code */ 657 goto ouch; 658 } 659 port = atoi(p); 660 if (port < 1 || port > 65535) { 661 /* port number is out of range */ 662 /* XXX should set an error code */ 663 goto ouch; 664 } 665 } 666 667 if (!port) { 668#if 0 669 /* 670 * commented out, since there is currently no service name 671 * for HTTP proxies 672 */ 673 struct servent *se; 674 675 if ((se = getservbyname("xxxx", "tcp")) != NULL) 676 port = ntohs(se->s_port); 677 else 678#endif 679 port = 3128; 680 } 681 DEBUG(fprintf(stderr, "proxy port: %d\n", port)); 682 683 /* connect */ 684 if ((fd = _fetch_connect(hostname, port, af, verbose)) == -1) 685 _fetch_syserr(); 686 return fd; 687 688 ouch: 689 if (hostname) 690 free(hostname); 691 return -1; 692} 693 694/* 695 * Connect to the correct HTTP server or proxy. 696 */ 697static int 698_http_connect(struct url *URL, int *proxy, char *flags) 699{ 700 int direct, verbose; 701 int af, fd; 702 char *p; 703 704#ifdef INET6 705 af = AF_UNSPEC; 706#else 707 af = AF_INET; 708#endif 709 710 direct = (flags && strchr(flags, 'd')); 711 verbose = (flags && strchr(flags, 'v')); 712 if (flags && strchr(flags, '4')) 713 af = AF_INET; 714 else if (flags && strchr(flags, '6')) 715 af = AF_INET6; 716 717 /* check port */ 718 if (!URL->port) { 719 struct servent *se; 720 721 /* Scheme can be ftp if we're using a proxy */ 722 if (strcasecmp(URL->scheme, "ftp") == 0) 723 if ((se = getservbyname("ftp", "tcp")) != NULL) 724 URL->port = ntohs(se->s_port); 725 else 726 URL->port = 21; 727 else 728 if ((se = getservbyname("http", "tcp")) != NULL) 729 URL->port = ntohs(se->s_port); 730 else 731 URL->port = 80; 732 } 733 734 if (!direct && (p = getenv("HTTP_PROXY")) != NULL && *p != '\0') { 735 /* attempt to connect to proxy server */ 736 if ((fd = _http_proxy_connect(p, af, verbose)) == -1) 737 return -1; 738 *proxy = 1; 739 } else { 740 /* if no proxy is configured, try direct */ 741 if (strcasecmp(URL->scheme, "ftp") == 0) { 742 /* can't talk http to an ftp server */ 743 /* XXX should set an error code */ 744 return -1; 745 } 746 if ((fd = _fetch_connect(URL->host, URL->port, af, verbose)) == -1) 747 /* _fetch_connect() has already set an error code */ 748 return -1; 749 *proxy = 0; 750 } 751 752 return fd; 753} 754 755 756/***************************************************************************** 757 * Core 758 */ 759 760/* 761 * Send a request and process the reply 762 */ 763static FILE * 764_http_request(struct url *URL, char *op, struct url_stat *us, char *flags) 765{ 766 struct url *url, *new; 767 int chunked, need_auth, noredirect, proxy, verbose; 768 int code, fd, i, n; 769 off_t offset; 770 size_t clength, length, size; 771 time_t mtime; 772 char *p; 773 FILE *f; 774 hdr h; 775 char *host; 776#ifdef INET6 777 char hbuf[MAXHOSTNAMELEN + 1]; 778#endif 779 780 noredirect = (flags && strchr(flags, 'A')); 781 verbose = (flags && strchr(flags, 'v')); 782 783 /* try the provided URL first */ 784 url = URL; 785 786 /* if the A flag is set, we only get one try */ 787 n = noredirect ? 1 : MAX_REDIRECT; 788 i = 0; 789 790 do { 791 new = NULL; 792 chunked = 0; 793 need_auth = 0; 794 offset = 0; 795 clength = -1; 796 length = -1; 797 size = -1; 798 mtime = 0; 799 retry: 800 /* connect to server or proxy */ 801 if ((fd = _http_connect(url, &proxy, flags)) == -1) 802 goto ouch; 803 804 host = url->host; 805#ifdef INET6 806 if (strchr(url->host, ':')) { 807 snprintf(hbuf, sizeof(hbuf), "[%s]", url->host); 808 host = hbuf; 809 } 810#endif 811 812 /* send request */ 813 if (verbose) 814 _fetch_info("requesting %s://%s:%d%s", 815 url->scheme, host, url->port, url->doc); 816 if (proxy) { 817 _http_cmd(fd, "%s %s://%s:%d%s HTTP/1.1", 818 op, url->scheme, host, url->port, url->doc); 819 } else { 820 _http_cmd(fd, "%s %s HTTP/1.1", 821 op, url->doc); 822 } 823 824 /* proxy authorization */ 825 if (proxy && (p = getenv("HTTP_PROXY_AUTH")) != NULL && *p != '\0') 826 _http_authorize(fd, "Proxy-Authorization", p); 827 828 /* server authorization */ 829 if (need_auth) { 830 if (*url->user || *url->pwd) 831 _http_basic_auth(fd, "Authorization", 832 url->user ? url->user : "", 833 url->pwd ? url->pwd : ""); 834 else if ((p = getenv("HTTP_AUTH")) != NULL && *p != '\0') 835 _http_authorize(fd, "Authorization", p); 836 else { 837 _http_seterr(HTTP_NEED_AUTH); 838 goto ouch; 839 } 840 } 841 842 /* other headers */ 843 _http_cmd(fd, "Host: %s:%d", host, url->port); 844 _http_cmd(fd, "User-Agent: %s " _LIBFETCH_VER, __progname); 845 if (url->offset) 846 _http_cmd(fd, "Range: bytes=%lld-", url->offset); 847 _http_cmd(fd, "Connection: close"); 848 _http_cmd(fd, ""); 849 850 /* get reply */ 851 switch ((code = _http_get_reply(fd))) { 852 case HTTP_OK: 853 case HTTP_PARTIAL: 854 /* fine */ 855 break; 856 case HTTP_MOVED_PERM: 857 case HTTP_MOVED_TEMP: 858 /* 859 * Not so fine, but we still have to read the headers to 860 * get the new location. 861 */ 862 break; 863 case HTTP_NEED_AUTH: 864 if (need_auth) { 865 /* 866 * We already sent out authorization code, so there's 867 * nothing more we can do. 868 */ 869 _http_seterr(code); 870 goto ouch; 871 } 872 /* try again, but send the password this time */ 873 if (verbose) 874 _fetch_info("server requires authorization"); 875 need_auth = 1; 876 close(fd); 877 goto retry; 878 case HTTP_NEED_PROXY_AUTH: 879 /* 880 * If we're talking to a proxy, we already sent our proxy 881 * authorization code, so there's nothing more we can do. 882 */ 883 _http_seterr(code); 884 goto ouch; 885 case HTTP_PROTOCOL_ERROR: 886 /* fall through */ 887 case -1: 888 _fetch_syserr(); 889 goto ouch; 890 default: 891 _http_seterr(code); 892 goto ouch; 893 } 894 895 /* get headers */ 896 do { 897 switch ((h = _http_next_header(fd, &p))) { 898 case hdr_syserror: 899 _fetch_syserr(); 900 goto ouch; 901 case hdr_error: 902 _http_seterr(HTTP_PROTOCOL_ERROR); 903 goto ouch; 904 case hdr_content_length: 905 _http_parse_length(p, &clength); 906 break; 907 case hdr_content_range: 908 _http_parse_range(p, &offset, &length, &size); 909 break; 910 case hdr_last_modified: 911 _http_parse_mtime(p, &mtime); 912 break; 913 case hdr_location: 914 if (!HTTP_REDIRECT(code)) 915 break; 916 if (new) 917 free(new); 918 if (verbose) 919 _fetch_info("%d redirect to %s", code, p); 920 if (*p == '/') 921 /* absolute path */ 922 new = fetchMakeURL(url->scheme, url->host, url->port, p, 923 url->user, url->pwd); 924 else 925 new = fetchParseURL(p); 926 if (new == NULL) { 927 /* XXX should set an error code */ 928 DEBUG(fprintf(stderr, "failed to parse new URL\n")); 929 goto ouch; 930 } 931 if (!*new->user && !*new->pwd) { 932 strcpy(new->user, url->user); 933 strcpy(new->pwd, url->pwd); 934 } 935 new->offset = url->offset; 936 new->length = url->length; 937 break; 938 case hdr_transfer_encoding: 939 /* XXX weak test*/ 940 chunked = (strcasecmp(p, "chunked") == 0); 941 break; 942 case hdr_end: 943 /* fall through */ 944 case hdr_unknown: 945 /* ignore */ 946 break; 947 } 948 } while (h > hdr_end); 949 950 /* we either have a hit, or a redirect with no Location: header */ 951 if (code == HTTP_OK || code == HTTP_PARTIAL || !new) 952 break; 953 954 /* we have a redirect */ 955 close(fd); 956 fd = -1; 957 if (url != URL) 958 fetchFreeURL(url); 959 url = new; 960 } while (++i < n); 961 962 /* no success */ 963 if (fd == -1) { 964 _http_seterr(code); 965 goto ouch; 966 } 967 968 DEBUG(fprintf(stderr, "offset: %lld, length: %d, size: %d, clength: %d\n", 969 offset, length, size, clength)); 970 971 /* check for inconsistencies */ 972 if (clength != -1 && length != -1 && clength != length) { 973 _http_seterr(HTTP_PROTOCOL_ERROR); 974 goto ouch; 975 } 976 if (clength == -1) 977 clength = length; 978 if (clength != -1) 979 length = offset + clength; 980 if (length != -1 && size != -1 && length != size) { 981 _http_seterr(HTTP_PROTOCOL_ERROR); 982 goto ouch; 983 } 984 if (size == -1) 985 size = length; 986 987 /* fill in stats */ 988 if (us) { 989 us->size = size; 990 us->atime = us->mtime = mtime; 991 } 992 993 /* too far? */ 994 if (offset > URL->offset) { 995 _http_seterr(HTTP_PROTOCOL_ERROR); 996 goto ouch; 997 } 998 999 /* report back real offset and size */ 1000 URL->offset = offset; 1001 URL->length = clength; 1002 1003 /* wrap it up in a FILE */ 1004 if ((f = chunked ? _http_funopen(fd) : fdopen(fd, "r")) == NULL) { 1005 _fetch_syserr(); 1006 goto ouch; 1007 } 1008 1009 if (url != URL) 1010 fetchFreeURL(url); 1011 1012 return f; 1013 1014 ouch: 1015 if (url != URL) 1016 fetchFreeURL(url); 1017 if (fd != -1) 1018 close(fd); 1019 return NULL; 1020} 1021 1022 1023/***************************************************************************** 1024 * Entry points 1025 */ 1026 1027/* 1028 * Retrieve and stat a file by HTTP 1029 */ 1030FILE * 1031fetchXGetHTTP(struct url *URL, struct url_stat *us, char *flags) 1032{ 1033 return _http_request(URL, "GET", us, flags); 1034} 1035 1036/* 1037 * Retrieve a file by HTTP 1038 */ 1039FILE * 1040fetchGetHTTP(struct url *URL, char *flags) 1041{ 1042 return fetchXGetHTTP(URL, NULL, flags); 1043} 1044 1045/* 1046 * Store a file by HTTP 1047 */ 1048FILE * 1049fetchPutHTTP(struct url *URL, char *flags) 1050{ 1051 warnx("fetchPutHTTP(): not implemented"); 1052 return NULL; 1053} 1054 1055/* 1056 * Get an HTTP document's metadata 1057 */ 1058int 1059fetchStatHTTP(struct url *URL, struct url_stat *us, char *flags) 1060{ 1061 FILE *f; 1062 1063 if ((f = _http_request(URL, "HEAD", us, flags)) == NULL) 1064 return -1; 1065 fclose(f); 1066 return 0; 1067} 1068 1069/* 1070 * List a directory 1071 */ 1072struct url_ent * 1073fetchListHTTP(struct url *url, char *flags) 1074{ 1075 warnx("fetchListHTTP(): not implemented"); 1076 return NULL; 1077} 1078