http.c revision 67043
128219Smsmith/*- 238061Smsmith * Copyright (c) 2000 Dag-Erling Co�dan Sm�rgrav 328219Smsmith * All rights reserved. 428219Smsmith * 528219Smsmith * Redistribution and use in source and binary forms, with or without 628219Smsmith * modification, are permitted provided that the following conditions 728219Smsmith * are met: 828219Smsmith * 1. Redistributions of source code must retain the above copyright 928219Smsmith * notice, this list of conditions and the following disclaimer 1028219Smsmith * in this position and unchanged. 1128219Smsmith * 2. Redistributions in binary form must reproduce the above copyright 1228219Smsmith * notice, this list of conditions and the following disclaimer in the 1328219Smsmith * documentation and/or other materials provided with the distribution. 1428219Smsmith * 3. The name of the author may not be used to endorse or promote products 1528219Smsmith * derived from this software without specific prior written permission. 1628219Smsmith * 1728219Smsmith * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1828219Smsmith * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1928219Smsmith * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 2028219Smsmith * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 2128219Smsmith * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2228219Smsmith * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2328219Smsmith * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2428219Smsmith * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2528219Smsmith * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2639134Snsouch * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2728219Smsmith * 2828219Smsmith * $FreeBSD: head/lib/libfetch/http.c 67043 2000-10-12 22:10:26Z des $ 2928219Smsmith */ 3028219Smsmith 3128219Smsmith/* 3228219Smsmith * The following copyright applies to the base64 code: 3328219Smsmith * 3428219Smsmith *- 3528219Smsmith * Copyright 1997 Massachusetts Institute of Technology 3628219Smsmith * 3728219Smsmith * Permission to use, copy, modify, and distribute this software and 3828257Smsmith * its documentation for any purpose and without fee is hereby 3928219Smsmith * granted, provided that both the above copyright notice and this 4038061Smsmith * permission notice appear in all copies, that both the above 4128219Smsmith * copyright notice and this permission notice appear in all 4228219Smsmith * supporting documentation, and that the name of M.I.T. not be used 4328219Smsmith * in advertising or publicity pertaining to distribution of the 4428219Smsmith * software without specific, written prior permission. M.I.T. makes 4528219Smsmith * no representations about the suitability of this software for any 4628219Smsmith * purpose. It is provided "as is" without express or implied 4728219Smsmith * warranty. 4828219Smsmith * 4928219Smsmith * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS 5028219Smsmith * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE, 5128219Smsmith * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 5228219Smsmith * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT 5328219Smsmith * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 5428219Smsmith * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 5528219Smsmith * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF 5628219Smsmith * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 5728219Smsmith * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 5828219Smsmith * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 5928219Smsmith * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 6028219Smsmith * SUCH DAMAGE. 6128219Smsmith */ 6228219Smsmith 6328219Smsmith#include <sys/param.h> 6428219Smsmith#include <sys/socket.h> 6528219Smsmith 6628219Smsmith#include <ctype.h> 6728219Smsmith#include <err.h> 6828219Smsmith#include <errno.h> 6928219Smsmith#include <locale.h> 7028219Smsmith#include <netdb.h> 7128219Smsmith#include <stdarg.h> 7228219Smsmith#include <stdio.h> 7328219Smsmith#include <stdlib.h> 7428219Smsmith#include <string.h> 7528219Smsmith#include <time.h> 7628219Smsmith#include <unistd.h> 7728219Smsmith 7828219Smsmith#include "fetch.h" 7928219Smsmith#include "common.h" 8028219Smsmith#include "httperr.h" 8128219Smsmith 8228219Smsmithextern char *__progname; /* XXX not portable */ 8328257Smsmith 8428257Smsmith/* Maximum number of redirects to follow */ 8528257Smsmith#define MAX_REDIRECT 5 8628257Smsmith 8728257Smsmith/* Symbolic names for reply codes we care about */ 8828257Smsmith#define HTTP_OK 200 8928257Smsmith#define HTTP_PARTIAL 206 9028257Smsmith#define HTTP_MOVED_PERM 301 9128257Smsmith#define HTTP_MOVED_TEMP 302 9228257Smsmith#define HTTP_SEE_OTHER 303 9328219Smsmith#define HTTP_NEED_AUTH 401 9428257Smsmith#define HTTP_NEED_PROXY_AUTH 403 9528257Smsmith#define HTTP_PROTOCOL_ERROR 999 9628257Smsmith 9739134Snsouch#define HTTP_REDIRECT(xyz) ((xyz) == HTTP_MOVED_PERM \ 9839134Snsouch || (xyz) == HTTP_MOVED_TEMP \ 9928257Smsmith || (xyz) == HTTP_SEE_OTHER) 10028257Smsmith 10128257Smsmith 10228257Smsmith 10328257Smsmith/***************************************************************************** 10428257Smsmith * I/O functions for decoding chunked streams 10528257Smsmith */ 10628257Smsmith 10728257Smsmithstruct cookie 10828257Smsmith{ 10928257Smsmith int fd; 11028257Smsmith char *buf; 11128257Smsmith size_t b_size; 11228257Smsmith size_t b_len; 11328257Smsmith int b_pos; 11428257Smsmith int eof; 11528257Smsmith int error; 11628257Smsmith long chunksize; 11728257Smsmith#ifndef NDEBUG 11828257Smsmith long total; 11928257Smsmith#endif 12028257Smsmith}; 12128257Smsmith 12228257Smsmith/* 12328257Smsmith * Get next chunk header 12428257Smsmith */ 12528257Smsmithstatic int 12628257Smsmith_http_new_chunk(struct cookie *c) 12728257Smsmith{ 12828257Smsmith char *p; 12928257Smsmith 13028257Smsmith if (_fetch_getln(c->fd, &c->buf, &c->b_size, &c->b_len) == -1) 13128257Smsmith return -1; 13228257Smsmith 13328257Smsmith if (c->b_len < 2 || !ishexnumber(*c->buf)) 13428257Smsmith return -1; 13528257Smsmith 13628257Smsmith for (p = c->buf; !isspace(*p) && *p != ';' && p < c->buf + c->b_len; ++p) 13728257Smsmith if (!ishexnumber(*p)) 13828257Smsmith return -1; 13928257Smsmith else if (isdigit(*p)) 14028257Smsmith c->chunksize = c->chunksize * 16 + *p - '0'; 14138061Smsmith else 14228257Smsmith c->chunksize = c->chunksize * 16 + 10 + tolower(*p) - 'a'; 14328257Smsmith 14428257Smsmith#ifndef NDEBUG 14528257Smsmith c->total += c->chunksize; 14628257Smsmith if (c->chunksize == 0) 14728257Smsmith fprintf(stderr, "\033[1m_http_fillbuf(): " 14828257Smsmith "end of last chunk\033[m\n"); 14928257Smsmith else 15038061Smsmith fprintf(stderr, "\033[1m_http_fillbuf(): " 15138061Smsmith "new chunk: %ld (%ld)\033[m\n", c->chunksize, c->total); 15238061Smsmith#endif 15328257Smsmith 15438061Smsmith return c->chunksize; 15538061Smsmith} 15638061Smsmith 15738061Smsmith/* 15838061Smsmith * Fill the input buffer, do chunk decoding on the fly 15938061Smsmith */ 16039134Snsouchstatic int 16139134Snsouch_http_fillbuf(struct cookie *c) 16239134Snsouch{ 16328257Smsmith if (c->error) 16438061Smsmith return -1; 16528257Smsmith if (c->eof) 16639134Snsouch return 0; 16728257Smsmith 16839134Snsouch if (c->chunksize == 0) { 16928257Smsmith switch (_http_new_chunk(c)) { 17039134Snsouch case -1: 17139134Snsouch c->error = 1; 17239134Snsouch return -1; 17338061Smsmith case 0: 17438061Smsmith c->eof = 1; 17539134Snsouch return 0; 17638061Smsmith } 17728257Smsmith } 17839134Snsouch 17928257Smsmith if (c->b_size < c->chunksize) { 18028257Smsmith char *tmp; 18138061Smsmith 18228257Smsmith if ((tmp = realloc(c->buf, c->chunksize)) == NULL) 18328257Smsmith return -1; 18428257Smsmith c->buf = tmp; 18528257Smsmith c->b_size = c->chunksize; 18628257Smsmith } 18728257Smsmith 18838061Smsmith if ((c->b_len = read(c->fd, c->buf, c->chunksize)) == -1) 18938061Smsmith return -1; 19038061Smsmith c->chunksize -= c->b_len; 19138061Smsmith 19238061Smsmith if (c->chunksize == 0) { 19338061Smsmith char endl[2]; 19428257Smsmith read(c->fd, endl, 2); 19528257Smsmith } 19628257Smsmith 19728257Smsmith c->b_pos = 0; 19828257Smsmith 19928257Smsmith return c->b_len; 20028257Smsmith} 20128257Smsmith 20228257Smsmith/* 20328257Smsmith * Read function 20428257Smsmith */ 20528257Smsmithstatic int 20628257Smsmith_http_readfn(void *v, char *buf, int len) 20728257Smsmith{ 20828257Smsmith struct cookie *c = (struct cookie *)v; 20928257Smsmith int l, pos; 21028257Smsmith 21128257Smsmith if (c->error) 21228257Smsmith return -1; 21328257Smsmith if (c->eof) 21428257Smsmith return 0; 21528257Smsmith 21628257Smsmith for (pos = 0; len > 0; pos += l, len -= l) { 21728257Smsmith /* empty buffer */ 21828257Smsmith if (!c->buf || c->b_pos == c->b_len) 21928257Smsmith if (_http_fillbuf(c) < 1) 22028257Smsmith break; 22128257Smsmith l = c->b_len - c->b_pos; 22228257Smsmith if (len < l) 22328257Smsmith l = len; 22428257Smsmith bcopy(c->buf + c->b_pos, buf + pos, l); 22528257Smsmith c->b_pos += l; 22628257Smsmith } 22728257Smsmith 22828257Smsmith if (!pos && c->error) 22928257Smsmith return -1; 23028257Smsmith return pos; 23128257Smsmith} 23228257Smsmith 23328257Smsmith/* 23428257Smsmith * Write function 23528257Smsmith */ 23638061Smsmithstatic int 23738061Smsmith_http_writefn(void *v, const char *buf, int len) 23828257Smsmith{ 23928257Smsmith struct cookie *c = (struct cookie *)v; 24028257Smsmith 24138061Smsmith return write(c->fd, buf, len); 24238061Smsmith} 24338061Smsmith 24439134Snsouch/* 24539134Snsouch * Close function 24639134Snsouch */ 24738061Smsmithstatic int 24838061Smsmith_http_closefn(void *v) 24928257Smsmith{ 25028257Smsmith struct cookie *c = (struct cookie *)v; 25128257Smsmith int r; 25228219Smsmith 25328219Smsmith r = close(c->fd); 25428219Smsmith if (c->buf) 25528219Smsmith free(c->buf); 25628219Smsmith free(c); 25728219Smsmith return r; 25828219Smsmith} 25928219Smsmith 26028219Smsmith/* 26128219Smsmith * Wrap a file descriptor up 26228219Smsmith */ 26328257Smsmithstatic FILE * 26428219Smsmith_http_funopen(int fd) 26528219Smsmith{ 26628257Smsmith struct cookie *c; 26728257Smsmith FILE *f; 26828257Smsmith 26928219Smsmith if ((c = calloc(1, sizeof *c)) == NULL) { 27028219Smsmith _fetch_syserr(); 27128219Smsmith return NULL; 27228219Smsmith } 27328219Smsmith c->fd = fd; 27428219Smsmith if (!(f = funopen(c, _http_readfn, _http_writefn, NULL, _http_closefn))) { 27528219Smsmith _fetch_syserr(); 27628219Smsmith free(c); 27728219Smsmith return NULL; 27828219Smsmith } 27928219Smsmith return f; 28028219Smsmith} 28128219Smsmith 28228219Smsmith 28328219Smsmith/***************************************************************************** 28428219Smsmith * Helper functions for talking to the server and parsing its replies 28528219Smsmith */ 28628219Smsmith 28728219Smsmith/* Header types */ 28828219Smsmithtypedef enum { 28928219Smsmith hdr_syserror = -2, 29028257Smsmith hdr_error = -1, 29128219Smsmith hdr_end = 0, 29228257Smsmith hdr_unknown = 1, 29328219Smsmith hdr_content_length, 29428257Smsmith hdr_content_range, 29528257Smsmith hdr_last_modified, 29628219Smsmith hdr_location, 29728219Smsmith hdr_transfer_encoding 29828257Smsmith} hdr; 29928257Smsmith 30028219Smsmith/* Names of interesting headers */ 30128257Smsmithstatic struct { 30228257Smsmith hdr num; 30328219Smsmith char *name; 30428257Smsmith} hdr_names[] = { 30528257Smsmith { hdr_content_length, "Content-Length" }, 30628257Smsmith { hdr_content_range, "Content-Range" }, 30728257Smsmith { hdr_last_modified, "Last-Modified" }, 30828257Smsmith { hdr_location, "Location" }, 30928257Smsmith { hdr_transfer_encoding, "Transfer-Encoding" }, 31028257Smsmith { hdr_unknown, NULL }, 31128257Smsmith}; 31228257Smsmith 31328219Smsmithstatic char *reply_buf; 31428257Smsmithstatic size_t reply_size; 31528257Smsmithstatic size_t reply_length; 31628257Smsmith 31728219Smsmith/* 31828257Smsmith * Send a formatted line; optionally echo to terminal 31928219Smsmith */ 32028219Smsmithstatic int 32128219Smsmith_http_cmd(int fd, char *fmt, ...) 32238061Smsmith{ 32338061Smsmith va_list ap; 32438061Smsmith size_t len; 32538061Smsmith char *msg; 32638061Smsmith int r; 32738061Smsmith 32838061Smsmith va_start(ap, fmt); 32938061Smsmith len = vasprintf(&msg, fmt, ap); 33038061Smsmith va_end(ap); 33138061Smsmith 33238061Smsmith if (msg == NULL) { 33338061Smsmith errno = ENOMEM; 33438061Smsmith _fetch_syserr(); 33538061Smsmith return -1; 33638061Smsmith } 33738061Smsmith 33838061Smsmith r = _fetch_putln(fd, msg, len); 33938061Smsmith free(msg); 34028257Smsmith 34128219Smsmith if (r == -1) { 34228257Smsmith _fetch_syserr(); 34328219Smsmith return -1; 34428219Smsmith } 34528257Smsmith 34628219Smsmith return 0; 34728219Smsmith} 34828219Smsmith 34928257Smsmith/* 35028257Smsmith * Get and parse status line 35128219Smsmith */ 35228219Smsmithstatic int 35328219Smsmith_http_get_reply(int fd) 35428219Smsmith{ 35528219Smsmith char *p; 35628257Smsmith 35728219Smsmith if (_fetch_getln(fd, &reply_buf, &reply_size, &reply_length) == -1) 35828257Smsmith return -1; 35928219Smsmith /* 36028219Smsmith * A valid status line looks like "HTTP/m.n xyz reason" where m 36128257Smsmith * and n are the major and minor protocol version numbers and xyz 36228219Smsmith * is the reply code. 36328219Smsmith * Unfortunately, there are servers out there (NCSA 1.5.1, to name 36428257Smsmith * just one) that do not send a version number, so we can't rely 36528257Smsmith * on finding one, but if we do, insist on it being 1.0 or 1.1. 36628219Smsmith * We don't care about the reason phrase. 36728219Smsmith */ 36828219Smsmith if (strncmp(reply_buf, "HTTP", 4) != 0) 36928219Smsmith return HTTP_PROTOCOL_ERROR; 37028219Smsmith p = reply_buf + 4; 37128257Smsmith if (*p == '/') { 37228219Smsmith if (p[1] != '1' || p[2] != '.' || (p[3] != '0' && p[3] != '1')) 37328257Smsmith return HTTP_PROTOCOL_ERROR; 37428257Smsmith p += 4; 37528257Smsmith } 37628219Smsmith if (*p != ' ' 37728219Smsmith || !isdigit(p[1]) 37828257Smsmith || !isdigit(p[2]) 37928219Smsmith || !isdigit(p[3])) 38028257Smsmith return HTTP_PROTOCOL_ERROR; 38128219Smsmith 38228219Smsmith return ((p[1] - '0') * 100 + (p[2] - '0') * 10 + (p[3] - '0')); 38328257Smsmith} 38428257Smsmith 38528257Smsmith/* 38628257Smsmith * Check a header; if the type matches the given string, return a 38728219Smsmith * pointer to the beginning of the value. 38828257Smsmith */ 38928257Smsmithstatic char * 39028257Smsmith_http_match(char *str, char *hdr) 39128257Smsmith{ 39228219Smsmith while (*str && *hdr && tolower(*str++) == tolower(*hdr++)) 39328257Smsmith /* nothing */; 39428257Smsmith if (*str || *hdr != ':') 39528257Smsmith return NULL; 39628219Smsmith while (*hdr && isspace(*++hdr)) 39728257Smsmith /* nothing */; 39828257Smsmith return hdr; 39928257Smsmith} 40028257Smsmith 40128219Smsmith/* 40228257Smsmith * Get the next header and return the appropriate symbolic code. 40328257Smsmith */ 40428219Smsmithstatic hdr 40538061Smsmith_http_next_header(int fd, char **p) 40638061Smsmith{ 40738061Smsmith int i; 40838061Smsmith 40938061Smsmith if (_fetch_getln(fd, &reply_buf, &reply_size, &reply_length) == -1) 41038061Smsmith return hdr_syserror; 41138061Smsmith while (reply_length && isspace(reply_buf[reply_length-1])) 41238061Smsmith reply_length--; 41338061Smsmith reply_buf[reply_length] = 0; 41428257Smsmith if (reply_length == 0) 41528257Smsmith return hdr_end; 41628257Smsmith /* 41728257Smsmith * We could check for malformed headers but we don't really care. 41828219Smsmith * A valid header starts with a token immediately followed by a 41928257Smsmith * colon; a token is any sequence of non-control, non-whitespace 42028219Smsmith * characters except "()<>@,;:\\\"{}". 42128219Smsmith */ 42228219Smsmith for (i = 0; hdr_names[i].num != hdr_unknown; i++) 42328257Smsmith if ((*p = _http_match(hdr_names[i].name, reply_buf)) != NULL) 42428219Smsmith return hdr_names[i].num; 42528257Smsmith return hdr_unknown; 42628219Smsmith} 42728219Smsmith 42828257Smsmith/* 42928219Smsmith * Parse a last-modified header 43028257Smsmith */ 43128219Smsmithstatic int 43228219Smsmith_http_parse_mtime(char *p, time_t *mtime) 43328257Smsmith{ 43428257Smsmith char locale[64], *r; 43528257Smsmith struct tm tm; 43628219Smsmith 43728257Smsmith strncpy(locale, setlocale(LC_TIME, NULL), sizeof locale); 43828219Smsmith setlocale(LC_TIME, "C"); 43928257Smsmith r = strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm); 44028257Smsmith /* XXX should add support for date-2 and date-3 */ 44128219Smsmith setlocale(LC_TIME, locale); 44238061Smsmith if (r == NULL) 44338061Smsmith return -1; 44438061Smsmith DEBUG(fprintf(stderr, "last modified: [\033[1m%04d-%02d-%02d " 44538061Smsmith "%02d:%02d:%02d\033[m]\n", 44638061Smsmith tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, 44738061Smsmith tm.tm_hour, tm.tm_min, tm.tm_sec)); 44828257Smsmith *mtime = timegm(&tm); 44928257Smsmith return 0; 45028219Smsmith} 45128219Smsmith 45228219Smsmith/* 453 * Parse a content-length header 454 */ 455static int 456_http_parse_length(char *p, off_t *length) 457{ 458 off_t len; 459 460 for (len = 0; *p && isdigit(*p); ++p) 461 len = len * 10 + (*p - '0'); 462 DEBUG(fprintf(stderr, "content length: [\033[1m%lld\033[m]\n", len)); 463 *length = len; 464 return 0; 465} 466 467/* 468 * Parse a content-range header 469 */ 470static int 471_http_parse_range(char *p, off_t *offset, off_t *length, off_t *size) 472{ 473 int first, last, len; 474 475 if (strncasecmp(p, "bytes ", 6) != 0) 476 return -1; 477 for (first = 0, p += 6; *p && isdigit(*p); ++p) 478 first = first * 10 + *p - '0'; 479 if (*p != '-') 480 return -1; 481 for (last = 0, ++p; *p && isdigit(*p); ++p) 482 last = last * 10 + *p - '0'; 483 if (first > last || *p != '/') 484 return -1; 485 for (len = 0, ++p; *p && isdigit(*p); ++p) 486 len = len * 10 + *p - '0'; 487 if (len < last - first + 1) 488 return -1; 489 DEBUG(fprintf(stderr, "content range: [\033[1m%d-%d/%d\033[m]\n", 490 first, last, len)); 491 *offset = first; 492 *length = last - first + 1; 493 *size = len; 494 return 0; 495} 496 497 498/***************************************************************************** 499 * Helper functions for authorization 500 */ 501 502/* 503 * Base64 encoding 504 */ 505static char * 506_http_base64(char *src) 507{ 508 static const char base64[] = 509 "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 510 "abcdefghijklmnopqrstuvwxyz" 511 "0123456789+/"; 512 char *str, *dst; 513 size_t l; 514 int t, r; 515 516 l = strlen(src); 517 if ((str = malloc(((l + 2) / 3) * 4)) == NULL) 518 return NULL; 519 dst = str; 520 r = 0; 521 522 while (l >= 3) { 523 t = (src[0] << 16) | (src[1] << 8) | src[2]; 524 dst[0] = base64[(t >> 18) & 0x3f]; 525 dst[1] = base64[(t >> 12) & 0x3f]; 526 dst[2] = base64[(t >> 6) & 0x3f]; 527 dst[3] = base64[(t >> 0) & 0x3f]; 528 src += 3; l -= 3; 529 dst += 4; r += 4; 530 } 531 532 switch (l) { 533 case 2: 534 t = (src[0] << 16) | (src[1] << 8); 535 dst[0] = base64[(t >> 18) & 0x3f]; 536 dst[1] = base64[(t >> 12) & 0x3f]; 537 dst[2] = base64[(t >> 6) & 0x3f]; 538 dst[3] = '='; 539 dst += 4; 540 r += 4; 541 break; 542 case 1: 543 t = src[0] << 16; 544 dst[0] = base64[(t >> 18) & 0x3f]; 545 dst[1] = base64[(t >> 12) & 0x3f]; 546 dst[2] = dst[3] = '='; 547 dst += 4; 548 r += 4; 549 break; 550 case 0: 551 break; 552 } 553 554 *dst = 0; 555 return str; 556} 557 558/* 559 * Encode username and password 560 */ 561static int 562_http_basic_auth(int fd, char *hdr, char *usr, char *pwd) 563{ 564 char *upw, *auth; 565 int r; 566 567 if (asprintf(&upw, "%s:%s", usr, pwd) == -1) 568 return -1; 569 auth = _http_base64(upw); 570 free(upw); 571 if (auth == NULL) 572 return -1; 573 r = _http_cmd(fd, "%s: Basic %s", hdr, auth); 574 free(auth); 575 return r; 576} 577 578/* 579 * Send an authorization header 580 */ 581static int 582_http_authorize(int fd, char *hdr, char *p) 583{ 584 /* basic authorization */ 585 if (strncasecmp(p, "basic:", 6) == 0) { 586 char *user, *pwd, *str; 587 int r; 588 589 /* skip realm */ 590 for (p += 6; *p && *p != ':'; ++p) 591 /* nothing */ ; 592 if (!*p || strchr(++p, ':') == NULL) 593 return -1; 594 if ((str = strdup(p)) == NULL) 595 return -1; /* XXX */ 596 user = str; 597 pwd = strchr(str, ':'); 598 *pwd++ = '\0'; 599 r = _http_basic_auth(fd, hdr, user, pwd); 600 free(str); 601 return r; 602 } 603 return -1; 604} 605 606 607/***************************************************************************** 608 * Helper functions for connecting to a server or proxy 609 */ 610 611/* 612 * Return the default port for this scheme 613 */ 614static int 615_http_default_port(char *scheme) 616{ 617 struct servent *se; 618 619 if ((se = getservbyname(scheme, "tcp")) != NULL) 620 return ntohs(se->s_port); 621 if (strcasecmp(scheme, SCHEME_FTP) == 0) 622 return FTP_DEFAULT_PORT; 623 if (strcasecmp(scheme, SCHEME_HTTP) == 0) 624 return HTTP_DEFAULT_PORT; 625 return 0; 626} 627 628/* 629 * Connect to the correct HTTP server or proxy. 630 */ 631static int 632_http_connect(struct url *URL, struct url *purl, char *flags) 633{ 634 int verbose; 635 int af, fd; 636 637#ifdef INET6 638 af = AF_UNSPEC; 639#else 640 af = AF_INET; 641#endif 642 643 verbose = (flags && strchr(flags, 'v')); 644 if (flags && strchr(flags, '4')) 645 af = AF_INET; 646#ifdef INET6 647 else if (flags && strchr(flags, '6')) 648 af = AF_INET6; 649#endif 650 651 if (purl) { 652 URL = purl; 653 } else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0) { 654 /* can't talk http to an ftp server */ 655 /* XXX should set an error code */ 656 return -1; 657 } 658 659 if ((fd = _fetch_connect(URL->host, URL->port, af, verbose)) == -1) 660 /* _fetch_connect() has already set an error code */ 661 return -1; 662 return fd; 663} 664 665static struct url * 666_http_get_proxy() 667{ 668 struct url *purl; 669 char *p; 670 671 if ((p = getenv("HTTP_PROXY")) && (purl = fetchParseURL(p))) { 672 if (!*purl->scheme) 673 strcpy(purl->scheme, SCHEME_HTTP); 674 if (!purl->port) 675 purl->port = _http_default_port(SCHEME_HTTP); 676 if (strcasecmp(purl->scheme, SCHEME_HTTP) == 0) 677 return purl; 678 fetchFreeURL(purl); 679 } 680 return NULL; 681} 682 683 684/***************************************************************************** 685 * Core 686 */ 687 688/* 689 * Send a request and process the reply 690 */ 691FILE * 692_http_request(struct url *URL, char *op, struct url_stat *us, 693 struct url *purl, char *flags) 694{ 695 struct url *url, *new; 696 int chunked, direct, need_auth, noredirect, verbose; 697 int code, fd, i, n; 698 off_t offset, clength, length, size; 699 time_t mtime; 700 char *p; 701 FILE *f; 702 hdr h; 703 char *host; 704#ifdef INET6 705 char hbuf[MAXHOSTNAMELEN + 1]; 706#endif 707 708 direct = (flags && strchr(flags, 'd')); 709 noredirect = (flags && strchr(flags, 'A')); 710 verbose = (flags && strchr(flags, 'v')); 711 712 if (direct && purl) { 713 fetchFreeURL(purl); 714 purl = NULL; 715 } 716 717 /* try the provided URL first */ 718 url = URL; 719 720 /* if the A flag is set, we only get one try */ 721 n = noredirect ? 1 : MAX_REDIRECT; 722 i = 0; 723 724 do { 725 new = NULL; 726 chunked = 0; 727 need_auth = 0; 728 offset = 0; 729 clength = -1; 730 length = -1; 731 size = -1; 732 mtime = 0; 733 retry: 734 /* check port */ 735 if (!url->port) 736 url->port = _http_default_port(url->scheme); 737 738 /* connect to server or proxy */ 739 if ((fd = _http_connect(url, purl, flags)) == -1) 740 goto ouch; 741 742 host = url->host; 743#ifdef INET6 744 if (strchr(url->host, ':')) { 745 snprintf(hbuf, sizeof(hbuf), "[%s]", url->host); 746 host = hbuf; 747 } 748#endif 749 750 /* send request */ 751 if (verbose) 752 _fetch_info("requesting %s://%s:%d%s", 753 url->scheme, host, url->port, url->doc); 754 if (purl) { 755 _http_cmd(fd, "%s %s://%s:%d%s HTTP/1.1", 756 op, url->scheme, host, url->port, url->doc); 757 } else { 758 _http_cmd(fd, "%s %s HTTP/1.1", 759 op, url->doc); 760 } 761 762 /* proxy authorization */ 763 if (purl) { 764 if (*purl->user || *purl->pwd) 765 _http_basic_auth(fd, "Proxy-Authorization", 766 purl->user, purl->pwd); 767 else if ((p = getenv("HTTP_PROXY_AUTH")) != NULL && *p != '\0') 768 _http_authorize(fd, "Proxy-Authorization", p); 769 } 770 771 /* server authorization */ 772 if (need_auth) { 773 if (*url->user || *url->pwd) 774 _http_basic_auth(fd, "Authorization", url->user, url->pwd); 775 else if ((p = getenv("HTTP_AUTH")) != NULL && *p != '\0') 776 _http_authorize(fd, "Authorization", p); 777 else { 778 _http_seterr(HTTP_NEED_AUTH); 779 goto ouch; 780 } 781 } 782 783 /* other headers */ 784 if (url->port == _http_default_port(url->scheme)) 785 _http_cmd(fd, "Host: %s", host); 786 else 787 _http_cmd(fd, "Host: %s:%d", host, url->port); 788 _http_cmd(fd, "User-Agent: %s " _LIBFETCH_VER, __progname); 789 if (url->offset) 790 _http_cmd(fd, "Range: bytes=%lld-", url->offset); 791 _http_cmd(fd, "Connection: close"); 792 _http_cmd(fd, ""); 793 794 /* get reply */ 795 switch ((code = _http_get_reply(fd))) { 796 case HTTP_OK: 797 case HTTP_PARTIAL: 798 /* fine */ 799 break; 800 case HTTP_MOVED_PERM: 801 case HTTP_MOVED_TEMP: 802 /* 803 * Not so fine, but we still have to read the headers to 804 * get the new location. 805 */ 806 break; 807 case HTTP_NEED_AUTH: 808 if (need_auth) { 809 /* 810 * We already sent out authorization code, so there's 811 * nothing more we can do. 812 */ 813 _http_seterr(code); 814 goto ouch; 815 } 816 /* try again, but send the password this time */ 817 if (verbose) 818 _fetch_info("server requires authorization"); 819 need_auth = 1; 820 close(fd); 821 goto retry; 822 case HTTP_NEED_PROXY_AUTH: 823 /* 824 * If we're talking to a proxy, we already sent our proxy 825 * authorization code, so there's nothing more we can do. 826 */ 827 _http_seterr(code); 828 goto ouch; 829 case HTTP_PROTOCOL_ERROR: 830 /* fall through */ 831 case -1: 832 _fetch_syserr(); 833 goto ouch; 834 default: 835 _http_seterr(code); 836 goto ouch; 837 } 838 839 /* get headers */ 840 do { 841 switch ((h = _http_next_header(fd, &p))) { 842 case hdr_syserror: 843 _fetch_syserr(); 844 goto ouch; 845 case hdr_error: 846 _http_seterr(HTTP_PROTOCOL_ERROR); 847 goto ouch; 848 case hdr_content_length: 849 _http_parse_length(p, &clength); 850 break; 851 case hdr_content_range: 852 _http_parse_range(p, &offset, &length, &size); 853 break; 854 case hdr_last_modified: 855 _http_parse_mtime(p, &mtime); 856 break; 857 case hdr_location: 858 if (!HTTP_REDIRECT(code)) 859 break; 860 if (new) 861 free(new); 862 if (verbose) 863 _fetch_info("%d redirect to %s", code, p); 864 if (*p == '/') 865 /* absolute path */ 866 new = fetchMakeURL(url->scheme, url->host, url->port, p, 867 url->user, url->pwd); 868 else 869 new = fetchParseURL(p); 870 if (new == NULL) { 871 /* XXX should set an error code */ 872 DEBUG(fprintf(stderr, "failed to parse new URL\n")); 873 goto ouch; 874 } 875 if (!*new->user && !*new->pwd) { 876 strcpy(new->user, url->user); 877 strcpy(new->pwd, url->pwd); 878 } 879 new->offset = url->offset; 880 new->length = url->length; 881 break; 882 case hdr_transfer_encoding: 883 /* XXX weak test*/ 884 chunked = (strcasecmp(p, "chunked") == 0); 885 break; 886 case hdr_end: 887 /* fall through */ 888 case hdr_unknown: 889 /* ignore */ 890 break; 891 } 892 } while (h > hdr_end); 893 894 /* we either have a hit, or a redirect with no Location: header */ 895 if (code == HTTP_OK || code == HTTP_PARTIAL || !new) 896 break; 897 898 /* we have a redirect */ 899 close(fd); 900 fd = -1; 901 if (url != URL) 902 fetchFreeURL(url); 903 url = new; 904 } while (++i < n); 905 906 /* no success */ 907 if (fd == -1) { 908 _http_seterr(code); 909 goto ouch; 910 } 911 912 DEBUG(fprintf(stderr, "offset %lld, length %lld, size %lld, clength %lld\n", 913 offset, length, size, clength)); 914 915 /* check for inconsistencies */ 916 if (clength != -1 && length != -1 && clength != length) { 917 _http_seterr(HTTP_PROTOCOL_ERROR); 918 goto ouch; 919 } 920 if (clength == -1) 921 clength = length; 922 if (clength != -1) 923 length = offset + clength; 924 if (length != -1 && size != -1 && length != size) { 925 _http_seterr(HTTP_PROTOCOL_ERROR); 926 goto ouch; 927 } 928 if (size == -1) 929 size = length; 930 931 /* fill in stats */ 932 if (us) { 933 us->size = size; 934 us->atime = us->mtime = mtime; 935 } 936 937 /* too far? */ 938 if (offset > URL->offset) { 939 _http_seterr(HTTP_PROTOCOL_ERROR); 940 goto ouch; 941 } 942 943 /* report back real offset and size */ 944 URL->offset = offset; 945 URL->length = clength; 946 947 /* wrap it up in a FILE */ 948 if ((f = chunked ? _http_funopen(fd) : fdopen(fd, "r")) == NULL) { 949 _fetch_syserr(); 950 goto ouch; 951 } 952 953 if (url != URL) 954 fetchFreeURL(url); 955 if (purl) 956 fetchFreeURL(purl); 957 958 return f; 959 960 ouch: 961 if (url != URL) 962 fetchFreeURL(url); 963 if (purl) 964 fetchFreeURL(purl); 965 if (fd != -1) 966 close(fd); 967 return NULL; 968} 969 970 971/***************************************************************************** 972 * Entry points 973 */ 974 975/* 976 * Retrieve and stat a file by HTTP 977 */ 978FILE * 979fetchXGetHTTP(struct url *URL, struct url_stat *us, char *flags) 980{ 981 return _http_request(URL, "GET", us, _http_get_proxy(), flags); 982} 983 984/* 985 * Retrieve a file by HTTP 986 */ 987FILE * 988fetchGetHTTP(struct url *URL, char *flags) 989{ 990 return fetchXGetHTTP(URL, NULL, flags); 991} 992 993/* 994 * Store a file by HTTP 995 */ 996FILE * 997fetchPutHTTP(struct url *URL, char *flags) 998{ 999 warnx("fetchPutHTTP(): not implemented"); 1000 return NULL; 1001} 1002 1003/* 1004 * Get an HTTP document's metadata 1005 */ 1006int 1007fetchStatHTTP(struct url *URL, struct url_stat *us, char *flags) 1008{ 1009 FILE *f; 1010 1011 if ((f = _http_request(URL, "HEAD", us, _http_get_proxy(), flags)) == NULL) 1012 return -1; 1013 fclose(f); 1014 return 0; 1015} 1016 1017/* 1018 * List a directory 1019 */ 1020struct url_ent * 1021fetchListHTTP(struct url *url, char *flags) 1022{ 1023 warnx("fetchListHTTP(): not implemented"); 1024 return NULL; 1025} 1026