http.c revision 242289
137535Sdes/*- 2236103Sdes * Copyright (c) 2000-2011 Dag-Erling Sm��rgrav 337535Sdes * All rights reserved. 437535Sdes * 537535Sdes * Redistribution and use in source and binary forms, with or without 637535Sdes * modification, are permitted provided that the following conditions 737535Sdes * are met: 837535Sdes * 1. Redistributions of source code must retain the above copyright 937535Sdes * notice, this list of conditions and the following disclaimer 1037535Sdes * in this position and unchanged. 1137535Sdes * 2. Redistributions in binary form must reproduce the above copyright 1237535Sdes * notice, this list of conditions and the following disclaimer in the 1337535Sdes * documentation and/or other materials provided with the distribution. 1437535Sdes * 3. The name of the author may not be used to endorse or promote products 1563012Sdes * derived from this software without specific prior written permission. 1637535Sdes * 1737535Sdes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1837535Sdes * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1937535Sdes * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 2037535Sdes * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 2137535Sdes * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2237535Sdes * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2337535Sdes * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2437535Sdes * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2537535Sdes * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2637535Sdes * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2737535Sdes */ 2837535Sdes 2984203Sdillon#include <sys/cdefs.h> 3084203Sdillon__FBSDID("$FreeBSD: stable/9/lib/libfetch/http.c 242289 2012-10-29 04:16:52Z eadler $"); 3184203Sdillon 3263236Sdes/* 3363236Sdes * The following copyright applies to the base64 code: 3463236Sdes * 3563236Sdes *- 3663236Sdes * Copyright 1997 Massachusetts Institute of Technology 3763236Sdes * 3863236Sdes * Permission to use, copy, modify, and distribute this software and 3963236Sdes * its documentation for any purpose and without fee is hereby 4063236Sdes * granted, provided that both the above copyright notice and this 4163236Sdes * permission notice appear in all copies, that both the above 4263236Sdes * copyright notice and this permission notice appear in all 4363236Sdes * supporting documentation, and that the name of M.I.T. not be used 4463236Sdes * in advertising or publicity pertaining to distribution of the 4563236Sdes * software without specific, written prior permission. M.I.T. makes 4663236Sdes * no representations about the suitability of this software for any 4763236Sdes * purpose. It is provided "as is" without express or implied 4863236Sdes * warranty. 4990267Sdes * 5063236Sdes * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS 5163236Sdes * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE, 5263236Sdes * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 5363236Sdes * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT 5463236Sdes * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 5563236Sdes * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 5663236Sdes * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF 5763236Sdes * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 5863236Sdes * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 5963236Sdes * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 6063236Sdes * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 6163236Sdes * SUCH DAMAGE. 6263236Sdes */ 6363236Sdes 6437535Sdes#include <sys/param.h> 6560737Sume#include <sys/socket.h> 66186124Smurray#include <sys/time.h> 6737535Sdes 6863012Sdes#include <ctype.h> 6937535Sdes#include <err.h> 7063012Sdes#include <errno.h> 7160376Sdes#include <locale.h> 7260189Sdes#include <netdb.h> 7337608Sdes#include <stdarg.h> 7437535Sdes#include <stdio.h> 7537535Sdes#include <stdlib.h> 7637535Sdes#include <string.h> 7760376Sdes#include <time.h> 7837535Sdes#include <unistd.h> 79202613Sdes#include <md5.h> 8037535Sdes 81141958Skbyanc#include <netinet/in.h> 82141958Skbyanc#include <netinet/tcp.h> 83141958Skbyanc 8437535Sdes#include "fetch.h" 8540939Sdes#include "common.h" 8641862Sdes#include "httperr.h" 8737535Sdes 8863012Sdes/* Maximum number of redirects to follow */ 89242032Seadler#define MAX_REDIRECT 20 9037535Sdes 9163012Sdes/* Symbolic names for reply codes we care about */ 9263012Sdes#define HTTP_OK 200 9363012Sdes#define HTTP_PARTIAL 206 9463012Sdes#define HTTP_MOVED_PERM 301 9563012Sdes#define HTTP_MOVED_TEMP 302 9663012Sdes#define HTTP_SEE_OTHER 303 97186124Smurray#define HTTP_NOT_MODIFIED 304 98169386Sdes#define HTTP_TEMP_REDIRECT 307 99242289Seadler#define HTTP_PERM_REDIRECT 308 10063012Sdes#define HTTP_NEED_AUTH 401 10187317Sdes#define HTTP_NEED_PROXY_AUTH 407 102125696Sdes#define HTTP_BAD_RANGE 416 10363012Sdes#define HTTP_PROTOCOL_ERROR 999 10460196Sdes 10563012Sdes#define HTTP_REDIRECT(xyz) ((xyz) == HTTP_MOVED_PERM \ 10690267Sdes || (xyz) == HTTP_MOVED_TEMP \ 107169386Sdes || (xyz) == HTTP_TEMP_REDIRECT \ 10890267Sdes || (xyz) == HTTP_SEE_OTHER) 10963012Sdes 11088771Sdes#define HTTP_ERROR(xyz) ((xyz) > 400 && (xyz) < 599) 11163012Sdes 11290267Sdes 11363012Sdes/***************************************************************************** 11463012Sdes * I/O functions for decoding chunked streams 11563012Sdes */ 11663012Sdes 11797859Sdesstruct httpio 11837535Sdes{ 11997858Sdes conn_t *conn; /* connection */ 12097866Sdes int chunked; /* chunked mode */ 12197858Sdes char *buf; /* chunk buffer */ 12297866Sdes size_t bufsize; /* size of chunk buffer */ 12397866Sdes ssize_t buflen; /* amount of data currently in buffer */ 12497866Sdes int bufpos; /* current read offset in buffer */ 12597858Sdes int eof; /* end-of-file flag */ 12697858Sdes int error; /* error flag */ 12797858Sdes size_t chunksize; /* remaining size of current chunk */ 12863281Sdes#ifndef NDEBUG 12990267Sdes size_t total; 13063012Sdes#endif 13137535Sdes}; 13237535Sdes 13337608Sdes/* 13463012Sdes * Get next chunk header 13537608Sdes */ 13637608Sdesstatic int 137174588Sdeshttp_new_chunk(struct httpio *io) 13837608Sdes{ 13990267Sdes char *p; 14090267Sdes 141174588Sdes if (fetch_getln(io->conn) == -1) 14290267Sdes return (-1); 14390267Sdes 144174761Sdes if (io->conn->buflen < 2 || !isxdigit((unsigned char)*io->conn->buf)) 14590267Sdes return (-1); 14690267Sdes 147174761Sdes for (p = io->conn->buf; *p && !isspace((unsigned char)*p); ++p) { 14890267Sdes if (*p == ';') 14990267Sdes break; 150174761Sdes if (!isxdigit((unsigned char)*p)) 15190267Sdes return (-1); 152174761Sdes if (isdigit((unsigned char)*p)) { 15397859Sdes io->chunksize = io->chunksize * 16 + 15490267Sdes *p - '0'; 15590267Sdes } else { 15697859Sdes io->chunksize = io->chunksize * 16 + 157176036Sdes 10 + tolower((unsigned char)*p) - 'a'; 15890267Sdes } 15990267Sdes } 16090267Sdes 16163281Sdes#ifndef NDEBUG 16290267Sdes if (fetchDebug) { 16397859Sdes io->total += io->chunksize; 16497859Sdes if (io->chunksize == 0) 165106207Sdes fprintf(stderr, "%s(): end of last chunk\n", __func__); 16690267Sdes else 167106207Sdes fprintf(stderr, "%s(): new chunk: %lu (%lu)\n", 168106207Sdes __func__, (unsigned long)io->chunksize, 169106207Sdes (unsigned long)io->total); 17090267Sdes } 17163012Sdes#endif 17290267Sdes 17397859Sdes return (io->chunksize); 17437608Sdes} 17537608Sdes 17637608Sdes/* 17797866Sdes * Grow the input buffer to at least len bytes 17897866Sdes */ 17997866Sdesstatic inline int 180174588Sdeshttp_growbuf(struct httpio *io, size_t len) 18197866Sdes{ 18297866Sdes char *tmp; 18397866Sdes 18497866Sdes if (io->bufsize >= len) 18597866Sdes return (0); 18697866Sdes 18797866Sdes if ((tmp = realloc(io->buf, len)) == NULL) 18897866Sdes return (-1); 18997866Sdes io->buf = tmp; 19097866Sdes io->bufsize = len; 191106044Sdes return (0); 19297866Sdes} 19397866Sdes 19497866Sdes/* 19537608Sdes * Fill the input buffer, do chunk decoding on the fly 19637608Sdes */ 19763012Sdesstatic int 198174588Sdeshttp_fillbuf(struct httpio *io, size_t len) 19937535Sdes{ 200231247Sbapt ssize_t nbytes; 201231247Sbapt 20297859Sdes if (io->error) 20390267Sdes return (-1); 20497859Sdes if (io->eof) 20590267Sdes return (0); 20690267Sdes 20797866Sdes if (io->chunked == 0) { 208174588Sdes if (http_growbuf(io, len) == -1) 20997866Sdes return (-1); 210231247Sbapt if ((nbytes = fetch_read(io->conn, io->buf, len)) == -1) { 211231247Sbapt io->error = errno; 21297866Sdes return (-1); 213106185Sdes } 214231247Sbapt io->buflen = nbytes; 21597866Sdes io->bufpos = 0; 21697866Sdes return (io->buflen); 21797866Sdes } 21897866Sdes 21997859Sdes if (io->chunksize == 0) { 220174588Sdes switch (http_new_chunk(io)) { 22190267Sdes case -1: 22297859Sdes io->error = 1; 22390267Sdes return (-1); 22490267Sdes case 0: 22597859Sdes io->eof = 1; 22690267Sdes return (0); 22790267Sdes } 22837535Sdes } 22963012Sdes 23097866Sdes if (len > io->chunksize) 23197866Sdes len = io->chunksize; 232174588Sdes if (http_growbuf(io, len) == -1) 23390267Sdes return (-1); 234231247Sbapt if ((nbytes = fetch_read(io->conn, io->buf, len)) == -1) { 235231247Sbapt io->error = errno; 23697866Sdes return (-1); 237106185Sdes } 238231247Sbapt io->buflen = nbytes; 23997866Sdes io->chunksize -= io->buflen; 24090267Sdes 24197859Sdes if (io->chunksize == 0) { 24297856Sdes char endl[2]; 24397856Sdes 244174588Sdes if (fetch_read(io->conn, endl, 2) != 2 || 24597856Sdes endl[0] != '\r' || endl[1] != '\n') 24690267Sdes return (-1); 24790267Sdes } 24890267Sdes 24997866Sdes io->bufpos = 0; 25090267Sdes 25197866Sdes return (io->buflen); 25237535Sdes} 25337535Sdes 25437608Sdes/* 25537608Sdes * Read function 25637608Sdes */ 25737535Sdesstatic int 258174588Sdeshttp_readfn(void *v, char *buf, int len) 25937535Sdes{ 26097859Sdes struct httpio *io = (struct httpio *)v; 26190267Sdes int l, pos; 26263012Sdes 26397859Sdes if (io->error) 26490267Sdes return (-1); 26597859Sdes if (io->eof) 26690267Sdes return (0); 26763012Sdes 26890267Sdes for (pos = 0; len > 0; pos += l, len -= l) { 26990267Sdes /* empty buffer */ 27097866Sdes if (!io->buf || io->bufpos == io->buflen) 271174588Sdes if (http_fillbuf(io, len) < 1) 27290267Sdes break; 27397866Sdes l = io->buflen - io->bufpos; 27490267Sdes if (len < l) 27590267Sdes l = len; 276176105Sdes memcpy(buf + pos, io->buf + io->bufpos, l); 27797866Sdes io->bufpos += l; 27890267Sdes } 27937535Sdes 280231247Sbapt if (!pos && io->error) { 281231247Sbapt if (io->error == EINTR) 282231247Sbapt io->error = 0; 28390267Sdes return (-1); 284231247Sbapt } 28590267Sdes return (pos); 28637535Sdes} 28737535Sdes 28837608Sdes/* 28937608Sdes * Write function 29037608Sdes */ 29137535Sdesstatic int 292174588Sdeshttp_writefn(void *v, const char *buf, int len) 29337535Sdes{ 29497859Sdes struct httpio *io = (struct httpio *)v; 29590267Sdes 296174588Sdes return (fetch_write(io->conn, buf, len)); 29737535Sdes} 29837535Sdes 29937608Sdes/* 30037608Sdes * Close function 30137608Sdes */ 30237535Sdesstatic int 303174588Sdeshttp_closefn(void *v) 30437535Sdes{ 30597859Sdes struct httpio *io = (struct httpio *)v; 30690267Sdes int r; 30763012Sdes 308174588Sdes r = fetch_close(io->conn); 30997859Sdes if (io->buf) 31097859Sdes free(io->buf); 31197859Sdes free(io); 31290267Sdes return (r); 31337535Sdes} 31437535Sdes 31537608Sdes/* 31663012Sdes * Wrap a file descriptor up 31737608Sdes */ 31863012Sdesstatic FILE * 319174588Sdeshttp_funopen(conn_t *conn, int chunked) 32037535Sdes{ 32197859Sdes struct httpio *io; 32290267Sdes FILE *f; 32363012Sdes 324109967Sdes if ((io = calloc(1, sizeof(*io))) == NULL) { 325174588Sdes fetch_syserr(); 32690267Sdes return (NULL); 32790267Sdes } 32897859Sdes io->conn = conn; 32997866Sdes io->chunked = chunked; 330174588Sdes f = funopen(io, http_readfn, http_writefn, NULL, http_closefn); 33190267Sdes if (f == NULL) { 332174588Sdes fetch_syserr(); 33397859Sdes free(io); 33490267Sdes return (NULL); 33590267Sdes } 33690267Sdes return (f); 33763012Sdes} 33863012Sdes 33990267Sdes 34063012Sdes/***************************************************************************** 34163012Sdes * Helper functions for talking to the server and parsing its replies 34263012Sdes */ 34363012Sdes 34463012Sdes/* Header types */ 34563012Sdestypedef enum { 34690267Sdes hdr_syserror = -2, 34790267Sdes hdr_error = -1, 34890267Sdes hdr_end = 0, 34990267Sdes hdr_unknown = 1, 35090267Sdes hdr_content_length, 35190267Sdes hdr_content_range, 35290267Sdes hdr_last_modified, 35390267Sdes hdr_location, 35490267Sdes hdr_transfer_encoding, 355202613Sdes hdr_www_authenticate, 356202613Sdes hdr_proxy_authenticate, 35785093Sdes} hdr_t; 35863012Sdes 35963012Sdes/* Names of interesting headers */ 36063012Sdesstatic struct { 36190267Sdes hdr_t num; 36290267Sdes const char *name; 36363012Sdes} hdr_names[] = { 36490267Sdes { hdr_content_length, "Content-Length" }, 36590267Sdes { hdr_content_range, "Content-Range" }, 36690267Sdes { hdr_last_modified, "Last-Modified" }, 36790267Sdes { hdr_location, "Location" }, 36890267Sdes { hdr_transfer_encoding, "Transfer-Encoding" }, 36990267Sdes { hdr_www_authenticate, "WWW-Authenticate" }, 370202613Sdes { hdr_proxy_authenticate, "Proxy-Authenticate" }, 37190267Sdes { hdr_unknown, NULL }, 37263012Sdes}; 37363012Sdes 37463012Sdes/* 37563012Sdes * Send a formatted line; optionally echo to terminal 37663012Sdes */ 37763012Sdesstatic int 378174588Sdeshttp_cmd(conn_t *conn, const char *fmt, ...) 37963012Sdes{ 38090267Sdes va_list ap; 38190267Sdes size_t len; 38290267Sdes char *msg; 38390267Sdes int r; 38463012Sdes 38590267Sdes va_start(ap, fmt); 38690267Sdes len = vasprintf(&msg, fmt, ap); 38790267Sdes va_end(ap); 38890267Sdes 38990267Sdes if (msg == NULL) { 39090267Sdes errno = ENOMEM; 391174588Sdes fetch_syserr(); 39290267Sdes return (-1); 39390267Sdes } 39490267Sdes 395174588Sdes r = fetch_putln(conn, msg, len); 39690267Sdes free(msg); 39790267Sdes 39890267Sdes if (r == -1) { 399174588Sdes fetch_syserr(); 40090267Sdes return (-1); 40190267Sdes } 40290267Sdes 40390267Sdes return (0); 40463012Sdes} 40563012Sdes 40663012Sdes/* 40763012Sdes * Get and parse status line 40863012Sdes */ 40963012Sdesstatic int 410174588Sdeshttp_get_reply(conn_t *conn) 41163012Sdes{ 41290267Sdes char *p; 41390267Sdes 414174588Sdes if (fetch_getln(conn) == -1) 41590267Sdes return (-1); 41690267Sdes /* 41790267Sdes * A valid status line looks like "HTTP/m.n xyz reason" where m 41890267Sdes * and n are the major and minor protocol version numbers and xyz 41990267Sdes * is the reply code. 42090267Sdes * Unfortunately, there are servers out there (NCSA 1.5.1, to name 42190267Sdes * just one) that do not send a version number, so we can't rely 42290267Sdes * on finding one, but if we do, insist on it being 1.0 or 1.1. 42390267Sdes * We don't care about the reason phrase. 42490267Sdes */ 42597856Sdes if (strncmp(conn->buf, "HTTP", 4) != 0) 42690267Sdes return (HTTP_PROTOCOL_ERROR); 42797856Sdes p = conn->buf + 4; 42890267Sdes if (*p == '/') { 42990267Sdes if (p[1] != '1' || p[2] != '.' || (p[3] != '0' && p[3] != '1')) 43090267Sdes return (HTTP_PROTOCOL_ERROR); 43190267Sdes p += 4; 43290267Sdes } 433174761Sdes if (*p != ' ' || 434174761Sdes !isdigit((unsigned char)p[1]) || 435174761Sdes !isdigit((unsigned char)p[2]) || 436174761Sdes !isdigit((unsigned char)p[3])) 43790267Sdes return (HTTP_PROTOCOL_ERROR); 43890267Sdes 43997856Sdes conn->err = (p[1] - '0') * 100 + (p[2] - '0') * 10 + (p[3] - '0'); 44097856Sdes return (conn->err); 44137535Sdes} 44237535Sdes 44337608Sdes/* 44490267Sdes * Check a header; if the type matches the given string, return a pointer 44590267Sdes * to the beginning of the value. 44663012Sdes */ 44775891Sarchiestatic const char * 448174588Sdeshttp_match(const char *str, const char *hdr) 44963012Sdes{ 450176036Sdes while (*str && *hdr && 451176036Sdes tolower((unsigned char)*str++) == tolower((unsigned char)*hdr++)) 45290267Sdes /* nothing */; 45390267Sdes if (*str || *hdr != ':') 45490267Sdes return (NULL); 455174761Sdes while (*hdr && isspace((unsigned char)*++hdr)) 45690267Sdes /* nothing */; 45790267Sdes return (hdr); 45863012Sdes} 45963012Sdes 460202613Sdes 46163012Sdes/* 462202613Sdes * Get the next header and return the appropriate symbolic code. We 463202613Sdes * need to read one line ahead for checking for a continuation line 464202613Sdes * belonging to the current header (continuation lines start with 465221821Sdes * white space). 466202613Sdes * 467202613Sdes * We get called with a fresh line already in the conn buffer, either 468202613Sdes * from the previous http_next_header() invocation, or, the first 469202613Sdes * time, from a fetch_getln() performed by our caller. 470202613Sdes * 471202613Sdes * This stops when we encounter an empty line (we dont read beyond the header 472202613Sdes * area). 473221821Sdes * 474202613Sdes * Note that the "headerbuf" is just a place to return the result. Its 475202613Sdes * contents are not used for the next call. This means that no cleanup 476202613Sdes * is needed when ie doing another connection, just call the cleanup when 477202613Sdes * fully done to deallocate memory. 47863012Sdes */ 479202613Sdes 480202613Sdes/* Limit the max number of continuation lines to some reasonable value */ 481202613Sdes#define HTTP_MAX_CONT_LINES 10 482202613Sdes 483202613Sdes/* Place into which to build a header from one or several lines */ 484202613Sdestypedef struct { 485202613Sdes char *buf; /* buffer */ 486202613Sdes size_t bufsize; /* buffer size */ 487202613Sdes size_t buflen; /* length of buffer contents */ 488202613Sdes} http_headerbuf_t; 489202613Sdes 490202613Sdesstatic void 491202613Sdesinit_http_headerbuf(http_headerbuf_t *buf) 49263012Sdes{ 493202613Sdes buf->buf = NULL; 494202613Sdes buf->bufsize = 0; 495202613Sdes buf->buflen = 0; 496202613Sdes} 49790267Sdes 498221821Sdesstatic void 499202613Sdesclean_http_headerbuf(http_headerbuf_t *buf) 500202613Sdes{ 501202613Sdes if (buf->buf) 502202613Sdes free(buf->buf); 503202613Sdes init_http_headerbuf(buf); 504202613Sdes} 505202613Sdes 506202613Sdes/* Remove whitespace at the end of the buffer */ 507221821Sdesstatic void 508202613Sdeshttp_conn_trimright(conn_t *conn) 509202613Sdes{ 510221821Sdes while (conn->buflen && 511202613Sdes isspace((unsigned char)conn->buf[conn->buflen - 1])) 51297856Sdes conn->buflen--; 51397856Sdes conn->buf[conn->buflen] = '\0'; 514202613Sdes} 515202613Sdes 516202613Sdesstatic hdr_t 517202613Sdeshttp_next_header(conn_t *conn, http_headerbuf_t *hbuf, const char **p) 518202613Sdes{ 519221820Sdes unsigned int i, len; 520202613Sdes 521221821Sdes /* 522202613Sdes * Have to do the stripping here because of the first line. So 523221821Sdes * it's done twice for the subsequent lines. No big deal 524202613Sdes */ 525202613Sdes http_conn_trimright(conn); 52697856Sdes if (conn->buflen == 0) 52797856Sdes return (hdr_end); 528202613Sdes 529202613Sdes /* Copy the line to the headerbuf */ 530202613Sdes if (hbuf->bufsize < conn->buflen + 1) { 531202613Sdes if ((hbuf->buf = realloc(hbuf->buf, conn->buflen + 1)) == NULL) 532202613Sdes return (hdr_syserror); 533202613Sdes hbuf->bufsize = conn->buflen + 1; 534202613Sdes } 535202613Sdes strcpy(hbuf->buf, conn->buf); 536202613Sdes hbuf->buflen = conn->buflen; 537202613Sdes 538221821Sdes /* 539202613Sdes * Fetch possible continuation lines. Stop at 1st non-continuation 540221821Sdes * and leave it in the conn buffer 541221821Sdes */ 542202613Sdes for (i = 0; i < HTTP_MAX_CONT_LINES; i++) { 543202613Sdes if (fetch_getln(conn) == -1) 544202613Sdes return (hdr_syserror); 545202613Sdes 546221821Sdes /* 547202613Sdes * Note: we carry on the idea from the previous version 548202613Sdes * that a pure whitespace line is equivalent to an empty 549202613Sdes * one (so it's not continuation and will be handled when 550221821Sdes * we are called next) 551202613Sdes */ 552202613Sdes http_conn_trimright(conn); 553202613Sdes if (conn->buf[0] != ' ' && conn->buf[0] != "\t"[0]) 554202613Sdes break; 555202613Sdes 556202613Sdes /* Got a continuation line. Concatenate to previous */ 557202613Sdes len = hbuf->buflen + conn->buflen; 558202613Sdes if (hbuf->bufsize < len + 1) { 559202613Sdes len *= 2; 560202613Sdes if ((hbuf->buf = realloc(hbuf->buf, len + 1)) == NULL) 561202613Sdes return (hdr_syserror); 562202613Sdes hbuf->bufsize = len + 1; 563202613Sdes } 564202613Sdes strcpy(hbuf->buf + hbuf->buflen, conn->buf); 565202613Sdes hbuf->buflen += conn->buflen; 566221821Sdes } 567202613Sdes 56890267Sdes /* 56990267Sdes * We could check for malformed headers but we don't really care. 57090267Sdes * A valid header starts with a token immediately followed by a 57190267Sdes * colon; a token is any sequence of non-control, non-whitespace 57290267Sdes * characters except "()<>@,;:\\\"{}". 57390267Sdes */ 57490267Sdes for (i = 0; hdr_names[i].num != hdr_unknown; i++) 575202613Sdes if ((*p = http_match(hdr_names[i].name, hbuf->buf)) != NULL) 57690267Sdes return (hdr_names[i].num); 577202613Sdes 57890267Sdes return (hdr_unknown); 57963012Sdes} 58063012Sdes 581202613Sdes/************************** 582202613Sdes * [Proxy-]Authenticate header parsing 583202613Sdes */ 584202613Sdes 585221821Sdes/* 586221821Sdes * Read doublequote-delimited string into output buffer obuf (allocated 587202613Sdes * by caller, whose responsibility it is to ensure that it's big enough) 588202613Sdes * cp points to the first char after the initial '"' 589221821Sdes * Handles \ quoting 590221821Sdes * Returns pointer to the first char after the terminating double quote, or 591202613Sdes * NULL for error. 592202613Sdes */ 593202613Sdesstatic const char * 594202613Sdeshttp_parse_headerstring(const char *cp, char *obuf) 595202613Sdes{ 596202613Sdes for (;;) { 597202613Sdes switch (*cp) { 598202613Sdes case 0: /* Unterminated string */ 599202613Sdes *obuf = 0; 600202613Sdes return (NULL); 601202613Sdes case '"': /* Ending quote */ 602202613Sdes *obuf = 0; 603202613Sdes return (++cp); 604202613Sdes case '\\': 605202613Sdes if (*++cp == 0) { 606202613Sdes *obuf = 0; 607202613Sdes return (NULL); 608202613Sdes } 609202613Sdes /* FALLTHROUGH */ 610202613Sdes default: 611202613Sdes *obuf++ = *cp++; 612202613Sdes } 613202613Sdes } 614202613Sdes} 615202613Sdes 616202613Sdes/* Http auth challenge schemes */ 617202613Sdestypedef enum {HTTPAS_UNKNOWN, HTTPAS_BASIC,HTTPAS_DIGEST} http_auth_schemes_t; 618202613Sdes 619202613Sdes/* Data holder for a Basic or Digest challenge. */ 620202613Sdestypedef struct { 621202613Sdes http_auth_schemes_t scheme; 622202613Sdes char *realm; 623202613Sdes char *qop; 624202613Sdes char *nonce; 625202613Sdes char *opaque; 626202613Sdes char *algo; 627202613Sdes int stale; 628202613Sdes int nc; /* Nonce count */ 629202613Sdes} http_auth_challenge_t; 630202613Sdes 631221821Sdesstatic void 632202613Sdesinit_http_auth_challenge(http_auth_challenge_t *b) 633202613Sdes{ 634202613Sdes b->scheme = HTTPAS_UNKNOWN; 635202613Sdes b->realm = b->qop = b->nonce = b->opaque = b->algo = NULL; 636202613Sdes b->stale = b->nc = 0; 637202613Sdes} 638202613Sdes 639221821Sdesstatic void 640202613Sdesclean_http_auth_challenge(http_auth_challenge_t *b) 641202613Sdes{ 642221821Sdes if (b->realm) 643202613Sdes free(b->realm); 644221821Sdes if (b->qop) 645202613Sdes free(b->qop); 646221821Sdes if (b->nonce) 647202613Sdes free(b->nonce); 648221821Sdes if (b->opaque) 649202613Sdes free(b->opaque); 650221821Sdes if (b->algo) 651202613Sdes free(b->algo); 652202613Sdes init_http_auth_challenge(b); 653202613Sdes} 654202613Sdes 655202613Sdes/* Data holder for an array of challenges offered in an http response. */ 656202613Sdes#define MAX_CHALLENGES 10 657202613Sdestypedef struct { 658202613Sdes http_auth_challenge_t *challenges[MAX_CHALLENGES]; 659202613Sdes int count; /* Number of parsed challenges in the array */ 660202613Sdes int valid; /* We did parse an authenticate header */ 661202613Sdes} http_auth_challenges_t; 662202613Sdes 663221821Sdesstatic void 664202613Sdesinit_http_auth_challenges(http_auth_challenges_t *cs) 665202613Sdes{ 666202613Sdes int i; 667202613Sdes for (i = 0; i < MAX_CHALLENGES; i++) 668202613Sdes cs->challenges[i] = NULL; 669202613Sdes cs->count = cs->valid = 0; 670202613Sdes} 671202613Sdes 672221821Sdesstatic void 673202613Sdesclean_http_auth_challenges(http_auth_challenges_t *cs) 674202613Sdes{ 675202613Sdes int i; 676202613Sdes /* We rely on non-zero pointers being allocated, not on the count */ 677202613Sdes for (i = 0; i < MAX_CHALLENGES; i++) { 678202613Sdes if (cs->challenges[i] != NULL) { 679202613Sdes clean_http_auth_challenge(cs->challenges[i]); 680202613Sdes free(cs->challenges[i]); 681202613Sdes } 682202613Sdes } 683202613Sdes init_http_auth_challenges(cs); 684202613Sdes} 685202613Sdes 686221821Sdes/* 687202613Sdes * Enumeration for lexical elements. Separators will be returned as their own 688202613Sdes * ascii value 689202613Sdes */ 690202613Sdestypedef enum {HTTPHL_WORD=256, HTTPHL_STRING=257, HTTPHL_END=258, 691202613Sdes HTTPHL_ERROR = 259} http_header_lex_t; 692202613Sdes 693221821Sdes/* 694202613Sdes * Determine what kind of token comes next and return possible value 695202613Sdes * in buf, which is supposed to have been allocated big enough by 696221821Sdes * caller. Advance input pointer and return element type. 697202613Sdes */ 698221821Sdesstatic int 699202613Sdeshttp_header_lex(const char **cpp, char *buf) 700202613Sdes{ 701202613Sdes size_t l; 702202613Sdes /* Eat initial whitespace */ 703202613Sdes *cpp += strspn(*cpp, " \t"); 704202613Sdes if (**cpp == 0) 705202613Sdes return (HTTPHL_END); 706202613Sdes 707202613Sdes /* Separator ? */ 708202613Sdes if (**cpp == ',' || **cpp == '=') 709202613Sdes return (*((*cpp)++)); 710202613Sdes 711202613Sdes /* String ? */ 712202613Sdes if (**cpp == '"') { 713202613Sdes *cpp = http_parse_headerstring(++*cpp, buf); 714202613Sdes if (*cpp == NULL) 715202613Sdes return (HTTPHL_ERROR); 716202613Sdes return (HTTPHL_STRING); 717202613Sdes } 718202613Sdes 719202613Sdes /* Read other token, until separator or whitespace */ 720202613Sdes l = strcspn(*cpp, " \t,="); 721202613Sdes memcpy(buf, *cpp, l); 722202613Sdes buf[l] = 0; 723202613Sdes *cpp += l; 724202613Sdes return (HTTPHL_WORD); 725202613Sdes} 726202613Sdes 727221821Sdes/* 728202613Sdes * Read challenges from http xxx-authenticate header and accumulate them 729202613Sdes * in the challenges list structure. 730202613Sdes * 731202613Sdes * Headers with multiple challenges are specified by rfc2617, but 732202613Sdes * servers (ie: squid) often send them in separate headers instead, 733202613Sdes * which in turn is forbidden by the http spec (multiple headers with 734202613Sdes * the same name are only allowed for pure comma-separated lists, see 735202613Sdes * rfc2616 sec 4.2). 736202613Sdes * 737202613Sdes * We support both approaches anyway 738202613Sdes */ 739221821Sdesstatic int 740202613Sdeshttp_parse_authenticate(const char *cp, http_auth_challenges_t *cs) 741202613Sdes{ 742202613Sdes int ret = -1; 743202613Sdes http_header_lex_t lex; 744202613Sdes char *key = malloc(strlen(cp) + 1); 745202613Sdes char *value = malloc(strlen(cp) + 1); 746202613Sdes char *buf = malloc(strlen(cp) + 1); 747202613Sdes 748202613Sdes if (key == NULL || value == NULL || buf == NULL) { 749202613Sdes fetch_syserr(); 750202613Sdes goto out; 751202613Sdes } 752202613Sdes 753202613Sdes /* In any case we've seen the header and we set the valid bit */ 754202613Sdes cs->valid = 1; 755202613Sdes 756202613Sdes /* Need word first */ 757202613Sdes lex = http_header_lex(&cp, key); 758202613Sdes if (lex != HTTPHL_WORD) 759202613Sdes goto out; 760202613Sdes 761202613Sdes /* Loop on challenges */ 762202613Sdes for (; cs->count < MAX_CHALLENGES; cs->count++) { 763221821Sdes cs->challenges[cs->count] = 764202613Sdes malloc(sizeof(http_auth_challenge_t)); 765202613Sdes if (cs->challenges[cs->count] == NULL) { 766202613Sdes fetch_syserr(); 767202613Sdes goto out; 768202613Sdes } 769202613Sdes init_http_auth_challenge(cs->challenges[cs->count]); 770202613Sdes if (!strcasecmp(key, "basic")) { 771202613Sdes cs->challenges[cs->count]->scheme = HTTPAS_BASIC; 772202613Sdes } else if (!strcasecmp(key, "digest")) { 773202613Sdes cs->challenges[cs->count]->scheme = HTTPAS_DIGEST; 774202613Sdes } else { 775202613Sdes cs->challenges[cs->count]->scheme = HTTPAS_UNKNOWN; 776221821Sdes /* 777221821Sdes * Continue parsing as basic or digest may 778202613Sdes * follow, and the syntax is the same for 779202613Sdes * all. We'll just ignore this one when 780202613Sdes * looking at the list 781202613Sdes */ 782202613Sdes } 783221821Sdes 784202613Sdes /* Loop on attributes */ 785202613Sdes for (;;) { 786202613Sdes /* Key */ 787202613Sdes lex = http_header_lex(&cp, key); 788202613Sdes if (lex != HTTPHL_WORD) 789202613Sdes goto out; 790202613Sdes 791202613Sdes /* Equal sign */ 792202613Sdes lex = http_header_lex(&cp, buf); 793202613Sdes if (lex != '=') 794202613Sdes goto out; 795202613Sdes 796202613Sdes /* Value */ 797202613Sdes lex = http_header_lex(&cp, value); 798202613Sdes if (lex != HTTPHL_WORD && lex != HTTPHL_STRING) 799202613Sdes goto out; 800202613Sdes 801202613Sdes if (!strcasecmp(key, "realm")) 802221821Sdes cs->challenges[cs->count]->realm = 803202613Sdes strdup(value); 804202613Sdes else if (!strcasecmp(key, "qop")) 805221821Sdes cs->challenges[cs->count]->qop = 806202613Sdes strdup(value); 807202613Sdes else if (!strcasecmp(key, "nonce")) 808221821Sdes cs->challenges[cs->count]->nonce = 809202613Sdes strdup(value); 810202613Sdes else if (!strcasecmp(key, "opaque")) 811221821Sdes cs->challenges[cs->count]->opaque = 812202613Sdes strdup(value); 813202613Sdes else if (!strcasecmp(key, "algorithm")) 814221821Sdes cs->challenges[cs->count]->algo = 815202613Sdes strdup(value); 816202613Sdes else if (!strcasecmp(key, "stale")) 817221821Sdes cs->challenges[cs->count]->stale = 818202613Sdes strcasecmp(value, "no"); 819202613Sdes /* Else ignore unknown attributes */ 820202613Sdes 821202613Sdes /* Comma or Next challenge or End */ 822202613Sdes lex = http_header_lex(&cp, key); 823221821Sdes /* 824221821Sdes * If we get a word here, this is the beginning of the 825221821Sdes * next challenge. Break the attributes loop 826221821Sdes */ 827202613Sdes if (lex == HTTPHL_WORD) 828202613Sdes break; 829202613Sdes 830202613Sdes if (lex == HTTPHL_END) { 831202613Sdes /* End while looking for ',' is normal exit */ 832202613Sdes cs->count++; 833202613Sdes ret = 0; 834202613Sdes goto out; 835202613Sdes } 836202613Sdes /* Anything else is an error */ 837202613Sdes if (lex != ',') 838202613Sdes goto out; 839202613Sdes 840202613Sdes } /* End attributes loop */ 841202613Sdes } /* End challenge loop */ 842202613Sdes 843221821Sdes /* 844221821Sdes * Challenges max count exceeded. This really can't happen 845221821Sdes * with normal data, something's fishy -> error 846221821Sdes */ 847202613Sdes 848202613Sdesout: 849202613Sdes if (key) 850202613Sdes free(key); 851202613Sdes if (value) 852202613Sdes free(value); 853202613Sdes if (buf) 854202613Sdes free(buf); 855202613Sdes return (ret); 856202613Sdes} 857202613Sdes 858202613Sdes 85963012Sdes/* 86063012Sdes * Parse a last-modified header 86163012Sdes */ 86263716Sdesstatic int 863174588Sdeshttp_parse_mtime(const char *p, time_t *mtime) 86463012Sdes{ 86590267Sdes char locale[64], *r; 86690267Sdes struct tm tm; 86763012Sdes 868109967Sdes strncpy(locale, setlocale(LC_TIME, NULL), sizeof(locale)); 86990267Sdes setlocale(LC_TIME, "C"); 87090267Sdes r = strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm); 87190267Sdes /* XXX should add support for date-2 and date-3 */ 87290267Sdes setlocale(LC_TIME, locale); 87390267Sdes if (r == NULL) 87490267Sdes return (-1); 87590267Sdes DEBUG(fprintf(stderr, "last modified: [%04d-%02d-%02d " 87688769Sdes "%02d:%02d:%02d]\n", 87763012Sdes tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, 87863012Sdes tm.tm_hour, tm.tm_min, tm.tm_sec)); 87990267Sdes *mtime = timegm(&tm); 88090267Sdes return (0); 88163012Sdes} 88263012Sdes 88363012Sdes/* 88463012Sdes * Parse a content-length header 88563012Sdes */ 88663716Sdesstatic int 887174588Sdeshttp_parse_length(const char *p, off_t *length) 88863012Sdes{ 88990267Sdes off_t len; 89090267Sdes 891174761Sdes for (len = 0; *p && isdigit((unsigned char)*p); ++p) 89290267Sdes len = len * 10 + (*p - '0'); 89390267Sdes if (*p) 89490267Sdes return (-1); 89590267Sdes DEBUG(fprintf(stderr, "content length: [%lld]\n", 89690267Sdes (long long)len)); 89790267Sdes *length = len; 89890267Sdes return (0); 89963012Sdes} 90063012Sdes 90163012Sdes/* 90263012Sdes * Parse a content-range header 90363012Sdes */ 90463716Sdesstatic int 905174588Sdeshttp_parse_range(const char *p, off_t *offset, off_t *length, off_t *size) 90663012Sdes{ 90790267Sdes off_t first, last, len; 90863716Sdes 90990267Sdes if (strncasecmp(p, "bytes ", 6) != 0) 91090267Sdes return (-1); 911125696Sdes p += 6; 912125696Sdes if (*p == '*') { 913125696Sdes first = last = -1; 914125696Sdes ++p; 915125696Sdes } else { 916174761Sdes for (first = 0; *p && isdigit((unsigned char)*p); ++p) 917125696Sdes first = first * 10 + *p - '0'; 918125696Sdes if (*p != '-') 919125696Sdes return (-1); 920174761Sdes for (last = 0, ++p; *p && isdigit((unsigned char)*p); ++p) 921125696Sdes last = last * 10 + *p - '0'; 922125696Sdes } 92390267Sdes if (first > last || *p != '/') 92490267Sdes return (-1); 925174761Sdes for (len = 0, ++p; *p && isdigit((unsigned char)*p); ++p) 92690267Sdes len = len * 10 + *p - '0'; 92790267Sdes if (*p || len < last - first + 1) 92890267Sdes return (-1); 929125696Sdes if (first == -1) { 930125696Sdes DEBUG(fprintf(stderr, "content range: [*/%lld]\n", 931125696Sdes (long long)len)); 932125696Sdes *length = 0; 933125696Sdes } else { 934125696Sdes DEBUG(fprintf(stderr, "content range: [%lld-%lld/%lld]\n", 935125696Sdes (long long)first, (long long)last, (long long)len)); 936125696Sdes *length = last - first + 1; 937125696Sdes } 93890267Sdes *offset = first; 93990267Sdes *size = len; 94090267Sdes return (0); 94163012Sdes} 94263012Sdes 94390267Sdes 94463012Sdes/***************************************************************************** 94563012Sdes * Helper functions for authorization 94663012Sdes */ 94763012Sdes 94863012Sdes/* 94937608Sdes * Base64 encoding 95037608Sdes */ 95162965Sdesstatic char * 952174588Sdeshttp_base64(const char *src) 95337608Sdes{ 95490267Sdes static const char base64[] = 95590267Sdes "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 95690267Sdes "abcdefghijklmnopqrstuvwxyz" 95790267Sdes "0123456789+/"; 95890267Sdes char *str, *dst; 95990267Sdes size_t l; 96090267Sdes int t, r; 96162965Sdes 96290267Sdes l = strlen(src); 963133280Sdes if ((str = malloc(((l + 2) / 3) * 4 + 1)) == NULL) 96490267Sdes return (NULL); 96590267Sdes dst = str; 96690267Sdes r = 0; 96737608Sdes 96890267Sdes while (l >= 3) { 96990267Sdes t = (src[0] << 16) | (src[1] << 8) | src[2]; 97090267Sdes dst[0] = base64[(t >> 18) & 0x3f]; 97190267Sdes dst[1] = base64[(t >> 12) & 0x3f]; 97290267Sdes dst[2] = base64[(t >> 6) & 0x3f]; 97390267Sdes dst[3] = base64[(t >> 0) & 0x3f]; 97490267Sdes src += 3; l -= 3; 97590267Sdes dst += 4; r += 4; 97690267Sdes } 97737608Sdes 97890267Sdes switch (l) { 97990267Sdes case 2: 98090267Sdes t = (src[0] << 16) | (src[1] << 8); 98190267Sdes dst[0] = base64[(t >> 18) & 0x3f]; 98290267Sdes dst[1] = base64[(t >> 12) & 0x3f]; 98390267Sdes dst[2] = base64[(t >> 6) & 0x3f]; 98490267Sdes dst[3] = '='; 98590267Sdes dst += 4; 98690267Sdes r += 4; 98790267Sdes break; 98890267Sdes case 1: 98990267Sdes t = src[0] << 16; 99090267Sdes dst[0] = base64[(t >> 18) & 0x3f]; 99190267Sdes dst[1] = base64[(t >> 12) & 0x3f]; 99290267Sdes dst[2] = dst[3] = '='; 99390267Sdes dst += 4; 99490267Sdes r += 4; 99590267Sdes break; 99690267Sdes case 0: 99790267Sdes break; 99890267Sdes } 99990267Sdes 100090267Sdes *dst = 0; 100190267Sdes return (str); 100237608Sdes} 100337608Sdes 1004202613Sdes 100537608Sdes/* 1006202613Sdes * Extract authorization parameters from environment value. 1007202613Sdes * The value is like scheme:realm:user:pass 1008202613Sdes */ 1009202613Sdestypedef struct { 1010202613Sdes char *scheme; 1011202613Sdes char *realm; 1012202613Sdes char *user; 1013202613Sdes char *password; 1014202613Sdes} http_auth_params_t; 1015202613Sdes 1016202613Sdesstatic void 1017202613Sdesinit_http_auth_params(http_auth_params_t *s) 1018202613Sdes{ 1019202613Sdes s->scheme = s->realm = s->user = s->password = 0; 1020202613Sdes} 1021202613Sdes 1022221821Sdesstatic void 1023202613Sdesclean_http_auth_params(http_auth_params_t *s) 1024202613Sdes{ 1025221821Sdes if (s->scheme) 1026202613Sdes free(s->scheme); 1027221821Sdes if (s->realm) 1028202613Sdes free(s->realm); 1029221821Sdes if (s->user) 1030202613Sdes free(s->user); 1031221821Sdes if (s->password) 1032202613Sdes free(s->password); 1033202613Sdes init_http_auth_params(s); 1034202613Sdes} 1035202613Sdes 1036202613Sdesstatic int 1037202613Sdeshttp_authfromenv(const char *p, http_auth_params_t *parms) 1038202613Sdes{ 1039202613Sdes int ret = -1; 1040202613Sdes char *v, *ve; 1041202613Sdes char *str = strdup(p); 1042202613Sdes 1043202613Sdes if (str == NULL) { 1044202613Sdes fetch_syserr(); 1045202613Sdes return (-1); 1046202613Sdes } 1047202613Sdes v = str; 1048202613Sdes 1049202613Sdes if ((ve = strchr(v, ':')) == NULL) 1050202613Sdes goto out; 1051202613Sdes 1052202613Sdes *ve = 0; 1053202613Sdes if ((parms->scheme = strdup(v)) == NULL) { 1054202613Sdes fetch_syserr(); 1055202613Sdes goto out; 1056202613Sdes } 1057202613Sdes v = ve + 1; 1058202613Sdes 1059202613Sdes if ((ve = strchr(v, ':')) == NULL) 1060202613Sdes goto out; 1061202613Sdes 1062202613Sdes *ve = 0; 1063202613Sdes if ((parms->realm = strdup(v)) == NULL) { 1064202613Sdes fetch_syserr(); 1065202613Sdes goto out; 1066202613Sdes } 1067202613Sdes v = ve + 1; 1068202613Sdes 1069202613Sdes if ((ve = strchr(v, ':')) == NULL) 1070202613Sdes goto out; 1071202613Sdes 1072202613Sdes *ve = 0; 1073202613Sdes if ((parms->user = strdup(v)) == NULL) { 1074202613Sdes fetch_syserr(); 1075202613Sdes goto out; 1076202613Sdes } 1077202613Sdes v = ve + 1; 1078202613Sdes 1079202613Sdes 1080202613Sdes if ((parms->password = strdup(v)) == NULL) { 1081202613Sdes fetch_syserr(); 1082202613Sdes goto out; 1083202613Sdes } 1084202613Sdes ret = 0; 1085202613Sdesout: 1086221821Sdes if (ret == -1) 1087202613Sdes clean_http_auth_params(parms); 1088202613Sdes if (str) 1089202613Sdes free(str); 1090202613Sdes return (ret); 1091202613Sdes} 1092202613Sdes 1093202613Sdes 1094221821Sdes/* 1095202613Sdes * Digest response: the code to compute the digest is taken from the 1096221821Sdes * sample implementation in RFC2616 1097202613Sdes */ 1098221822Sdes#define IN const 1099202613Sdes#define OUT 1100202613Sdes 1101202613Sdes#define HASHLEN 16 1102202613Sdestypedef char HASH[HASHLEN]; 1103202613Sdes#define HASHHEXLEN 32 1104202613Sdestypedef char HASHHEX[HASHHEXLEN+1]; 1105202613Sdes 1106202613Sdesstatic const char *hexchars = "0123456789abcdef"; 1107221821Sdesstatic void 1108202613SdesCvtHex(IN HASH Bin, OUT HASHHEX Hex) 1109202613Sdes{ 1110202613Sdes unsigned short i; 1111202613Sdes unsigned char j; 1112202613Sdes 1113202613Sdes for (i = 0; i < HASHLEN; i++) { 1114202613Sdes j = (Bin[i] >> 4) & 0xf; 1115202613Sdes Hex[i*2] = hexchars[j]; 1116202613Sdes j = Bin[i] & 0xf; 1117202613Sdes Hex[i*2+1] = hexchars[j]; 1118202613Sdes }; 1119202613Sdes Hex[HASHHEXLEN] = '\0'; 1120202613Sdes}; 1121202613Sdes 1122202613Sdes/* calculate H(A1) as per spec */ 1123221821Sdesstatic void 1124202613SdesDigestCalcHA1( 1125202613Sdes IN char * pszAlg, 1126202613Sdes IN char * pszUserName, 1127202613Sdes IN char * pszRealm, 1128202613Sdes IN char * pszPassword, 1129202613Sdes IN char * pszNonce, 1130202613Sdes IN char * pszCNonce, 1131202613Sdes OUT HASHHEX SessionKey 1132202613Sdes ) 1133202613Sdes{ 1134202613Sdes MD5_CTX Md5Ctx; 1135202613Sdes HASH HA1; 1136202613Sdes 1137202613Sdes MD5Init(&Md5Ctx); 1138202613Sdes MD5Update(&Md5Ctx, pszUserName, strlen(pszUserName)); 1139202613Sdes MD5Update(&Md5Ctx, ":", 1); 1140202613Sdes MD5Update(&Md5Ctx, pszRealm, strlen(pszRealm)); 1141202613Sdes MD5Update(&Md5Ctx, ":", 1); 1142202613Sdes MD5Update(&Md5Ctx, pszPassword, strlen(pszPassword)); 1143202613Sdes MD5Final(HA1, &Md5Ctx); 1144202613Sdes if (strcasecmp(pszAlg, "md5-sess") == 0) { 1145202613Sdes 1146202613Sdes MD5Init(&Md5Ctx); 1147202613Sdes MD5Update(&Md5Ctx, HA1, HASHLEN); 1148202613Sdes MD5Update(&Md5Ctx, ":", 1); 1149202613Sdes MD5Update(&Md5Ctx, pszNonce, strlen(pszNonce)); 1150202613Sdes MD5Update(&Md5Ctx, ":", 1); 1151202613Sdes MD5Update(&Md5Ctx, pszCNonce, strlen(pszCNonce)); 1152202613Sdes MD5Final(HA1, &Md5Ctx); 1153202613Sdes }; 1154202613Sdes CvtHex(HA1, SessionKey); 1155202613Sdes} 1156202613Sdes 1157202613Sdes/* calculate request-digest/response-digest as per HTTP Digest spec */ 1158221821Sdesstatic void 1159202613SdesDigestCalcResponse( 1160202613Sdes IN HASHHEX HA1, /* H(A1) */ 1161202613Sdes IN char * pszNonce, /* nonce from server */ 1162202613Sdes IN char * pszNonceCount, /* 8 hex digits */ 1163202613Sdes IN char * pszCNonce, /* client nonce */ 1164202613Sdes IN char * pszQop, /* qop-value: "", "auth", "auth-int" */ 1165202613Sdes IN char * pszMethod, /* method from the request */ 1166202613Sdes IN char * pszDigestUri, /* requested URL */ 1167202613Sdes IN HASHHEX HEntity, /* H(entity body) if qop="auth-int" */ 1168202613Sdes OUT HASHHEX Response /* request-digest or response-digest */ 1169202613Sdes ) 1170202613Sdes{ 1171221821Sdes/* DEBUG(fprintf(stderr, 1172202613Sdes "Calc: HA1[%s] Nonce[%s] qop[%s] method[%s] URI[%s]\n", 1173202613Sdes HA1, pszNonce, pszQop, pszMethod, pszDigestUri));*/ 1174202613Sdes MD5_CTX Md5Ctx; 1175202613Sdes HASH HA2; 1176202613Sdes HASH RespHash; 1177202613Sdes HASHHEX HA2Hex; 1178202613Sdes 1179202613Sdes // calculate H(A2) 1180202613Sdes MD5Init(&Md5Ctx); 1181202613Sdes MD5Update(&Md5Ctx, pszMethod, strlen(pszMethod)); 1182202613Sdes MD5Update(&Md5Ctx, ":", 1); 1183202613Sdes MD5Update(&Md5Ctx, pszDigestUri, strlen(pszDigestUri)); 1184202613Sdes if (strcasecmp(pszQop, "auth-int") == 0) { 1185202613Sdes MD5Update(&Md5Ctx, ":", 1); 1186202613Sdes MD5Update(&Md5Ctx, HEntity, HASHHEXLEN); 1187202613Sdes }; 1188202613Sdes MD5Final(HA2, &Md5Ctx); 1189202613Sdes CvtHex(HA2, HA2Hex); 1190202613Sdes 1191202613Sdes // calculate response 1192202613Sdes MD5Init(&Md5Ctx); 1193202613Sdes MD5Update(&Md5Ctx, HA1, HASHHEXLEN); 1194202613Sdes MD5Update(&Md5Ctx, ":", 1); 1195202613Sdes MD5Update(&Md5Ctx, pszNonce, strlen(pszNonce)); 1196202613Sdes MD5Update(&Md5Ctx, ":", 1); 1197202613Sdes if (*pszQop) { 1198202613Sdes MD5Update(&Md5Ctx, pszNonceCount, strlen(pszNonceCount)); 1199202613Sdes MD5Update(&Md5Ctx, ":", 1); 1200202613Sdes MD5Update(&Md5Ctx, pszCNonce, strlen(pszCNonce)); 1201202613Sdes MD5Update(&Md5Ctx, ":", 1); 1202202613Sdes MD5Update(&Md5Ctx, pszQop, strlen(pszQop)); 1203202613Sdes MD5Update(&Md5Ctx, ":", 1); 1204202613Sdes }; 1205202613Sdes MD5Update(&Md5Ctx, HA2Hex, HASHHEXLEN); 1206202613Sdes MD5Final(RespHash, &Md5Ctx); 1207202613Sdes CvtHex(RespHash, Response); 1208202613Sdes} 1209202613Sdes 1210221821Sdes/* 1211221821Sdes * Generate/Send a Digest authorization header 1212202613Sdes * This looks like: [Proxy-]Authorization: credentials 1213202613Sdes * 1214202613Sdes * credentials = "Digest" digest-response 1215202613Sdes * digest-response = 1#( username | realm | nonce | digest-uri 1216202613Sdes * | response | [ algorithm ] | [cnonce] | 1217202613Sdes * [opaque] | [message-qop] | 1218202613Sdes * [nonce-count] | [auth-param] ) 1219202613Sdes * username = "username" "=" username-value 1220202613Sdes * username-value = quoted-string 1221202613Sdes * digest-uri = "uri" "=" digest-uri-value 1222202613Sdes * digest-uri-value = request-uri ; As specified by HTTP/1.1 1223202613Sdes * message-qop = "qop" "=" qop-value 1224202613Sdes * cnonce = "cnonce" "=" cnonce-value 1225202613Sdes * cnonce-value = nonce-value 1226202613Sdes * nonce-count = "nc" "=" nc-value 1227202613Sdes * nc-value = 8LHEX 1228202613Sdes * response = "response" "=" request-digest 1229202613Sdes * request-digest = <"> 32LHEX <"> 1230202613Sdes */ 1231202613Sdesstatic int 1232202613Sdeshttp_digest_auth(conn_t *conn, const char *hdr, http_auth_challenge_t *c, 1233202613Sdes http_auth_params_t *parms, struct url *url) 1234202613Sdes{ 1235202613Sdes int r; 1236202613Sdes char noncecount[10]; 1237202613Sdes char cnonce[40]; 1238202613Sdes char *options = 0; 1239202613Sdes 1240202613Sdes if (!c->realm || !c->nonce) { 1241202613Sdes DEBUG(fprintf(stderr, "realm/nonce not set in challenge\n")); 1242202613Sdes return(-1); 1243202613Sdes } 1244221821Sdes if (!c->algo) 1245202613Sdes c->algo = strdup(""); 1246202613Sdes 1247221821Sdes if (asprintf(&options, "%s%s%s%s", 1248202613Sdes *c->algo? ",algorithm=" : "", c->algo, 1249202613Sdes c->opaque? ",opaque=" : "", c->opaque?c->opaque:"")== -1) 1250202613Sdes return (-1); 1251202613Sdes 1252202613Sdes if (!c->qop) { 1253202613Sdes c->qop = strdup(""); 1254202613Sdes *noncecount = 0; 1255202613Sdes *cnonce = 0; 1256202613Sdes } else { 1257202613Sdes c->nc++; 1258202613Sdes sprintf(noncecount, "%08x", c->nc); 1259202613Sdes /* We don't try very hard with the cnonce ... */ 1260202613Sdes sprintf(cnonce, "%x%lx", getpid(), (unsigned long)time(0)); 1261202613Sdes } 1262202613Sdes 1263202613Sdes HASHHEX HA1; 1264202613Sdes DigestCalcHA1(c->algo, parms->user, c->realm, 1265202613Sdes parms->password, c->nonce, cnonce, HA1); 1266202613Sdes DEBUG(fprintf(stderr, "HA1: [%s]\n", HA1)); 1267202613Sdes HASHHEX digest; 1268202613Sdes DigestCalcResponse(HA1, c->nonce, noncecount, cnonce, c->qop, 1269202613Sdes "GET", url->doc, "", digest); 1270202613Sdes 1271202613Sdes if (c->qop[0]) { 1272202613Sdes r = http_cmd(conn, "%s: Digest username=\"%s\",realm=\"%s\"," 1273202613Sdes "nonce=\"%s\",uri=\"%s\",response=\"%s\"," 1274202613Sdes "qop=\"auth\", cnonce=\"%s\", nc=%s%s", 1275221821Sdes hdr, parms->user, c->realm, 1276202613Sdes c->nonce, url->doc, digest, 1277202613Sdes cnonce, noncecount, options); 1278202613Sdes } else { 1279202613Sdes r = http_cmd(conn, "%s: Digest username=\"%s\",realm=\"%s\"," 1280202613Sdes "nonce=\"%s\",uri=\"%s\",response=\"%s\"%s", 1281221821Sdes hdr, parms->user, c->realm, 1282202613Sdes c->nonce, url->doc, digest, options); 1283202613Sdes } 1284202613Sdes if (options) 1285202613Sdes free(options); 1286202613Sdes return (r); 1287202613Sdes} 1288202613Sdes 1289202613Sdes/* 129037608Sdes * Encode username and password 129137608Sdes */ 129262965Sdesstatic int 1293174588Sdeshttp_basic_auth(conn_t *conn, const char *hdr, const char *usr, const char *pwd) 129437608Sdes{ 129590267Sdes char *upw, *auth; 129690267Sdes int r; 129737608Sdes 1298202613Sdes DEBUG(fprintf(stderr, "basic: usr: [%s]\n", usr)); 1299202613Sdes DEBUG(fprintf(stderr, "basic: pwd: [%s]\n", pwd)); 130090267Sdes if (asprintf(&upw, "%s:%s", usr, pwd) == -1) 130190267Sdes return (-1); 1302174588Sdes auth = http_base64(upw); 130390267Sdes free(upw); 130490267Sdes if (auth == NULL) 130590267Sdes return (-1); 1306174588Sdes r = http_cmd(conn, "%s: Basic %s", hdr, auth); 130790267Sdes free(auth); 130890267Sdes return (r); 130962965Sdes} 131062965Sdes 131162965Sdes/* 1312221821Sdes * Chose the challenge to answer and call the appropriate routine to 1313202613Sdes * produce the header. 131462965Sdes */ 131562965Sdesstatic int 1316202613Sdeshttp_authorize(conn_t *conn, const char *hdr, http_auth_challenges_t *cs, 1317202613Sdes http_auth_params_t *parms, struct url *url) 131862965Sdes{ 1319202613Sdes http_auth_challenge_t *basic = NULL; 1320202613Sdes http_auth_challenge_t *digest = NULL; 1321202613Sdes int i; 132262965Sdes 1323202613Sdes /* If user or pass are null we're not happy */ 1324202613Sdes if (!parms->user || !parms->password) { 1325202613Sdes DEBUG(fprintf(stderr, "NULL usr or pass\n")); 1326202613Sdes return (-1); 132790267Sdes } 1328202613Sdes 1329202613Sdes /* Look for a Digest and a Basic challenge */ 1330202613Sdes for (i = 0; i < cs->count; i++) { 1331202613Sdes if (cs->challenges[i]->scheme == HTTPAS_BASIC) 1332202613Sdes basic = cs->challenges[i]; 1333202613Sdes if (cs->challenges[i]->scheme == HTTPAS_DIGEST) 1334202613Sdes digest = cs->challenges[i]; 1335202613Sdes } 1336202613Sdes 1337202613Sdes /* Error if "Digest" was specified and there is no Digest challenge */ 1338221821Sdes if (!digest && (parms->scheme && 1339202613Sdes !strcasecmp(parms->scheme, "digest"))) { 1340221821Sdes DEBUG(fprintf(stderr, 1341202613Sdes "Digest auth in env, not supported by peer\n")); 1342202613Sdes return (-1); 1343202613Sdes } 1344221821Sdes /* 1345221821Sdes * If "basic" was specified in the environment, or there is no Digest 1346202613Sdes * challenge, do the basic thing. Don't need a challenge for this, 1347221821Sdes * so no need to check basic!=NULL 1348202613Sdes */ 1349202613Sdes if (!digest || (parms->scheme && !strcasecmp(parms->scheme,"basic"))) 1350202613Sdes return (http_basic_auth(conn,hdr,parms->user,parms->password)); 1351202613Sdes 1352202613Sdes /* Else, prefer digest. We just checked that it's not NULL */ 1353202613Sdes return (http_digest_auth(conn, hdr, digest, parms, url)); 135437608Sdes} 135537608Sdes 135663012Sdes/***************************************************************************** 135763012Sdes * Helper functions for connecting to a server or proxy 135863012Sdes */ 135963012Sdes 136037608Sdes/* 136190267Sdes * Connect to the correct HTTP server or proxy. 136263012Sdes */ 136397856Sdesstatic conn_t * 1364174588Sdeshttp_connect(struct url *URL, struct url *purl, const char *flags) 136563012Sdes{ 136697856Sdes conn_t *conn; 136790267Sdes int verbose; 1368141958Skbyanc int af, val; 136990267Sdes 137063012Sdes#ifdef INET6 137190267Sdes af = AF_UNSPEC; 137260737Sume#else 137390267Sdes af = AF_INET; 137460737Sume#endif 137590267Sdes 137690267Sdes verbose = CHECK_FLAG('v'); 137790267Sdes if (CHECK_FLAG('4')) 137890267Sdes af = AF_INET; 137967043Sdes#ifdef INET6 138090267Sdes else if (CHECK_FLAG('6')) 138190267Sdes af = AF_INET6; 138267043Sdes#endif 138367043Sdes 138497868Sdes if (purl && strcasecmp(URL->scheme, SCHEME_HTTPS) != 0) { 138590267Sdes URL = purl; 138690267Sdes } else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0) { 138790267Sdes /* can't talk http to an ftp server */ 138890267Sdes /* XXX should set an error code */ 138997856Sdes return (NULL); 139090267Sdes } 139190267Sdes 1392174588Sdes if ((conn = fetch_connect(URL->host, URL->port, af, verbose)) == NULL) 1393174588Sdes /* fetch_connect() has already set an error code */ 139497856Sdes return (NULL); 139597868Sdes if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && 1396174588Sdes fetch_ssl(conn, verbose) == -1) { 1397174588Sdes fetch_close(conn); 139897891Sdes /* grrr */ 139997891Sdes errno = EAUTH; 1400174588Sdes fetch_syserr(); 140197868Sdes return (NULL); 140297868Sdes } 1403141958Skbyanc 1404141958Skbyanc val = 1; 1405141958Skbyanc setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val, sizeof(val)); 1406141958Skbyanc 140797856Sdes return (conn); 140867043Sdes} 140967043Sdes 141067043Sdesstatic struct url * 1411174752Sdeshttp_get_proxy(struct url * url, const char *flags) 141267043Sdes{ 141390267Sdes struct url *purl; 141490267Sdes char *p; 141590267Sdes 1416112797Sdes if (flags != NULL && strchr(flags, 'd') != NULL) 1417112081Sdes return (NULL); 1418174752Sdes if (fetch_no_proxy_match(url->host)) 1419174752Sdes return (NULL); 142090267Sdes if (((p = getenv("HTTP_PROXY")) || (p = getenv("http_proxy"))) && 1421149414Sdes *p && (purl = fetchParseURL(p))) { 142290267Sdes if (!*purl->scheme) 142390267Sdes strcpy(purl->scheme, SCHEME_HTTP); 142490267Sdes if (!purl->port) 1425174588Sdes purl->port = fetch_default_proxy_port(purl->scheme); 142690267Sdes if (strcasecmp(purl->scheme, SCHEME_HTTP) == 0) 142790267Sdes return (purl); 142890267Sdes fetchFreeURL(purl); 142990267Sdes } 143090267Sdes return (NULL); 143160376Sdes} 143260376Sdes 143388771Sdesstatic void 1434174588Sdeshttp_print_html(FILE *out, FILE *in) 143588771Sdes{ 143690267Sdes size_t len; 143790267Sdes char *line, *p, *q; 143890267Sdes int comment, tag; 143988771Sdes 144090267Sdes comment = tag = 0; 144190267Sdes while ((line = fgetln(in, &len)) != NULL) { 1442174761Sdes while (len && isspace((unsigned char)line[len - 1])) 144390267Sdes --len; 144490267Sdes for (p = q = line; q < line + len; ++q) { 144590267Sdes if (comment && *q == '-') { 144690267Sdes if (q + 2 < line + len && 144790267Sdes strcmp(q, "-->") == 0) { 144890267Sdes tag = comment = 0; 144990267Sdes q += 2; 145090267Sdes } 145190267Sdes } else if (tag && !comment && *q == '>') { 145290267Sdes p = q + 1; 145390267Sdes tag = 0; 145490267Sdes } else if (!tag && *q == '<') { 145590267Sdes if (q > p) 145690267Sdes fwrite(p, q - p, 1, out); 145790267Sdes tag = 1; 145890267Sdes if (q + 3 < line + len && 145990267Sdes strcmp(q, "<!--") == 0) { 146090267Sdes comment = 1; 146190267Sdes q += 3; 146290267Sdes } 146390267Sdes } 146488771Sdes } 146590267Sdes if (!tag && q > p) 146690267Sdes fwrite(p, q - p, 1, out); 146790267Sdes fputc('\n', out); 146888771Sdes } 146988771Sdes} 147088771Sdes 147190267Sdes 147263012Sdes/***************************************************************************** 147363012Sdes * Core 147460954Sdes */ 147560954Sdes 147660954Sdes/* 147763012Sdes * Send a request and process the reply 147897866Sdes * 147997866Sdes * XXX This function is way too long, the do..while loop should be split 148097866Sdes * XXX off into a separate function. 148160376Sdes */ 148267043SdesFILE * 1483174588Sdeshttp_request(struct url *URL, const char *op, struct url_stat *us, 1484202613Sdes struct url *purl, const char *flags) 148560376Sdes{ 1486186124Smurray char timebuf[80]; 1487186124Smurray char hbuf[MAXHOSTNAMELEN + 7], *host; 148897856Sdes conn_t *conn; 148990267Sdes struct url *url, *new; 1490202613Sdes int chunked, direct, ims, noredirect, verbose; 1491143049Skbyanc int e, i, n, val; 149290267Sdes off_t offset, clength, length, size; 149390267Sdes time_t mtime; 149490267Sdes const char *p; 149590267Sdes FILE *f; 149690267Sdes hdr_t h; 1497186124Smurray struct tm *timestruct; 1498202613Sdes http_headerbuf_t headerbuf; 1499202613Sdes http_auth_challenges_t server_challenges; 1500202613Sdes http_auth_challenges_t proxy_challenges; 150163012Sdes 1502202613Sdes /* The following calls don't allocate anything */ 1503221821Sdes init_http_headerbuf(&headerbuf); 1504202613Sdes init_http_auth_challenges(&server_challenges); 1505202613Sdes init_http_auth_challenges(&proxy_challenges); 1506202613Sdes 150790267Sdes direct = CHECK_FLAG('d'); 150890267Sdes noredirect = CHECK_FLAG('A'); 150990267Sdes verbose = CHECK_FLAG('v'); 1510186124Smurray ims = CHECK_FLAG('i'); 151160737Sume 151290267Sdes if (direct && purl) { 151390267Sdes fetchFreeURL(purl); 151490267Sdes purl = NULL; 151590267Sdes } 151663716Sdes 151790267Sdes /* try the provided URL first */ 151890267Sdes url = URL; 151963012Sdes 1520242289Seadler n = MAX_REDIRECT; 152190267Sdes i = 0; 152263012Sdes 152398422Sdes e = HTTP_PROTOCOL_ERROR; 152490267Sdes do { 152590267Sdes new = NULL; 152690267Sdes chunked = 0; 152790267Sdes offset = 0; 152890267Sdes clength = -1; 152990267Sdes length = -1; 153090267Sdes size = -1; 153190267Sdes mtime = 0; 153290267Sdes 153390267Sdes /* check port */ 153490267Sdes if (!url->port) 1535174588Sdes url->port = fetch_default_port(url->scheme); 153690267Sdes 153790267Sdes /* were we redirected to an FTP URL? */ 153890267Sdes if (purl == NULL && strcmp(url->scheme, SCHEME_FTP) == 0) { 153990267Sdes if (strcmp(op, "GET") == 0) 1540174588Sdes return (ftp_request(url, "RETR", us, purl, flags)); 154190267Sdes else if (strcmp(op, "HEAD") == 0) 1542174588Sdes return (ftp_request(url, "STAT", us, purl, flags)); 154390267Sdes } 154490267Sdes 154590267Sdes /* connect to server or proxy */ 1546174588Sdes if ((conn = http_connect(url, purl, flags)) == NULL) 154790267Sdes goto ouch; 154890267Sdes 154990267Sdes host = url->host; 155060737Sume#ifdef INET6 155190267Sdes if (strchr(url->host, ':')) { 155290267Sdes snprintf(hbuf, sizeof(hbuf), "[%s]", url->host); 155390267Sdes host = hbuf; 155490267Sdes } 155560737Sume#endif 1556174588Sdes if (url->port != fetch_default_port(url->scheme)) { 1557107372Sdes if (host != hbuf) { 1558107372Sdes strcpy(hbuf, host); 1559107372Sdes host = hbuf; 1560107372Sdes } 1561107372Sdes snprintf(hbuf + strlen(hbuf), 1562107372Sdes sizeof(hbuf) - strlen(hbuf), ":%d", url->port); 1563107372Sdes } 156437535Sdes 156590267Sdes /* send request */ 156690267Sdes if (verbose) 1567174588Sdes fetch_info("requesting %s://%s%s", 1568107372Sdes url->scheme, host, url->doc); 156990267Sdes if (purl) { 1570174588Sdes http_cmd(conn, "%s %s://%s%s HTTP/1.1", 1571107372Sdes op, url->scheme, host, url->doc); 157290267Sdes } else { 1573174588Sdes http_cmd(conn, "%s %s HTTP/1.1", 157490267Sdes op, url->doc); 157590267Sdes } 157637535Sdes 1577186124Smurray if (ims && url->ims_time) { 1578186124Smurray timestruct = gmtime((time_t *)&url->ims_time); 1579186124Smurray (void)strftime(timebuf, 80, "%a, %d %b %Y %T GMT", 1580186124Smurray timestruct); 1581186124Smurray if (verbose) 1582186124Smurray fetch_info("If-Modified-Since: %s", timebuf); 1583186124Smurray http_cmd(conn, "If-Modified-Since: %s", timebuf); 1584186124Smurray } 158590267Sdes /* virtual host */ 1586174588Sdes http_cmd(conn, "Host: %s", host); 158790267Sdes 1588221821Sdes /* 1589221821Sdes * Proxy authorization: we only send auth after we received 1590221821Sdes * a 407 error. We do not first try basic anyway (changed 1591221821Sdes * when support was added for digest-auth) 1592221821Sdes */ 1593202613Sdes if (purl && proxy_challenges.valid) { 1594202613Sdes http_auth_params_t aparams; 1595202613Sdes init_http_auth_params(&aparams); 1596202613Sdes if (*purl->user || *purl->pwd) { 1597221821Sdes aparams.user = purl->user ? 1598202613Sdes strdup(purl->user) : strdup(""); 1599202613Sdes aparams.password = purl->pwd? 1600202613Sdes strdup(purl->pwd) : strdup(""); 1601221821Sdes } else if ((p = getenv("HTTP_PROXY_AUTH")) != NULL && 1602202613Sdes *p != '\0') { 1603202613Sdes if (http_authfromenv(p, &aparams) < 0) { 1604202613Sdes http_seterr(HTTP_NEED_PROXY_AUTH); 1605202613Sdes goto ouch; 1606202613Sdes } 1607202613Sdes } 1608221821Sdes http_authorize(conn, "Proxy-Authorization", 1609202613Sdes &proxy_challenges, &aparams, url); 1610202613Sdes clean_http_auth_params(&aparams); 161190267Sdes } 161290267Sdes 1613221821Sdes /* 1614221821Sdes * Server authorization: we never send "a priori" 1615202613Sdes * Basic auth, which used to be done if user/pass were 1616202613Sdes * set in the url. This would be weird because we'd send the 1617221821Sdes * password in the clear even if Digest is finally to be 1618202613Sdes * used (it would have made more sense for the 1619221821Sdes * pre-digest version to do this when Basic was specified 1620221821Sdes * in the environment) 1621221821Sdes */ 1622202613Sdes if (server_challenges.valid) { 1623202613Sdes http_auth_params_t aparams; 1624202613Sdes init_http_auth_params(&aparams); 1625202613Sdes if (*url->user || *url->pwd) { 1626221821Sdes aparams.user = url->user ? 1627202613Sdes strdup(url->user) : strdup(""); 1628221821Sdes aparams.password = url->pwd ? 1629202613Sdes strdup(url->pwd) : strdup(""); 1630221821Sdes } else if ((p = getenv("HTTP_AUTH")) != NULL && 1631202613Sdes *p != '\0') { 1632202613Sdes if (http_authfromenv(p, &aparams) < 0) { 1633202613Sdes http_seterr(HTTP_NEED_AUTH); 1634202613Sdes goto ouch; 1635202613Sdes } 1636221821Sdes } else if (fetchAuthMethod && 1637202613Sdes fetchAuthMethod(url) == 0) { 1638221821Sdes aparams.user = url->user ? 1639202613Sdes strdup(url->user) : strdup(""); 1640221821Sdes aparams.password = url->pwd ? 1641202613Sdes strdup(url->pwd) : strdup(""); 164290267Sdes } else { 1643174588Sdes http_seterr(HTTP_NEED_AUTH); 164490267Sdes goto ouch; 164590267Sdes } 1646221821Sdes http_authorize(conn, "Authorization", 1647202613Sdes &server_challenges, &aparams, url); 1648202613Sdes clean_http_auth_params(&aparams); 164990267Sdes } 165090267Sdes 165190267Sdes /* other headers */ 1652107372Sdes if ((p = getenv("HTTP_REFERER")) != NULL && *p != '\0') { 1653107372Sdes if (strcasecmp(p, "auto") == 0) 1654174588Sdes http_cmd(conn, "Referer: %s://%s%s", 1655107372Sdes url->scheme, host, url->doc); 1656107372Sdes else 1657174588Sdes http_cmd(conn, "Referer: %s", p); 1658107372Sdes } 165990267Sdes if ((p = getenv("HTTP_USER_AGENT")) != NULL && *p != '\0') 1660174588Sdes http_cmd(conn, "User-Agent: %s", p); 166190267Sdes else 1662174588Sdes http_cmd(conn, "User-Agent: %s " _LIBFETCH_VER, getprogname()); 1663109693Sdes if (url->offset > 0) 1664174588Sdes http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset); 1665174588Sdes http_cmd(conn, "Connection: close"); 1666174588Sdes http_cmd(conn, ""); 166790267Sdes 1668143049Skbyanc /* 1669143049Skbyanc * Force the queued request to be dispatched. Normally, one 1670143049Skbyanc * would do this with shutdown(2) but squid proxies can be 1671143049Skbyanc * configured to disallow such half-closed connections. To 1672143049Skbyanc * be compatible with such configurations, fiddle with socket 1673143049Skbyanc * options to force the pending data to be written. 1674143049Skbyanc */ 1675143049Skbyanc val = 0; 1676143049Skbyanc setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val, 1677143049Skbyanc sizeof(val)); 1678143049Skbyanc val = 1; 1679143049Skbyanc setsockopt(conn->sd, IPPROTO_TCP, TCP_NODELAY, &val, 1680143049Skbyanc sizeof(val)); 1681143049Skbyanc 168290267Sdes /* get reply */ 1683174588Sdes switch (http_get_reply(conn)) { 168490267Sdes case HTTP_OK: 168590267Sdes case HTTP_PARTIAL: 1686186124Smurray case HTTP_NOT_MODIFIED: 168790267Sdes /* fine */ 168890267Sdes break; 168990267Sdes case HTTP_MOVED_PERM: 169090267Sdes case HTTP_MOVED_TEMP: 169190267Sdes case HTTP_SEE_OTHER: 169290267Sdes /* 1693125695Sdes * Not so fine, but we still have to read the 1694125695Sdes * headers to get the new location. 169590267Sdes */ 169690267Sdes break; 169790267Sdes case HTTP_NEED_AUTH: 1698202613Sdes if (server_challenges.valid) { 169990267Sdes /* 1700125695Sdes * We already sent out authorization code, 1701125695Sdes * so there's nothing more we can do. 170290267Sdes */ 1703174588Sdes http_seterr(conn->err); 170490267Sdes goto ouch; 170590267Sdes } 170690267Sdes /* try again, but send the password this time */ 170790267Sdes if (verbose) 1708174588Sdes fetch_info("server requires authorization"); 170990267Sdes break; 171090267Sdes case HTTP_NEED_PROXY_AUTH: 1711202613Sdes if (proxy_challenges.valid) { 1712202613Sdes /* 1713202613Sdes * We already sent our proxy 1714202613Sdes * authorization code, so there's 1715202613Sdes * nothing more we can do. */ 1716202613Sdes http_seterr(conn->err); 1717202613Sdes goto ouch; 1718202613Sdes } 1719202613Sdes /* try again, but send the password this time */ 1720202613Sdes if (verbose) 1721202613Sdes fetch_info("proxy requires authorization"); 1722202613Sdes break; 1723125696Sdes case HTTP_BAD_RANGE: 1724125696Sdes /* 1725125696Sdes * This can happen if we ask for 0 bytes because 1726125696Sdes * we already have the whole file. Consider this 1727125696Sdes * a success for now, and check sizes later. 1728125696Sdes */ 1729125696Sdes break; 173090267Sdes case HTTP_PROTOCOL_ERROR: 173190267Sdes /* fall through */ 173290267Sdes case -1: 1733174588Sdes fetch_syserr(); 173490267Sdes goto ouch; 173590267Sdes default: 1736174588Sdes http_seterr(conn->err); 173790267Sdes if (!verbose) 173890267Sdes goto ouch; 173990267Sdes /* fall through so we can get the full error message */ 174090267Sdes } 174190267Sdes 1742202613Sdes /* get headers. http_next_header expects one line readahead */ 1743202613Sdes if (fetch_getln(conn) == -1) { 1744202613Sdes fetch_syserr(); 1745202613Sdes goto ouch; 1746202613Sdes } 174790267Sdes do { 1748202613Sdes switch ((h = http_next_header(conn, &headerbuf, &p))) { 174990267Sdes case hdr_syserror: 1750174588Sdes fetch_syserr(); 175190267Sdes goto ouch; 175290267Sdes case hdr_error: 1753174588Sdes http_seterr(HTTP_PROTOCOL_ERROR); 175490267Sdes goto ouch; 175590267Sdes case hdr_content_length: 1756174588Sdes http_parse_length(p, &clength); 175790267Sdes break; 175890267Sdes case hdr_content_range: 1759174588Sdes http_parse_range(p, &offset, &length, &size); 176090267Sdes break; 176190267Sdes case hdr_last_modified: 1762174588Sdes http_parse_mtime(p, &mtime); 176390267Sdes break; 176490267Sdes case hdr_location: 176597856Sdes if (!HTTP_REDIRECT(conn->err)) 176690267Sdes break; 1767242289Seadler /* 1768242289Seadler * if the A flag is set, we don't follow 1769242289Seadler * temporary redirects. 1770242289Seadler */ 1771242289Seadler if (noredirect && 1772242289Seadler conn->err != HTTP_MOVED_PERM && 1773242289Seadler conn->err != HTTP_PERM_REDIRECT) { 1774242289Seadler n = 1; 1775242289Seadler break; 1776242289Seadler } 177790267Sdes if (new) 177890267Sdes free(new); 177990267Sdes if (verbose) 1780174588Sdes fetch_info("%d redirect to %s", conn->err, p); 178190267Sdes if (*p == '/') 178290267Sdes /* absolute path */ 178390267Sdes new = fetchMakeURL(url->scheme, url->host, url->port, p, 178490267Sdes url->user, url->pwd); 178590267Sdes else 178690267Sdes new = fetchParseURL(p); 178790267Sdes if (new == NULL) { 178890267Sdes /* XXX should set an error code */ 178990267Sdes DEBUG(fprintf(stderr, "failed to parse new URL\n")); 179090267Sdes goto ouch; 179190267Sdes } 1792236108Sdes 1793236108Sdes /* Only copy credentials if the host matches */ 1794236108Sdes if (!strcmp(new->host, url->host) && !*new->user && !*new->pwd) { 179590267Sdes strcpy(new->user, url->user); 179690267Sdes strcpy(new->pwd, url->pwd); 179790267Sdes } 179890267Sdes new->offset = url->offset; 179990267Sdes new->length = url->length; 180090267Sdes break; 180190267Sdes case hdr_transfer_encoding: 180290267Sdes /* XXX weak test*/ 180390267Sdes chunked = (strcasecmp(p, "chunked") == 0); 180490267Sdes break; 180590267Sdes case hdr_www_authenticate: 180697856Sdes if (conn->err != HTTP_NEED_AUTH) 180790267Sdes break; 1808210563Sdes if (http_parse_authenticate(p, &server_challenges) == 0) 1809209632Sdes ++n; 181090267Sdes break; 1811202613Sdes case hdr_proxy_authenticate: 1812202613Sdes if (conn->err != HTTP_NEED_PROXY_AUTH) 1813202613Sdes break; 1814210563Sdes if (http_parse_authenticate(p, &proxy_challenges) == 0) 1815209632Sdes ++n; 1816202613Sdes break; 181790267Sdes case hdr_end: 181890267Sdes /* fall through */ 181990267Sdes case hdr_unknown: 182090267Sdes /* ignore */ 182190267Sdes break; 182290267Sdes } 182390267Sdes } while (h > hdr_end); 182490267Sdes 182590267Sdes /* we need to provide authentication */ 1826221821Sdes if (conn->err == HTTP_NEED_AUTH || 1827202613Sdes conn->err == HTTP_NEED_PROXY_AUTH) { 182898422Sdes e = conn->err; 1829221821Sdes if ((conn->err == HTTP_NEED_AUTH && 1830221821Sdes !server_challenges.valid) || 1831221821Sdes (conn->err == HTTP_NEED_PROXY_AUTH && 1832202613Sdes !proxy_challenges.valid)) { 1833202613Sdes /* 401/7 but no www/proxy-authenticate ?? */ 1834202613Sdes DEBUG(fprintf(stderr, "401/7 and no auth header\n")); 1835202613Sdes goto ouch; 1836202613Sdes } 1837174588Sdes fetch_close(conn); 183897856Sdes conn = NULL; 183990267Sdes continue; 184090267Sdes } 184190267Sdes 1842125696Sdes /* requested range not satisfiable */ 1843125696Sdes if (conn->err == HTTP_BAD_RANGE) { 1844125696Sdes if (url->offset == size && url->length == 0) { 1845125696Sdes /* asked for 0 bytes; fake it */ 1846125696Sdes offset = url->offset; 1847184222Sru clength = -1; 1848125696Sdes conn->err = HTTP_OK; 1849125696Sdes break; 1850125696Sdes } else { 1851174588Sdes http_seterr(conn->err); 1852125696Sdes goto ouch; 1853125696Sdes } 1854125696Sdes } 1855125696Sdes 1856104404Sru /* we have a hit or an error */ 1857186124Smurray if (conn->err == HTTP_OK 1858186124Smurray || conn->err == HTTP_NOT_MODIFIED 1859186124Smurray || conn->err == HTTP_PARTIAL 1860186124Smurray || HTTP_ERROR(conn->err)) 1861104404Sru break; 1862104404Sru 186390267Sdes /* all other cases: we got a redirect */ 186498422Sdes e = conn->err; 1865202613Sdes clean_http_auth_challenges(&server_challenges); 1866174588Sdes fetch_close(conn); 186797856Sdes conn = NULL; 186890267Sdes if (!new) { 186990267Sdes DEBUG(fprintf(stderr, "redirect with no new location\n")); 187090267Sdes break; 187190267Sdes } 187290267Sdes if (url != URL) 187390267Sdes fetchFreeURL(url); 187490267Sdes url = new; 187590267Sdes } while (++i < n); 187690267Sdes 187790267Sdes /* we failed, or ran out of retries */ 187897856Sdes if (conn == NULL) { 1879174588Sdes http_seterr(e); 188063012Sdes goto ouch; 188163012Sdes } 188260376Sdes 188390267Sdes DEBUG(fprintf(stderr, "offset %lld, length %lld," 188490267Sdes " size %lld, clength %lld\n", 188590267Sdes (long long)offset, (long long)length, 188690267Sdes (long long)size, (long long)clength)); 188760376Sdes 1888186124Smurray if (conn->err == HTTP_NOT_MODIFIED) { 1889186124Smurray http_seterr(HTTP_NOT_MODIFIED); 1890186124Smurray return (NULL); 1891186124Smurray } 1892186124Smurray 189390267Sdes /* check for inconsistencies */ 189490267Sdes if (clength != -1 && length != -1 && clength != length) { 1895174588Sdes http_seterr(HTTP_PROTOCOL_ERROR); 189663012Sdes goto ouch; 189763012Sdes } 189890267Sdes if (clength == -1) 189990267Sdes clength = length; 190090267Sdes if (clength != -1) 190190267Sdes length = offset + clength; 190290267Sdes if (length != -1 && size != -1 && length != size) { 1903174588Sdes http_seterr(HTTP_PROTOCOL_ERROR); 190463012Sdes goto ouch; 190590267Sdes } 190690267Sdes if (size == -1) 190790267Sdes size = length; 190860376Sdes 190990267Sdes /* fill in stats */ 191090267Sdes if (us) { 191190267Sdes us->size = size; 191290267Sdes us->atime = us->mtime = mtime; 191390267Sdes } 191463069Sdes 191590267Sdes /* too far? */ 1916109693Sdes if (URL->offset > 0 && offset > URL->offset) { 1917174588Sdes http_seterr(HTTP_PROTOCOL_ERROR); 191890267Sdes goto ouch; 191977238Sdes } 192060376Sdes 192190267Sdes /* report back real offset and size */ 192290267Sdes URL->offset = offset; 192390267Sdes URL->length = clength; 192437535Sdes 192590267Sdes /* wrap it up in a FILE */ 1926174588Sdes if ((f = http_funopen(conn, chunked)) == NULL) { 1927174588Sdes fetch_syserr(); 192890267Sdes goto ouch; 192990267Sdes } 193063716Sdes 193190267Sdes if (url != URL) 193290267Sdes fetchFreeURL(url); 193390267Sdes if (purl) 193490267Sdes fetchFreeURL(purl); 193563567Sdes 193697856Sdes if (HTTP_ERROR(conn->err)) { 1937174588Sdes http_print_html(stderr, f); 193890267Sdes fclose(f); 193990267Sdes f = NULL; 194090267Sdes } 1941202613Sdes clean_http_headerbuf(&headerbuf); 1942202613Sdes clean_http_auth_challenges(&server_challenges); 1943202613Sdes clean_http_auth_challenges(&proxy_challenges); 194490267Sdes return (f); 194588771Sdes 194690267Sdesouch: 194790267Sdes if (url != URL) 194890267Sdes fetchFreeURL(url); 194990267Sdes if (purl) 195090267Sdes fetchFreeURL(purl); 195197856Sdes if (conn != NULL) 1952174588Sdes fetch_close(conn); 1953202613Sdes clean_http_headerbuf(&headerbuf); 1954202613Sdes clean_http_auth_challenges(&server_challenges); 1955202613Sdes clean_http_auth_challenges(&proxy_challenges); 195690267Sdes return (NULL); 195763012Sdes} 195860189Sdes 195990267Sdes 196063012Sdes/***************************************************************************** 196163012Sdes * Entry points 196263012Sdes */ 196363012Sdes 196463012Sdes/* 196563340Sdes * Retrieve and stat a file by HTTP 196663340Sdes */ 196763340SdesFILE * 196875891SarchiefetchXGetHTTP(struct url *URL, struct url_stat *us, const char *flags) 196963340Sdes{ 1970174752Sdes return (http_request(URL, "GET", us, http_get_proxy(URL, flags), flags)); 197163340Sdes} 197263340Sdes 197363340Sdes/* 197463012Sdes * Retrieve a file by HTTP 197563012Sdes */ 197663012SdesFILE * 197775891SarchiefetchGetHTTP(struct url *URL, const char *flags) 197863012Sdes{ 197990267Sdes return (fetchXGetHTTP(URL, NULL, flags)); 198037535Sdes} 198137535Sdes 198263340Sdes/* 198363340Sdes * Store a file by HTTP 198463340Sdes */ 198537535SdesFILE * 198685093SdesfetchPutHTTP(struct url *URL __unused, const char *flags __unused) 198737535Sdes{ 198890267Sdes warnx("fetchPutHTTP(): not implemented"); 198990267Sdes return (NULL); 199037535Sdes} 199140975Sdes 199240975Sdes/* 199340975Sdes * Get an HTTP document's metadata 199440975Sdes */ 199540975Sdesint 199675891SarchiefetchStatHTTP(struct url *URL, struct url_stat *us, const char *flags) 199740975Sdes{ 199890267Sdes FILE *f; 199990267Sdes 2000174752Sdes f = http_request(URL, "HEAD", us, http_get_proxy(URL, flags), flags); 2001112081Sdes if (f == NULL) 200290267Sdes return (-1); 200390267Sdes fclose(f); 200490267Sdes return (0); 200540975Sdes} 200641989Sdes 200741989Sdes/* 200841989Sdes * List a directory 200941989Sdes */ 201041989Sdesstruct url_ent * 201185093SdesfetchListHTTP(struct url *url __unused, const char *flags __unused) 201241989Sdes{ 201390267Sdes warnx("fetchListHTTP(): not implemented"); 201490267Sdes return (NULL); 201541989Sdes} 2016