portsnap.sh revision 148871
1148871Scperciva#!/bin/sh 2148871Scperciva 3148871Scperciva#- 4148871Scperciva# Copyright 2004-2005 Colin Percival 5148871Scperciva# All rights reserved 6148871Scperciva# 7148871Scperciva# Redistribution and use in source and binary forms, with or without 8148871Scperciva# modification, are permitted providing that the following conditions 9148871Scperciva# are met: 10148871Scperciva# 1. Redistributions of source code must retain the above copyright 11148871Scperciva# notice, this list of conditions and the following disclaimer. 12148871Scperciva# 2. Redistributions in binary form must reproduce the above copyright 13148871Scperciva# notice, this list of conditions and the following disclaimer in the 14148871Scperciva# documentation and/or other materials provided with the distribution. 15148871Scperciva# 16148871Scperciva# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17148871Scperciva# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18148871Scperciva# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19148871Scperciva# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 20148871Scperciva# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21148871Scperciva# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22148871Scperciva# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23148871Scperciva# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24148871Scperciva# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25148871Scperciva# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26148871Scperciva# POSSIBILITY OF SUCH DAMAGE. 27148871Scperciva 28148871Scperciva# $FreeBSD: head/usr.sbin/portsnap/portsnap/portsnap.sh 148871 2005-08-08 20:10:06Z cperciva $ 29148871Scperciva 30148871Scperciva#### Usage function -- called from command-line handling code. 31148871Scperciva 32148871Scperciva# Usage instructions. Options not listed: 33148871Scperciva# --debug -- don't filter output from utilities 34148871Scperciva# --no-stats -- don't show progress statistics while fetching files 35148871Scpercivausage() { 36148871Scperciva cat <<EOF 37148871Scpercivausage: `basename $0` [options] command [path] 38148871Scperciva 39148871ScpercivaOptions: 40148871Scperciva -d workdir -- Store working files in workdir 41148871Scperciva (default: /var/db/portsnap/) 42148871Scperciva -f conffile -- Read configuration options from conffile 43148871Scperciva (default: /etc/portsnap.conf) 44148871Scperciva -I -- Update INDEX only. (update command only) 45148871Scperciva -k KEY -- Trust an RSA key with SHA256 hash of KEY 46148871Scperciva -p portsdir -- Location of uncompressed ports tree 47148871Scperciva (default: /usr/ports/) 48148871Scperciva -s server -- Server from which to fetch updates. 49148871Scperciva (default: portsnap.FreeBSD.org) 50148871Scperciva path -- Extract only parts of the tree starting with the given 51148871Scperciva string. (extract command only) 52148871ScpercivaCommands: 53148871Scperciva fetch -- Fetch a compressed snapshot of the ports tree, 54148871Scperciva or update an existing snapshot. 55148871Scperciva cron -- Sleep rand(3600) seconds, and then fetch updates. 56148871Scperciva extract -- Extract snapshot of ports tree, replacing existing 57148871Scperciva files and directories. 58148871Scperciva update -- Update ports tree to match current snapshot, replacing 59148871Scperciva files and directories which have changed. 60148871ScpercivaEOF 61148871Scperciva exit 0 62148871Scperciva} 63148871Scperciva 64148871Scperciva#### Parameter handling functions. 65148871Scperciva 66148871Scperciva# Initialize parameters to null, just in case they're 67148871Scperciva# set in the environment. 68148871Scpercivainit_params() { 69148871Scperciva KEYPRINT="" 70148871Scperciva EXTRACTPATH="" 71148871Scperciva WORKDIR="" 72148871Scperciva PORTSDIR="" 73148871Scperciva CONFFILE="" 74148871Scperciva COMMAND="" 75148871Scperciva QUIETREDIR="" 76148871Scperciva QUIETFLAG="" 77148871Scperciva STATSREDIR="" 78148871Scperciva NDEBUG="" 79148871Scperciva DDSTATS="" 80148871Scperciva INDEXONLY="" 81148871Scperciva SERVERNAME="" 82148871Scperciva} 83148871Scperciva 84148871Scperciva# Parse the command line 85148871Scpercivaparse_cmdline() { 86148871Scperciva while [ $# -gt 0 ]; do 87148871Scperciva case "$1" in 88148871Scperciva -d) 89148871Scperciva if [ $# -eq 1 ]; then usage; fi 90148871Scperciva if [ ! -z "${WORKDIR}" ]; then usage; fi 91148871Scperciva shift; WORKDIR="$1" 92148871Scperciva ;; 93148871Scperciva --debug) 94148871Scperciva QUIETREDIR="/dev/stderr" 95148871Scperciva STATSREDIR="/dev/stderr" 96148871Scperciva QUIETFLAG=" " 97148871Scperciva NDEBUG=" " 98148871Scperciva DDSTATS=".." 99148871Scperciva ;; 100148871Scperciva -f) 101148871Scperciva if [ $# -eq 1 ]; then usage; fi 102148871Scperciva if [ ! -z "${CONFFILE}" ]; then usage; fi 103148871Scperciva shift; CONFFILE="$1" 104148871Scperciva ;; 105148871Scperciva -h | --help | help) 106148871Scperciva usage 107148871Scperciva ;; 108148871Scperciva -I) 109148871Scperciva INDEXONLY="YES" 110148871Scperciva ;; 111148871Scperciva -k) 112148871Scperciva if [ $# -eq 1 ]; then usage; fi 113148871Scperciva if [ ! -z "${KEYPRINT}" ]; then usage; fi 114148871Scperciva shift; KEYPRINT="$1" 115148871Scperciva ;; 116148871Scperciva --no-stats) 117148871Scperciva if [ -z "${STATSREDIR}" ]; then 118148871Scperciva STATSREDIR="/dev/null" 119148871Scperciva DDSTATS=".. " 120148871Scperciva fi 121148871Scperciva ;; 122148871Scperciva -p) 123148871Scperciva if [ $# -eq 1 ]; then usage; fi 124148871Scperciva if [ ! -z "${PORTSDIR}" ]; then usage; fi 125148871Scperciva shift; PORTSDIR="$1" 126148871Scperciva ;; 127148871Scperciva -s) 128148871Scperciva if [ $# -eq 1 ]; then usage; fi 129148871Scperciva if [ ! -z "${SERVERNAME}" ]; then usage; fi 130148871Scperciva shift; SERVERNAME="$1" 131148871Scperciva ;; 132148871Scperciva cron | extract | fetch | update) 133148871Scperciva if [ ! -z "${COMMAND}" ]; then usage; fi 134148871Scperciva COMMAND="$1" 135148871Scperciva ;; 136148871Scperciva *) 137148871Scperciva if [ $# -gt 1 ]; then usage; fi 138148871Scperciva if [ "${COMMAND}" = "extract" ]; then usage; fi 139148871Scperciva EXTRACTPATH="$1" 140148871Scperciva ;; 141148871Scperciva esac 142148871Scperciva shift 143148871Scperciva done 144148871Scperciva 145148871Scperciva if [ -z "${COMMAND}" ]; then 146148871Scperciva usage 147148871Scperciva fi 148148871Scperciva} 149148871Scperciva 150148871Scperciva# If CONFFILE was specified at the command-line, make 151148871Scperciva# sure that it exists and is readable. 152148871Scpercivasanity_conffile() { 153148871Scperciva if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then 154148871Scperciva echo -n "File does not exist " 155148871Scperciva echo -n "or is not readable: " 156148871Scperciva echo ${CONFFILE} 157148871Scperciva exit 1 158148871Scperciva fi 159148871Scperciva} 160148871Scperciva 161148871Scperciva# If a configuration file hasn't been specified, use 162148871Scperciva# the default value (/etc/portsnap.conf) 163148871Scpercivadefault_conffile() { 164148871Scperciva if [ -z "${CONFFILE}" ]; then 165148871Scperciva CONFFILE="/etc/portsnap.conf" 166148871Scperciva fi 167148871Scperciva} 168148871Scperciva 169148871Scperciva# Read {KEYPRINT, SERVERNAME, WORKDIR, PORTSDIR} from the configuration 170148871Scperciva# file if they haven't already been set. If the configuration 171148871Scperciva# file doesn't exist, do nothing. 172148871Scpercivaparse_conffile() { 173148871Scperciva if [ -r "${CONFFILE}" ]; then 174148871Scperciva for X in KEYPRINT WORKDIR PORTSDIR SERVERNAME; do 175148871Scperciva eval _=\$${X} 176148871Scperciva if [ -z "${_}" ]; then 177148871Scperciva eval ${X}=`grep "^${X}=" "${CONFFILE}" | 178148871Scperciva cut -f 2- -d '=' | tail -1` 179148871Scperciva fi 180148871Scperciva done 181148871Scperciva fi 182148871Scperciva} 183148871Scperciva 184148871Scperciva# If parameters have not been set, use default values 185148871Scpercivadefault_params() { 186148871Scperciva _QUIETREDIR="/dev/null" 187148871Scperciva _QUIETFLAG="-q" 188148871Scperciva _STATSREDIR="/dev/stdout" 189148871Scperciva _WORKDIR="/var/db/portsnap" 190148871Scperciva _PORTSDIR="/usr/ports" 191148871Scperciva _NDEBUG="-n" 192148871Scperciva for X in QUIETREDIR QUIETFLAG STATSREDIR WORKDIR PORTSDIR NDEBUG; do 193148871Scperciva eval _=\$${X} 194148871Scperciva eval __=\$_${X} 195148871Scperciva if [ -z "${_}" ]; then 196148871Scperciva eval ${X}=${__} 197148871Scperciva fi 198148871Scperciva done 199148871Scperciva} 200148871Scperciva 201148871Scperciva# Perform sanity checks and set some final parameters 202148871Scperciva# in preparation for fetching files. Also chdir into 203148871Scperciva# the working directory. 204148871Scpercivafetch_check_params() { 205148871Scperciva export HTTP_USER_AGENT="portsnap (${COMMAND}, `uname -r`)" 206148871Scperciva 207148871Scperciva _SERVERNAME_z=\ 208148871Scperciva"SERVERNAME must be given via command line or configuration file." 209148871Scperciva _KEYPRINT_z="Key must be given via -k option or configuration file." 210148871Scperciva _KEYPRINT_bad="Invalid key fingerprint: " 211148871Scperciva _WORKDIR_bad="Directory does not exist or is not writable: " 212148871Scperciva 213148871Scperciva if [ -z "${SERVERNAME}" ]; then 214148871Scperciva echo -n "`basename $0`: " 215148871Scperciva echo "${_SERVERNAME_z}" 216148871Scperciva exit 1 217148871Scperciva fi 218148871Scperciva if [ -z "${KEYPRINT}" ]; then 219148871Scperciva echo -n "`basename $0`: " 220148871Scperciva echo "${_KEYPRINT_z}" 221148871Scperciva exit 1 222148871Scperciva fi 223148871Scperciva if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then 224148871Scperciva echo -n "`basename $0`: " 225148871Scperciva echo -n "${_KEYPRINT_bad}" 226148871Scperciva echo ${KEYPRINT} 227148871Scperciva exit 1 228148871Scperciva fi 229148871Scperciva if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then 230148871Scperciva echo -n "`basename $0`: " 231148871Scperciva echo -n "${_WORKDIR_bad}" 232148871Scperciva echo ${WORKDIR} 233148871Scperciva exit 1 234148871Scperciva fi 235148871Scperciva cd ${WORKDIR} || exit 1 236148871Scperciva 237148871Scperciva BSPATCH=/usr/bin/bspatch 238148871Scperciva SHA256=/sbin/sha256 239148871Scperciva PHTTPGET=/usr/libexec/phttpget 240148871Scperciva} 241148871Scperciva 242148871Scperciva# Perform sanity checks and set some final parameters 243148871Scperciva# in preparation for extracting or updating ${PORTSDIR} 244148871Scpercivaextract_check_params() { 245148871Scperciva _WORKDIR_bad="Directory does not exist: " 246148871Scperciva _PORTSDIR_bad="Directory does not exist or is not writable: " 247148871Scperciva 248148871Scperciva if ! [ -d "${WORKDIR}" ]; then 249148871Scperciva echo -n "`basename $0`: " 250148871Scperciva echo -n "${_WORKDIR_bad}" 251148871Scperciva echo ${WORKDIR} 252148871Scperciva exit 1 253148871Scperciva fi 254148871Scperciva if ! [ -d "${PORTSDIR}" -a -w "${PORTSDIR}" ]; then 255148871Scperciva echo -n "`basename $0`: " 256148871Scperciva echo -n "${_PORTSDIR_bad}" 257148871Scperciva echo ${PORTSDIR} 258148871Scperciva exit 1 259148871Scperciva fi 260148871Scperciva 261148871Scperciva if ! [ -d "${WORKDIR}/files" -a -r "${WORKDIR}/tag" \ 262148871Scperciva -a -r "${WORKDIR}/INDEX" -a -r "${WORKDIR}/tINDEX" ]; then 263148871Scperciva echo "No snapshot available. Try running" 264148871Scperciva echo "# `basename $0` fetch" 265148871Scperciva exit 1 266148871Scperciva fi 267148871Scperciva 268148871Scperciva MKINDEX=/usr/libexec/make_index 269148871Scperciva} 270148871Scperciva 271148871Scperciva# Perform sanity checks and set some final parameters 272148871Scperciva# in preparation for updating ${PORTSDIR} 273148871Scpercivaupdate_check_params() { 274148871Scperciva extract_check_params 275148871Scperciva 276148871Scperciva if ! [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 277148871Scperciva echo "${PORTSDIR} was not created by portsnap." 278148871Scperciva echo -n "You must run '`basename $0` extract' before " 279148871Scperciva echo "running '`basename $0` update'." 280148871Scperciva exit 1 281148871Scperciva fi 282148871Scperciva 283148871Scperciva} 284148871Scperciva 285148871Scperciva#### Core functionality -- the actual work gets done here 286148871Scperciva 287148871Scperciva# Use an SRV query to pick a server. If the SRV query doesn't provide 288148871Scperciva# a useful answer, use the server name specified by the user. 289148871Scperciva# Put another way... look up _http._tcp.${SERVERNAME} and pick a server 290148871Scperciva# from that; or if no servers are returned, use ${SERVERNAME}. 291148871Scperciva# This allows a user to specify "portsnap.freebsd.org" (in which case 292148871Scperciva# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org" 293148871Scperciva# (in which case portsnap will use that particular server, since there 294148871Scperciva# won't be an SRV entry for that name). 295148871Scperciva# 296148871Scperciva# We don't implement the recommendations from RFC 2782 completely, since 297148871Scperciva# we are only looking to pick a single server -- the recommendations are 298148871Scperciva# targetted at applications which obtain a list of servers and then try 299148871Scperciva# each in turn, but we are instead just going to pick one server and let 300148871Scperciva# the user re-run portsnap if a broken server was selected. 301148871Scperciva# 302148871Scperciva# We also ignore the Port field, since we are always going to use port 80. 303148871Scpercivafetch_pick_server() { 304148871Scperciva echo -n "Looking up ${SERVERNAME} mirrors..." 305148871Scperciva 306148871Scperciva# Issue the SRV query and pull out the Priority, Weight, and Target fields. 307148871Scperciva host -t srv "_http._tcp.${SERVERNAME}" | 308148871Scperciva grep -E "^_http._tcp.${SERVERNAME} has SRV record" | 309148871Scperciva cut -f 5,6,8 -d ' ' > serverlist 310148871Scperciva 311148871Scperciva# If no records, give up -- we'll just use the server name we were given. 312148871Scperciva if [ `wc -l < serverlist` -eq 0 ]; then 313148871Scperciva echo " none found." 314148871Scperciva return 315148871Scperciva fi 316148871Scperciva 317148871Scperciva# Find the highest priority level (lowest numeric value). 318148871Scperciva SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1` 319148871Scperciva 320148871Scperciva# Add up the weights of the response lines at that priority level. 321148871Scperciva SRV_WSUM=0; 322148871Scperciva while read X; do 323148871Scperciva case "$X" in 324148871Scperciva ${SRV_PRIORITY}\ *) 325148871Scperciva SRV_W=`echo $X | cut -f 2 -d ' '` 326148871Scperciva SRV_WSUM=$(($SRV_WSUM + $SRV_W)) 327148871Scperciva ;; 328148871Scperciva esac 329148871Scperciva done < serverlist 330148871Scperciva 331148871Scperciva# If all the weights are 0, pretend that they are all 1 instead. 332148871Scperciva if [ ${SRV_WSUM} -eq 0 ]; then 333148871Scperciva SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l` 334148871Scperciva SRV_W_ADD=1 335148871Scperciva else 336148871Scperciva SRV_W_ADD=0 337148871Scperciva fi 338148871Scperciva 339148871Scperciva# Pick a random value between 1 and the sum of the weights 340148871Scperciva SRV_RND=`jot -r 1 1 ${SRV_WSUM}` 341148871Scperciva 342148871Scperciva# Read through the list of mirrors and set SERVERNAME 343148871Scperciva while read X; do 344148871Scperciva case "$X" in 345148871Scperciva ${SRV_PRIORITY}\ *) 346148871Scperciva SRV_W=`echo $X | cut -f 2 -d ' '` 347148871Scperciva SRV_W=$(($SRV_W + $SRV_W_ADD)) 348148871Scperciva if [ $SRV_RND -le $SRV_W ]; then 349148871Scperciva SERVERNAME=`echo $X | cut -f 3 -d ' '` 350148871Scperciva break 351148871Scperciva else 352148871Scperciva SRV_RND=$(($SRV_RND - $SRV_W)) 353148871Scperciva fi 354148871Scperciva ;; 355148871Scperciva esac 356148871Scperciva done < serverlist 357148871Scperciva 358148871Scperciva echo " using ${SERVERNAME}" 359148871Scperciva} 360148871Scperciva 361148871Scperciva# Check that we have a public key with an appropriate hash, or 362148871Scperciva# fetch the key if it doesn't exist. 363148871Scpercivafetch_key() { 364148871Scperciva if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 365148871Scperciva return 366148871Scperciva fi 367148871Scperciva 368148871Scperciva echo -n "Fetching public key... " 369148871Scperciva rm -f pub.ssl 370148871Scperciva fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \ 371148871Scperciva 2>${QUIETREDIR} || true 372148871Scperciva if ! [ -r pub.ssl ]; then 373148871Scperciva echo "failed." 374148871Scperciva return 1 375148871Scperciva fi 376148871Scperciva if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 377148871Scperciva echo "key has incorrect hash." 378148871Scperciva rm -f pub.ssl 379148871Scperciva return 1 380148871Scperciva fi 381148871Scperciva echo "done." 382148871Scperciva} 383148871Scperciva 384148871Scperciva# Fetch a snapshot tag 385148871Scpercivafetch_tag() { 386148871Scperciva rm -f snapshot.ssl tag.new 387148871Scperciva 388148871Scperciva echo ${NDEBUG} "Fetching snapshot tag... " 389148871Scperciva fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl 390148871Scperciva 2>${QUIETREDIR} || true 391148871Scperciva if ! [ -r $1.ssl ]; then 392148871Scperciva echo "failed." 393148871Scperciva return 1 394148871Scperciva fi 395148871Scperciva 396148871Scperciva openssl rsautl -pubin -inkey pub.ssl -verify \ 397148871Scperciva < $1.ssl > tag.new 2>${QUIETREDIR} || true 398148871Scperciva rm $1.ssl 399148871Scperciva 400148871Scperciva if ! [ `wc -l < tag.new` = 1 ] || 401148871Scperciva ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then 402148871Scperciva echo "invalid snapshot tag." 403148871Scperciva return 1 404148871Scperciva fi 405148871Scperciva 406148871Scperciva echo "done." 407148871Scperciva 408148871Scperciva SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new` 409148871Scperciva SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new` 410148871Scperciva} 411148871Scperciva 412148871Scperciva# Sanity-check the date on a snapshot tag 413148871Scpercivafetch_snapshot_tagsanity() { 414148871Scperciva if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then 415148871Scperciva echo "Snapshot appears to be more than a year old!" 416148871Scperciva echo "(Is the system clock correct?)" 417148871Scperciva echo "Cowarly refusing to proceed any further." 418148871Scperciva return 1 419148871Scperciva fi 420148871Scperciva if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then 421148871Scperciva echo -n "Snapshot appears to have been created more than " 422148871Scperciva echo "one day into the future!" 423148871Scperciva echo "(Is the system clock correct?)" 424148871Scperciva echo "Cowardly refusing to proceed any further." 425148871Scperciva return 1 426148871Scperciva fi 427148871Scperciva} 428148871Scperciva 429148871Scperciva# Sanity-check the date on a snapshot update tag 430148871Scpercivafetch_update_tagsanity() { 431148871Scperciva fetch_snapshot_tagsanity || return 1 432148871Scperciva 433148871Scperciva if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then 434148871Scperciva echo -n "Latest snapshot on server is " 435148871Scperciva echo "older than what we already have!" 436148871Scperciva echo -n "Cowardly refusing to downgrade from " 437148871Scperciva date -r ${OLDSNAPSHOTDATE} 438148871Scperciva echo -n "to `date -r ${SNAPSHOTDATE}`." 439148871Scperciva return 1 440148871Scperciva fi 441148871Scperciva} 442148871Scperciva 443148871Scperciva# Compare old and new tags; return 1 if update is unnecessary 444148871Scpercivafetch_update_neededp() { 445148871Scperciva if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then 446148871Scperciva echo -n "Latest snapshot on server matches " 447148871Scperciva echo "what we already have." 448148871Scperciva echo "No updates needed." 449148871Scperciva rm tag.new 450148871Scperciva return 1 451148871Scperciva fi 452148871Scperciva if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then 453148871Scperciva echo -n "Ports tree hasn't changed since " 454148871Scperciva echo "last snapshot." 455148871Scperciva echo "No updates needed." 456148871Scperciva rm tag.new 457148871Scperciva return 1 458148871Scperciva fi 459148871Scperciva 460148871Scperciva return 0 461148871Scperciva} 462148871Scperciva 463148871Scperciva# Fetch snapshot metadata file 464148871Scpercivafetch_metadata() { 465148871Scperciva rm -f ${SNAPSHOTHASH} tINDEX.new 466148871Scperciva 467148871Scperciva echo ${NDEBUG} "Fetching snapshot metadata... " 468148871Scperciva fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} 469148871Scperciva 2>${QUIETREDIR} || return 470148871Scperciva if [ `${SHA256} -q ${SNAPSHOTHASH}` != ${SNAPSHOTHASH} ]; then 471148871Scperciva echo "snapshot metadata corrupt." 472148871Scperciva return 1 473148871Scperciva fi 474148871Scperciva mv ${SNAPSHOTHASH} tINDEX.new 475148871Scperciva echo "done." 476148871Scperciva} 477148871Scperciva 478148871Scperciva# Warn user about bogus metadata 479148871Scpercivafetch_metadata_freakout() { 480148871Scperciva echo 481148871Scperciva echo "Portsnap metadata is correctly signed, but contains" 482148871Scperciva echo "at least one line which appears bogus." 483148871Scperciva echo "Cowardly refusing to proceed any further." 484148871Scperciva} 485148871Scperciva 486148871Scperciva# Sanity-check a snapshot metadata file 487148871Scpercivafetch_metadata_sanity() { 488148871Scperciva if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then 489148871Scperciva fetch_metadata_freakout 490148871Scperciva return 1 491148871Scperciva fi 492148871Scperciva if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then 493148871Scperciva echo 494148871Scperciva echo "Portsnap metadata appears bogus." 495148871Scperciva echo "Cowardly refusing to proceed any further." 496148871Scperciva return 1 497148871Scperciva fi 498148871Scperciva} 499148871Scperciva 500148871Scperciva# Take a list of ${oldhash}|${newhash} and output a list of needed patches 501148871Scpercivafetch_make_patchlist() { 502148871Scperciva grep -vE "^([0-9a-f]{64})\|\1$" | 503148871Scperciva while read LINE; do 504148871Scperciva X=`echo ${LINE} | cut -f 1 -d '|'` 505148871Scperciva Y=`echo ${LINE} | cut -f 2 -d '|'` 506148871Scperciva if [ -f "files/${Y}.gz" ]; then continue; fi 507148871Scperciva if [ ! -f "files/${X}.gz" ]; then continue; fi 508148871Scperciva echo "${LINE}" 509148871Scperciva done 510148871Scperciva} 511148871Scperciva 512148871Scperciva# Print user-friendly progress statistics 513148871Scpercivafetch_progress() { 514148871Scperciva LNC=0 515148871Scperciva while read x; do 516148871Scperciva LNC=$(($LNC + 1)) 517148871Scperciva if [ $(($LNC % 10)) = 0 ]; then 518148871Scperciva echo -n $FN2 519148871Scperciva elif [ $(($LNC % 2)) = 0 ]; then 520148871Scperciva echo -n . 521148871Scperciva fi 522148871Scperciva done 523148871Scperciva echo -n " " 524148871Scperciva} 525148871Scperciva 526148871Scperciva# Sanity-check an index file 527148871Scpercivafetch_index_sanity() { 528148871Scperciva if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new || 529148871Scperciva fgrep -q "./" INDEX.new; then 530148871Scperciva fetch_metadata_freakout 531148871Scperciva return 1 532148871Scperciva fi 533148871Scperciva} 534148871Scperciva 535148871Scperciva# Verify a list of files 536148871Scpercivafetch_snapshot_verify() { 537148871Scperciva while read F; do 538148871Scperciva if [ `gunzip -c snap/${F} | ${SHA256} -q` != ${F} ]; then 539148871Scperciva echo "snapshot corrupt." 540148871Scperciva return 1 541148871Scperciva fi 542148871Scperciva done 543148871Scperciva return 0 544148871Scperciva} 545148871Scperciva 546148871Scperciva# Fetch a snapshot tarball, extract, and verify. 547148871Scpercivafetch_snapshot() { 548148871Scperciva fetch_tag snapshot || return 1 549148871Scperciva fetch_snapshot_tagsanity || return 1 550148871Scperciva fetch_metadata || return 1 551148871Scperciva fetch_metadata_sanity || return 1 552148871Scperciva 553148871Scperciva rm -f ${SNAPSHOTHASH}.tgz 554148871Scperciva rm -rf snap/ 555148871Scperciva 556148871Scperciva# Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will 557148871Scperciva# probably take a while, so the progrees reports that fetch(1) generates 558148871Scperciva# will be useful for keeping the users' attention from drifting. 559148871Scperciva echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:" 560148871Scperciva fetch http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1 561148871Scperciva 562148871Scperciva echo -n "Extracting snapshot... " 563148871Scperciva tar -xzf ${SNAPSHOTHASH}.tgz snap/ || return 1 564148871Scperciva rm ${SNAPSHOTHASH}.tgz 565148871Scperciva echo "done." 566148871Scperciva 567148871Scperciva echo -n "Verifying snapshot integrity... " 568148871Scperciva# Verify the metadata files 569148871Scperciva cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1 570148871Scperciva# Extract the index 571148871Scperciva rm -f INDEX.new 572148871Scperciva gunzip -c snap/`look INDEX tINDEX.new | 573148871Scperciva cut -f 2 -d '|'`.gz > INDEX.new 574148871Scperciva fetch_index_sanity || return 1 575148871Scperciva# Verify the snapshot contents 576148871Scperciva cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1 577148871Scperciva echo "done." 578148871Scperciva 579148871Scperciva# Move files into their proper locations 580148871Scperciva rm -f tag INDEX tINDEX 581148871Scperciva rm -rf files 582148871Scperciva mv tag.new tag 583148871Scperciva mv tINDEX.new tINDEX 584148871Scperciva mv INDEX.new INDEX 585148871Scperciva mv snap/ files/ 586148871Scperciva 587148871Scperciva return 0 588148871Scperciva} 589148871Scperciva 590148871Scperciva# Update a compressed snapshot 591148871Scpercivafetch_update() { 592148871Scperciva rm -f patchlist diff OLD NEW filelist INDEX.new 593148871Scperciva 594148871Scperciva OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag` 595148871Scperciva OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag` 596148871Scperciva 597148871Scperciva fetch_tag latest || return 1 598148871Scperciva fetch_update_tagsanity || return 1 599148871Scperciva fetch_update_neededp || return 0 600148871Scperciva fetch_metadata || return 1 601148871Scperciva fetch_metadata_sanity || return 1 602148871Scperciva 603148871Scperciva echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` " 604148871Scperciva echo "to `date -r ${SNAPSHOTDATE}`." 605148871Scperciva 606148871Scperciva# Generate a list of wanted metadata patches 607148871Scperciva join -t '|' -o 1.2,2.2 tINDEX tINDEX.new | 608148871Scperciva fetch_make_patchlist > patchlist 609148871Scperciva 610148871Scperciva# Attempt to fetch metadata patches 611148871Scperciva echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 612148871Scperciva echo ${NDEBUG} "metadata patches.${DDSTATS}" 613148871Scperciva tr '|' '-' < patchlist | 614148871Scperciva lam -s "tp/" - -s ".gz" | 615148871Scperciva xargs ${PHTTPGET} ${SERVERNAME} \ 616148871Scperciva 2>${STATSREDIR} | fetch_progress 617148871Scperciva echo "done." 618148871Scperciva 619148871Scperciva# Attempt to apply metadata patches 620148871Scperciva echo -n "Applying metadata patches... " 621148871Scperciva while read LINE; do 622148871Scperciva X=`echo ${LINE} | cut -f 1 -d '|'` 623148871Scperciva Y=`echo ${LINE} | cut -f 2 -d '|'` 624148871Scperciva if [ ! -f "${X}-${Y}.gz" ]; then continue; fi 625148871Scperciva gunzip -c < ${X}-${Y}.gz > diff 626148871Scperciva gunzip -c < files/${X}.gz > OLD 627148871Scperciva cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp 628148871Scperciva grep '^\+' diff | cut -c 2- | 629148871Scperciva sort -k 1,1 -t '|' -m - ptmp > NEW 630148871Scperciva if [ `${SHA256} -q NEW` = ${Y} ]; then 631148871Scperciva mv NEW files/${Y} 632148871Scperciva gzip -n files/${Y} 633148871Scperciva fi 634148871Scperciva rm -f diff OLD NEW ${X}-${Y}.gz ptmp 635148871Scperciva done < patchlist 2>${QUIETREDIR} 636148871Scperciva echo "done." 637148871Scperciva 638148871Scperciva# Update metadata without patches 639148871Scperciva join -t '|' -v 2 tINDEX tINDEX.new | 640148871Scperciva cut -f 2 -d '|' /dev/stdin patchlist | 641148871Scperciva while read Y; do 642148871Scperciva if [ ! -f "files/${Y}.gz" ]; then 643148871Scperciva echo ${Y}; 644148871Scperciva fi 645148871Scperciva done > filelist 646148871Scperciva echo -n "Fetching `wc -l < filelist | tr -d ' '` " 647148871Scperciva echo ${NDEBUG} "metadata files... " 648148871Scperciva lam -s "f/" - -s ".gz" < filelist | 649148871Scperciva xargs ${PHTTPGET} ${SERVERNAME} \ 650148871Scperciva 2>${QUIETREDIR} 651148871Scperciva 652148871Scperciva while read Y; do 653148871Scperciva if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 654148871Scperciva mv ${Y}.gz files/${Y}.gz 655148871Scperciva else 656148871Scperciva echo "metadata is corrupt." 657148871Scperciva return 1 658148871Scperciva fi 659148871Scperciva done < filelist 660148871Scperciva echo "done." 661148871Scperciva 662148871Scperciva# Extract the index 663148871Scperciva gunzip -c files/`look INDEX tINDEX.new | 664148871Scperciva cut -f 2 -d '|'`.gz > INDEX.new 665148871Scperciva fetch_index_sanity || return 1 666148871Scperciva 667148871Scperciva# Generate a list of wanted ports patches 668148871Scperciva join -t '|' -o 1.2,2.2 INDEX INDEX.new | 669148871Scperciva fetch_make_patchlist > patchlist 670148871Scperciva 671148871Scperciva# Attempt to fetch ports patches 672148871Scperciva echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 673148871Scperciva echo ${NDEBUG} "patches.${DDSTATS}" 674148871Scperciva tr '|' '-' < patchlist | lam -s "bp/" - | 675148871Scperciva xargs ${PHTTPGET} ${SERVERNAME} \ 676148871Scperciva 2>${STATSREDIR} | fetch_progress 677148871Scperciva echo "done." 678148871Scperciva 679148871Scperciva# Attempt to apply ports patches 680148871Scperciva echo -n "Applying patches... " 681148871Scperciva while read LINE; do 682148871Scperciva X=`echo ${LINE} | cut -f 1 -d '|'` 683148871Scperciva Y=`echo ${LINE} | cut -f 2 -d '|'` 684148871Scperciva if [ ! -f "${X}-${Y}" ]; then continue; fi 685148871Scperciva gunzip -c < files/${X}.gz > OLD 686148871Scperciva ${BSPATCH} OLD NEW ${X}-${Y} 687148871Scperciva if [ `${SHA256} -q NEW` = ${Y} ]; then 688148871Scperciva mv NEW files/${Y} 689148871Scperciva gzip -n files/${Y} 690148871Scperciva fi 691148871Scperciva rm -f diff OLD NEW ${X}-${Y} 692148871Scperciva done < patchlist 2>${QUIETREDIR} 693148871Scperciva echo "done." 694148871Scperciva 695148871Scperciva# Update ports without patches 696148871Scperciva join -t '|' -v 2 INDEX INDEX.new | 697148871Scperciva cut -f 2 -d '|' /dev/stdin patchlist | 698148871Scperciva while read Y; do 699148871Scperciva if [ ! -f "files/${Y}.gz" ]; then 700148871Scperciva echo ${Y}; 701148871Scperciva fi 702148871Scperciva done > filelist 703148871Scperciva echo -n "Fetching `wc -l < filelist | tr -d ' '` " 704148871Scperciva echo ${NDEBUG} "new ports or files... " 705148871Scperciva lam -s "f/" - -s ".gz" < filelist | 706148871Scperciva xargs ${PHTTPGET} ${SERVERNAME} \ 707148871Scperciva 2>${QUIETREDIR} 708148871Scperciva 709148871Scperciva while read Y; do 710148871Scperciva if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 711148871Scperciva mv ${Y}.gz files/${Y}.gz 712148871Scperciva else 713148871Scperciva echo "snapshot is corrupt." 714148871Scperciva return 1 715148871Scperciva fi 716148871Scperciva done < filelist 717148871Scperciva echo "done." 718148871Scperciva 719148871Scperciva# Remove files which are no longer needed 720148871Scperciva cut -f 2 -d '|' tINDEX INDEX | sort > oldfiles 721148871Scperciva cut -f 2 -d '|' tINDEX.new INDEX.new | sort | comm -13 - oldfiles | 722148871Scperciva lam -s "files/" - -s ".gz" | xargs rm -f 723148871Scperciva rm patchlist filelist oldfiles 724148871Scperciva 725148871Scperciva# We're done! 726148871Scperciva mv INDEX.new INDEX 727148871Scperciva mv tINDEX.new tINDEX 728148871Scperciva mv tag.new tag 729148871Scperciva 730148871Scperciva return 0 731148871Scperciva} 732148871Scperciva 733148871Scperciva# Do the actual work involved in "fetch" / "cron". 734148871Scpercivafetch_run() { 735148871Scperciva fetch_pick_server 736148871Scperciva 737148871Scperciva fetch_key || return 1 738148871Scperciva 739148871Scperciva if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then 740148871Scperciva fetch_snapshot || return 1 741148871Scperciva fi 742148871Scperciva fetch_update || return 1 743148871Scperciva} 744148871Scperciva 745148871Scperciva# Build a ports INDEX file 746148871Scpercivaextract_make_index() { 747148871Scperciva gunzip -c "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX | 748148871Scperciva cut -f 2 -d '|'`.gz" | ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2 749148871Scperciva} 750148871Scperciva 751148871Scperciva# Create INDEX, INDEX-5, INDEX-6 752148871Scpercivaextract_indices() { 753148871Scperciva echo -n "Building new INDEX files... " 754148871Scperciva extract_make_index DESCRIBE.4 INDEX || return 1 755148871Scperciva extract_make_index DESCRIBE.5 INDEX-5 || return 1 756148871Scperciva extract_make_index DESCRIBE.6 INDEX-6 || return 1 757148871Scperciva echo "done." 758148871Scperciva} 759148871Scperciva 760148871Scperciva# Create .portsnap.INDEX 761148871Scpercivaextract_metadata() { 762148871Scperciva sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX 763148871Scperciva} 764148871Scperciva 765148871Scperciva# Do the actual work involved in "extract" 766148871Scpercivaextract_run() { 767148871Scperciva grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX | while read LINE; do 768148871Scperciva FILE=`echo ${LINE} | cut -f 1 -d '|'` 769148871Scperciva HASH=`echo ${LINE} | cut -f 2 -d '|'` 770148871Scperciva echo ${PORTSDIR}/${FILE} 771148871Scperciva if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 772148871Scperciva echo "files/${HASH}.gz not found -- snapshot corrupt." 773148871Scperciva return 1 774148871Scperciva fi 775148871Scperciva case ${FILE} in 776148871Scperciva */) 777148871Scperciva rm -rf ${PORTSDIR}/${FILE} 778148871Scperciva mkdir -p ${PORTSDIR}/${FILE} 779148871Scperciva tar -xzf ${WORKDIR}/files/${HASH}.gz \ 780148871Scperciva -C ${PORTSDIR}/${FILE} 781148871Scperciva ;; 782148871Scperciva *) 783148871Scperciva rm -f ${PORTSDIR}/${FILE} 784148871Scperciva tar -xzf ${WORKDIR}/files/${HASH}.gz \ 785148871Scperciva -C ${PORTSDIR} ${FILE} 786148871Scperciva ;; 787148871Scperciva esac 788148871Scperciva done 789148871Scperciva if [ ! -z "${EXTRACTPATH}" ]; then 790148871Scperciva return 0; 791148871Scperciva fi 792148871Scperciva 793148871Scperciva extract_metadata 794148871Scperciva extract_indices 795148871Scperciva} 796148871Scperciva 797148871Scperciva# Do the actual work involved in "update" 798148871Scpercivaupdate_run() { 799148871Scperciva if ! [ -z "${INDEXONLY}" ]; then 800148871Scperciva extract_indices >/dev/null || return 1 801148871Scperciva return 0 802148871Scperciva fi 803148871Scperciva 804148871Scperciva echo -n "Removing old files and directories... " 805148871Scperciva sort ${WORKDIR}/INDEX | comm -23 ${PORTSDIR}/.portsnap.INDEX - | 806148871Scperciva cut -f 1 -d '|' | lam -s "${PORTSDIR}/" - | xargs rm -rf 807148871Scperciva echo "done." 808148871Scperciva 809148871Scperciva# Install new files 810148871Scperciva echo "Extracting new files:" 811148871Scperciva sort ${WORKDIR}/INDEX | comm -13 ${PORTSDIR}/.portsnap.INDEX - | 812148871Scperciva while read LINE; do 813148871Scperciva FILE=`echo ${LINE} | cut -f 1 -d '|'` 814148871Scperciva HASH=`echo ${LINE} | cut -f 2 -d '|'` 815148871Scperciva echo ${PORTSDIR}/${FILE} 816148871Scperciva if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 817148871Scperciva echo "files/${HASH}.gz not found -- snapshot corrupt." 818148871Scperciva return 1 819148871Scperciva fi 820148871Scperciva case ${FILE} in 821148871Scperciva */) 822148871Scperciva mkdir -p ${PORTSDIR}/${FILE} 823148871Scperciva tar -xzf ${WORKDIR}/files/${HASH}.gz \ 824148871Scperciva -C ${PORTSDIR}/${FILE} 825148871Scperciva ;; 826148871Scperciva *) 827148871Scperciva tar -xzf ${WORKDIR}/files/${HASH}.gz \ 828148871Scperciva -C ${PORTSDIR} ${FILE} 829148871Scperciva ;; 830148871Scperciva esac 831148871Scperciva done 832148871Scperciva 833148871Scperciva extract_metadata 834148871Scperciva extract_indices 835148871Scperciva} 836148871Scperciva 837148871Scperciva#### Main functions -- call parameter-handling and core functions 838148871Scperciva 839148871Scperciva# Using the command line, configuration file, and defaults, 840148871Scperciva# set all the parameters which are needed later. 841148871Scpercivaget_params() { 842148871Scperciva init_params 843148871Scperciva parse_cmdline $@ 844148871Scperciva sanity_conffile 845148871Scperciva default_conffile 846148871Scperciva parse_conffile 847148871Scperciva default_params 848148871Scperciva} 849148871Scperciva 850148871Scperciva# Fetch command. Make sure that we're being called 851148871Scperciva# interactively, then run fetch_check_params and fetch_run 852148871Scpercivacmd_fetch() { 853148871Scperciva if [ ! -t 0 ]; then 854148871Scperciva echo -n "`basename $0` fetch should not " 855148871Scperciva echo "be run non-interactively." 856148871Scperciva echo "Run `basename $0` cron instead." 857148871Scperciva exit 1 858148871Scperciva fi 859148871Scperciva fetch_check_params 860148871Scperciva fetch_run || exit 1 861148871Scperciva} 862148871Scperciva 863148871Scperciva# Cron command. Make sure the parameters are sensible; wait 864148871Scperciva# rand(3600) seconds; then fetch updates. While fetching updates, 865148871Scperciva# send output to a temporary file; only print that file if the 866148871Scperciva# fetching failed. 867148871Scpercivacmd_cron() { 868148871Scperciva fetch_check_params 869148871Scperciva sleep `jot -r 1 0 3600` 870148871Scperciva 871148871Scperciva TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1 872148871Scperciva if ! fetch_run >> ${TMPFILE}; then 873148871Scperciva cat ${TMPFILE} 874148871Scperciva rm ${TMPFILE} 875148871Scperciva exit 1 876148871Scperciva fi 877148871Scperciva 878148871Scperciva rm ${TMPFILE} 879148871Scperciva} 880148871Scperciva 881148871Scperciva# Extract command. Make sure the parameters are sensible, 882148871Scperciva# then extract the ports tree (or part thereof). 883148871Scpercivacmd_extract() { 884148871Scperciva extract_check_params 885148871Scperciva extract_run || exit 1 886148871Scperciva} 887148871Scperciva 888148871Scperciva# Update command. Make sure the parameters are sensible, 889148871Scperciva# then update the ports tree. 890148871Scpercivacmd_update() { 891148871Scperciva update_check_params 892148871Scperciva update_run || exit 1 893148871Scperciva} 894148871Scperciva 895148871Scperciva#### Entry point 896148871Scperciva 897148871Scperciva# Make sure we find utilities from the base system 898148871Scpercivaexport PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH} 899148871Scperciva 900148871Scpercivaget_params $@ 901148871Scpercivacmd_${COMMAND} 902