portsnap.sh revision 158274
11558Srgrimes#!/bin/sh 21558Srgrimes 31558Srgrimes#- 41558Srgrimes# Copyright 2004-2005 Colin Percival 51558Srgrimes# All rights reserved 61558Srgrimes# 71558Srgrimes# Redistribution and use in source and binary forms, with or without 81558Srgrimes# modification, are permitted providing that the following conditions 91558Srgrimes# are met: 101558Srgrimes# 1. Redistributions of source code must retain the above copyright 111558Srgrimes# notice, this list of conditions and the following disclaimer. 121558Srgrimes# 2. Redistributions in binary form must reproduce the above copyright 131558Srgrimes# notice, this list of conditions and the following disclaimer in the 141558Srgrimes# documentation and/or other materials provided with the distribution. 151558Srgrimes# 161558Srgrimes# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 171558Srgrimes# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 181558Srgrimes# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 191558Srgrimes# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 201558Srgrimes# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 211558Srgrimes# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 221558Srgrimes# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 231558Srgrimes# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 241558Srgrimes# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 251558Srgrimes# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 261558Srgrimes# POSSIBILITY OF SUCH DAMAGE. 271558Srgrimes 2823681Speter# $FreeBSD: head/usr.sbin/portsnap/portsnap/portsnap.sh 158274 2006-05-03 21:29:01Z cperciva $ 2950476Speter 301558Srgrimes#### Usage function -- called from command-line handling code. 31270255Srmacklem 321558Srgrimes# Usage instructions. Options not listed: 331558Srgrimes# --debug -- don't filter output from utilities 341558Srgrimes# --no-stats -- don't show progress statistics while fetching files 351558Srgrimesusage() { 361558Srgrimes cat <<EOF 371558Srgrimesusage: `basename $0` [options] command ... [path] 381558Srgrimes 391558SrgrimesOptions: 4068960Sru -d workdir -- Store working files in workdir 411558Srgrimes (default: /var/db/portsnap/) 421558Srgrimes -f conffile -- Read configuration options from conffile 4337663Scharnier (default: /etc/portsnap.conf) 441558Srgrimes -I -- Update INDEX only. (update command only) 451558Srgrimes -k KEY -- Trust an RSA key with SHA256 hash of KEY 461558Srgrimes -p portsdir -- Location of uncompressed ports tree 471558Srgrimes (default: /usr/ports/) 481558Srgrimes -s server -- Server from which to fetch updates. 4970152Sru (default: portsnap.FreeBSD.org) 5070152Sru path -- Extract only parts of the tree starting with the given 5170152Sru string. (extract command only) 5270152SruCommands: 531558Srgrimes fetch -- Fetch a compressed snapshot of the ports tree, 541558Srgrimes or update an existing snapshot. 551558Srgrimes cron -- Sleep rand(3600) seconds, and then fetch updates. 561558Srgrimes extract -- Extract snapshot of ports tree, replacing existing 57192934Srmacklem files and directories. 5879118Sdd update -- Update ports tree to match current snapshot, replacing 5979213Sru files and directories which have changed. 6079213SruEOF 61192934Srmacklem exit 0 621558Srgrimes} 63107788Sru 64180071Sdanger#### Parameter handling functions. 65180071Sdanger 66180071Sdanger# Initialize parameters to null, just in case they're 67107788Sru# set in the environment. 681558Srgrimesinit_params() { 691558Srgrimes KEYPRINT="" 70107788Sru EXTRACTPATH="" 711558Srgrimes WORKDIR="" 72192934Srmacklem PORTSDIR="" 731558Srgrimes CONFFILE="" 741558Srgrimes COMMAND="" 75223954Srmacklem COMMANDS="" 76233648Seadler QUIETREDIR="" 77223954Srmacklem QUIETFLAG="" 78223954Srmacklem STATSREDIR="" 79223954Srmacklem XARGST="" 80223954Srmacklem NDEBUG="" 81223954Srmacklem DDSTATS="" 82107788Sru INDEXONLY="" 831558Srgrimes SERVERNAME="" 841558Srgrimes REFUSE="" 851558Srgrimes} 86107788Sru 8723681Speter# Parse the command line 8823681Speterparse_cmdline() { 8947594Skris while [ $# -gt 0 ]; do 9047594Skris case "$1" in 91223954Srmacklem -d) 92223954Srmacklem if [ $# -eq 1 ]; then usage; fi 93233648Seadler if [ ! -z "${WORKDIR}" ]; then usage; fi 94270255Srmacklem shift; WORKDIR="$1" 95270255Srmacklem ;; 96223954Srmacklem --debug) 97270255Srmacklem QUIETREDIR="/dev/stderr" 98192934Srmacklem STATSREDIR="/dev/stderr" 99209926Smaxim QUIETFLAG=" " 100223954Srmacklem NDEBUG=" " 101223954Srmacklem XARGST="-t" 102223954Srmacklem DDSTATS=".." 103223954Srmacklem ;; 104241330Sjoel -f) 105223954Srmacklem if [ $# -eq 1 ]; then usage; fi 1061558Srgrimes if [ ! -z "${CONFFILE}" ]; then usage; fi 107180071Sdanger shift; CONFFILE="$1" 108180112Sdanger ;; 109180071Sdanger -h | --help | help) 110180112Sdanger usage 111180071Sdanger ;; 112107788Sru -I) 1131558Srgrimes INDEXONLY="YES" 1141558Srgrimes ;; 115107788Sru -k) 1161558Srgrimes if [ $# -eq 1 ]; then usage; fi 117107788Sru if [ ! -z "${KEYPRINT}" ]; then usage; fi 118180112Sdanger shift; KEYPRINT="$1" 1191558Srgrimes ;; 120192934Srmacklem --no-stats) 121192934Srmacklem if [ -z "${STATSREDIR}" ]; then 122192934Srmacklem STATSREDIR="/dev/null" 1231558Srgrimes DDSTATS=".. " 1241558Srgrimes fi 1251558Srgrimes ;; 1261558Srgrimes -p) 127180154Sdanger if [ $# -eq 1 ]; then usage; fi 1281558Srgrimes if [ ! -z "${PORTSDIR}" ]; then usage; fi 1291558Srgrimes shift; PORTSDIR="$1" 1301558Srgrimes ;; 1311558Srgrimes -s) 13271099Sru if [ $# -eq 1 ]; then usage; fi 1331558Srgrimes if [ ! -z "${SERVERNAME}" ]; then usage; fi 1341558Srgrimes shift; SERVERNAME="$1" 1351558Srgrimes ;; 136180154Sdanger cron | extract | fetch | update) 1371558Srgrimes COMMANDS="${COMMANDS} $1" 1381558Srgrimes ;; 1391558Srgrimes *) 1401558Srgrimes if [ $# -gt 1 ]; then usage; fi 1411558Srgrimes if echo ${COMMANDS} | grep -vq extract; then 1421558Srgrimes usage 1431558Srgrimes fi 1441558Srgrimes EXTRACTPATH="$1" 145180154Sdanger ;; 1461558Srgrimes esac 1471558Srgrimes shift 1481558Srgrimes done 149180154Sdanger 1501558Srgrimes if [ -z "${COMMANDS}" ]; then 151180112Sdanger usage 1521558Srgrimes fi 1531558Srgrimes} 1541558Srgrimes 1551558Srgrimes# If CONFFILE was specified at the command-line, make 1561558Srgrimes# sure that it exists and is readable. 1571558Srgrimessanity_conffile() { 1581558Srgrimes if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then 1591558Srgrimes echo -n "File does not exist " 1601558Srgrimes echo -n "or is not readable: " 1611558Srgrimes echo ${CONFFILE} 1621558Srgrimes exit 1 1631558Srgrimes fi 1641558Srgrimes} 1651558Srgrimes 1661558Srgrimes# If a configuration file hasn't been specified, use 1671558Srgrimes# the default value (/etc/portsnap.conf) 1681558Srgrimesdefault_conffile() { 1691558Srgrimes if [ -z "${CONFFILE}" ]; then 1701558Srgrimes CONFFILE="/etc/portsnap.conf" 1711558Srgrimes fi 1721558Srgrimes} 1731558Srgrimes 1741558Srgrimes# Read {KEYPRINT, SERVERNAME, WORKDIR, PORTSDIR} from the configuration 1751558Srgrimes# file if they haven't already been set. If the configuration 1761558Srgrimes# file doesn't exist, do nothing. 177184588Sdfr# Also read REFUSE (which cannot be set via the command line) if it is 178184588Sdfr# present in the configuration file. 179184588Sdfrparse_conffile() { 180184588Sdfr if [ -r "${CONFFILE}" ]; then 181184588Sdfr for X in KEYPRINT WORKDIR PORTSDIR SERVERNAME; do 182184588Sdfr eval _=\$${X} 183184588Sdfr if [ -z "${_}" ]; then 184184588Sdfr eval ${X}=`grep "^${X}=" "${CONFFILE}" | 185184588Sdfr cut -f 2- -d '=' | tail -1` 186184588Sdfr fi 187184588Sdfr done 1881558Srgrimes 1891558Srgrimes if grep -qE "^REFUSE[[:space:]]" ${CONFFILE}; then 190107788Sru REFUSE="^(` 1911558Srgrimes grep -E "^REFUSE[[:space:]]" "${CONFFILE}" | 1921558Srgrimes cut -c 7- | xargs echo | tr ' ' '|' 1931558Srgrimes `)" 1941558Srgrimes fi 1951558Srgrimes fi 1961558Srgrimes} 1971558Srgrimes 19837663Scharnier# If parameters have not been set, use default values 19937663Scharnierdefault_params() { 20027447Sdfr _QUIETREDIR="/dev/null" 20127447Sdfr _QUIETFLAG="-q" 20257669Ssheldonh _STATSREDIR="/dev/stdout" 20357669Ssheldonh _WORKDIR="/var/db/portsnap" 204180112Sdanger _PORTSDIR="/usr/ports" 20557669Ssheldonh _NDEBUG="-n" 20627447Sdfr for X in QUIETREDIR QUIETFLAG STATSREDIR WORKDIR PORTSDIR NDEBUG; do 20737663Scharnier eval _=\$${X} 20837663Scharnier eval __=\$_${X} 20937663Scharnier if [ -z "${_}" ]; then 21027447Sdfr eval ${X}=${__} 21127447Sdfr fi 21227447Sdfr done 21327447Sdfr} 21427447Sdfr 21527447Sdfr# Perform sanity checks and set some final parameters 21627447Sdfr# in preparation for fetching files. Also chdir into 21727447Sdfr# the working directory. 21827447Sdfrfetch_check_params() { 219127480Sceri export HTTP_USER_AGENT="portsnap (${COMMAND}, `uname -r`)" 220127317Sceri 221127317Sceri _SERVERNAME_z=\ 22227447Sdfr"SERVERNAME must be given via command line or configuration file." 22327447Sdfr _KEYPRINT_z="Key must be given via -k option or configuration file." 22427447Sdfr _KEYPRINT_bad="Invalid key fingerprint: " 225154990Sjkoshy _WORKDIR_bad="Directory does not exist or is not writable: " 22637663Scharnier 22727447Sdfr if [ -z "${SERVERNAME}" ]; then 22881462Sru echo -n "`basename $0`: " 22981462Sru echo "${_SERVERNAME_z}" 23057669Ssheldonh exit 1 23157669Ssheldonh fi 23227447Sdfr if [ -z "${KEYPRINT}" ]; then 23327447Sdfr echo -n "`basename $0`: " 23427447Sdfr echo "${_KEYPRINT_z}" 23527447Sdfr exit 1 23627447Sdfr fi 23727447Sdfr if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then 23827447Sdfr echo -n "`basename $0`: " 23927447Sdfr echo -n "${_KEYPRINT_bad}" 24027447Sdfr echo ${KEYPRINT} 24127447Sdfr exit 1 242100336Sjoerg fi 243100336Sjoerg if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then 244100336Sjoerg echo -n "`basename $0`: " 245100336Sjoerg echo -n "${_WORKDIR_bad}" 246100336Sjoerg echo ${WORKDIR} 247100336Sjoerg exit 1 248100336Sjoerg fi 249100336Sjoerg cd ${WORKDIR} || exit 1 250100336Sjoerg 2511558Srgrimes BSPATCH=/usr/bin/bspatch 2521558Srgrimes SHA256=/sbin/sha256 2531558Srgrimes PHTTPGET=/usr/libexec/phttpget 254180071Sdanger} 255180071Sdanger 256180071Sdanger# Perform sanity checks and set some final parameters 257180071Sdanger# in preparation for extracting or updating ${PORTSDIR} 258180071Sdanger# Complain if ${PORTSDIR} exists but is not writable, 259180112Sdanger# but don't complain if ${PORTSDIR} doesn't exist. 260180112Sdangerextract_check_params() { 261180112Sdanger _WORKDIR_bad="Directory does not exist: " 26271099Sru _PORTSDIR_bad="Directory is not writable: " 26337663Scharnier 2641558Srgrimes if ! [ -d "${WORKDIR}" ]; then 2651558Srgrimes echo -n "`basename $0`: " 2661558Srgrimes echo -n "${_WORKDIR_bad}" 2671558Srgrimes echo ${WORKDIR} 2681558Srgrimes exit 1 2691558Srgrimes fi 2701558Srgrimes if [ -d "${PORTSDIR}" ] && ! [ -w "${PORTSDIR}" ]; then 2711558Srgrimes echo -n "`basename $0`: " 272180071Sdanger echo -n "${_PORTSDIR_bad}" 273180071Sdanger echo ${PORTSDIR} 274180071Sdanger exit 1 2751558Srgrimes fi 2761558Srgrimes 2771558Srgrimes if ! [ -d "${WORKDIR}/files" -a -r "${WORKDIR}/tag" \ 2781558Srgrimes -a -r "${WORKDIR}/INDEX" -a -r "${WORKDIR}/tINDEX" ]; then 2791558Srgrimes echo "No snapshot available. Try running" 280180154Sdanger echo "# `basename $0` fetch" 2811558Srgrimes exit 1 2821558Srgrimes fi 2831558Srgrimes 2841558Srgrimes MKINDEX=/usr/libexec/make_index 2851558Srgrimes} 286180071Sdanger 287180071Sdanger# Perform sanity checks and set some final parameters 288180071Sdanger# in preparation for updating ${PORTSDIR} 289180071Sdangerupdate_check_params() { 290180071Sdanger extract_check_params 291180071Sdanger 292180071Sdanger if ! [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 2931558Srgrimes echo "${PORTSDIR} was not created by portsnap." 2941558Srgrimes echo -n "You must run '`basename $0` extract' before " 29571099Sru echo "running '`basename $0` update'." 296100336Sjoerg exit 1 297100336Sjoerg fi 298100336Sjoerg 2991558Srgrimes} 300180071Sdanger 301180071Sdanger#### Core functionality -- the actual work gets done here 302180071Sdanger 303180112Sdanger# Use an SRV query to pick a server. If the SRV query doesn't provide 304180112Sdanger# a useful answer, use the server name specified by the user. 305180112Sdanger# Put another way... look up _http._tcp.${SERVERNAME} and pick a server 306180112Sdanger# from that; or if no servers are returned, use ${SERVERNAME}. 307180112Sdanger# This allows a user to specify "portsnap.freebsd.org" (in which case 308180112Sdanger# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org" 309180071Sdanger# (in which case portsnap will use that particular server, since there 310192934Srmacklem# won't be an SRV entry for that name). 311192934Srmacklem# 312192934Srmacklem# We ignore the Port field, since we are always going to use port 80. 313270255Srmacklem 314270255Srmacklem# Fetch the mirror list, but do not pick a mirror yet. Returns 1 if 315244689Srmacklem# no mirrors are available for any reason. 316244689Srmacklemfetch_pick_server_init() { 317192934Srmacklem : > serverlist_tried 318192934Srmacklem 319192934Srmacklem# Check that host(1) exists (i.e., that the system wasn't built with the 320192934Srmacklem# WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist. 321192934Srmacklem if ! which -s host; then 322244689Srmacklem : > serverlist_full 323244689Srmacklem return 1 324192934Srmacklem fi 325192934Srmacklem 326192934Srmacklem echo -n "Looking up ${SERVERNAME} mirrors... " 327192934Srmacklem 328100336Sjoerg# Issue the SRV query and pull out the Priority, Weight, and Target fields. 329192934Srmacklem# BIND 9 prints "$name has SRV record ..." while BIND 8 prints 330192934Srmacklem# "$name server selection ..."; we allow either format. 331192934Srmacklem MLIST="_http._tcp.${SERVERNAME}" 332192934Srmacklem host -t srv "${MLIST}" | 333192934Srmacklem sed -nE "s/${MLIST} (has SRV record|server selection) //p" | 334192934Srmacklem cut -f 1,2,4 -d ' ' | 335192934Srmacklem sed -e 's/\.$//' | 336192934Srmacklem sort > serverlist_full 337192934Srmacklem 338192934Srmacklem# If no records, give up -- we'll just use the server name we were given. 339192934Srmacklem if [ `wc -l < serverlist_full` -eq 0 ]; then 340192934Srmacklem echo "none found." 341100336Sjoerg return 1 342100336Sjoerg fi 343100336Sjoerg 344100336Sjoerg# Report how many mirrors we found. 3451558Srgrimes echo `wc -l < serverlist_full` "mirrors found." 346180071Sdanger 347100336Sjoerg# Generate a random seed for use in picking mirrors. If HTTP_PROXY 348100336Sjoerg# is set, this will be used to generate the seed; otherwise, the seed 349100336Sjoerg# will be random. 350100336Sjoerg if [ -z "${HTTP_PROXY}" ]; then 351100336Sjoerg RANDVALUE=`sha256 -qs "${HTTP_PROXY}" | 352100336Sjoerg tr -d 'a-f' | 353100336Sjoerg cut -c 1-9` 354100336Sjoerg else 355100336Sjoerg RANDVALUE=`jot -r 1 0 999999999` 356100336Sjoerg fi 357100336Sjoerg echo "XXXdebug: HTTP_PROXY=${HTTP_PROXY}" 358100336Sjoerg echo "XXXdebug: RANDVALUE=${RANDVALUE}" 359100336Sjoerg} 360100336Sjoerg 361100336Sjoerg# Pick a mirror. Returns 1 if we have run out of mirrors to try. 362100336Sjoergfetch_pick_server() { 363100336Sjoerg# Generate a list of not-yet-tried mirrors 364100336Sjoerg sort serverlist_tried | 3651558Srgrimes comm -23 serverlist_full - > serverlist 3661558Srgrimes 3671558Srgrimes# Have we run out of mirrors? 3681558Srgrimes if [ `wc -l < serverlist` -eq 0 ]; then 369180071Sdanger echo "No mirrors remaining, giving up." 370180071Sdanger return 1 3711558Srgrimes fi 372103716Smarkm 373100336Sjoerg# Find the highest priority level (lowest numeric value). 374184588Sdfr SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1` 375184588Sdfr 376192934Srmacklem# Add up the weights of the response lines at that priority level. 377192934Srmacklem SRV_WSUM=0; 3781558Srgrimes while read X; do 3791558Srgrimes case "$X" in 3801558Srgrimes ${SRV_PRIORITY}\ *) 381180154Sdanger SRV_W=`echo $X | cut -f 2 -d ' '` 3821558Srgrimes SRV_WSUM=$(($SRV_WSUM + $SRV_W)) 383154990Sjkoshy ;; 3841558Srgrimes esac 385107788Sru done < serverlist 386124034Sobrien 387180112Sdanger# If all the weights are 0, pretend that they are all 1 instead. 388154990Sjkoshy if [ ${SRV_WSUM} -eq 0 ]; then 3891558Srgrimes SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l` 3901558Srgrimes SRV_W_ADD=1 3911558Srgrimes else 3921558Srgrimes SRV_W_ADD=0 393180112Sdanger fi 394180071Sdanger 395180071Sdanger# Pick a value between 0 and the sum of the weights - 1 396180071Sdanger SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}` 397180071Sdanger 398180071Sdanger# Read through the list of mirrors and set SERVERNAME. Write the line 399180071Sdanger# corresponding to the mirror we selected into serverlist_tried so that 4001558Srgrimes# we won't try it again. 4011558Srgrimes while read X; do 4021558Srgrimes case "$X" in 4031558Srgrimes ${SRV_PRIORITY}\ *) 4041558Srgrimes SRV_W=`echo $X | cut -f 2 -d ' '` 405180071Sdanger SRV_W=$(($SRV_W + $SRV_W_ADD)) 406180071Sdanger if [ $SRV_RND -lt $SRV_W ]; then 4071558Srgrimes SERVERNAME=`echo $X | cut -f 3 -d ' '` 408180071Sdanger echo "$X" >> serverlist_tried 409180071Sdanger break 4101558Srgrimes else 411180112Sdanger SRV_RND=$(($SRV_RND - $SRV_W)) 412154990Sjkoshy fi 4131558Srgrimes ;; 4141558Srgrimes esac 415180112Sdanger done < serverlist 416180071Sdanger} 417180071Sdanger 4181558Srgrimes# Check that we have a public key with an appropriate hash, or 419180112Sdanger# fetch the key if it doesn't exist. Returns 1 if the key has 420154990Sjkoshy# not yet been fetched. 421180071Sdangerfetch_key() { 422180071Sdanger if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 423180112Sdanger return 0 424180071Sdanger fi 425180071Sdanger 426180071Sdanger echo -n "Fetching public key from ${SERVERNAME}... " 427180071Sdanger rm -f pub.ssl 428180071Sdanger fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \ 429103716Smarkm 2>${QUIETREDIR} || true 43062459Ssheldonh if ! [ -r pub.ssl ]; then 431180112Sdanger echo "failed." 432180071Sdanger return 1 433180071Sdanger fi 434180112Sdanger if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 435180112Sdanger echo "key has incorrect hash." 436180112Sdanger rm -f pub.ssl 437180112Sdanger return 1 438180112Sdanger fi 439180112Sdanger echo "done." 440180112Sdanger} 441180071Sdanger 442180112Sdanger# Fetch a snapshot tag 443180071Sdangerfetch_tag() { 444180112Sdanger rm -f snapshot.ssl tag.new 445180112Sdanger 446180112Sdanger echo ${NDEBUG} "Fetching snapshot tag from ${SERVERNAME}... " 447180071Sdanger fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl \ 448180071Sdanger 2>${QUIETREDIR} || true 449180112Sdanger if ! [ -r $1.ssl ]; then 450180112Sdanger echo "failed." 451180112Sdanger return 1 452180071Sdanger fi 453107788Sru 454154990Sjkoshy openssl rsautl -pubin -inkey pub.ssl -verify \ 455180112Sdanger < $1.ssl > tag.new 2>${QUIETREDIR} || true 456100336Sjoerg rm $1.ssl 457100336Sjoerg 458154990Sjkoshy if ! [ `wc -l < tag.new` = 1 ] || 459100336Sjoerg ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then 460100336Sjoerg echo "invalid snapshot tag." 461107788Sru return 1 462100336Sjoerg fi 463100336Sjoerg 464100336Sjoerg echo "done." 46562459Ssheldonh 466100336Sjoerg SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new` 467100336Sjoerg SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new` 468100336Sjoerg} 469100336Sjoerg 470100336Sjoerg# Sanity-check the date on a snapshot tag 471100336Sjoergfetch_snapshot_tagsanity() { 47262459Ssheldonh if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then 473100336Sjoerg echo "Snapshot appears to be more than a year old!" 474154990Sjkoshy echo "(Is the system clock correct?)" 475108317Sschweikh echo "Cowardly refusing to proceed any further." 476100336Sjoerg return 1 477100336Sjoerg fi 478100336Sjoerg if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then 479100336Sjoerg echo -n "Snapshot appears to have been created more than " 480154990Sjkoshy echo "one day into the future!" 481100336Sjoerg echo "(Is the system clock correct?)" 482154990Sjkoshy echo "Cowardly refusing to proceed any further." 483107788Sru return 1 484184588Sdfr fi 485184588Sdfr} 486184588Sdfr 487184588Sdfr# Sanity-check the date on a snapshot update tag 488184588Sdfrfetch_update_tagsanity() { 489184588Sdfr fetch_snapshot_tagsanity || return 1 490184588Sdfr 491184588Sdfr if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then 492184588Sdfr echo -n "Latest snapshot on server is " 493192934Srmacklem echo "older than what we already have!" 494192934Srmacklem echo -n "Cowardly refusing to downgrade from " 495192934Srmacklem date -r ${OLDSNAPSHOTDATE} 496192934Srmacklem echo "to `date -r ${SNAPSHOTDATE}`." 497192934Srmacklem return 1 498192934Srmacklem fi 4991558Srgrimes} 500200076Strasz 5011558Srgrimes# Compare old and new tags; return 1 if update is unnecessary 5021558Srgrimesfetch_update_neededp() { 5031558Srgrimes if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then 5041558Srgrimes echo -n "Latest snapshot on server matches " 5051558Srgrimes echo "what we already have." 5061558Srgrimes echo "No updates needed." 5071558Srgrimes rm tag.new 5081558Srgrimes return 1 5091558Srgrimes fi 510107788Sru if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then 5111558Srgrimes echo -n "Ports tree hasn't changed since " 5121558Srgrimes echo "last snapshot." 5131558Srgrimes echo "No updates needed." 514 rm tag.new 515 return 1 516 fi 517 518 return 0 519} 520 521# Fetch snapshot metadata file 522fetch_metadata() { 523 rm -f ${SNAPSHOTHASH} tINDEX.new 524 525 echo ${NDEBUG} "Fetching snapshot metadata... " 526 fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} 527 2>${QUIETREDIR} || return 528 if [ `${SHA256} -q ${SNAPSHOTHASH}` != ${SNAPSHOTHASH} ]; then 529 echo "snapshot metadata corrupt." 530 return 1 531 fi 532 mv ${SNAPSHOTHASH} tINDEX.new 533 echo "done." 534} 535 536# Warn user about bogus metadata 537fetch_metadata_freakout() { 538 echo 539 echo "Portsnap metadata is correctly signed, but contains" 540 echo "at least one line which appears bogus." 541 echo "Cowardly refusing to proceed any further." 542} 543 544# Sanity-check a snapshot metadata file 545fetch_metadata_sanity() { 546 if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then 547 fetch_metadata_freakout 548 return 1 549 fi 550 if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then 551 echo 552 echo "Portsnap metadata appears bogus." 553 echo "Cowardly refusing to proceed any further." 554 return 1 555 fi 556} 557 558# Take a list of ${oldhash}|${newhash} and output a list of needed patches 559fetch_make_patchlist() { 560 grep -vE "^([0-9a-f]{64})\|\1$" | 561 while read LINE; do 562 X=`echo ${LINE} | cut -f 1 -d '|'` 563 Y=`echo ${LINE} | cut -f 2 -d '|'` 564 if [ -f "files/${Y}.gz" ]; then continue; fi 565 if [ ! -f "files/${X}.gz" ]; then continue; fi 566 echo "${LINE}" 567 done 568} 569 570# Print user-friendly progress statistics 571fetch_progress() { 572 LNC=0 573 while read x; do 574 LNC=$(($LNC + 1)) 575 if [ $(($LNC % 10)) = 0 ]; then 576 echo -n $LNC 577 elif [ $(($LNC % 2)) = 0 ]; then 578 echo -n . 579 fi 580 done 581 echo -n " " 582} 583 584# Sanity-check an index file 585fetch_index_sanity() { 586 if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new || 587 fgrep -q "./" INDEX.new; then 588 fetch_metadata_freakout 589 return 1 590 fi 591} 592 593# Verify a list of files 594fetch_snapshot_verify() { 595 while read F; do 596 if [ `gunzip -c snap/${F} | ${SHA256} -q` != ${F} ]; then 597 echo "snapshot corrupt." 598 return 1 599 fi 600 done 601 return 0 602} 603 604# Fetch a snapshot tarball, extract, and verify. 605fetch_snapshot() { 606 while ! fetch_tag snapshot; do 607 fetch_pick_server || return 1 608 done 609 fetch_snapshot_tagsanity || return 1 610 fetch_metadata || return 1 611 fetch_metadata_sanity || return 1 612 613 rm -rf snap/ 614 615# Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will 616# probably take a while, so the progrees reports that fetch(1) generates 617# will be useful for keeping the users' attention from drifting. 618 echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:" 619 fetch -r http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1 620 621 echo -n "Extracting snapshot... " 622 tar -xzf ${SNAPSHOTHASH}.tgz snap/ || return 1 623 rm ${SNAPSHOTHASH}.tgz 624 echo "done." 625 626 echo -n "Verifying snapshot integrity... " 627# Verify the metadata files 628 cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1 629# Extract the index 630 rm -f INDEX.new 631 gunzip -c snap/`look INDEX tINDEX.new | 632 cut -f 2 -d '|'`.gz > INDEX.new 633 fetch_index_sanity || return 1 634# Verify the snapshot contents 635 cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1 636 echo "done." 637 638# Move files into their proper locations 639 rm -f tag INDEX tINDEX 640 rm -rf files 641 mv tag.new tag 642 mv tINDEX.new tINDEX 643 mv INDEX.new INDEX 644 mv snap/ files/ 645 646 return 0 647} 648 649# Update a compressed snapshot 650fetch_update() { 651 rm -f patchlist diff OLD NEW filelist INDEX.new 652 653 OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag` 654 OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag` 655 656 while ! fetch_tag latest; do 657 fetch_pick_server || return 1 658 done 659 fetch_update_tagsanity || return 1 660 fetch_update_neededp || return 0 661 fetch_metadata || return 1 662 fetch_metadata_sanity || return 1 663 664 echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` " 665 echo "to `date -r ${SNAPSHOTDATE}`." 666 667# Generate a list of wanted metadata patches 668 join -t '|' -o 1.2,2.2 tINDEX tINDEX.new | 669 fetch_make_patchlist > patchlist 670 671# Attempt to fetch metadata patches 672 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 673 echo ${NDEBUG} "metadata patches.${DDSTATS}" 674 tr '|' '-' < patchlist | 675 lam -s "tp/" - -s ".gz" | 676 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 677 2>${STATSREDIR} | fetch_progress 678 echo "done." 679 680# Attempt to apply metadata patches 681 echo -n "Applying metadata patches... " 682 while read LINE; do 683 X=`echo ${LINE} | cut -f 1 -d '|'` 684 Y=`echo ${LINE} | cut -f 2 -d '|'` 685 if [ ! -f "${X}-${Y}.gz" ]; then continue; fi 686 gunzip -c < ${X}-${Y}.gz > diff 687 gunzip -c < files/${X}.gz > OLD 688 cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp 689 grep '^\+' diff | cut -c 2- | 690 sort -k 1,1 -t '|' -m - ptmp > NEW 691 if [ `${SHA256} -q NEW` = ${Y} ]; then 692 mv NEW files/${Y} 693 gzip -n files/${Y} 694 fi 695 rm -f diff OLD NEW ${X}-${Y}.gz ptmp 696 done < patchlist 2>${QUIETREDIR} 697 echo "done." 698 699# Update metadata without patches 700 join -t '|' -v 2 tINDEX tINDEX.new | 701 cut -f 2 -d '|' /dev/stdin patchlist | 702 while read Y; do 703 if [ ! -f "files/${Y}.gz" ]; then 704 echo ${Y}; 705 fi 706 done > filelist 707 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 708 echo ${NDEBUG} "metadata files... " 709 lam -s "f/" - -s ".gz" < filelist | 710 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 711 2>${QUIETREDIR} 712 713 while read Y; do 714 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 715 mv ${Y}.gz files/${Y}.gz 716 else 717 echo "metadata is corrupt." 718 return 1 719 fi 720 done < filelist 721 echo "done." 722 723# Extract the index 724 gunzip -c files/`look INDEX tINDEX.new | 725 cut -f 2 -d '|'`.gz > INDEX.new 726 fetch_index_sanity || return 1 727 728# If we have decided to refuse certain updates, construct a hybrid index which 729# is equal to the old index for parts of the tree which we don't want to 730# update, and equal to the new index for parts of the tree which gets updates. 731# This means that we should always have a "complete snapshot" of the ports 732# tree -- with the caveat that it isn't actually a snapshot. 733 if [ ! -z "${REFUSE}" ]; then 734 echo "Refusing to download updates for ${REFUSE}" \ 735 >${QUIETREDIR} 736 737 grep -Ev "${REFUSE}" INDEX.new > INDEX.tmp 738 grep -E "${REFUSE}" INDEX | 739 sort -m -k 1,1 -t '|' - INDEX.tmp > INDEX.new 740 rm -f INDEX.tmp 741 fi 742 743# Generate a list of wanted ports patches 744 join -t '|' -o 1.2,2.2 INDEX INDEX.new | 745 fetch_make_patchlist > patchlist 746 747# Attempt to fetch ports patches 748 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 749 echo ${NDEBUG} "patches.${DDSTATS}" 750 tr '|' '-' < patchlist | lam -s "bp/" - | 751 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 752 2>${STATSREDIR} | fetch_progress 753 echo "done." 754 755# Attempt to apply ports patches 756 echo -n "Applying patches... " 757 while read LINE; do 758 X=`echo ${LINE} | cut -f 1 -d '|'` 759 Y=`echo ${LINE} | cut -f 2 -d '|'` 760 if [ ! -f "${X}-${Y}" ]; then continue; fi 761 gunzip -c < files/${X}.gz > OLD 762 ${BSPATCH} OLD NEW ${X}-${Y} 763 if [ `${SHA256} -q NEW` = ${Y} ]; then 764 mv NEW files/${Y} 765 gzip -n files/${Y} 766 fi 767 rm -f diff OLD NEW ${X}-${Y} 768 done < patchlist 2>${QUIETREDIR} 769 echo "done." 770 771# Update ports without patches 772 join -t '|' -v 2 INDEX INDEX.new | 773 cut -f 2 -d '|' /dev/stdin patchlist | 774 while read Y; do 775 if [ ! -f "files/${Y}.gz" ]; then 776 echo ${Y}; 777 fi 778 done > filelist 779 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 780 echo ${NDEBUG} "new ports or files... " 781 lam -s "f/" - -s ".gz" < filelist | 782 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 783 2>${QUIETREDIR} 784 785 while read Y; do 786 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 787 mv ${Y}.gz files/${Y}.gz 788 else 789 echo "snapshot is corrupt." 790 return 1 791 fi 792 done < filelist 793 echo "done." 794 795# Remove files which are no longer needed 796 cut -f 2 -d '|' tINDEX INDEX | sort > oldfiles 797 cut -f 2 -d '|' tINDEX.new INDEX.new | sort | comm -13 - oldfiles | 798 lam -s "files/" - -s ".gz" | xargs rm -f 799 rm patchlist filelist oldfiles 800 801# We're done! 802 mv INDEX.new INDEX 803 mv tINDEX.new tINDEX 804 mv tag.new tag 805 806 return 0 807} 808 809# Do the actual work involved in "fetch" / "cron". 810fetch_run() { 811 fetch_pick_server_init && fetch_pick_server 812 813 while ! fetch_key; do 814 fetch_pick_server || return 1 815 done 816 817 if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then 818 fetch_snapshot || return 1 819 fi 820 fetch_update || return 1 821} 822 823# Build a ports INDEX file 824extract_make_index() { 825 gunzip -c "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX | 826 cut -f 2 -d '|'`.gz" | ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2 827} 828 829# Create INDEX, INDEX-5, INDEX-6 830extract_indices() { 831 echo -n "Building new INDEX files... " 832 extract_make_index DESCRIBE.4 INDEX || return 1 833 extract_make_index DESCRIBE.5 INDEX-5 || return 1 834 extract_make_index DESCRIBE.6 INDEX-6 || return 1 835 echo "done." 836} 837 838# Create .portsnap.INDEX; if we are REFUSEing to touch certain directories, 839# merge the values from any exiting .portsnap.INDEX file. 840extract_metadata() { 841 if [ -z "${REFUSE}" ]; then 842 sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX 843 elif [ -f ${PORTSDIR}/.portsnap.INDEX ]; then 844 grep -E "${REFUSE}" ${PORTSDIR}/.portsnap.INDEX \ 845 > ${PORTSDIR}/.portsnap.INDEX.tmp 846 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort | 847 sort -m - ${PORTSDIR}/.portsnap.INDEX.tmp \ 848 > ${PORTSDIR}/.portsnap.INDEX 849 rm -f ${PORTSDIR}/.portsnap.INDEX.tmp 850 else 851 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort \ 852 > ${PORTSDIR}/.portsnap.INDEX 853 fi 854} 855 856# Do the actual work involved in "extract" 857extract_run() { 858 mkdir -p ${PORTSDIR} || return 1 859 860 if ! 861 if ! [ -z "${EXTRACTPATH}" ]; then 862 grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX 863 elif ! [ -z "${REFUSE}" ]; then 864 grep -vE "${REFUSE}" ${WORKDIR}/INDEX 865 else 866 cat ${WORKDIR}/INDEX 867 fi | while read LINE; do 868 FILE=`echo ${LINE} | cut -f 1 -d '|'` 869 HASH=`echo ${LINE} | cut -f 2 -d '|'` 870 echo ${PORTSDIR}/${FILE} 871 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 872 echo "files/${HASH}.gz not found -- snapshot corrupt." 873 return 1 874 fi 875 case ${FILE} in 876 */) 877 rm -rf ${PORTSDIR}/${FILE} 878 mkdir -p ${PORTSDIR}/${FILE} 879 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 880 -C ${PORTSDIR}/${FILE} 881 ;; 882 *) 883 rm -f ${PORTSDIR}/${FILE} 884 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 885 -C ${PORTSDIR} ${FILE} 886 ;; 887 esac 888 done; then 889 return 1 890 fi 891 if [ ! -z "${EXTRACTPATH}" ]; then 892 return 0; 893 fi 894 895 extract_metadata 896 extract_indices 897} 898 899# Do the actual work involved in "update" 900update_run() { 901 if ! [ -z "${INDEXONLY}" ]; then 902 extract_indices >/dev/null || return 1 903 return 0 904 fi 905 906 if sort ${WORKDIR}/INDEX | 907 cmp -s ${PORTSDIR}/.portsnap.INDEX -; then 908 echo "Ports tree is already up to date." 909 return 0 910 fi 911 912# If we are REFUSEing to touch certain directories, don't remove files 913# from those directories (even if they are out of date) 914 echo -n "Removing old files and directories... " 915 if ! [ -z "${REFUSE}" ]; then 916 sort ${WORKDIR}/INDEX | 917 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 918 grep -vE "${REFUSE}" | 919 lam -s "${PORTSDIR}/" - | xargs rm -rf 920 else 921 sort ${WORKDIR}/INDEX | 922 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 923 lam -s "${PORTSDIR}/" - | xargs rm -rf 924 fi 925 echo "done." 926 927# Install new files 928 echo "Extracting new files:" 929 if ! 930 if ! [ -z "${REFUSE}" ]; then 931 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort 932 else 933 sort ${WORKDIR}/INDEX 934 fi | 935 comm -13 ${PORTSDIR}/.portsnap.INDEX - | 936 while read LINE; do 937 FILE=`echo ${LINE} | cut -f 1 -d '|'` 938 HASH=`echo ${LINE} | cut -f 2 -d '|'` 939 echo ${PORTSDIR}/${FILE} 940 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 941 echo "files/${HASH}.gz not found -- snapshot corrupt." 942 return 1 943 fi 944 case ${FILE} in 945 */) 946 mkdir -p ${PORTSDIR}/${FILE} 947 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 948 -C ${PORTSDIR}/${FILE} 949 ;; 950 *) 951 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 952 -C ${PORTSDIR} ${FILE} 953 ;; 954 esac 955 done; then 956 return 1 957 fi 958 959 extract_metadata 960 extract_indices 961} 962 963#### Main functions -- call parameter-handling and core functions 964 965# Using the command line, configuration file, and defaults, 966# set all the parameters which are needed later. 967get_params() { 968 init_params 969 parse_cmdline $@ 970 sanity_conffile 971 default_conffile 972 parse_conffile 973 default_params 974} 975 976# Fetch command. Make sure that we're being called 977# interactively, then run fetch_check_params and fetch_run 978cmd_fetch() { 979 if [ ! -t 0 ]; then 980 echo -n "`basename $0` fetch should not " 981 echo "be run non-interactively." 982 echo "Run `basename $0` cron instead." 983 exit 1 984 fi 985 fetch_check_params 986 fetch_run || exit 1 987} 988 989# Cron command. Make sure the parameters are sensible; wait 990# rand(3600) seconds; then fetch updates. While fetching updates, 991# send output to a temporary file; only print that file if the 992# fetching failed. 993cmd_cron() { 994 fetch_check_params 995 sleep `jot -r 1 0 3600` 996 997 TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1 998 if ! fetch_run >> ${TMPFILE}; then 999 cat ${TMPFILE} 1000 rm ${TMPFILE} 1001 exit 1 1002 fi 1003 1004 rm ${TMPFILE} 1005} 1006 1007# Extract command. Make sure the parameters are sensible, 1008# then extract the ports tree (or part thereof). 1009cmd_extract() { 1010 extract_check_params 1011 extract_run || exit 1 1012} 1013 1014# Update command. Make sure the parameters are sensible, 1015# then update the ports tree. 1016cmd_update() { 1017 update_check_params 1018 update_run || exit 1 1019} 1020 1021#### Entry point 1022 1023# Make sure we find utilities from the base system 1024export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH} 1025 1026get_params $@ 1027for COMMAND in ${COMMANDS}; do 1028 cmd_${COMMAND} 1029done 1030