portsnap.sh revision 306379
1#!/bin/sh 2 3#- 4# Copyright 2004-2005 Colin Percival 5# All rights reserved 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted providing that the following conditions 9# are met: 10# 1. Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# 2. Redistributions in binary form must reproduce the above copyright 13# notice, this list of conditions and the following disclaimer in the 14# documentation and/or other materials provided with the distribution. 15# 16# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 20# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26# POSSIBILITY OF SUCH DAMAGE. 27 28# $FreeBSD: releng/11.0/usr.sbin/portsnap/portsnap/portsnap.sh 306379 2016-09-27 19:36:12Z emaste $ 29 30#### Usage function -- called from command-line handling code. 31 32# Usage instructions. Options not listed: 33# --debug -- don't filter output from utilities 34# --no-stats -- don't show progress statistics while fetching files 35usage() { 36 cat <<EOF 37usage: `basename $0` [options] command ... [path] 38 39Options: 40 -d workdir -- Store working files in workdir 41 (default: /var/db/portsnap/) 42 -f conffile -- Read configuration options from conffile 43 (default: /etc/portsnap.conf) 44 -I -- Update INDEX only. (update command only) 45 -k KEY -- Trust an RSA key with SHA256 hash of KEY 46 -l descfile -- Merge the specified local describes file into the INDEX. 47 -p portsdir -- Location of uncompressed ports tree 48 (default: /usr/ports/) 49 -s server -- Server from which to fetch updates. 50 (default: portsnap.FreeBSD.org) 51 --interactive -- interactive: override auto-detection of calling process 52 (use this when calling portsnap from an interactive, non- 53 terminal application AND NEVER ELSE). 54 path -- Extract only parts of the tree starting with the given 55 string. (extract command only) 56Commands: 57 fetch -- Fetch a compressed snapshot of the ports tree, 58 or update an existing snapshot. 59 cron -- Sleep rand(3600) seconds, and then fetch updates. 60 extract -- Extract snapshot of ports tree, replacing existing 61 files and directories. 62 update -- Update ports tree to match current snapshot, replacing 63 files and directories which have changed. 64 auto -- Fetch updates, and either extract a new ports tree or 65 update an existing tree. 66EOF 67 exit 0 68} 69 70#### Parameter handling functions. 71 72# Initialize parameters to null, just in case they're 73# set in the environment. 74init_params() { 75 KEYPRINT="" 76 EXTRACTPATH="" 77 WORKDIR="" 78 PORTSDIR="" 79 CONFFILE="" 80 COMMAND="" 81 COMMANDS="" 82 QUIETREDIR="" 83 QUIETFLAG="" 84 STATSREDIR="" 85 XARGST="" 86 NDEBUG="" 87 DDSTATS="" 88 INDEXONLY="" 89 SERVERNAME="" 90 REFUSE="" 91 LOCALDESC="" 92 INTERACTIVE="" 93} 94 95# Parse the command line 96parse_cmdline() { 97 while [ $# -gt 0 ]; do 98 case "$1" in 99 -d) 100 if [ $# -eq 1 ]; then usage; fi 101 if [ ! -z "${WORKDIR}" ]; then usage; fi 102 shift; WORKDIR="$1" 103 ;; 104 --debug) 105 QUIETREDIR="/dev/stderr" 106 STATSREDIR="/dev/stderr" 107 QUIETFLAG=" " 108 NDEBUG=" " 109 XARGST="-t" 110 DDSTATS=".." 111 ;; 112 --interactive) 113 INTERACTIVE="YES" 114 ;; 115 -f) 116 if [ $# -eq 1 ]; then usage; fi 117 if [ ! -z "${CONFFILE}" ]; then usage; fi 118 shift; CONFFILE="$1" 119 ;; 120 -h | --help | help) 121 usage 122 ;; 123 -I) 124 INDEXONLY="YES" 125 ;; 126 -k) 127 if [ $# -eq 1 ]; then usage; fi 128 if [ ! -z "${KEYPRINT}" ]; then usage; fi 129 shift; KEYPRINT="$1" 130 ;; 131 -l) 132 if [ $# -eq 1 ]; then usage; fi 133 if [ ! -z "${LOCALDESC}" ]; then usage; fi 134 shift; LOCALDESC="$1" 135 ;; 136 --no-stats) 137 if [ -z "${STATSREDIR}" ]; then 138 STATSREDIR="/dev/null" 139 DDSTATS=".. " 140 fi 141 ;; 142 -p) 143 if [ $# -eq 1 ]; then usage; fi 144 if [ ! -z "${PORTSDIR}" ]; then usage; fi 145 shift; PORTSDIR="$1" 146 ;; 147 -s) 148 if [ $# -eq 1 ]; then usage; fi 149 if [ ! -z "${SERVERNAME}" ]; then usage; fi 150 shift; SERVERNAME="$1" 151 ;; 152 cron | extract | fetch | update | auto) 153 COMMANDS="${COMMANDS} $1" 154 ;; 155 up) 156 COMMANDS="${COMMANDS} update" 157 ;; 158 alfred) 159 COMMANDS="${COMMANDS} auto" 160 ;; 161 *) 162 if [ $# -gt 1 ]; then usage; fi 163 if echo ${COMMANDS} | grep -vq extract; then 164 usage 165 fi 166 EXTRACTPATH="$1" 167 ;; 168 esac 169 shift 170 done 171 172 if [ -z "${COMMANDS}" ]; then 173 usage 174 fi 175} 176 177# If CONFFILE was specified at the command-line, make 178# sure that it exists and is readable. 179sanity_conffile() { 180 if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then 181 echo -n "File does not exist " 182 echo -n "or is not readable: " 183 echo ${CONFFILE} 184 exit 1 185 fi 186} 187 188# If a configuration file hasn't been specified, use 189# the default value (/etc/portsnap.conf) 190default_conffile() { 191 if [ -z "${CONFFILE}" ]; then 192 CONFFILE="/etc/portsnap.conf" 193 fi 194} 195 196# Read {KEYPRINT, SERVERNAME, WORKDIR, PORTSDIR} from the configuration 197# file if they haven't already been set. If the configuration 198# file doesn't exist, do nothing. 199# Also read REFUSE (which cannot be set via the command line) if it is 200# present in the configuration file. 201parse_conffile() { 202 if [ -r "${CONFFILE}" ]; then 203 for X in KEYPRINT WORKDIR PORTSDIR SERVERNAME; do 204 eval _=\$${X} 205 if [ -z "${_}" ]; then 206 eval ${X}=`grep "^${X}=" "${CONFFILE}" | 207 cut -f 2- -d '=' | tail -1` 208 fi 209 done 210 211 if grep -qE "^REFUSE[[:space:]]" ${CONFFILE}; then 212 REFUSE="^(` 213 grep -E "^REFUSE[[:space:]]" "${CONFFILE}" | 214 cut -c 7- | xargs echo | tr ' ' '|' 215 `)" 216 fi 217 218 if grep -qE "^INDEX[[:space:]]" ${CONFFILE}; then 219 INDEXPAIRS="` 220 grep -E "^INDEX[[:space:]]" "${CONFFILE}" | 221 cut -c 7- | tr ' ' '|' | xargs echo`" 222 fi 223 fi 224} 225 226# If parameters have not been set, use default values 227default_params() { 228 _QUIETREDIR="/dev/null" 229 _QUIETFLAG="-q" 230 _STATSREDIR="/dev/stdout" 231 _WORKDIR="/var/db/portsnap" 232 _PORTSDIR="/usr/ports" 233 _NDEBUG="-n" 234 _LOCALDESC="/dev/null" 235 for X in QUIETREDIR QUIETFLAG STATSREDIR WORKDIR PORTSDIR \ 236 NDEBUG LOCALDESC; do 237 eval _=\$${X} 238 eval __=\$_${X} 239 if [ -z "${_}" ]; then 240 eval ${X}=${__} 241 fi 242 done 243 if [ -z "${INTERACTIVE}" ]; then 244 if [ -t 0 ]; then 245 INTERACTIVE="YES" 246 else 247 INTERACTIVE="NO" 248 fi 249 fi 250} 251 252# Perform sanity checks and set some final parameters 253# in preparation for fetching files. Also chdir into 254# the working directory. 255fetch_check_params() { 256 export HTTP_USER_AGENT="portsnap (${COMMAND}, `uname -r`)" 257 258 _SERVERNAME_z=\ 259"SERVERNAME must be given via command line or configuration file." 260 _KEYPRINT_z="Key must be given via -k option or configuration file." 261 _KEYPRINT_bad="Invalid key fingerprint: " 262 _WORKDIR_bad="Directory does not exist or is not writable: " 263 264 if [ -z "${SERVERNAME}" ]; then 265 echo -n "`basename $0`: " 266 echo "${_SERVERNAME_z}" 267 exit 1 268 fi 269 if [ -z "${KEYPRINT}" ]; then 270 echo -n "`basename $0`: " 271 echo "${_KEYPRINT_z}" 272 exit 1 273 fi 274 if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then 275 echo -n "`basename $0`: " 276 echo -n "${_KEYPRINT_bad}" 277 echo ${KEYPRINT} 278 exit 1 279 fi 280 if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then 281 echo -n "`basename $0`: " 282 echo -n "${_WORKDIR_bad}" 283 echo ${WORKDIR} 284 exit 1 285 fi 286 cd ${WORKDIR} || exit 1 287 288 BSPATCH=/usr/bin/bspatch 289 SHA256=/sbin/sha256 290 PHTTPGET=/usr/libexec/phttpget 291} 292 293# Perform sanity checks and set some final parameters 294# in preparation for extracting or updating ${PORTSDIR} 295# Complain if ${PORTSDIR} exists but is not writable, 296# but don't complain if ${PORTSDIR} doesn't exist. 297extract_check_params() { 298 _WORKDIR_bad="Directory does not exist: " 299 _PORTSDIR_bad="Directory is not writable: " 300 301 if ! [ -d "${WORKDIR}" ]; then 302 echo -n "`basename $0`: " 303 echo -n "${_WORKDIR_bad}" 304 echo ${WORKDIR} 305 exit 1 306 fi 307 if [ -d "${PORTSDIR}" ] && ! [ -w "${PORTSDIR}" ]; then 308 echo -n "`basename $0`: " 309 echo -n "${_PORTSDIR_bad}" 310 echo ${PORTSDIR} 311 exit 1 312 fi 313 314 if ! [ -d "${WORKDIR}/files" -a -r "${WORKDIR}/tag" \ 315 -a -r "${WORKDIR}/INDEX" -a -r "${WORKDIR}/tINDEX" ]; then 316 echo "No snapshot available. Try running" 317 echo "# `basename $0` fetch" 318 exit 1 319 fi 320 321 MKINDEX=/usr/libexec/make_index 322} 323 324# Perform sanity checks and set some final parameters 325# in preparation for updating ${PORTSDIR} 326update_check_params() { 327 extract_check_params 328 329 if ! [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 330 echo "${PORTSDIR} was not created by portsnap." 331 echo -n "You must run '`basename $0` extract' before " 332 echo "running '`basename $0` update'." 333 exit 1 334 fi 335 336} 337 338#### Core functionality -- the actual work gets done here 339 340# Use an SRV query to pick a server. If the SRV query doesn't provide 341# a useful answer, use the server name specified by the user. 342# Put another way... look up _http._tcp.${SERVERNAME} and pick a server 343# from that; or if no servers are returned, use ${SERVERNAME}. 344# This allows a user to specify "portsnap.freebsd.org" (in which case 345# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org" 346# (in which case portsnap will use that particular server, since there 347# won't be an SRV entry for that name). 348# 349# We ignore the Port field, since we are always going to use port 80. 350 351# Fetch the mirror list, but do not pick a mirror yet. Returns 1 if 352# no mirrors are available for any reason. 353fetch_pick_server_init() { 354 : > serverlist_tried 355 356# Check that host(1) exists (i.e., that the system wasn't built with the 357# WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist. 358 if ! which -s host; then 359 : > serverlist_full 360 return 1 361 fi 362 363 echo -n "Looking up ${SERVERNAME} mirrors... " 364 365# Issue the SRV query and pull out the Priority, Weight, and Target fields. 366# BIND 9 prints "$name has SRV record ..." while BIND 8 prints 367# "$name server selection ..."; we allow either format. 368 MLIST="_http._tcp.${SERVERNAME}" 369 host -t srv "${MLIST}" | 370 sed -nE "s/${MLIST} (has SRV record|server selection) //Ip" | 371 cut -f 1,2,4 -d ' ' | 372 sed -e 's/\.$//' | 373 sort > serverlist_full 374 375# If no records, give up -- we'll just use the server name we were given. 376 if [ `wc -l < serverlist_full` -eq 0 ]; then 377 echo "none found." 378 return 1 379 fi 380 381# Report how many mirrors we found. 382 echo `wc -l < serverlist_full` "mirrors found." 383 384# Generate a random seed for use in picking mirrors. If HTTP_PROXY 385# is set, this will be used to generate the seed; otherwise, the seed 386# will be random. 387 if [ -n "${HTTP_PROXY}${http_proxy}" ]; then 388 RANDVALUE=`sha256 -qs "${HTTP_PROXY}${http_proxy}" | 389 tr -d 'a-f' | 390 cut -c 1-9` 391 else 392 RANDVALUE=`jot -r 1 0 999999999` 393 fi 394} 395 396# Pick a mirror. Returns 1 if we have run out of mirrors to try. 397fetch_pick_server() { 398# Generate a list of not-yet-tried mirrors 399 sort serverlist_tried | 400 comm -23 serverlist_full - > serverlist 401 402# Have we run out of mirrors? 403 if [ `wc -l < serverlist` -eq 0 ]; then 404 echo "No mirrors remaining, giving up." 405 return 1 406 fi 407 408# Find the highest priority level (lowest numeric value). 409 SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1` 410 411# Add up the weights of the response lines at that priority level. 412 SRV_WSUM=0; 413 while read X; do 414 case "$X" in 415 ${SRV_PRIORITY}\ *) 416 SRV_W=`echo $X | cut -f 2 -d ' '` 417 SRV_WSUM=$(($SRV_WSUM + $SRV_W)) 418 ;; 419 esac 420 done < serverlist 421 422# If all the weights are 0, pretend that they are all 1 instead. 423 if [ ${SRV_WSUM} -eq 0 ]; then 424 SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l` 425 SRV_W_ADD=1 426 else 427 SRV_W_ADD=0 428 fi 429 430# Pick a value between 0 and the sum of the weights - 1 431 SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}` 432 433# Read through the list of mirrors and set SERVERNAME. Write the line 434# corresponding to the mirror we selected into serverlist_tried so that 435# we won't try it again. 436 while read X; do 437 case "$X" in 438 ${SRV_PRIORITY}\ *) 439 SRV_W=`echo $X | cut -f 2 -d ' '` 440 SRV_W=$(($SRV_W + $SRV_W_ADD)) 441 if [ $SRV_RND -lt $SRV_W ]; then 442 SERVERNAME=`echo $X | cut -f 3 -d ' '` 443 echo "$X" >> serverlist_tried 444 break 445 else 446 SRV_RND=$(($SRV_RND - $SRV_W)) 447 fi 448 ;; 449 esac 450 done < serverlist 451} 452 453# Check that we have a public key with an appropriate hash, or 454# fetch the key if it doesn't exist. Returns 1 if the key has 455# not yet been fetched. 456fetch_key() { 457 if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 458 return 0 459 fi 460 461 echo -n "Fetching public key from ${SERVERNAME}... " 462 rm -f pub.ssl 463 fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \ 464 2>${QUIETREDIR} || true 465 if ! [ -r pub.ssl ]; then 466 echo "failed." 467 return 1 468 fi 469 if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 470 echo "key has incorrect hash." 471 rm -f pub.ssl 472 return 1 473 fi 474 echo "done." 475} 476 477# Fetch a snapshot tag 478fetch_tag() { 479 rm -f snapshot.ssl tag.new 480 481 echo ${NDEBUG} "Fetching snapshot tag from ${SERVERNAME}... " 482 fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl \ 483 2>${QUIETREDIR} || true 484 if ! [ -r $1.ssl ]; then 485 echo "failed." 486 return 1 487 fi 488 489 openssl rsautl -pubin -inkey pub.ssl -verify \ 490 < $1.ssl > tag.new 2>${QUIETREDIR} || true 491 rm $1.ssl 492 493 if ! [ `wc -l < tag.new` = 1 ] || 494 ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then 495 echo "invalid snapshot tag." 496 return 1 497 fi 498 499 echo "done." 500 501 SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new` 502 SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new` 503} 504 505# Sanity-check the date on a snapshot tag 506fetch_snapshot_tagsanity() { 507 if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then 508 echo "Snapshot appears to be more than a year old!" 509 echo "(Is the system clock correct?)" 510 echo "Cowardly refusing to proceed any further." 511 return 1 512 fi 513 if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then 514 echo -n "Snapshot appears to have been created more than " 515 echo "one day into the future!" 516 echo "(Is the system clock correct?)" 517 echo "Cowardly refusing to proceed any further." 518 return 1 519 fi 520} 521 522# Sanity-check the date on a snapshot update tag 523fetch_update_tagsanity() { 524 fetch_snapshot_tagsanity || return 1 525 526 if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then 527 echo -n "Latest snapshot on server is " 528 echo "older than what we already have!" 529 echo -n "Cowardly refusing to downgrade from " 530 date -r ${OLDSNAPSHOTDATE} 531 echo "to `date -r ${SNAPSHOTDATE}`." 532 return 1 533 fi 534} 535 536# Compare old and new tags; return 1 if update is unnecessary 537fetch_update_neededp() { 538 if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then 539 echo -n "Latest snapshot on server matches " 540 echo "what we already have." 541 echo "No updates needed." 542 rm tag.new 543 return 1 544 fi 545 if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then 546 echo -n "Ports tree hasn't changed since " 547 echo "last snapshot." 548 echo "No updates needed." 549 rm tag.new 550 return 1 551 fi 552 553 return 0 554} 555 556# Fetch snapshot metadata file 557fetch_metadata() { 558 rm -f ${SNAPSHOTHASH} tINDEX.new 559 560 echo ${NDEBUG} "Fetching snapshot metadata... " 561 fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} \ 562 2>${QUIETREDIR} || return 563 if [ "`${SHA256} -q ${SNAPSHOTHASH}`" != ${SNAPSHOTHASH} ]; then 564 echo "snapshot metadata corrupt." 565 return 1 566 fi 567 mv ${SNAPSHOTHASH} tINDEX.new 568 echo "done." 569} 570 571# Warn user about bogus metadata 572fetch_metadata_freakout() { 573 echo 574 echo "Portsnap metadata is correctly signed, but contains" 575 echo "at least one line which appears bogus." 576 echo "Cowardly refusing to proceed any further." 577} 578 579# Sanity-check a snapshot metadata file 580fetch_metadata_sanity() { 581 if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then 582 fetch_metadata_freakout 583 return 1 584 fi 585 if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then 586 echo 587 echo "Portsnap metadata appears bogus." 588 echo "Cowardly refusing to proceed any further." 589 return 1 590 fi 591} 592 593# Take a list of ${oldhash}|${newhash} and output a list of needed patches 594fetch_make_patchlist() { 595 local IFS='|' 596 echo "" 1>${QUIETREDIR} 597 grep -vE "^([0-9a-f]{64})\|\1$" | 598 while read X Y; do 599 printf "Processing: $X $Y ...\r" 1>${QUIETREDIR} 600 if [ -f "files/${Y}.gz" -o ! -f "files/${X}.gz" ]; then continue; fi 601 echo "${X}|${Y}" 602 done 603 echo "" 1>${QUIETREDIR} 604} 605 606# Print user-friendly progress statistics 607fetch_progress() { 608 LNC=0 609 while read x; do 610 LNC=$(($LNC + 1)) 611 if [ $(($LNC % 10)) = 0 ]; then 612 echo -n $LNC 613 elif [ $(($LNC % 2)) = 0 ]; then 614 echo -n . 615 fi 616 done 617 echo -n " " 618} 619 620pct_fmt() 621{ 622 printf " \r" 623 printf "($1/$2) %02.2f%% " `echo "scale=4;$LNC / $TOTAL * 100"|bc` 624} 625 626fetch_progress_percent() { 627 TOTAL=$1 628 LNC=0 629 pct_fmt $LNC $TOTAL 630 while read x; do 631 LNC=$(($LNC + 1)) 632 if [ $(($LNC % 100)) = 0 ]; then 633 pct_fmt $LNC $TOTAL 634 elif [ $(($LNC % 10)) = 0 ]; then 635 echo -n . 636 fi 637 done 638 pct_fmt $LNC $TOTAL 639 echo " done. " 640} 641 642# Sanity-check an index file 643fetch_index_sanity() { 644 if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new || 645 fgrep -q "./" INDEX.new; then 646 fetch_metadata_freakout 647 return 1 648 fi 649} 650 651# Verify a list of files 652fetch_snapshot_verify() { 653 while read F; do 654 if [ "`gunzip -c < snap/${F}.gz | ${SHA256} -q`" != ${F} ]; then 655 echo "snapshot corrupt." 656 return 1 657 fi 658 done 659 return 0 660} 661 662# Fetch a snapshot tarball, extract, and verify. 663fetch_snapshot() { 664 while ! fetch_tag snapshot; do 665 fetch_pick_server || return 1 666 done 667 fetch_snapshot_tagsanity || return 1 668 fetch_metadata || return 1 669 fetch_metadata_sanity || return 1 670 671 rm -rf snap/ 672 673# Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will 674# probably take a while, so the progrees reports that fetch(1) generates 675# will be useful for keeping the users' attention from drifting. 676 echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:" 677 fetch -r http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1 678 679 echo -n "Extracting snapshot... " 680 tar -xz --numeric-owner -f ${SNAPSHOTHASH}.tgz snap/ || return 1 681 rm ${SNAPSHOTHASH}.tgz 682 echo "done." 683 684 echo -n "Verifying snapshot integrity... " 685# Verify the metadata files 686 cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1 687# Extract the index 688 rm -f INDEX.new 689 gunzip -c < snap/`look INDEX tINDEX.new | 690 cut -f 2 -d '|'`.gz > INDEX.new 691 fetch_index_sanity || return 1 692# Verify the snapshot contents 693 cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1 694 echo "done." 695 696# Move files into their proper locations 697 rm -f tag INDEX tINDEX 698 rm -rf files 699 mv tag.new tag 700 mv tINDEX.new tINDEX 701 mv INDEX.new INDEX 702 mv snap/ files/ 703 704 return 0 705} 706 707# Update a compressed snapshot 708fetch_update() { 709 rm -f patchlist diff OLD NEW filelist INDEX.new 710 711 OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag` 712 OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag` 713 714 while ! fetch_tag latest; do 715 fetch_pick_server || return 1 716 done 717 fetch_update_tagsanity || return 1 718 fetch_update_neededp || return 0 719 fetch_metadata || return 1 720 fetch_metadata_sanity || return 1 721 722 echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` " 723 echo "to `date -r ${SNAPSHOTDATE}`." 724 725# Generate a list of wanted metadata patches 726 join -t '|' -o 1.2,2.2 tINDEX tINDEX.new | 727 fetch_make_patchlist > patchlist 728 729# Attempt to fetch metadata patches 730 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 731 echo ${NDEBUG} "metadata patches.${DDSTATS}" 732 tr '|' '-' < patchlist | 733 lam -s "tp/" - -s ".gz" | 734 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 735 2>${STATSREDIR} | fetch_progress 736 echo "done." 737 738# Attempt to apply metadata patches 739 echo -n "Applying metadata patches... " 740 local oldifs="$IFS" IFS='|' 741 while read X Y; do 742 if [ ! -f "${X}-${Y}.gz" ]; then continue; fi 743 gunzip -c < ${X}-${Y}.gz > diff 744 gunzip -c < files/${X}.gz > OLD 745 cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp 746 grep '^\+' diff | cut -c 2- | 747 sort -k 1,1 -t '|' -m - ptmp > NEW 748 if [ `${SHA256} -q NEW` = ${Y} ]; then 749 mv NEW files/${Y} 750 gzip -n files/${Y} 751 fi 752 rm -f diff OLD NEW ${X}-${Y}.gz ptmp 753 done < patchlist 2>${QUIETREDIR} 754 IFS="$oldifs" 755 echo "done." 756 757# Update metadata without patches 758 join -t '|' -v 2 tINDEX tINDEX.new | 759 cut -f 2 -d '|' /dev/stdin patchlist | 760 while read Y; do 761 if [ ! -f "files/${Y}.gz" ]; then 762 echo ${Y}; 763 fi 764 done > filelist 765 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 766 echo ${NDEBUG} "metadata files... " 767 lam -s "f/" - -s ".gz" < filelist | 768 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 769 2>${QUIETREDIR} 770 771 while read Y; do 772 echo -n "Verifying ${Y}... " 1>${QUIETREDIR} 773 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 774 mv ${Y}.gz files/${Y}.gz 775 else 776 echo "metadata is corrupt." 777 return 1 778 fi 779 echo "ok." 1>${QUIETREDIR} 780 done < filelist 781 echo "done." 782 783# Extract the index 784 echo -n "Extracting index... " 1>${QUIETREDIR} 785 gunzip -c < files/`look INDEX tINDEX.new | 786 cut -f 2 -d '|'`.gz > INDEX.new 787 fetch_index_sanity || return 1 788 789# If we have decided to refuse certain updates, construct a hybrid index which 790# is equal to the old index for parts of the tree which we don't want to 791# update, and equal to the new index for parts of the tree which gets updates. 792# This means that we should always have a "complete snapshot" of the ports 793# tree -- with the caveat that it isn't actually a snapshot. 794 if [ ! -z "${REFUSE}" ]; then 795 echo "Refusing to download updates for ${REFUSE}" \ 796 >${QUIETREDIR} 797 798 grep -Ev "${REFUSE}" INDEX.new > INDEX.tmp 799 grep -E "${REFUSE}" INDEX | 800 sort -m -k 1,1 -t '|' - INDEX.tmp > INDEX.new 801 rm -f INDEX.tmp 802 fi 803 804# Generate a list of wanted ports patches 805 echo -n "Generating list of wanted patches..." 1>${QUIETREDIR} 806 join -t '|' -o 1.2,2.2 INDEX INDEX.new | 807 fetch_make_patchlist > patchlist 808 echo " done." 1>${QUIETREDIR} 809 810# Attempt to fetch ports patches 811 patchcnt=`wc -l < patchlist | tr -d ' '` 812 echo -n "Fetching $patchcnt " 813 echo ${NDEBUG} "patches.${DDSTATS}" 814 echo " " 815 tr '|' '-' < patchlist | lam -s "bp/" - | 816 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 817 2>${STATSREDIR} | fetch_progress_percent $patchcnt 818 echo "done." 819 820# Attempt to apply ports patches 821 PATCHCNT=`wc -l patchlist` 822 echo "Applying patches... " 823 local oldifs="$IFS" IFS='|' 824 I=0 825 while read X Y; do 826 I=$(($I + 1)) 827 F="${X}-${Y}" 828 if [ ! -f "${F}" ]; then 829 printf " Skipping ${F} (${I} of ${PATCHCNT}).\r" 830 continue; 831 fi 832 echo " Processing ${F}..." 1>${QUIETREDIR} 833 gunzip -c < files/${X}.gz > OLD 834 ${BSPATCH} OLD NEW ${X}-${Y} 835 if [ `${SHA256} -q NEW` = ${Y} ]; then 836 mv NEW files/${Y} 837 gzip -n files/${Y} 838 fi 839 rm -f diff OLD NEW ${X}-${Y} 840 done < patchlist 2>${QUIETREDIR} 841 IFS="$oldifs" 842 echo "done." 843 844# Update ports without patches 845 join -t '|' -v 2 INDEX INDEX.new | 846 cut -f 2 -d '|' /dev/stdin patchlist | 847 while read Y; do 848 if [ ! -f "files/${Y}.gz" ]; then 849 echo ${Y}; 850 fi 851 done > filelist 852 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 853 echo ${NDEBUG} "new ports or files... " 854 lam -s "f/" - -s ".gz" < filelist | 855 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 856 2>${QUIETREDIR} 857 858 I=0 859 while read Y; do 860 I=$(($I + 1)) 861 printf " Processing ${Y} (${I} of ${PATCHCNT}).\r" 1>${QUIETREDIR} 862 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 863 mv ${Y}.gz files/${Y}.gz 864 else 865 echo "snapshot is corrupt." 866 return 1 867 fi 868 done < filelist 869 echo "done." 870 871# Remove files which are no longer needed 872 cut -f 2 -d '|' tINDEX INDEX | sort -u > oldfiles 873 cut -f 2 -d '|' tINDEX.new INDEX.new | sort -u | comm -13 - oldfiles | 874 lam -s "files/" - -s ".gz" | xargs rm -f 875 rm patchlist filelist oldfiles 876 877# We're done! 878 mv INDEX.new INDEX 879 mv tINDEX.new tINDEX 880 mv tag.new tag 881 882 return 0 883} 884 885# Do the actual work involved in "fetch" / "cron". 886fetch_run() { 887 fetch_pick_server_init && fetch_pick_server 888 889 while ! fetch_key; do 890 fetch_pick_server || return 1 891 done 892 893 if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then 894 fetch_snapshot || return 1 895 fi 896 fetch_update || return 1 897} 898 899# Build a ports INDEX file 900extract_make_index() { 901 if ! look $1 ${WORKDIR}/tINDEX > /dev/null; then 902 echo -n "$1 not provided by portsnap server; " 903 echo "$2 not being generated." 904 else 905 gunzip -c < "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX | 906 cut -f 2 -d '|'`.gz" | 907 cat - ${LOCALDESC} | 908 ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2 909 fi 910} 911 912# Create INDEX, INDEX-5, INDEX-6 913extract_indices() { 914 echo -n "Building new INDEX files... " 915 for PAIR in ${INDEXPAIRS}; do 916 INDEXFILE=`echo ${PAIR} | cut -f 1 -d '|'` 917 DESCRIBEFILE=`echo ${PAIR} | cut -f 2 -d '|'` 918 extract_make_index ${DESCRIBEFILE} ${INDEXFILE} || return 1 919 done 920 echo "done." 921} 922 923# Create .portsnap.INDEX; if we are REFUSEing to touch certain directories, 924# merge the values from any exiting .portsnap.INDEX file. 925extract_metadata() { 926 if [ -z "${REFUSE}" ]; then 927 sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX 928 elif [ -f ${PORTSDIR}/.portsnap.INDEX ]; then 929 grep -E "${REFUSE}" ${PORTSDIR}/.portsnap.INDEX \ 930 > ${PORTSDIR}/.portsnap.INDEX.tmp 931 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort | 932 sort -m - ${PORTSDIR}/.portsnap.INDEX.tmp \ 933 > ${PORTSDIR}/.portsnap.INDEX 934 rm -f ${PORTSDIR}/.portsnap.INDEX.tmp 935 else 936 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort \ 937 > ${PORTSDIR}/.portsnap.INDEX 938 fi 939} 940 941# Do the actual work involved in "extract" 942extract_run() { 943 local oldifs="$IFS" IFS='|' 944 mkdir -p ${PORTSDIR} || return 1 945 946 if ! 947 if ! [ -z "${EXTRACTPATH}" ]; then 948 grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX 949 elif ! [ -z "${REFUSE}" ]; then 950 grep -vE "${REFUSE}" ${WORKDIR}/INDEX 951 else 952 cat ${WORKDIR}/INDEX 953 fi | while read FILE HASH; do 954 echo ${PORTSDIR}/${FILE} 955 if ! [ -s "${WORKDIR}/files/${HASH}.gz" ]; then 956 echo "files/${HASH}.gz not found -- snapshot corrupt." 957 return 1 958 fi 959 case ${FILE} in 960 */) 961 rm -rf ${PORTSDIR}/${FILE%/} 962 mkdir -p ${PORTSDIR}/${FILE} 963 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 964 -C ${PORTSDIR}/${FILE} 965 ;; 966 *) 967 rm -f ${PORTSDIR}/${FILE} 968 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 969 -C ${PORTSDIR} ${FILE} 970 ;; 971 esac 972 done; then 973 return 1 974 fi 975 if [ ! -z "${EXTRACTPATH}" ]; then 976 return 0; 977 fi 978 979 IFS="$oldifs" 980 981 extract_metadata 982 extract_indices 983} 984 985update_run_extract() { 986 local IFS='|' 987 988# Install new files 989 echo "Extracting new files:" 990 if ! 991 if ! [ -z "${REFUSE}" ]; then 992 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort 993 else 994 sort ${WORKDIR}/INDEX 995 fi | 996 comm -13 ${PORTSDIR}/.portsnap.INDEX - | 997 while read FILE HASH; do 998 echo ${PORTSDIR}/${FILE} 999 if ! [ -s "${WORKDIR}/files/${HASH}.gz" ]; then 1000 echo "files/${HASH}.gz not found -- snapshot corrupt." 1001 return 1 1002 fi 1003 case ${FILE} in 1004 */) 1005 mkdir -p ${PORTSDIR}/${FILE} 1006 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 1007 -C ${PORTSDIR}/${FILE} 1008 ;; 1009 *) 1010 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 1011 -C ${PORTSDIR} ${FILE} 1012 ;; 1013 esac 1014 done; then 1015 return 1 1016 fi 1017} 1018 1019# Do the actual work involved in "update" 1020update_run() { 1021 if ! [ -z "${INDEXONLY}" ]; then 1022 extract_indices >/dev/null || return 1 1023 return 0 1024 fi 1025 1026 if sort ${WORKDIR}/INDEX | 1027 cmp -s ${PORTSDIR}/.portsnap.INDEX -; then 1028 echo "Ports tree is already up to date." 1029 return 0 1030 fi 1031 1032# If we are REFUSEing to touch certain directories, don't remove files 1033# from those directories (even if they are out of date) 1034 echo -n "Removing old files and directories... " 1035 if ! [ -z "${REFUSE}" ]; then 1036 sort ${WORKDIR}/INDEX | 1037 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 1038 grep -vE "${REFUSE}" | 1039 lam -s "${PORTSDIR}/" - | 1040 sed -e 's|/$||' | xargs rm -rf 1041 else 1042 sort ${WORKDIR}/INDEX | 1043 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 1044 lam -s "${PORTSDIR}/" - | 1045 sed -e 's|/$||' | xargs rm -rf 1046 fi 1047 echo "done." 1048 1049 update_run_extract || return 1 1050 extract_metadata 1051 extract_indices 1052} 1053 1054#### Main functions -- call parameter-handling and core functions 1055 1056# Using the command line, configuration file, and defaults, 1057# set all the parameters which are needed later. 1058get_params() { 1059 init_params 1060 parse_cmdline $@ 1061 sanity_conffile 1062 default_conffile 1063 parse_conffile 1064 default_params 1065} 1066 1067# Fetch command. Make sure that we're being called 1068# interactively, then run fetch_check_params and fetch_run 1069cmd_fetch() { 1070 if [ "${INTERACTIVE}" != "YES" ]; then 1071 echo -n "`basename $0` fetch should not " 1072 echo "be run non-interactively." 1073 echo "Run `basename $0` cron instead" 1074 exit 1 1075 fi 1076 fetch_check_params 1077 fetch_run || exit 1 1078} 1079 1080# Cron command. Make sure the parameters are sensible; wait 1081# rand(3600) seconds; then fetch updates. While fetching updates, 1082# send output to a temporary file; only print that file if the 1083# fetching failed. 1084cmd_cron() { 1085 fetch_check_params 1086 sleep `jot -r 1 0 3600` 1087 1088 TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1 1089 if ! fetch_run >> ${TMPFILE}; then 1090 cat ${TMPFILE} 1091 rm ${TMPFILE} 1092 exit 1 1093 fi 1094 1095 rm ${TMPFILE} 1096} 1097 1098# Extract command. Make sure the parameters are sensible, 1099# then extract the ports tree (or part thereof). 1100cmd_extract() { 1101 extract_check_params 1102 extract_run || exit 1 1103} 1104 1105# Update command. Make sure the parameters are sensible, 1106# then update the ports tree. 1107cmd_update() { 1108 update_check_params 1109 update_run || exit 1 1110} 1111 1112# Auto command. Run 'fetch' or 'cron' depending on 1113# whether stdin is a terminal; then run 'update' or 1114# 'extract' depending on whether ${PORTSDIR} exists. 1115cmd_auto() { 1116 if [ "${INTERACTIVE}" = "YES" ]; then 1117 cmd_fetch 1118 else 1119 cmd_cron 1120 fi 1121 if [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 1122 cmd_update 1123 else 1124 cmd_extract 1125 fi 1126} 1127 1128#### Entry point 1129 1130# Make sure we find utilities from the base system 1131export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH} 1132 1133# Set LC_ALL in order to avoid problems with character ranges like [A-Z]. 1134export LC_ALL=C 1135 1136get_params $@ 1137for COMMAND in ${COMMANDS}; do 1138 cmd_${COMMAND} 1139done 1140