portsnap.sh revision 306419
1#!/bin/sh 2 3#- 4# Copyright 2004-2005 Colin Percival 5# All rights reserved 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted providing that the following conditions 9# are met: 10# 1. Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# 2. Redistributions in binary form must reproduce the above copyright 13# notice, this list of conditions and the following disclaimer in the 14# documentation and/or other materials provided with the distribution. 15# 16# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 20# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26# POSSIBILITY OF SUCH DAMAGE. 27 28# $FreeBSD: releng/11.0/usr.sbin/portsnap/portsnap/portsnap.sh 306419 2016-09-28 22:04:07Z emaste $ 29 30#### Usage function -- called from command-line handling code. 31 32# Usage instructions. Options not listed: 33# --debug -- don't filter output from utilities 34# --no-stats -- don't show progress statistics while fetching files 35usage() { 36 cat <<EOF 37usage: `basename $0` [options] command ... [path] 38 39Options: 40 -d workdir -- Store working files in workdir 41 (default: /var/db/portsnap/) 42 -f conffile -- Read configuration options from conffile 43 (default: /etc/portsnap.conf) 44 -I -- Update INDEX only. (update command only) 45 -k KEY -- Trust an RSA key with SHA256 hash of KEY 46 -l descfile -- Merge the specified local describes file into the INDEX. 47 -p portsdir -- Location of uncompressed ports tree 48 (default: /usr/ports/) 49 -s server -- Server from which to fetch updates. 50 (default: portsnap.FreeBSD.org) 51 --interactive -- interactive: override auto-detection of calling process 52 (use this when calling portsnap from an interactive, non- 53 terminal application AND NEVER ELSE). 54 path -- Extract only parts of the tree starting with the given 55 string. (extract command only) 56Commands: 57 fetch -- Fetch a compressed snapshot of the ports tree, 58 or update an existing snapshot. 59 cron -- Sleep rand(3600) seconds, and then fetch updates. 60 extract -- Extract snapshot of ports tree, replacing existing 61 files and directories. 62 update -- Update ports tree to match current snapshot, replacing 63 files and directories which have changed. 64 auto -- Fetch updates, and either extract a new ports tree or 65 update an existing tree. 66EOF 67 exit 0 68} 69 70#### Parameter handling functions. 71 72# Initialize parameters to null, just in case they're 73# set in the environment. 74init_params() { 75 KEYPRINT="" 76 EXTRACTPATH="" 77 WORKDIR="" 78 PORTSDIR="" 79 CONFFILE="" 80 COMMAND="" 81 COMMANDS="" 82 QUIETREDIR="" 83 QUIETFLAG="" 84 STATSREDIR="" 85 XARGST="" 86 NDEBUG="" 87 DDSTATS="" 88 INDEXONLY="" 89 SERVERNAME="" 90 REFUSE="" 91 LOCALDESC="" 92 INTERACTIVE="" 93} 94 95# Parse the command line 96parse_cmdline() { 97 while [ $# -gt 0 ]; do 98 case "$1" in 99 -d) 100 if [ $# -eq 1 ]; then usage; fi 101 if [ ! -z "${WORKDIR}" ]; then usage; fi 102 shift; WORKDIR="$1" 103 ;; 104 --debug) 105 QUIETREDIR="/dev/stderr" 106 STATSREDIR="/dev/stderr" 107 QUIETFLAG=" " 108 NDEBUG=" " 109 XARGST="-t" 110 DDSTATS=".." 111 ;; 112 --interactive) 113 INTERACTIVE="YES" 114 ;; 115 -f) 116 if [ $# -eq 1 ]; then usage; fi 117 if [ ! -z "${CONFFILE}" ]; then usage; fi 118 shift; CONFFILE="$1" 119 ;; 120 -h | --help | help) 121 usage 122 ;; 123 -I) 124 INDEXONLY="YES" 125 ;; 126 -k) 127 if [ $# -eq 1 ]; then usage; fi 128 if [ ! -z "${KEYPRINT}" ]; then usage; fi 129 shift; KEYPRINT="$1" 130 ;; 131 -l) 132 if [ $# -eq 1 ]; then usage; fi 133 if [ ! -z "${LOCALDESC}" ]; then usage; fi 134 shift; LOCALDESC="$1" 135 ;; 136 --no-stats) 137 if [ -z "${STATSREDIR}" ]; then 138 STATSREDIR="/dev/null" 139 DDSTATS=".. " 140 fi 141 ;; 142 -p) 143 if [ $# -eq 1 ]; then usage; fi 144 if [ ! -z "${PORTSDIR}" ]; then usage; fi 145 shift; PORTSDIR="$1" 146 ;; 147 -s) 148 if [ $# -eq 1 ]; then usage; fi 149 if [ ! -z "${SERVERNAME}" ]; then usage; fi 150 shift; SERVERNAME="$1" 151 ;; 152 cron | extract | fetch | update | auto) 153 COMMANDS="${COMMANDS} $1" 154 ;; 155 up) 156 COMMANDS="${COMMANDS} update" 157 ;; 158 alfred) 159 COMMANDS="${COMMANDS} auto" 160 ;; 161 *) 162 if [ $# -gt 1 ]; then usage; fi 163 if echo ${COMMANDS} | grep -vq extract; then 164 usage 165 fi 166 EXTRACTPATH="$1" 167 ;; 168 esac 169 shift 170 done 171 172 if [ -z "${COMMANDS}" ]; then 173 usage 174 fi 175} 176 177# If CONFFILE was specified at the command-line, make 178# sure that it exists and is readable. 179sanity_conffile() { 180 if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then 181 echo -n "File does not exist " 182 echo -n "or is not readable: " 183 echo ${CONFFILE} 184 exit 1 185 fi 186} 187 188# If a configuration file hasn't been specified, use 189# the default value (/etc/portsnap.conf) 190default_conffile() { 191 if [ -z "${CONFFILE}" ]; then 192 CONFFILE="/etc/portsnap.conf" 193 fi 194} 195 196# Read {KEYPRINT, SERVERNAME, WORKDIR, PORTSDIR} from the configuration 197# file if they haven't already been set. If the configuration 198# file doesn't exist, do nothing. 199# Also read REFUSE (which cannot be set via the command line) if it is 200# present in the configuration file. 201parse_conffile() { 202 if [ -r "${CONFFILE}" ]; then 203 for X in KEYPRINT WORKDIR PORTSDIR SERVERNAME; do 204 eval _=\$${X} 205 if [ -z "${_}" ]; then 206 eval ${X}=`grep "^${X}=" "${CONFFILE}" | 207 cut -f 2- -d '=' | tail -1` 208 fi 209 done 210 211 if grep -qE "^REFUSE[[:space:]]" ${CONFFILE}; then 212 REFUSE="^(` 213 grep -E "^REFUSE[[:space:]]" "${CONFFILE}" | 214 cut -c 7- | xargs echo | tr ' ' '|' 215 `)" 216 fi 217 218 if grep -qE "^INDEX[[:space:]]" ${CONFFILE}; then 219 INDEXPAIRS="` 220 grep -E "^INDEX[[:space:]]" "${CONFFILE}" | 221 cut -c 7- | tr ' ' '|' | xargs echo`" 222 fi 223 fi 224} 225 226# If parameters have not been set, use default values 227default_params() { 228 _QUIETREDIR="/dev/null" 229 _QUIETFLAG="-q" 230 _STATSREDIR="/dev/stdout" 231 _WORKDIR="/var/db/portsnap" 232 _PORTSDIR="/usr/ports" 233 _NDEBUG="-n" 234 _LOCALDESC="/dev/null" 235 for X in QUIETREDIR QUIETFLAG STATSREDIR WORKDIR PORTSDIR \ 236 NDEBUG LOCALDESC; do 237 eval _=\$${X} 238 eval __=\$_${X} 239 if [ -z "${_}" ]; then 240 eval ${X}=${__} 241 fi 242 done 243 if [ -z "${INTERACTIVE}" ]; then 244 if [ -t 0 ]; then 245 INTERACTIVE="YES" 246 else 247 INTERACTIVE="NO" 248 fi 249 fi 250} 251 252# Perform sanity checks and set some final parameters 253# in preparation for fetching files. Also chdir into 254# the working directory. 255fetch_check_params() { 256 export HTTP_USER_AGENT="portsnap (${COMMAND}, `uname -r`)" 257 258 _SERVERNAME_z=\ 259"SERVERNAME must be given via command line or configuration file." 260 _KEYPRINT_z="Key must be given via -k option or configuration file." 261 _KEYPRINT_bad="Invalid key fingerprint: " 262 _WORKDIR_bad="Directory does not exist or is not writable: " 263 264 if [ -z "${SERVERNAME}" ]; then 265 echo -n "`basename $0`: " 266 echo "${_SERVERNAME_z}" 267 exit 1 268 fi 269 if [ -z "${KEYPRINT}" ]; then 270 echo -n "`basename $0`: " 271 echo "${_KEYPRINT_z}" 272 exit 1 273 fi 274 if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then 275 echo -n "`basename $0`: " 276 echo -n "${_KEYPRINT_bad}" 277 echo ${KEYPRINT} 278 exit 1 279 fi 280 if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then 281 echo -n "`basename $0`: " 282 echo -n "${_WORKDIR_bad}" 283 echo ${WORKDIR} 284 exit 1 285 fi 286 cd ${WORKDIR} || exit 1 287 288 BSPATCH=/usr/bin/bspatch 289 SHA256=/sbin/sha256 290 PHTTPGET=/usr/libexec/phttpget 291} 292 293# Perform sanity checks and set some final parameters 294# in preparation for extracting or updating ${PORTSDIR} 295# Complain if ${PORTSDIR} exists but is not writable, 296# but don't complain if ${PORTSDIR} doesn't exist. 297extract_check_params() { 298 _WORKDIR_bad="Directory does not exist: " 299 _PORTSDIR_bad="Directory is not writable: " 300 301 if ! [ -d "${WORKDIR}" ]; then 302 echo -n "`basename $0`: " 303 echo -n "${_WORKDIR_bad}" 304 echo ${WORKDIR} 305 exit 1 306 fi 307 if [ -d "${PORTSDIR}" ] && ! [ -w "${PORTSDIR}" ]; then 308 echo -n "`basename $0`: " 309 echo -n "${_PORTSDIR_bad}" 310 echo ${PORTSDIR} 311 exit 1 312 fi 313 314 if ! [ -d "${WORKDIR}/files" -a -r "${WORKDIR}/tag" \ 315 -a -r "${WORKDIR}/INDEX" -a -r "${WORKDIR}/tINDEX" ]; then 316 echo "No snapshot available. Try running" 317 echo "# `basename $0` fetch" 318 exit 1 319 fi 320 321 MKINDEX=/usr/libexec/make_index 322} 323 324# Perform sanity checks and set some final parameters 325# in preparation for updating ${PORTSDIR} 326update_check_params() { 327 extract_check_params 328 329 if ! [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 330 echo "${PORTSDIR} was not created by portsnap." 331 echo -n "You must run '`basename $0` extract' before " 332 echo "running '`basename $0` update'." 333 exit 1 334 fi 335 336} 337 338#### Core functionality -- the actual work gets done here 339 340# Use an SRV query to pick a server. If the SRV query doesn't provide 341# a useful answer, use the server name specified by the user. 342# Put another way... look up _http._tcp.${SERVERNAME} and pick a server 343# from that; or if no servers are returned, use ${SERVERNAME}. 344# This allows a user to specify "portsnap.freebsd.org" (in which case 345# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org" 346# (in which case portsnap will use that particular server, since there 347# won't be an SRV entry for that name). 348# 349# We ignore the Port field, since we are always going to use port 80. 350 351# Fetch the mirror list, but do not pick a mirror yet. Returns 1 if 352# no mirrors are available for any reason. 353fetch_pick_server_init() { 354 : > serverlist_tried 355 356# Check that host(1) exists (i.e., that the system wasn't built with the 357# WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist. 358 if ! which -s host; then 359 : > serverlist_full 360 return 1 361 fi 362 363 echo -n "Looking up ${SERVERNAME} mirrors... " 364 365# Issue the SRV query and pull out the Priority, Weight, and Target fields. 366# BIND 9 prints "$name has SRV record ..." while BIND 8 prints 367# "$name server selection ..."; we allow either format. 368 MLIST="_http._tcp.${SERVERNAME}" 369 host -t srv "${MLIST}" | 370 sed -nE "s/${MLIST} (has SRV record|server selection) //Ip" | 371 cut -f 1,2,4 -d ' ' | 372 sed -e 's/\.$//' | 373 sort > serverlist_full 374 375# If no records, give up -- we'll just use the server name we were given. 376 if [ `wc -l < serverlist_full` -eq 0 ]; then 377 echo "none found." 378 return 1 379 fi 380 381# Report how many mirrors we found. 382 echo `wc -l < serverlist_full` "mirrors found." 383 384# Generate a random seed for use in picking mirrors. If HTTP_PROXY 385# is set, this will be used to generate the seed; otherwise, the seed 386# will be random. 387 if [ -n "${HTTP_PROXY}${http_proxy}" ]; then 388 RANDVALUE=`sha256 -qs "${HTTP_PROXY}${http_proxy}" | 389 tr -d 'a-f' | 390 cut -c 1-9` 391 else 392 RANDVALUE=`jot -r 1 0 999999999` 393 fi 394} 395 396# Pick a mirror. Returns 1 if we have run out of mirrors to try. 397fetch_pick_server() { 398# Generate a list of not-yet-tried mirrors 399 sort serverlist_tried | 400 comm -23 serverlist_full - > serverlist 401 402# Have we run out of mirrors? 403 if [ `wc -l < serverlist` -eq 0 ]; then 404 echo "No mirrors remaining, giving up." 405 return 1 406 fi 407 408# Find the highest priority level (lowest numeric value). 409 SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1` 410 411# Add up the weights of the response lines at that priority level. 412 SRV_WSUM=0; 413 while read X; do 414 case "$X" in 415 ${SRV_PRIORITY}\ *) 416 SRV_W=`echo $X | cut -f 2 -d ' '` 417 SRV_WSUM=$(($SRV_WSUM + $SRV_W)) 418 ;; 419 esac 420 done < serverlist 421 422# If all the weights are 0, pretend that they are all 1 instead. 423 if [ ${SRV_WSUM} -eq 0 ]; then 424 SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l` 425 SRV_W_ADD=1 426 else 427 SRV_W_ADD=0 428 fi 429 430# Pick a value between 0 and the sum of the weights - 1 431 SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}` 432 433# Read through the list of mirrors and set SERVERNAME. Write the line 434# corresponding to the mirror we selected into serverlist_tried so that 435# we won't try it again. 436 while read X; do 437 case "$X" in 438 ${SRV_PRIORITY}\ *) 439 SRV_W=`echo $X | cut -f 2 -d ' '` 440 SRV_W=$(($SRV_W + $SRV_W_ADD)) 441 if [ $SRV_RND -lt $SRV_W ]; then 442 SERVERNAME=`echo $X | cut -f 3 -d ' '` 443 echo "$X" >> serverlist_tried 444 break 445 else 446 SRV_RND=$(($SRV_RND - $SRV_W)) 447 fi 448 ;; 449 esac 450 done < serverlist 451} 452 453# Check that we have a public key with an appropriate hash, or 454# fetch the key if it doesn't exist. Returns 1 if the key has 455# not yet been fetched. 456fetch_key() { 457 if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 458 return 0 459 fi 460 461 echo -n "Fetching public key from ${SERVERNAME}... " 462 rm -f pub.ssl 463 fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \ 464 2>${QUIETREDIR} || true 465 if ! [ -r pub.ssl ]; then 466 echo "failed." 467 return 1 468 fi 469 if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 470 echo "key has incorrect hash." 471 rm -f pub.ssl 472 return 1 473 fi 474 echo "done." 475} 476 477# Fetch a snapshot tag 478fetch_tag() { 479 rm -f snapshot.ssl tag.new 480 481 echo ${NDEBUG} "Fetching snapshot tag from ${SERVERNAME}... " 482 fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl \ 483 2>${QUIETREDIR} || true 484 if ! [ -r $1.ssl ]; then 485 echo "failed." 486 return 1 487 fi 488 489 openssl rsautl -pubin -inkey pub.ssl -verify \ 490 < $1.ssl > tag.new 2>${QUIETREDIR} || true 491 rm $1.ssl 492 493 if ! [ `wc -l < tag.new` = 1 ] || 494 ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then 495 echo "invalid snapshot tag." 496 return 1 497 fi 498 499 echo "done." 500 501 SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new` 502 SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new` 503} 504 505# Sanity-check the date on a snapshot tag 506fetch_snapshot_tagsanity() { 507 if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then 508 echo "Snapshot appears to be more than a year old!" 509 echo "(Is the system clock correct?)" 510 echo "Cowardly refusing to proceed any further." 511 return 1 512 fi 513 if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then 514 echo -n "Snapshot appears to have been created more than " 515 echo "one day into the future!" 516 echo "(Is the system clock correct?)" 517 echo "Cowardly refusing to proceed any further." 518 return 1 519 fi 520} 521 522# Sanity-check the date on a snapshot update tag 523fetch_update_tagsanity() { 524 fetch_snapshot_tagsanity || return 1 525 526 if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then 527 echo -n "Latest snapshot on server is " 528 echo "older than what we already have!" 529 echo -n "Cowardly refusing to downgrade from " 530 date -r ${OLDSNAPSHOTDATE} 531 echo "to `date -r ${SNAPSHOTDATE}`." 532 return 1 533 fi 534} 535 536# Compare old and new tags; return 1 if update is unnecessary 537fetch_update_neededp() { 538 if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then 539 echo -n "Latest snapshot on server matches " 540 echo "what we already have." 541 echo "No updates needed." 542 rm tag.new 543 return 1 544 fi 545 if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then 546 echo -n "Ports tree hasn't changed since " 547 echo "last snapshot." 548 echo "No updates needed." 549 rm tag.new 550 return 1 551 fi 552 553 return 0 554} 555 556# Fetch snapshot metadata file 557fetch_metadata() { 558 rm -f ${SNAPSHOTHASH} tINDEX.new 559 560 echo ${NDEBUG} "Fetching snapshot metadata... " 561 fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} \ 562 2>${QUIETREDIR} || return 563 if [ "`${SHA256} -q ${SNAPSHOTHASH}`" != ${SNAPSHOTHASH} ]; then 564 echo "snapshot metadata corrupt." 565 return 1 566 fi 567 mv ${SNAPSHOTHASH} tINDEX.new 568 echo "done." 569} 570 571# Warn user about bogus metadata 572fetch_metadata_freakout() { 573 echo 574 echo "Portsnap metadata is correctly signed, but contains" 575 echo "at least one line which appears bogus." 576 echo "Cowardly refusing to proceed any further." 577} 578 579# Sanity-check a snapshot metadata file 580fetch_metadata_sanity() { 581 if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then 582 fetch_metadata_freakout 583 return 1 584 fi 585 if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then 586 echo 587 echo "Portsnap metadata appears bogus." 588 echo "Cowardly refusing to proceed any further." 589 return 1 590 fi 591} 592 593# Take a list of ${oldhash}|${newhash} and output a list of needed patches 594fetch_make_patchlist() { 595 local IFS='|' 596 echo "" 1>${QUIETREDIR} 597 grep -vE "^([0-9a-f]{64})\|\1$" | 598 while read X Y; do 599 printf "Processing: $X $Y ...\r" 1>${QUIETREDIR} 600 if [ -f "files/${Y}.gz" -o ! -f "files/${X}.gz" ]; then continue; fi 601 echo "${X}|${Y}" 602 done 603 echo "" 1>${QUIETREDIR} 604} 605 606# Print user-friendly progress statistics 607fetch_progress() { 608 LNC=0 609 while read x; do 610 LNC=$(($LNC + 1)) 611 if [ $(($LNC % 10)) = 0 ]; then 612 echo -n $LNC 613 elif [ $(($LNC % 2)) = 0 ]; then 614 echo -n . 615 fi 616 done 617 echo -n " " 618} 619 620pct_fmt() 621{ 622 printf " \r" 623 printf "($1/$2) %02.2f%% " `echo "scale=4;$LNC / $TOTAL * 100"|bc` 624} 625 626fetch_progress_percent() { 627 TOTAL=$1 628 LNC=0 629 pct_fmt $LNC $TOTAL 630 while read x; do 631 LNC=$(($LNC + 1)) 632 if [ $(($LNC % 100)) = 0 ]; then 633 pct_fmt $LNC $TOTAL 634 elif [ $(($LNC % 10)) = 0 ]; then 635 echo -n . 636 fi 637 done 638 pct_fmt $LNC $TOTAL 639 echo " done. " 640} 641 642# Sanity-check an index file 643fetch_index_sanity() { 644 if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new || 645 fgrep -q "./" INDEX.new; then 646 fetch_metadata_freakout 647 return 1 648 fi 649} 650 651# Verify a list of files 652fetch_snapshot_verify() { 653 while read F; do 654 if [ "`gunzip -c < snap/${F}.gz | ${SHA256} -q`" != ${F} ]; then 655 echo "snapshot corrupt." 656 return 1 657 fi 658 done 659 return 0 660} 661 662# Fetch a snapshot tarball, extract, and verify. 663fetch_snapshot() { 664 while ! fetch_tag snapshot; do 665 fetch_pick_server || return 1 666 done 667 fetch_snapshot_tagsanity || return 1 668 fetch_metadata || return 1 669 fetch_metadata_sanity || return 1 670 671 rm -rf snap/ 672 673# Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will 674# probably take a while, so the progrees reports that fetch(1) generates 675# will be useful for keeping the users' attention from drifting. 676 echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:" 677 fetch -r http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1 678 679 echo -n "Extracting snapshot... " 680 tar -xz --numeric-owner -f ${SNAPSHOTHASH}.tgz snap/ || return 1 681 rm ${SNAPSHOTHASH}.tgz 682 echo "done." 683 684 echo -n "Verifying snapshot integrity... " 685# Verify the metadata files 686 cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1 687# Extract the index 688 rm -f INDEX.new 689 gunzip -c < snap/`look INDEX tINDEX.new | 690 cut -f 2 -d '|'`.gz > INDEX.new 691 fetch_index_sanity || return 1 692# Verify the snapshot contents 693 cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1 694 cut -f 2 -d '|' tINDEX.new INDEX.new | sort -u > files.expected 695 find snap -mindepth 1 | sed -E 's^snap/(.*)\.gz^\1^' | sort > files.snap 696 if ! cmp -s files.expected files.snap; then 697 echo "unexpected files in snapshot." 698 return 1 699 fi 700 rm files.expected files.snap 701 echo "done." 702 703# Move files into their proper locations 704 rm -f tag INDEX tINDEX 705 rm -rf files 706 mv tag.new tag 707 mv tINDEX.new tINDEX 708 mv INDEX.new INDEX 709 mv snap/ files/ 710 711 return 0 712} 713 714# Update a compressed snapshot 715fetch_update() { 716 rm -f patchlist diff OLD NEW filelist INDEX.new 717 718 OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag` 719 OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag` 720 721 while ! fetch_tag latest; do 722 fetch_pick_server || return 1 723 done 724 fetch_update_tagsanity || return 1 725 fetch_update_neededp || return 0 726 fetch_metadata || return 1 727 fetch_metadata_sanity || return 1 728 729 echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` " 730 echo "to `date -r ${SNAPSHOTDATE}`." 731 732# Generate a list of wanted metadata patches 733 join -t '|' -o 1.2,2.2 tINDEX tINDEX.new | 734 fetch_make_patchlist > patchlist 735 736# Attempt to fetch metadata patches 737 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 738 echo ${NDEBUG} "metadata patches.${DDSTATS}" 739 tr '|' '-' < patchlist | 740 lam -s "tp/" - -s ".gz" | 741 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 742 2>${STATSREDIR} | fetch_progress 743 echo "done." 744 745# Attempt to apply metadata patches 746 echo -n "Applying metadata patches... " 747 local oldifs="$IFS" IFS='|' 748 while read X Y; do 749 if [ ! -f "${X}-${Y}.gz" ]; then continue; fi 750 gunzip -c < ${X}-${Y}.gz > diff 751 gunzip -c < files/${X}.gz > OLD 752 cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp 753 grep '^\+' diff | cut -c 2- | 754 sort -k 1,1 -t '|' -m - ptmp > NEW 755 if [ `${SHA256} -q NEW` = ${Y} ]; then 756 mv NEW files/${Y} 757 gzip -n files/${Y} 758 fi 759 rm -f diff OLD NEW ${X}-${Y}.gz ptmp 760 done < patchlist 2>${QUIETREDIR} 761 IFS="$oldifs" 762 echo "done." 763 764# Update metadata without patches 765 join -t '|' -v 2 tINDEX tINDEX.new | 766 cut -f 2 -d '|' /dev/stdin patchlist | 767 while read Y; do 768 if [ ! -f "files/${Y}.gz" ]; then 769 echo ${Y}; 770 fi 771 done > filelist 772 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 773 echo ${NDEBUG} "metadata files... " 774 lam -s "f/" - -s ".gz" < filelist | 775 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 776 2>${QUIETREDIR} 777 778 while read Y; do 779 echo -n "Verifying ${Y}... " 1>${QUIETREDIR} 780 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 781 mv ${Y}.gz files/${Y}.gz 782 else 783 echo "metadata is corrupt." 784 return 1 785 fi 786 echo "ok." 1>${QUIETREDIR} 787 done < filelist 788 echo "done." 789 790# Extract the index 791 echo -n "Extracting index... " 1>${QUIETREDIR} 792 gunzip -c < files/`look INDEX tINDEX.new | 793 cut -f 2 -d '|'`.gz > INDEX.new 794 fetch_index_sanity || return 1 795 796# If we have decided to refuse certain updates, construct a hybrid index which 797# is equal to the old index for parts of the tree which we don't want to 798# update, and equal to the new index for parts of the tree which gets updates. 799# This means that we should always have a "complete snapshot" of the ports 800# tree -- with the caveat that it isn't actually a snapshot. 801 if [ ! -z "${REFUSE}" ]; then 802 echo "Refusing to download updates for ${REFUSE}" \ 803 >${QUIETREDIR} 804 805 grep -Ev "${REFUSE}" INDEX.new > INDEX.tmp 806 grep -E "${REFUSE}" INDEX | 807 sort -m -k 1,1 -t '|' - INDEX.tmp > INDEX.new 808 rm -f INDEX.tmp 809 fi 810 811# Generate a list of wanted ports patches 812 echo -n "Generating list of wanted patches..." 1>${QUIETREDIR} 813 join -t '|' -o 1.2,2.2 INDEX INDEX.new | 814 fetch_make_patchlist > patchlist 815 echo " done." 1>${QUIETREDIR} 816 817# Attempt to fetch ports patches 818 patchcnt=`wc -l < patchlist | tr -d ' '` 819 echo -n "Fetching $patchcnt " 820 echo ${NDEBUG} "patches.${DDSTATS}" 821 echo " " 822 tr '|' '-' < patchlist | lam -s "bp/" - | 823 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 824 2>${STATSREDIR} | fetch_progress_percent $patchcnt 825 echo "done." 826 827# Attempt to apply ports patches 828 PATCHCNT=`wc -l patchlist` 829 echo "Applying patches... " 830 local oldifs="$IFS" IFS='|' 831 I=0 832 while read X Y; do 833 I=$(($I + 1)) 834 F="${X}-${Y}" 835 if [ ! -f "${F}" ]; then 836 printf " Skipping ${F} (${I} of ${PATCHCNT}).\r" 837 continue; 838 fi 839 echo " Processing ${F}..." 1>${QUIETREDIR} 840 gunzip -c < files/${X}.gz > OLD 841 ${BSPATCH} OLD NEW ${X}-${Y} 842 if [ `${SHA256} -q NEW` = ${Y} ]; then 843 mv NEW files/${Y} 844 gzip -n files/${Y} 845 fi 846 rm -f diff OLD NEW ${X}-${Y} 847 done < patchlist 2>${QUIETREDIR} 848 IFS="$oldifs" 849 echo "done." 850 851# Update ports without patches 852 join -t '|' -v 2 INDEX INDEX.new | 853 cut -f 2 -d '|' /dev/stdin patchlist | 854 while read Y; do 855 if [ ! -f "files/${Y}.gz" ]; then 856 echo ${Y}; 857 fi 858 done > filelist 859 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 860 echo ${NDEBUG} "new ports or files... " 861 lam -s "f/" - -s ".gz" < filelist | 862 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 863 2>${QUIETREDIR} 864 865 I=0 866 while read Y; do 867 I=$(($I + 1)) 868 printf " Processing ${Y} (${I} of ${PATCHCNT}).\r" 1>${QUIETREDIR} 869 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 870 mv ${Y}.gz files/${Y}.gz 871 else 872 echo "snapshot is corrupt." 873 return 1 874 fi 875 done < filelist 876 echo "done." 877 878# Remove files which are no longer needed 879 cut -f 2 -d '|' tINDEX INDEX | sort -u > oldfiles 880 cut -f 2 -d '|' tINDEX.new INDEX.new | sort -u | comm -13 - oldfiles | 881 lam -s "files/" - -s ".gz" | xargs rm -f 882 rm patchlist filelist oldfiles 883 884# We're done! 885 mv INDEX.new INDEX 886 mv tINDEX.new tINDEX 887 mv tag.new tag 888 889 return 0 890} 891 892# Do the actual work involved in "fetch" / "cron". 893fetch_run() { 894 fetch_pick_server_init && fetch_pick_server 895 896 while ! fetch_key; do 897 fetch_pick_server || return 1 898 done 899 900 if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then 901 fetch_snapshot || return 1 902 fi 903 fetch_update || return 1 904} 905 906# Build a ports INDEX file 907extract_make_index() { 908 if ! look $1 ${WORKDIR}/tINDEX > /dev/null; then 909 echo -n "$1 not provided by portsnap server; " 910 echo "$2 not being generated." 911 else 912 gunzip -c < "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX | 913 cut -f 2 -d '|'`.gz" | 914 cat - ${LOCALDESC} | 915 ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2 916 fi 917} 918 919# Create INDEX, INDEX-5, INDEX-6 920extract_indices() { 921 echo -n "Building new INDEX files... " 922 for PAIR in ${INDEXPAIRS}; do 923 INDEXFILE=`echo ${PAIR} | cut -f 1 -d '|'` 924 DESCRIBEFILE=`echo ${PAIR} | cut -f 2 -d '|'` 925 extract_make_index ${DESCRIBEFILE} ${INDEXFILE} || return 1 926 done 927 echo "done." 928} 929 930# Create .portsnap.INDEX; if we are REFUSEing to touch certain directories, 931# merge the values from any exiting .portsnap.INDEX file. 932extract_metadata() { 933 if [ -z "${REFUSE}" ]; then 934 sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX 935 elif [ -f ${PORTSDIR}/.portsnap.INDEX ]; then 936 grep -E "${REFUSE}" ${PORTSDIR}/.portsnap.INDEX \ 937 > ${PORTSDIR}/.portsnap.INDEX.tmp 938 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort | 939 sort -m - ${PORTSDIR}/.portsnap.INDEX.tmp \ 940 > ${PORTSDIR}/.portsnap.INDEX 941 rm -f ${PORTSDIR}/.portsnap.INDEX.tmp 942 else 943 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort \ 944 > ${PORTSDIR}/.portsnap.INDEX 945 fi 946} 947 948# Do the actual work involved in "extract" 949extract_run() { 950 local oldifs="$IFS" IFS='|' 951 mkdir -p ${PORTSDIR} || return 1 952 953 if ! 954 if ! [ -z "${EXTRACTPATH}" ]; then 955 grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX 956 elif ! [ -z "${REFUSE}" ]; then 957 grep -vE "${REFUSE}" ${WORKDIR}/INDEX 958 else 959 cat ${WORKDIR}/INDEX 960 fi | while read FILE HASH; do 961 echo ${PORTSDIR}/${FILE} 962 if ! [ -s "${WORKDIR}/files/${HASH}.gz" ]; then 963 echo "files/${HASH}.gz not found -- snapshot corrupt." 964 return 1 965 fi 966 case ${FILE} in 967 */) 968 rm -rf ${PORTSDIR}/${FILE%/} 969 mkdir -p ${PORTSDIR}/${FILE} 970 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 971 -C ${PORTSDIR}/${FILE} 972 ;; 973 *) 974 rm -f ${PORTSDIR}/${FILE} 975 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 976 -C ${PORTSDIR} ${FILE} 977 ;; 978 esac 979 done; then 980 return 1 981 fi 982 if [ ! -z "${EXTRACTPATH}" ]; then 983 return 0; 984 fi 985 986 IFS="$oldifs" 987 988 extract_metadata 989 extract_indices 990} 991 992update_run_extract() { 993 local IFS='|' 994 995# Install new files 996 echo "Extracting new files:" 997 if ! 998 if ! [ -z "${REFUSE}" ]; then 999 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort 1000 else 1001 sort ${WORKDIR}/INDEX 1002 fi | 1003 comm -13 ${PORTSDIR}/.portsnap.INDEX - | 1004 while read FILE HASH; do 1005 echo ${PORTSDIR}/${FILE} 1006 if ! [ -s "${WORKDIR}/files/${HASH}.gz" ]; then 1007 echo "files/${HASH}.gz not found -- snapshot corrupt." 1008 return 1 1009 fi 1010 case ${FILE} in 1011 */) 1012 mkdir -p ${PORTSDIR}/${FILE} 1013 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 1014 -C ${PORTSDIR}/${FILE} 1015 ;; 1016 *) 1017 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 1018 -C ${PORTSDIR} ${FILE} 1019 ;; 1020 esac 1021 done; then 1022 return 1 1023 fi 1024} 1025 1026# Do the actual work involved in "update" 1027update_run() { 1028 if ! [ -z "${INDEXONLY}" ]; then 1029 extract_indices >/dev/null || return 1 1030 return 0 1031 fi 1032 1033 if sort ${WORKDIR}/INDEX | 1034 cmp -s ${PORTSDIR}/.portsnap.INDEX -; then 1035 echo "Ports tree is already up to date." 1036 return 0 1037 fi 1038 1039# If we are REFUSEing to touch certain directories, don't remove files 1040# from those directories (even if they are out of date) 1041 echo -n "Removing old files and directories... " 1042 if ! [ -z "${REFUSE}" ]; then 1043 sort ${WORKDIR}/INDEX | 1044 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 1045 grep -vE "${REFUSE}" | 1046 lam -s "${PORTSDIR}/" - | 1047 sed -e 's|/$||' | xargs rm -rf 1048 else 1049 sort ${WORKDIR}/INDEX | 1050 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 1051 lam -s "${PORTSDIR}/" - | 1052 sed -e 's|/$||' | xargs rm -rf 1053 fi 1054 echo "done." 1055 1056 update_run_extract || return 1 1057 extract_metadata 1058 extract_indices 1059} 1060 1061#### Main functions -- call parameter-handling and core functions 1062 1063# Using the command line, configuration file, and defaults, 1064# set all the parameters which are needed later. 1065get_params() { 1066 init_params 1067 parse_cmdline $@ 1068 sanity_conffile 1069 default_conffile 1070 parse_conffile 1071 default_params 1072} 1073 1074# Fetch command. Make sure that we're being called 1075# interactively, then run fetch_check_params and fetch_run 1076cmd_fetch() { 1077 if [ "${INTERACTIVE}" != "YES" ]; then 1078 echo -n "`basename $0` fetch should not " 1079 echo "be run non-interactively." 1080 echo "Run `basename $0` cron instead" 1081 exit 1 1082 fi 1083 fetch_check_params 1084 fetch_run || exit 1 1085} 1086 1087# Cron command. Make sure the parameters are sensible; wait 1088# rand(3600) seconds; then fetch updates. While fetching updates, 1089# send output to a temporary file; only print that file if the 1090# fetching failed. 1091cmd_cron() { 1092 fetch_check_params 1093 sleep `jot -r 1 0 3600` 1094 1095 TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1 1096 if ! fetch_run >> ${TMPFILE}; then 1097 cat ${TMPFILE} 1098 rm ${TMPFILE} 1099 exit 1 1100 fi 1101 1102 rm ${TMPFILE} 1103} 1104 1105# Extract command. Make sure the parameters are sensible, 1106# then extract the ports tree (or part thereof). 1107cmd_extract() { 1108 extract_check_params 1109 extract_run || exit 1 1110} 1111 1112# Update command. Make sure the parameters are sensible, 1113# then update the ports tree. 1114cmd_update() { 1115 update_check_params 1116 update_run || exit 1 1117} 1118 1119# Auto command. Run 'fetch' or 'cron' depending on 1120# whether stdin is a terminal; then run 'update' or 1121# 'extract' depending on whether ${PORTSDIR} exists. 1122cmd_auto() { 1123 if [ "${INTERACTIVE}" = "YES" ]; then 1124 cmd_fetch 1125 else 1126 cmd_cron 1127 fi 1128 if [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 1129 cmd_update 1130 else 1131 cmd_extract 1132 fi 1133} 1134 1135#### Entry point 1136 1137# Make sure we find utilities from the base system 1138export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH} 1139 1140# Set LC_ALL in order to avoid problems with character ranges like [A-Z]. 1141export LC_ALL=C 1142 1143get_params $@ 1144for COMMAND in ${COMMANDS}; do 1145 cmd_${COMMAND} 1146done 1147