portsnap.sh revision 305649
1#!/bin/sh 2 3#- 4# Copyright 2004-2005 Colin Percival 5# All rights reserved 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted providing that the following conditions 9# are met: 10# 1. Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# 2. Redistributions in binary form must reproduce the above copyright 13# notice, this list of conditions and the following disclaimer in the 14# documentation and/or other materials provided with the distribution. 15# 16# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 20# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26# POSSIBILITY OF SUCH DAMAGE. 27 28# $FreeBSD: stable/10/usr.sbin/portsnap/portsnap/portsnap.sh 305649 2016-09-09 06:59:49Z delphij $ 29 30#### Usage function -- called from command-line handling code. 31 32# Usage instructions. Options not listed: 33# --debug -- don't filter output from utilities 34# --no-stats -- don't show progress statistics while fetching files 35usage() { 36 cat <<EOF 37usage: `basename $0` [options] command ... [path] 38 39Options: 40 -d workdir -- Store working files in workdir 41 (default: /var/db/portsnap/) 42 -f conffile -- Read configuration options from conffile 43 (default: /etc/portsnap.conf) 44 -I -- Update INDEX only. (update command only) 45 -k KEY -- Trust an RSA key with SHA256 hash of KEY 46 -l descfile -- Merge the specified local describes file into the INDEX. 47 -p portsdir -- Location of uncompressed ports tree 48 (default: /usr/ports/) 49 -s server -- Server from which to fetch updates. 50 (default: portsnap.FreeBSD.org) 51 --interactive -- interactive: override auto-detection of calling process 52 (use this when calling portsnap from an interactive, non- 53 terminal application AND NEVER ELSE). 54 path -- Extract only parts of the tree starting with the given 55 string. (extract command only) 56Commands: 57 fetch -- Fetch a compressed snapshot of the ports tree, 58 or update an existing snapshot. 59 cron -- Sleep rand(3600) seconds, and then fetch updates. 60 extract -- Extract snapshot of ports tree, replacing existing 61 files and directories. 62 update -- Update ports tree to match current snapshot, replacing 63 files and directories which have changed. 64EOF 65 exit 0 66} 67 68#### Parameter handling functions. 69 70# Initialize parameters to null, just in case they're 71# set in the environment. 72init_params() { 73 KEYPRINT="" 74 EXTRACTPATH="" 75 WORKDIR="" 76 PORTSDIR="" 77 CONFFILE="" 78 COMMAND="" 79 COMMANDS="" 80 QUIETREDIR="" 81 QUIETFLAG="" 82 STATSREDIR="" 83 XARGST="" 84 NDEBUG="" 85 DDSTATS="" 86 INDEXONLY="" 87 SERVERNAME="" 88 REFUSE="" 89 LOCALDESC="" 90 INTERACTIVE="" 91} 92 93# Parse the command line 94parse_cmdline() { 95 while [ $# -gt 0 ]; do 96 case "$1" in 97 -d) 98 if [ $# -eq 1 ]; then usage; fi 99 if [ ! -z "${WORKDIR}" ]; then usage; fi 100 shift; WORKDIR="$1" 101 ;; 102 --debug) 103 QUIETREDIR="/dev/stderr" 104 STATSREDIR="/dev/stderr" 105 QUIETFLAG=" " 106 NDEBUG=" " 107 XARGST="-t" 108 DDSTATS=".." 109 ;; 110 --interactive) 111 INTERACTIVE="YES" 112 ;; 113 -f) 114 if [ $# -eq 1 ]; then usage; fi 115 if [ ! -z "${CONFFILE}" ]; then usage; fi 116 shift; CONFFILE="$1" 117 ;; 118 -h | --help | help) 119 usage 120 ;; 121 -I) 122 INDEXONLY="YES" 123 ;; 124 -k) 125 if [ $# -eq 1 ]; then usage; fi 126 if [ ! -z "${KEYPRINT}" ]; then usage; fi 127 shift; KEYPRINT="$1" 128 ;; 129 -l) 130 if [ $# -eq 1 ]; then usage; fi 131 if [ ! -z "${LOCALDESC}" ]; then usage; fi 132 shift; LOCALDESC="$1" 133 ;; 134 --no-stats) 135 if [ -z "${STATSREDIR}" ]; then 136 STATSREDIR="/dev/null" 137 DDSTATS=".. " 138 fi 139 ;; 140 -p) 141 if [ $# -eq 1 ]; then usage; fi 142 if [ ! -z "${PORTSDIR}" ]; then usage; fi 143 shift; PORTSDIR="$1" 144 ;; 145 -s) 146 if [ $# -eq 1 ]; then usage; fi 147 if [ ! -z "${SERVERNAME}" ]; then usage; fi 148 shift; SERVERNAME="$1" 149 ;; 150 cron | extract | fetch | update | alfred) 151 COMMANDS="${COMMANDS} $1" 152 ;; 153 up) 154 COMMANDS="${COMMANDS} update" 155 ;; 156 *) 157 if [ $# -gt 1 ]; then usage; fi 158 if echo ${COMMANDS} | grep -vq extract; then 159 usage 160 fi 161 EXTRACTPATH="$1" 162 ;; 163 esac 164 shift 165 done 166 167 if [ -z "${COMMANDS}" ]; then 168 usage 169 fi 170} 171 172# If CONFFILE was specified at the command-line, make 173# sure that it exists and is readable. 174sanity_conffile() { 175 if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then 176 echo -n "File does not exist " 177 echo -n "or is not readable: " 178 echo ${CONFFILE} 179 exit 1 180 fi 181} 182 183# If a configuration file hasn't been specified, use 184# the default value (/etc/portsnap.conf) 185default_conffile() { 186 if [ -z "${CONFFILE}" ]; then 187 CONFFILE="/etc/portsnap.conf" 188 fi 189} 190 191# Read {KEYPRINT, SERVERNAME, WORKDIR, PORTSDIR} from the configuration 192# file if they haven't already been set. If the configuration 193# file doesn't exist, do nothing. 194# Also read REFUSE (which cannot be set via the command line) if it is 195# present in the configuration file. 196parse_conffile() { 197 if [ -r "${CONFFILE}" ]; then 198 for X in KEYPRINT WORKDIR PORTSDIR SERVERNAME; do 199 eval _=\$${X} 200 if [ -z "${_}" ]; then 201 eval ${X}=`grep "^${X}=" "${CONFFILE}" | 202 cut -f 2- -d '=' | tail -1` 203 fi 204 done 205 206 if grep -qE "^REFUSE[[:space:]]" ${CONFFILE}; then 207 REFUSE="^(` 208 grep -E "^REFUSE[[:space:]]" "${CONFFILE}" | 209 cut -c 7- | xargs echo | tr ' ' '|' 210 `)" 211 fi 212 213 if grep -qE "^INDEX[[:space:]]" ${CONFFILE}; then 214 INDEXPAIRS="` 215 grep -E "^INDEX[[:space:]]" "${CONFFILE}" | 216 cut -c 7- | tr ' ' '|' | xargs echo`" 217 fi 218 fi 219} 220 221# If parameters have not been set, use default values 222default_params() { 223 _QUIETREDIR="/dev/null" 224 _QUIETFLAG="-q" 225 _STATSREDIR="/dev/stdout" 226 _WORKDIR="/var/db/portsnap" 227 _PORTSDIR="/usr/ports" 228 _NDEBUG="-n" 229 _LOCALDESC="/dev/null" 230 for X in QUIETREDIR QUIETFLAG STATSREDIR WORKDIR PORTSDIR \ 231 NDEBUG LOCALDESC; do 232 eval _=\$${X} 233 eval __=\$_${X} 234 if [ -z "${_}" ]; then 235 eval ${X}=${__} 236 fi 237 done 238 if [ -z "${INTERACTIVE}" ]; then 239 if [ -t 0 ]; then 240 INTERACTIVE="YES" 241 else 242 INTERACTIVE="NO" 243 fi 244 fi 245} 246 247# Perform sanity checks and set some final parameters 248# in preparation for fetching files. Also chdir into 249# the working directory. 250fetch_check_params() { 251 export HTTP_USER_AGENT="portsnap (${COMMAND}, `uname -r`)" 252 253 _SERVERNAME_z=\ 254"SERVERNAME must be given via command line or configuration file." 255 _KEYPRINT_z="Key must be given via -k option or configuration file." 256 _KEYPRINT_bad="Invalid key fingerprint: " 257 _WORKDIR_bad="Directory does not exist or is not writable: " 258 259 if [ -z "${SERVERNAME}" ]; then 260 echo -n "`basename $0`: " 261 echo "${_SERVERNAME_z}" 262 exit 1 263 fi 264 if [ -z "${KEYPRINT}" ]; then 265 echo -n "`basename $0`: " 266 echo "${_KEYPRINT_z}" 267 exit 1 268 fi 269 if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then 270 echo -n "`basename $0`: " 271 echo -n "${_KEYPRINT_bad}" 272 echo ${KEYPRINT} 273 exit 1 274 fi 275 if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then 276 echo -n "`basename $0`: " 277 echo -n "${_WORKDIR_bad}" 278 echo ${WORKDIR} 279 exit 1 280 fi 281 cd ${WORKDIR} || exit 1 282 283 BSPATCH=/usr/bin/bspatch 284 SHA256=/sbin/sha256 285 PHTTPGET=/usr/libexec/phttpget 286} 287 288# Perform sanity checks and set some final parameters 289# in preparation for extracting or updating ${PORTSDIR} 290# Complain if ${PORTSDIR} exists but is not writable, 291# but don't complain if ${PORTSDIR} doesn't exist. 292extract_check_params() { 293 _WORKDIR_bad="Directory does not exist: " 294 _PORTSDIR_bad="Directory is not writable: " 295 296 if ! [ -d "${WORKDIR}" ]; then 297 echo -n "`basename $0`: " 298 echo -n "${_WORKDIR_bad}" 299 echo ${WORKDIR} 300 exit 1 301 fi 302 if [ -d "${PORTSDIR}" ] && ! [ -w "${PORTSDIR}" ]; then 303 echo -n "`basename $0`: " 304 echo -n "${_PORTSDIR_bad}" 305 echo ${PORTSDIR} 306 exit 1 307 fi 308 309 if ! [ -d "${WORKDIR}/files" -a -r "${WORKDIR}/tag" \ 310 -a -r "${WORKDIR}/INDEX" -a -r "${WORKDIR}/tINDEX" ]; then 311 echo "No snapshot available. Try running" 312 echo "# `basename $0` fetch" 313 exit 1 314 fi 315 316 MKINDEX=/usr/libexec/make_index 317} 318 319# Perform sanity checks and set some final parameters 320# in preparation for updating ${PORTSDIR} 321update_check_params() { 322 extract_check_params 323 324 if ! [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 325 echo "${PORTSDIR} was not created by portsnap." 326 echo -n "You must run '`basename $0` extract' before " 327 echo "running '`basename $0` update'." 328 exit 1 329 fi 330 331} 332 333#### Core functionality -- the actual work gets done here 334 335# Use an SRV query to pick a server. If the SRV query doesn't provide 336# a useful answer, use the server name specified by the user. 337# Put another way... look up _http._tcp.${SERVERNAME} and pick a server 338# from that; or if no servers are returned, use ${SERVERNAME}. 339# This allows a user to specify "portsnap.freebsd.org" (in which case 340# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org" 341# (in which case portsnap will use that particular server, since there 342# won't be an SRV entry for that name). 343# 344# We ignore the Port field, since we are always going to use port 80. 345 346# Fetch the mirror list, but do not pick a mirror yet. Returns 1 if 347# no mirrors are available for any reason. 348fetch_pick_server_init() { 349 : > serverlist_tried 350 351# Check that host(1) exists (i.e., that the system wasn't built with the 352# WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist. 353 if ! which -s host; then 354 : > serverlist_full 355 return 1 356 fi 357 358 echo -n "Looking up ${SERVERNAME} mirrors... " 359 360# Issue the SRV query and pull out the Priority, Weight, and Target fields. 361# BIND 9 prints "$name has SRV record ..." while BIND 8 prints 362# "$name server selection ..."; we allow either format. 363 MLIST="_http._tcp.${SERVERNAME}" 364 host -t srv "${MLIST}" | 365 sed -nE "s/${MLIST} (has SRV record|server selection) //Ip" | 366 cut -f 1,2,4 -d ' ' | 367 sed -e 's/\.$//' | 368 sort > serverlist_full 369 370# If no records, give up -- we'll just use the server name we were given. 371 if [ `wc -l < serverlist_full` -eq 0 ]; then 372 echo "none found." 373 return 1 374 fi 375 376# Report how many mirrors we found. 377 echo `wc -l < serverlist_full` "mirrors found." 378 379# Generate a random seed for use in picking mirrors. If HTTP_PROXY 380# is set, this will be used to generate the seed; otherwise, the seed 381# will be random. 382 if [ -n "${HTTP_PROXY}${http_proxy}" ]; then 383 RANDVALUE=`sha256 -qs "${HTTP_PROXY}${http_proxy}" | 384 tr -d 'a-f' | 385 cut -c 1-9` 386 else 387 RANDVALUE=`jot -r 1 0 999999999` 388 fi 389} 390 391# Pick a mirror. Returns 1 if we have run out of mirrors to try. 392fetch_pick_server() { 393# Generate a list of not-yet-tried mirrors 394 sort serverlist_tried | 395 comm -23 serverlist_full - > serverlist 396 397# Have we run out of mirrors? 398 if [ `wc -l < serverlist` -eq 0 ]; then 399 echo "No mirrors remaining, giving up." 400 return 1 401 fi 402 403# Find the highest priority level (lowest numeric value). 404 SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1` 405 406# Add up the weights of the response lines at that priority level. 407 SRV_WSUM=0; 408 while read X; do 409 case "$X" in 410 ${SRV_PRIORITY}\ *) 411 SRV_W=`echo $X | cut -f 2 -d ' '` 412 SRV_WSUM=$(($SRV_WSUM + $SRV_W)) 413 ;; 414 esac 415 done < serverlist 416 417# If all the weights are 0, pretend that they are all 1 instead. 418 if [ ${SRV_WSUM} -eq 0 ]; then 419 SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l` 420 SRV_W_ADD=1 421 else 422 SRV_W_ADD=0 423 fi 424 425# Pick a value between 0 and the sum of the weights - 1 426 SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}` 427 428# Read through the list of mirrors and set SERVERNAME. Write the line 429# corresponding to the mirror we selected into serverlist_tried so that 430# we won't try it again. 431 while read X; do 432 case "$X" in 433 ${SRV_PRIORITY}\ *) 434 SRV_W=`echo $X | cut -f 2 -d ' '` 435 SRV_W=$(($SRV_W + $SRV_W_ADD)) 436 if [ $SRV_RND -lt $SRV_W ]; then 437 SERVERNAME=`echo $X | cut -f 3 -d ' '` 438 echo "$X" >> serverlist_tried 439 break 440 else 441 SRV_RND=$(($SRV_RND - $SRV_W)) 442 fi 443 ;; 444 esac 445 done < serverlist 446} 447 448# Check that we have a public key with an appropriate hash, or 449# fetch the key if it doesn't exist. Returns 1 if the key has 450# not yet been fetched. 451fetch_key() { 452 if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 453 return 0 454 fi 455 456 echo -n "Fetching public key from ${SERVERNAME}... " 457 rm -f pub.ssl 458 fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \ 459 2>${QUIETREDIR} || true 460 if ! [ -r pub.ssl ]; then 461 echo "failed." 462 return 1 463 fi 464 if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 465 echo "key has incorrect hash." 466 rm -f pub.ssl 467 return 1 468 fi 469 echo "done." 470} 471 472# Fetch a snapshot tag 473fetch_tag() { 474 rm -f snapshot.ssl tag.new 475 476 echo ${NDEBUG} "Fetching snapshot tag from ${SERVERNAME}... " 477 fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl \ 478 2>${QUIETREDIR} || true 479 if ! [ -r $1.ssl ]; then 480 echo "failed." 481 return 1 482 fi 483 484 openssl rsautl -pubin -inkey pub.ssl -verify \ 485 < $1.ssl > tag.new 2>${QUIETREDIR} || true 486 rm $1.ssl 487 488 if ! [ `wc -l < tag.new` = 1 ] || 489 ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then 490 echo "invalid snapshot tag." 491 return 1 492 fi 493 494 echo "done." 495 496 SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new` 497 SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new` 498} 499 500# Sanity-check the date on a snapshot tag 501fetch_snapshot_tagsanity() { 502 if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then 503 echo "Snapshot appears to be more than a year old!" 504 echo "(Is the system clock correct?)" 505 echo "Cowardly refusing to proceed any further." 506 return 1 507 fi 508 if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then 509 echo -n "Snapshot appears to have been created more than " 510 echo "one day into the future!" 511 echo "(Is the system clock correct?)" 512 echo "Cowardly refusing to proceed any further." 513 return 1 514 fi 515} 516 517# Sanity-check the date on a snapshot update tag 518fetch_update_tagsanity() { 519 fetch_snapshot_tagsanity || return 1 520 521 if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then 522 echo -n "Latest snapshot on server is " 523 echo "older than what we already have!" 524 echo -n "Cowardly refusing to downgrade from " 525 date -r ${OLDSNAPSHOTDATE} 526 echo "to `date -r ${SNAPSHOTDATE}`." 527 return 1 528 fi 529} 530 531# Compare old and new tags; return 1 if update is unnecessary 532fetch_update_neededp() { 533 if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then 534 echo -n "Latest snapshot on server matches " 535 echo "what we already have." 536 echo "No updates needed." 537 rm tag.new 538 return 1 539 fi 540 if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then 541 echo -n "Ports tree hasn't changed since " 542 echo "last snapshot." 543 echo "No updates needed." 544 rm tag.new 545 return 1 546 fi 547 548 return 0 549} 550 551# Fetch snapshot metadata file 552fetch_metadata() { 553 rm -f ${SNAPSHOTHASH} tINDEX.new 554 555 echo ${NDEBUG} "Fetching snapshot metadata... " 556 fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} \ 557 2>${QUIETREDIR} || return 558 if [ "`${SHA256} -q ${SNAPSHOTHASH}`" != ${SNAPSHOTHASH} ]; then 559 echo "snapshot metadata corrupt." 560 return 1 561 fi 562 mv ${SNAPSHOTHASH} tINDEX.new 563 echo "done." 564} 565 566# Warn user about bogus metadata 567fetch_metadata_freakout() { 568 echo 569 echo "Portsnap metadata is correctly signed, but contains" 570 echo "at least one line which appears bogus." 571 echo "Cowardly refusing to proceed any further." 572} 573 574# Sanity-check a snapshot metadata file 575fetch_metadata_sanity() { 576 if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then 577 fetch_metadata_freakout 578 return 1 579 fi 580 if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then 581 echo 582 echo "Portsnap metadata appears bogus." 583 echo "Cowardly refusing to proceed any further." 584 return 1 585 fi 586} 587 588# Take a list of ${oldhash}|${newhash} and output a list of needed patches 589fetch_make_patchlist() { 590 local IFS='|' 591 echo "" 1>${QUIETREDIR} 592 grep -vE "^([0-9a-f]{64})\|\1$" | 593 while read X Y; do 594 printf "Processing: $X $Y ...\r" 1>${QUIETREDIR} 595 if [ -f "files/${Y}.gz" -o ! -f "files/${X}.gz" ]; then continue; fi 596 echo "${X}|${Y}" 597 done 598 echo "" 1>${QUIETREDIR} 599} 600 601# Print user-friendly progress statistics 602fetch_progress() { 603 LNC=0 604 while read x; do 605 LNC=$(($LNC + 1)) 606 if [ $(($LNC % 10)) = 0 ]; then 607 echo -n $LNC 608 elif [ $(($LNC % 2)) = 0 ]; then 609 echo -n . 610 fi 611 done 612 echo -n " " 613} 614 615pct_fmt() 616{ 617 printf " \r" 618 printf "($1/$2) %02.2f%% " `echo "scale=4;$LNC / $TOTAL * 100"|bc` 619} 620 621fetch_progress_percent() { 622 TOTAL=$1 623 LNC=0 624 pct_fmt $LNC $TOTAL 625 while read x; do 626 LNC=$(($LNC + 1)) 627 if [ $(($LNC % 100)) = 0 ]; then 628 pct_fmt $LNC $TOTAL 629 elif [ $(($LNC % 10)) = 0 ]; then 630 echo -n . 631 fi 632 done 633 pct_fmt $LNC $TOTAL 634 echo " done. " 635} 636 637# Sanity-check an index file 638fetch_index_sanity() { 639 if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new || 640 fgrep -q "./" INDEX.new; then 641 fetch_metadata_freakout 642 return 1 643 fi 644} 645 646# Verify a list of files 647fetch_snapshot_verify() { 648 while read F; do 649 if [ "`gunzip -c < snap/${F}.gz | ${SHA256} -q`" != ${F} ]; then 650 echo "snapshot corrupt." 651 return 1 652 fi 653 done 654 return 0 655} 656 657# Fetch a snapshot tarball, extract, and verify. 658fetch_snapshot() { 659 while ! fetch_tag snapshot; do 660 fetch_pick_server || return 1 661 done 662 fetch_snapshot_tagsanity || return 1 663 fetch_metadata || return 1 664 fetch_metadata_sanity || return 1 665 666 rm -rf snap/ 667 668# Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will 669# probably take a while, so the progrees reports that fetch(1) generates 670# will be useful for keeping the users' attention from drifting. 671 echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:" 672 fetch -r http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1 673 674 echo -n "Extracting snapshot... " 675 tar -xz --numeric-owner -f ${SNAPSHOTHASH}.tgz snap/ || return 1 676 rm ${SNAPSHOTHASH}.tgz 677 echo "done." 678 679 echo -n "Verifying snapshot integrity... " 680# Verify the metadata files 681 cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1 682# Extract the index 683 rm -f INDEX.new 684 gunzip -c < snap/`look INDEX tINDEX.new | 685 cut -f 2 -d '|'`.gz > INDEX.new 686 fetch_index_sanity || return 1 687# Verify the snapshot contents 688 cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1 689 echo "done." 690 691# Move files into their proper locations 692 rm -f tag INDEX tINDEX 693 rm -rf files 694 mv tag.new tag 695 mv tINDEX.new tINDEX 696 mv INDEX.new INDEX 697 mv snap/ files/ 698 699 return 0 700} 701 702# Update a compressed snapshot 703fetch_update() { 704 rm -f patchlist diff OLD NEW filelist INDEX.new 705 706 OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag` 707 OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag` 708 709 while ! fetch_tag latest; do 710 fetch_pick_server || return 1 711 done 712 fetch_update_tagsanity || return 1 713 fetch_update_neededp || return 0 714 fetch_metadata || return 1 715 fetch_metadata_sanity || return 1 716 717 echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` " 718 echo "to `date -r ${SNAPSHOTDATE}`." 719 720# Generate a list of wanted metadata patches 721 join -t '|' -o 1.2,2.2 tINDEX tINDEX.new | 722 fetch_make_patchlist > patchlist 723 724# Attempt to fetch metadata patches 725 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 726 echo ${NDEBUG} "metadata patches.${DDSTATS}" 727 tr '|' '-' < patchlist | 728 lam -s "tp/" - -s ".gz" | 729 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 730 2>${STATSREDIR} | fetch_progress 731 echo "done." 732 733# Attempt to apply metadata patches 734 echo -n "Applying metadata patches... " 735 local oldifs="$IFS" IFS='|' 736 while read X Y; do 737 if [ ! -f "${X}-${Y}.gz" ]; then continue; fi 738 gunzip -c < ${X}-${Y}.gz > diff 739 gunzip -c < files/${X}.gz > OLD 740 cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp 741 grep '^\+' diff | cut -c 2- | 742 sort -k 1,1 -t '|' -m - ptmp > NEW 743 if [ `${SHA256} -q NEW` = ${Y} ]; then 744 mv NEW files/${Y} 745 gzip -n files/${Y} 746 fi 747 rm -f diff OLD NEW ${X}-${Y}.gz ptmp 748 done < patchlist 2>${QUIETREDIR} 749 IFS="$oldifs" 750 echo "done." 751 752# Update metadata without patches 753 join -t '|' -v 2 tINDEX tINDEX.new | 754 cut -f 2 -d '|' /dev/stdin patchlist | 755 while read Y; do 756 if [ ! -f "files/${Y}.gz" ]; then 757 echo ${Y}; 758 fi 759 done > filelist 760 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 761 echo ${NDEBUG} "metadata files... " 762 lam -s "f/" - -s ".gz" < filelist | 763 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 764 2>${QUIETREDIR} 765 766 while read Y; do 767 echo -n "Verifying ${Y}... " 1>${QUIETREDIR} 768 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 769 mv ${Y}.gz files/${Y}.gz 770 else 771 echo "metadata is corrupt." 772 return 1 773 fi 774 echo "ok." 1>${QUIETREDIR} 775 done < filelist 776 echo "done." 777 778# Extract the index 779 echo -n "Extracting index... " 1>${QUIETREDIR} 780 gunzip -c < files/`look INDEX tINDEX.new | 781 cut -f 2 -d '|'`.gz > INDEX.new 782 fetch_index_sanity || return 1 783 784# If we have decided to refuse certain updates, construct a hybrid index which 785# is equal to the old index for parts of the tree which we don't want to 786# update, and equal to the new index for parts of the tree which gets updates. 787# This means that we should always have a "complete snapshot" of the ports 788# tree -- with the caveat that it isn't actually a snapshot. 789 if [ ! -z "${REFUSE}" ]; then 790 echo "Refusing to download updates for ${REFUSE}" \ 791 >${QUIETREDIR} 792 793 grep -Ev "${REFUSE}" INDEX.new > INDEX.tmp 794 grep -E "${REFUSE}" INDEX | 795 sort -m -k 1,1 -t '|' - INDEX.tmp > INDEX.new 796 rm -f INDEX.tmp 797 fi 798 799# Generate a list of wanted ports patches 800 echo -n "Generating list of wanted patches..." 1>${QUIETREDIR} 801 join -t '|' -o 1.2,2.2 INDEX INDEX.new | 802 fetch_make_patchlist > patchlist 803 echo " done." 1>${QUIETREDIR} 804 805# Attempt to fetch ports patches 806 patchcnt=`wc -l < patchlist | tr -d ' '` 807 echo -n "Fetching $patchcnt " 808 echo ${NDEBUG} "patches.${DDSTATS}" 809 echo " " 810 tr '|' '-' < patchlist | lam -s "bp/" - | 811 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 812 2>${STATSREDIR} | fetch_progress_percent $patchcnt 813 echo "done." 814 815# Attempt to apply ports patches 816 PATCHCNT=`wc -l patchlist` 817 echo "Applying patches... " 818 local oldifs="$IFS" IFS='|' 819 I=0 820 while read X Y; do 821 I=$(($I + 1)) 822 F="${X}-${Y}" 823 if [ ! -f "${F}" ]; then 824 printf " Skipping ${F} (${I} of ${PATCHCNT}).\r" 825 continue; 826 fi 827 echo " Processing ${F}..." 1>${QUIETREDIR} 828 gunzip -c < files/${X}.gz > OLD 829 ${BSPATCH} OLD NEW ${X}-${Y} 830 if [ `${SHA256} -q NEW` = ${Y} ]; then 831 mv NEW files/${Y} 832 gzip -n files/${Y} 833 fi 834 rm -f diff OLD NEW ${X}-${Y} 835 done < patchlist 2>${QUIETREDIR} 836 IFS="$oldifs" 837 echo "done." 838 839# Update ports without patches 840 join -t '|' -v 2 INDEX INDEX.new | 841 cut -f 2 -d '|' /dev/stdin patchlist | 842 while read Y; do 843 if [ ! -f "files/${Y}.gz" ]; then 844 echo ${Y}; 845 fi 846 done > filelist 847 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 848 echo ${NDEBUG} "new ports or files... " 849 lam -s "f/" - -s ".gz" < filelist | 850 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 851 2>${QUIETREDIR} 852 853 I=0 854 while read Y; do 855 I=$(($I + 1)) 856 printf " Processing ${Y} (${I} of ${PATCHCNT}).\r" 1>${QUIETREDIR} 857 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 858 mv ${Y}.gz files/${Y}.gz 859 else 860 echo "snapshot is corrupt." 861 return 1 862 fi 863 done < filelist 864 echo "done." 865 866# Remove files which are no longer needed 867 cut -f 2 -d '|' tINDEX INDEX | sort -u > oldfiles 868 cut -f 2 -d '|' tINDEX.new INDEX.new | sort -u | comm -13 - oldfiles | 869 lam -s "files/" - -s ".gz" | xargs rm -f 870 rm patchlist filelist oldfiles 871 872# We're done! 873 mv INDEX.new INDEX 874 mv tINDEX.new tINDEX 875 mv tag.new tag 876 877 return 0 878} 879 880# Do the actual work involved in "fetch" / "cron". 881fetch_run() { 882 fetch_pick_server_init && fetch_pick_server 883 884 while ! fetch_key; do 885 fetch_pick_server || return 1 886 done 887 888 if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then 889 fetch_snapshot || return 1 890 fi 891 fetch_update || return 1 892} 893 894# Build a ports INDEX file 895extract_make_index() { 896 if ! look $1 ${WORKDIR}/tINDEX > /dev/null; then 897 echo -n "$1 not provided by portsnap server; " 898 echo "$2 not being generated." 899 else 900 gunzip -c < "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX | 901 cut -f 2 -d '|'`.gz" | 902 cat - ${LOCALDESC} | 903 ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2 904 fi 905} 906 907# Create INDEX, INDEX-5, INDEX-6 908extract_indices() { 909 echo -n "Building new INDEX files... " 910 for PAIR in ${INDEXPAIRS}; do 911 INDEXFILE=`echo ${PAIR} | cut -f 1 -d '|'` 912 DESCRIBEFILE=`echo ${PAIR} | cut -f 2 -d '|'` 913 extract_make_index ${DESCRIBEFILE} ${INDEXFILE} || return 1 914 done 915 echo "done." 916} 917 918# Create .portsnap.INDEX; if we are REFUSEing to touch certain directories, 919# merge the values from any exiting .portsnap.INDEX file. 920extract_metadata() { 921 if [ -z "${REFUSE}" ]; then 922 sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX 923 elif [ -f ${PORTSDIR}/.portsnap.INDEX ]; then 924 grep -E "${REFUSE}" ${PORTSDIR}/.portsnap.INDEX \ 925 > ${PORTSDIR}/.portsnap.INDEX.tmp 926 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort | 927 sort -m - ${PORTSDIR}/.portsnap.INDEX.tmp \ 928 > ${PORTSDIR}/.portsnap.INDEX 929 rm -f ${PORTSDIR}/.portsnap.INDEX.tmp 930 else 931 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort \ 932 > ${PORTSDIR}/.portsnap.INDEX 933 fi 934} 935 936# Do the actual work involved in "extract" 937extract_run() { 938 local oldifs="$IFS" IFS='|' 939 mkdir -p ${PORTSDIR} || return 1 940 941 if ! 942 if ! [ -z "${EXTRACTPATH}" ]; then 943 grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX 944 elif ! [ -z "${REFUSE}" ]; then 945 grep -vE "${REFUSE}" ${WORKDIR}/INDEX 946 else 947 cat ${WORKDIR}/INDEX 948 fi | while read FILE HASH; do 949 echo ${PORTSDIR}/${FILE} 950 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 951 echo "files/${HASH}.gz not found -- snapshot corrupt." 952 return 1 953 fi 954 case ${FILE} in 955 */) 956 rm -rf ${PORTSDIR}/${FILE%/} 957 mkdir -p ${PORTSDIR}/${FILE} 958 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 959 -C ${PORTSDIR}/${FILE} 960 ;; 961 *) 962 rm -f ${PORTSDIR}/${FILE} 963 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 964 -C ${PORTSDIR} ${FILE} 965 ;; 966 esac 967 done; then 968 return 1 969 fi 970 if [ ! -z "${EXTRACTPATH}" ]; then 971 return 0; 972 fi 973 974 IFS="$oldifs" 975 976 extract_metadata 977 extract_indices 978} 979 980update_run_extract() { 981 local IFS='|' 982 983# Install new files 984 echo "Extracting new files:" 985 if ! 986 if ! [ -z "${REFUSE}" ]; then 987 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort 988 else 989 sort ${WORKDIR}/INDEX 990 fi | 991 comm -13 ${PORTSDIR}/.portsnap.INDEX - | 992 while read FILE HASH; do 993 echo ${PORTSDIR}/${FILE} 994 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 995 echo "files/${HASH}.gz not found -- snapshot corrupt." 996 return 1 997 fi 998 case ${FILE} in 999 */) 1000 mkdir -p ${PORTSDIR}/${FILE} 1001 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 1002 -C ${PORTSDIR}/${FILE} 1003 ;; 1004 *) 1005 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 1006 -C ${PORTSDIR} ${FILE} 1007 ;; 1008 esac 1009 done; then 1010 return 1 1011 fi 1012} 1013 1014# Do the actual work involved in "update" 1015update_run() { 1016 if ! [ -z "${INDEXONLY}" ]; then 1017 extract_indices >/dev/null || return 1 1018 return 0 1019 fi 1020 1021 if sort ${WORKDIR}/INDEX | 1022 cmp -s ${PORTSDIR}/.portsnap.INDEX -; then 1023 echo "Ports tree is already up to date." 1024 return 0 1025 fi 1026 1027# If we are REFUSEing to touch certain directories, don't remove files 1028# from those directories (even if they are out of date) 1029 echo -n "Removing old files and directories... " 1030 if ! [ -z "${REFUSE}" ]; then 1031 sort ${WORKDIR}/INDEX | 1032 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 1033 grep -vE "${REFUSE}" | 1034 lam -s "${PORTSDIR}/" - | 1035 sed -e 's|/$||' | xargs rm -rf 1036 else 1037 sort ${WORKDIR}/INDEX | 1038 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 1039 lam -s "${PORTSDIR}/" - | 1040 sed -e 's|/$||' | xargs rm -rf 1041 fi 1042 echo "done." 1043 1044 update_run_extract || return 1 1045 extract_metadata 1046 extract_indices 1047} 1048 1049#### Main functions -- call parameter-handling and core functions 1050 1051# Using the command line, configuration file, and defaults, 1052# set all the parameters which are needed later. 1053get_params() { 1054 init_params 1055 parse_cmdline $@ 1056 sanity_conffile 1057 default_conffile 1058 parse_conffile 1059 default_params 1060} 1061 1062# Fetch command. Make sure that we're being called 1063# interactively, then run fetch_check_params and fetch_run 1064cmd_fetch() { 1065 if [ "${INTERACTIVE}" != "YES" ]; then 1066 echo -n "`basename $0` fetch should not " 1067 echo "be run non-interactively." 1068 echo "Run `basename $0` cron instead" 1069 exit 1 1070 fi 1071 fetch_check_params 1072 fetch_run || exit 1 1073} 1074 1075# Cron command. Make sure the parameters are sensible; wait 1076# rand(3600) seconds; then fetch updates. While fetching updates, 1077# send output to a temporary file; only print that file if the 1078# fetching failed. 1079cmd_cron() { 1080 fetch_check_params 1081 sleep `jot -r 1 0 3600` 1082 1083 TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1 1084 if ! fetch_run >> ${TMPFILE}; then 1085 cat ${TMPFILE} 1086 rm ${TMPFILE} 1087 exit 1 1088 fi 1089 1090 rm ${TMPFILE} 1091} 1092 1093# Extract command. Make sure the parameters are sensible, 1094# then extract the ports tree (or part thereof). 1095cmd_extract() { 1096 extract_check_params 1097 extract_run || exit 1 1098} 1099 1100# Update command. Make sure the parameters are sensible, 1101# then update the ports tree. 1102cmd_update() { 1103 update_check_params 1104 update_run || exit 1 1105} 1106 1107# Alfred command. Run 'fetch' or 'cron' depending on 1108# whether stdin is a terminal; then run 'update' or 1109# 'extract' depending on whether ${PORTSDIR} exists. 1110cmd_alfred() { 1111 if [ "${INTERACTIVE}" = "YES" ]; then 1112 cmd_fetch 1113 else 1114 cmd_cron 1115 fi 1116 if [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 1117 cmd_update 1118 else 1119 cmd_extract 1120 fi 1121} 1122 1123#### Entry point 1124 1125# Make sure we find utilities from the base system 1126export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH} 1127 1128# Set LC_ALL in order to avoid problems with character ranges like [A-Z]. 1129export LC_ALL=C 1130 1131get_params $@ 1132for COMMAND in ${COMMANDS}; do 1133 cmd_${COMMAND} 1134done 1135