portsnap.sh revision 216575
1#!/bin/sh 2 3#- 4# Copyright 2004-2005 Colin Percival 5# All rights reserved 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted providing that the following conditions 9# are met: 10# 1. Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# 2. Redistributions in binary form must reproduce the above copyright 13# notice, this list of conditions and the following disclaimer in the 14# documentation and/or other materials provided with the distribution. 15# 16# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 20# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26# POSSIBILITY OF SUCH DAMAGE. 27 28# $FreeBSD: head/usr.sbin/portsnap/portsnap/portsnap.sh 216575 2010-12-19 23:09:42Z simon $ 29 30#### Usage function -- called from command-line handling code. 31 32# Usage instructions. Options not listed: 33# --debug -- don't filter output from utilities 34# --no-stats -- don't show progress statistics while fetching files 35usage() { 36 cat <<EOF 37usage: `basename $0` [options] command ... [path] 38 39Options: 40 -d workdir -- Store working files in workdir 41 (default: /var/db/portsnap/) 42 -f conffile -- Read configuration options from conffile 43 (default: /etc/portsnap.conf) 44 -I -- Update INDEX only. (update command only) 45 -k KEY -- Trust an RSA key with SHA256 hash of KEY 46 -l descfile -- Merge the specified local describes file into the INDEX. 47 -p portsdir -- Location of uncompressed ports tree 48 (default: /usr/ports/) 49 -s server -- Server from which to fetch updates. 50 (default: portsnap.FreeBSD.org) 51 path -- Extract only parts of the tree starting with the given 52 string. (extract command only) 53Commands: 54 fetch -- Fetch a compressed snapshot of the ports tree, 55 or update an existing snapshot. 56 cron -- Sleep rand(3600) seconds, and then fetch updates. 57 extract -- Extract snapshot of ports tree, replacing existing 58 files and directories. 59 update -- Update ports tree to match current snapshot, replacing 60 files and directories which have changed. 61EOF 62 exit 0 63} 64 65#### Parameter handling functions. 66 67# Initialize parameters to null, just in case they're 68# set in the environment. 69init_params() { 70 KEYPRINT="" 71 EXTRACTPATH="" 72 WORKDIR="" 73 PORTSDIR="" 74 CONFFILE="" 75 COMMAND="" 76 COMMANDS="" 77 QUIETREDIR="" 78 QUIETFLAG="" 79 STATSREDIR="" 80 XARGST="" 81 NDEBUG="" 82 DDSTATS="" 83 INDEXONLY="" 84 SERVERNAME="" 85 REFUSE="" 86 LOCALDESC="" 87} 88 89# Parse the command line 90parse_cmdline() { 91 while [ $# -gt 0 ]; do 92 case "$1" in 93 -d) 94 if [ $# -eq 1 ]; then usage; fi 95 if [ ! -z "${WORKDIR}" ]; then usage; fi 96 shift; WORKDIR="$1" 97 ;; 98 --debug) 99 QUIETREDIR="/dev/stderr" 100 STATSREDIR="/dev/stderr" 101 QUIETFLAG=" " 102 NDEBUG=" " 103 XARGST="-t" 104 DDSTATS=".." 105 ;; 106 -f) 107 if [ $# -eq 1 ]; then usage; fi 108 if [ ! -z "${CONFFILE}" ]; then usage; fi 109 shift; CONFFILE="$1" 110 ;; 111 -h | --help | help) 112 usage 113 ;; 114 -I) 115 INDEXONLY="YES" 116 ;; 117 -k) 118 if [ $# -eq 1 ]; then usage; fi 119 if [ ! -z "${KEYPRINT}" ]; then usage; fi 120 shift; KEYPRINT="$1" 121 ;; 122 -l) 123 if [ $# -eq 1 ]; then usage; fi 124 if [ ! -z "${LOCALDESC}" ]; then usage; fi 125 shift; LOCALDESC="$1" 126 ;; 127 --no-stats) 128 if [ -z "${STATSREDIR}" ]; then 129 STATSREDIR="/dev/null" 130 DDSTATS=".. " 131 fi 132 ;; 133 -p) 134 if [ $# -eq 1 ]; then usage; fi 135 if [ ! -z "${PORTSDIR}" ]; then usage; fi 136 shift; PORTSDIR="$1" 137 ;; 138 -s) 139 if [ $# -eq 1 ]; then usage; fi 140 if [ ! -z "${SERVERNAME}" ]; then usage; fi 141 shift; SERVERNAME="$1" 142 ;; 143 cron | extract | fetch | update | alfred) 144 COMMANDS="${COMMANDS} $1" 145 ;; 146 *) 147 if [ $# -gt 1 ]; then usage; fi 148 if echo ${COMMANDS} | grep -vq extract; then 149 usage 150 fi 151 EXTRACTPATH="$1" 152 ;; 153 esac 154 shift 155 done 156 157 if [ -z "${COMMANDS}" ]; then 158 usage 159 fi 160} 161 162# If CONFFILE was specified at the command-line, make 163# sure that it exists and is readable. 164sanity_conffile() { 165 if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then 166 echo -n "File does not exist " 167 echo -n "or is not readable: " 168 echo ${CONFFILE} 169 exit 1 170 fi 171} 172 173# If a configuration file hasn't been specified, use 174# the default value (/etc/portsnap.conf) 175default_conffile() { 176 if [ -z "${CONFFILE}" ]; then 177 CONFFILE="/etc/portsnap.conf" 178 fi 179} 180 181# Read {KEYPRINT, SERVERNAME, WORKDIR, PORTSDIR} from the configuration 182# file if they haven't already been set. If the configuration 183# file doesn't exist, do nothing. 184# Also read REFUSE (which cannot be set via the command line) if it is 185# present in the configuration file. 186parse_conffile() { 187 if [ -r "${CONFFILE}" ]; then 188 for X in KEYPRINT WORKDIR PORTSDIR SERVERNAME; do 189 eval _=\$${X} 190 if [ -z "${_}" ]; then 191 eval ${X}=`grep "^${X}=" "${CONFFILE}" | 192 cut -f 2- -d '=' | tail -1` 193 fi 194 done 195 196 if grep -qE "^REFUSE[[:space:]]" ${CONFFILE}; then 197 REFUSE="^(` 198 grep -E "^REFUSE[[:space:]]" "${CONFFILE}" | 199 cut -c 7- | xargs echo | tr ' ' '|' 200 `)" 201 fi 202 203 if grep -qE "^INDEX[[:space:]]" ${CONFFILE}; then 204 INDEXPAIRS="` 205 grep -E "^INDEX[[:space:]]" "${CONFFILE}" | 206 cut -c 7- | tr ' ' '|' | xargs echo`" 207 fi 208 fi 209} 210 211# If parameters have not been set, use default values 212default_params() { 213 _QUIETREDIR="/dev/null" 214 _QUIETFLAG="-q" 215 _STATSREDIR="/dev/stdout" 216 _WORKDIR="/var/db/portsnap" 217 _PORTSDIR="/usr/ports" 218 _NDEBUG="-n" 219 _LOCALDESC="/dev/null" 220 for X in QUIETREDIR QUIETFLAG STATSREDIR WORKDIR PORTSDIR \ 221 NDEBUG LOCALDESC; do 222 eval _=\$${X} 223 eval __=\$_${X} 224 if [ -z "${_}" ]; then 225 eval ${X}=${__} 226 fi 227 done 228} 229 230# Perform sanity checks and set some final parameters 231# in preparation for fetching files. Also chdir into 232# the working directory. 233fetch_check_params() { 234 export HTTP_USER_AGENT="portsnap (${COMMAND}, `uname -r`)" 235 236 _SERVERNAME_z=\ 237"SERVERNAME must be given via command line or configuration file." 238 _KEYPRINT_z="Key must be given via -k option or configuration file." 239 _KEYPRINT_bad="Invalid key fingerprint: " 240 _WORKDIR_bad="Directory does not exist or is not writable: " 241 242 if [ -z "${SERVERNAME}" ]; then 243 echo -n "`basename $0`: " 244 echo "${_SERVERNAME_z}" 245 exit 1 246 fi 247 if [ -z "${KEYPRINT}" ]; then 248 echo -n "`basename $0`: " 249 echo "${_KEYPRINT_z}" 250 exit 1 251 fi 252 if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then 253 echo -n "`basename $0`: " 254 echo -n "${_KEYPRINT_bad}" 255 echo ${KEYPRINT} 256 exit 1 257 fi 258 if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then 259 echo -n "`basename $0`: " 260 echo -n "${_WORKDIR_bad}" 261 echo ${WORKDIR} 262 exit 1 263 fi 264 cd ${WORKDIR} || exit 1 265 266 BSPATCH=/usr/bin/bspatch 267 SHA256=/sbin/sha256 268 PHTTPGET=/usr/libexec/phttpget 269} 270 271# Perform sanity checks and set some final parameters 272# in preparation for extracting or updating ${PORTSDIR} 273# Complain if ${PORTSDIR} exists but is not writable, 274# but don't complain if ${PORTSDIR} doesn't exist. 275extract_check_params() { 276 _WORKDIR_bad="Directory does not exist: " 277 _PORTSDIR_bad="Directory is not writable: " 278 279 if ! [ -d "${WORKDIR}" ]; then 280 echo -n "`basename $0`: " 281 echo -n "${_WORKDIR_bad}" 282 echo ${WORKDIR} 283 exit 1 284 fi 285 if [ -d "${PORTSDIR}" ] && ! [ -w "${PORTSDIR}" ]; then 286 echo -n "`basename $0`: " 287 echo -n "${_PORTSDIR_bad}" 288 echo ${PORTSDIR} 289 exit 1 290 fi 291 292 if ! [ -d "${WORKDIR}/files" -a -r "${WORKDIR}/tag" \ 293 -a -r "${WORKDIR}/INDEX" -a -r "${WORKDIR}/tINDEX" ]; then 294 echo "No snapshot available. Try running" 295 echo "# `basename $0` fetch" 296 exit 1 297 fi 298 299 MKINDEX=/usr/libexec/make_index 300} 301 302# Perform sanity checks and set some final parameters 303# in preparation for updating ${PORTSDIR} 304update_check_params() { 305 extract_check_params 306 307 if ! [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 308 echo "${PORTSDIR} was not created by portsnap." 309 echo -n "You must run '`basename $0` extract' before " 310 echo "running '`basename $0` update'." 311 exit 1 312 fi 313 314} 315 316#### Core functionality -- the actual work gets done here 317 318# Use an SRV query to pick a server. If the SRV query doesn't provide 319# a useful answer, use the server name specified by the user. 320# Put another way... look up _http._tcp.${SERVERNAME} and pick a server 321# from that; or if no servers are returned, use ${SERVERNAME}. 322# This allows a user to specify "portsnap.freebsd.org" (in which case 323# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org" 324# (in which case portsnap will use that particular server, since there 325# won't be an SRV entry for that name). 326# 327# We ignore the Port field, since we are always going to use port 80. 328 329# Fetch the mirror list, but do not pick a mirror yet. Returns 1 if 330# no mirrors are available for any reason. 331fetch_pick_server_init() { 332 : > serverlist_tried 333 334# Check that host(1) exists (i.e., that the system wasn't built with the 335# WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist. 336 if ! which -s host; then 337 : > serverlist_full 338 return 1 339 fi 340 341 echo -n "Looking up ${SERVERNAME} mirrors... " 342 343# Issue the SRV query and pull out the Priority, Weight, and Target fields. 344# BIND 9 prints "$name has SRV record ..." while BIND 8 prints 345# "$name server selection ..."; we allow either format. 346 MLIST="_http._tcp.${SERVERNAME}" 347 host -t srv "${MLIST}" | 348 sed -nE "s/${MLIST} (has SRV record|server selection) //p" | 349 cut -f 1,2,4 -d ' ' | 350 sed -e 's/\.$//' | 351 sort > serverlist_full 352 353# If no records, give up -- we'll just use the server name we were given. 354 if [ `wc -l < serverlist_full` -eq 0 ]; then 355 echo "none found." 356 return 1 357 fi 358 359# Report how many mirrors we found. 360 echo `wc -l < serverlist_full` "mirrors found." 361 362# Generate a random seed for use in picking mirrors. If HTTP_PROXY 363# is set, this will be used to generate the seed; otherwise, the seed 364# will be random. 365 if [ -n "${HTTP_PROXY}${http_proxy}" ]; then 366 RANDVALUE=`sha256 -qs "${HTTP_PROXY}${http_proxy}" | 367 tr -d 'a-f' | 368 cut -c 1-9` 369 else 370 RANDVALUE=`jot -r 1 0 999999999` 371 fi 372} 373 374# Pick a mirror. Returns 1 if we have run out of mirrors to try. 375fetch_pick_server() { 376# Generate a list of not-yet-tried mirrors 377 sort serverlist_tried | 378 comm -23 serverlist_full - > serverlist 379 380# Have we run out of mirrors? 381 if [ `wc -l < serverlist` -eq 0 ]; then 382 echo "No mirrors remaining, giving up." 383 return 1 384 fi 385 386# Find the highest priority level (lowest numeric value). 387 SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1` 388 389# Add up the weights of the response lines at that priority level. 390 SRV_WSUM=0; 391 while read X; do 392 case "$X" in 393 ${SRV_PRIORITY}\ *) 394 SRV_W=`echo $X | cut -f 2 -d ' '` 395 SRV_WSUM=$(($SRV_WSUM + $SRV_W)) 396 ;; 397 esac 398 done < serverlist 399 400# If all the weights are 0, pretend that they are all 1 instead. 401 if [ ${SRV_WSUM} -eq 0 ]; then 402 SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l` 403 SRV_W_ADD=1 404 else 405 SRV_W_ADD=0 406 fi 407 408# Pick a value between 0 and the sum of the weights - 1 409 SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}` 410 411# Read through the list of mirrors and set SERVERNAME. Write the line 412# corresponding to the mirror we selected into serverlist_tried so that 413# we won't try it again. 414 while read X; do 415 case "$X" in 416 ${SRV_PRIORITY}\ *) 417 SRV_W=`echo $X | cut -f 2 -d ' '` 418 SRV_W=$(($SRV_W + $SRV_W_ADD)) 419 if [ $SRV_RND -lt $SRV_W ]; then 420 SERVERNAME=`echo $X | cut -f 3 -d ' '` 421 echo "$X" >> serverlist_tried 422 break 423 else 424 SRV_RND=$(($SRV_RND - $SRV_W)) 425 fi 426 ;; 427 esac 428 done < serverlist 429} 430 431# Check that we have a public key with an appropriate hash, or 432# fetch the key if it doesn't exist. Returns 1 if the key has 433# not yet been fetched. 434fetch_key() { 435 if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 436 return 0 437 fi 438 439 echo -n "Fetching public key from ${SERVERNAME}... " 440 rm -f pub.ssl 441 fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \ 442 2>${QUIETREDIR} || true 443 if ! [ -r pub.ssl ]; then 444 echo "failed." 445 return 1 446 fi 447 if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 448 echo "key has incorrect hash." 449 rm -f pub.ssl 450 return 1 451 fi 452 echo "done." 453} 454 455# Fetch a snapshot tag 456fetch_tag() { 457 rm -f snapshot.ssl tag.new 458 459 echo ${NDEBUG} "Fetching snapshot tag from ${SERVERNAME}... " 460 fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl \ 461 2>${QUIETREDIR} || true 462 if ! [ -r $1.ssl ]; then 463 echo "failed." 464 return 1 465 fi 466 467 openssl rsautl -pubin -inkey pub.ssl -verify \ 468 < $1.ssl > tag.new 2>${QUIETREDIR} || true 469 rm $1.ssl 470 471 if ! [ `wc -l < tag.new` = 1 ] || 472 ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then 473 echo "invalid snapshot tag." 474 return 1 475 fi 476 477 echo "done." 478 479 SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new` 480 SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new` 481} 482 483# Sanity-check the date on a snapshot tag 484fetch_snapshot_tagsanity() { 485 if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then 486 echo "Snapshot appears to be more than a year old!" 487 echo "(Is the system clock correct?)" 488 echo "Cowardly refusing to proceed any further." 489 return 1 490 fi 491 if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then 492 echo -n "Snapshot appears to have been created more than " 493 echo "one day into the future!" 494 echo "(Is the system clock correct?)" 495 echo "Cowardly refusing to proceed any further." 496 return 1 497 fi 498} 499 500# Sanity-check the date on a snapshot update tag 501fetch_update_tagsanity() { 502 fetch_snapshot_tagsanity || return 1 503 504 if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then 505 echo -n "Latest snapshot on server is " 506 echo "older than what we already have!" 507 echo -n "Cowardly refusing to downgrade from " 508 date -r ${OLDSNAPSHOTDATE} 509 echo "to `date -r ${SNAPSHOTDATE}`." 510 return 1 511 fi 512} 513 514# Compare old and new tags; return 1 if update is unnecessary 515fetch_update_neededp() { 516 if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then 517 echo -n "Latest snapshot on server matches " 518 echo "what we already have." 519 echo "No updates needed." 520 rm tag.new 521 return 1 522 fi 523 if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then 524 echo -n "Ports tree hasn't changed since " 525 echo "last snapshot." 526 echo "No updates needed." 527 rm tag.new 528 return 1 529 fi 530 531 return 0 532} 533 534# Fetch snapshot metadata file 535fetch_metadata() { 536 rm -f ${SNAPSHOTHASH} tINDEX.new 537 538 echo ${NDEBUG} "Fetching snapshot metadata... " 539 fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} 540 2>${QUIETREDIR} || return 541 if [ `${SHA256} -q ${SNAPSHOTHASH}` != ${SNAPSHOTHASH} ]; then 542 echo "snapshot metadata corrupt." 543 return 1 544 fi 545 mv ${SNAPSHOTHASH} tINDEX.new 546 echo "done." 547} 548 549# Warn user about bogus metadata 550fetch_metadata_freakout() { 551 echo 552 echo "Portsnap metadata is correctly signed, but contains" 553 echo "at least one line which appears bogus." 554 echo "Cowardly refusing to proceed any further." 555} 556 557# Sanity-check a snapshot metadata file 558fetch_metadata_sanity() { 559 if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then 560 fetch_metadata_freakout 561 return 1 562 fi 563 if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then 564 echo 565 echo "Portsnap metadata appears bogus." 566 echo "Cowardly refusing to proceed any further." 567 return 1 568 fi 569} 570 571# Take a list of ${oldhash}|${newhash} and output a list of needed patches 572fetch_make_patchlist() { 573 grep -vE "^([0-9a-f]{64})\|\1$" | 574 while read LINE; do 575 X=`echo ${LINE} | cut -f 1 -d '|'` 576 Y=`echo ${LINE} | cut -f 2 -d '|'` 577 if [ -f "files/${Y}.gz" ]; then continue; fi 578 if [ ! -f "files/${X}.gz" ]; then continue; fi 579 echo "${LINE}" 580 done 581} 582 583# Print user-friendly progress statistics 584fetch_progress() { 585 LNC=0 586 while read x; do 587 LNC=$(($LNC + 1)) 588 if [ $(($LNC % 10)) = 0 ]; then 589 echo -n $LNC 590 elif [ $(($LNC % 2)) = 0 ]; then 591 echo -n . 592 fi 593 done 594 echo -n " " 595} 596 597# Sanity-check an index file 598fetch_index_sanity() { 599 if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new || 600 fgrep -q "./" INDEX.new; then 601 fetch_metadata_freakout 602 return 1 603 fi 604} 605 606# Verify a list of files 607fetch_snapshot_verify() { 608 while read F; do 609 if [ `gunzip -c snap/${F} | ${SHA256} -q` != ${F} ]; then 610 echo "snapshot corrupt." 611 return 1 612 fi 613 done 614 return 0 615} 616 617# Fetch a snapshot tarball, extract, and verify. 618fetch_snapshot() { 619 while ! fetch_tag snapshot; do 620 fetch_pick_server || return 1 621 done 622 fetch_snapshot_tagsanity || return 1 623 fetch_metadata || return 1 624 fetch_metadata_sanity || return 1 625 626 rm -rf snap/ 627 628# Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will 629# probably take a while, so the progrees reports that fetch(1) generates 630# will be useful for keeping the users' attention from drifting. 631 echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:" 632 fetch -r http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1 633 634 echo -n "Extracting snapshot... " 635 tar -xz --numeric-owner -f ${SNAPSHOTHASH}.tgz snap/ || return 1 636 rm ${SNAPSHOTHASH}.tgz 637 echo "done." 638 639 echo -n "Verifying snapshot integrity... " 640# Verify the metadata files 641 cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1 642# Extract the index 643 rm -f INDEX.new 644 gunzip -c snap/`look INDEX tINDEX.new | 645 cut -f 2 -d '|'`.gz > INDEX.new 646 fetch_index_sanity || return 1 647# Verify the snapshot contents 648 cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1 649 echo "done." 650 651# Move files into their proper locations 652 rm -f tag INDEX tINDEX 653 rm -rf files 654 mv tag.new tag 655 mv tINDEX.new tINDEX 656 mv INDEX.new INDEX 657 mv snap/ files/ 658 659 return 0 660} 661 662# Update a compressed snapshot 663fetch_update() { 664 rm -f patchlist diff OLD NEW filelist INDEX.new 665 666 OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag` 667 OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag` 668 669 while ! fetch_tag latest; do 670 fetch_pick_server || return 1 671 done 672 fetch_update_tagsanity || return 1 673 fetch_update_neededp || return 0 674 fetch_metadata || return 1 675 fetch_metadata_sanity || return 1 676 677 echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` " 678 echo "to `date -r ${SNAPSHOTDATE}`." 679 680# Generate a list of wanted metadata patches 681 join -t '|' -o 1.2,2.2 tINDEX tINDEX.new | 682 fetch_make_patchlist > patchlist 683 684# Attempt to fetch metadata patches 685 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 686 echo ${NDEBUG} "metadata patches.${DDSTATS}" 687 tr '|' '-' < patchlist | 688 lam -s "tp/" - -s ".gz" | 689 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 690 2>${STATSREDIR} | fetch_progress 691 echo "done." 692 693# Attempt to apply metadata patches 694 echo -n "Applying metadata patches... " 695 while read LINE; do 696 X=`echo ${LINE} | cut -f 1 -d '|'` 697 Y=`echo ${LINE} | cut -f 2 -d '|'` 698 if [ ! -f "${X}-${Y}.gz" ]; then continue; fi 699 gunzip -c < ${X}-${Y}.gz > diff 700 gunzip -c < files/${X}.gz > OLD 701 cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp 702 grep '^\+' diff | cut -c 2- | 703 sort -k 1,1 -t '|' -m - ptmp > NEW 704 if [ `${SHA256} -q NEW` = ${Y} ]; then 705 mv NEW files/${Y} 706 gzip -n files/${Y} 707 fi 708 rm -f diff OLD NEW ${X}-${Y}.gz ptmp 709 done < patchlist 2>${QUIETREDIR} 710 echo "done." 711 712# Update metadata without patches 713 join -t '|' -v 2 tINDEX tINDEX.new | 714 cut -f 2 -d '|' /dev/stdin patchlist | 715 while read Y; do 716 if [ ! -f "files/${Y}.gz" ]; then 717 echo ${Y}; 718 fi 719 done > filelist 720 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 721 echo ${NDEBUG} "metadata files... " 722 lam -s "f/" - -s ".gz" < filelist | 723 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 724 2>${QUIETREDIR} 725 726 while read Y; do 727 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 728 mv ${Y}.gz files/${Y}.gz 729 else 730 echo "metadata is corrupt." 731 return 1 732 fi 733 done < filelist 734 echo "done." 735 736# Extract the index 737 gunzip -c files/`look INDEX tINDEX.new | 738 cut -f 2 -d '|'`.gz > INDEX.new 739 fetch_index_sanity || return 1 740 741# If we have decided to refuse certain updates, construct a hybrid index which 742# is equal to the old index for parts of the tree which we don't want to 743# update, and equal to the new index for parts of the tree which gets updates. 744# This means that we should always have a "complete snapshot" of the ports 745# tree -- with the caveat that it isn't actually a snapshot. 746 if [ ! -z "${REFUSE}" ]; then 747 echo "Refusing to download updates for ${REFUSE}" \ 748 >${QUIETREDIR} 749 750 grep -Ev "${REFUSE}" INDEX.new > INDEX.tmp 751 grep -E "${REFUSE}" INDEX | 752 sort -m -k 1,1 -t '|' - INDEX.tmp > INDEX.new 753 rm -f INDEX.tmp 754 fi 755 756# Generate a list of wanted ports patches 757 join -t '|' -o 1.2,2.2 INDEX INDEX.new | 758 fetch_make_patchlist > patchlist 759 760# Attempt to fetch ports patches 761 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 762 echo ${NDEBUG} "patches.${DDSTATS}" 763 tr '|' '-' < patchlist | lam -s "bp/" - | 764 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 765 2>${STATSREDIR} | fetch_progress 766 echo "done." 767 768# Attempt to apply ports patches 769 echo -n "Applying patches... " 770 while read LINE; do 771 X=`echo ${LINE} | cut -f 1 -d '|'` 772 Y=`echo ${LINE} | cut -f 2 -d '|'` 773 if [ ! -f "${X}-${Y}" ]; then continue; fi 774 gunzip -c < files/${X}.gz > OLD 775 ${BSPATCH} OLD NEW ${X}-${Y} 776 if [ `${SHA256} -q NEW` = ${Y} ]; then 777 mv NEW files/${Y} 778 gzip -n files/${Y} 779 fi 780 rm -f diff OLD NEW ${X}-${Y} 781 done < patchlist 2>${QUIETREDIR} 782 echo "done." 783 784# Update ports without patches 785 join -t '|' -v 2 INDEX INDEX.new | 786 cut -f 2 -d '|' /dev/stdin patchlist | 787 while read Y; do 788 if [ ! -f "files/${Y}.gz" ]; then 789 echo ${Y}; 790 fi 791 done > filelist 792 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 793 echo ${NDEBUG} "new ports or files... " 794 lam -s "f/" - -s ".gz" < filelist | 795 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 796 2>${QUIETREDIR} 797 798 while read Y; do 799 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 800 mv ${Y}.gz files/${Y}.gz 801 else 802 echo "snapshot is corrupt." 803 return 1 804 fi 805 done < filelist 806 echo "done." 807 808# Remove files which are no longer needed 809 cut -f 2 -d '|' tINDEX INDEX | sort > oldfiles 810 cut -f 2 -d '|' tINDEX.new INDEX.new | sort | comm -13 - oldfiles | 811 lam -s "files/" - -s ".gz" | xargs rm -f 812 rm patchlist filelist oldfiles 813 814# We're done! 815 mv INDEX.new INDEX 816 mv tINDEX.new tINDEX 817 mv tag.new tag 818 819 return 0 820} 821 822# Do the actual work involved in "fetch" / "cron". 823fetch_run() { 824 fetch_pick_server_init && fetch_pick_server 825 826 while ! fetch_key; do 827 fetch_pick_server || return 1 828 done 829 830 if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then 831 fetch_snapshot || return 1 832 fi 833 fetch_update || return 1 834} 835 836# Build a ports INDEX file 837extract_make_index() { 838 if ! look $1 ${WORKDIR}/tINDEX > /dev/null; then 839 echo -n "$1 not provided by portsnap server; " 840 echo "$2 not being generated." 841 else 842 gunzip -c "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX | 843 cut -f 2 -d '|'`.gz" | 844 cat - ${LOCALDESC} | 845 ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2 846 fi 847} 848 849# Create INDEX, INDEX-5, INDEX-6 850extract_indices() { 851 echo -n "Building new INDEX files... " 852 for PAIR in ${INDEXPAIRS}; do 853 INDEXFILE=`echo ${PAIR} | cut -f 1 -d '|'` 854 DESCRIBEFILE=`echo ${PAIR} | cut -f 2 -d '|'` 855 extract_make_index ${DESCRIBEFILE} ${INDEXFILE} || return 1 856 done 857 echo "done." 858} 859 860# Create .portsnap.INDEX; if we are REFUSEing to touch certain directories, 861# merge the values from any exiting .portsnap.INDEX file. 862extract_metadata() { 863 if [ -z "${REFUSE}" ]; then 864 sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX 865 elif [ -f ${PORTSDIR}/.portsnap.INDEX ]; then 866 grep -E "${REFUSE}" ${PORTSDIR}/.portsnap.INDEX \ 867 > ${PORTSDIR}/.portsnap.INDEX.tmp 868 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort | 869 sort -m - ${PORTSDIR}/.portsnap.INDEX.tmp \ 870 > ${PORTSDIR}/.portsnap.INDEX 871 rm -f ${PORTSDIR}/.portsnap.INDEX.tmp 872 else 873 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort \ 874 > ${PORTSDIR}/.portsnap.INDEX 875 fi 876} 877 878# Do the actual work involved in "extract" 879extract_run() { 880 mkdir -p ${PORTSDIR} || return 1 881 882 if ! 883 if ! [ -z "${EXTRACTPATH}" ]; then 884 grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX 885 elif ! [ -z "${REFUSE}" ]; then 886 grep -vE "${REFUSE}" ${WORKDIR}/INDEX 887 else 888 cat ${WORKDIR}/INDEX 889 fi | tr '|' ' ' | while read FILE HASH; do 890 echo ${PORTSDIR}/${FILE} 891 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 892 echo "files/${HASH}.gz not found -- snapshot corrupt." 893 return 1 894 fi 895 case ${FILE} in 896 */) 897 rm -rf ${PORTSDIR}/${FILE%/} 898 mkdir -p ${PORTSDIR}/${FILE} 899 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 900 -C ${PORTSDIR}/${FILE} 901 ;; 902 *) 903 rm -f ${PORTSDIR}/${FILE} 904 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 905 -C ${PORTSDIR} ${FILE} 906 ;; 907 esac 908 done; then 909 return 1 910 fi 911 if [ ! -z "${EXTRACTPATH}" ]; then 912 return 0; 913 fi 914 915 extract_metadata 916 extract_indices 917} 918 919# Do the actual work involved in "update" 920update_run() { 921 if ! [ -z "${INDEXONLY}" ]; then 922 extract_indices >/dev/null || return 1 923 return 0 924 fi 925 926 if sort ${WORKDIR}/INDEX | 927 cmp -s ${PORTSDIR}/.portsnap.INDEX -; then 928 echo "Ports tree is already up to date." 929 return 0 930 fi 931 932# If we are REFUSEing to touch certain directories, don't remove files 933# from those directories (even if they are out of date) 934 echo -n "Removing old files and directories... " 935 if ! [ -z "${REFUSE}" ]; then 936 sort ${WORKDIR}/INDEX | 937 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 938 grep -vE "${REFUSE}" | 939 lam -s "${PORTSDIR}/" - | 940 sed -e 's|/$||' | xargs rm -rf 941 else 942 sort ${WORKDIR}/INDEX | 943 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 944 lam -s "${PORTSDIR}/" - | 945 sed -e 's|/$||' | xargs rm -rf 946 fi 947 echo "done." 948 949# Install new files 950 echo "Extracting new files:" 951 if ! 952 if ! [ -z "${REFUSE}" ]; then 953 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort 954 else 955 sort ${WORKDIR}/INDEX 956 fi | 957 comm -13 ${PORTSDIR}/.portsnap.INDEX - | 958 while read LINE; do 959 FILE=`echo ${LINE} | cut -f 1 -d '|'` 960 HASH=`echo ${LINE} | cut -f 2 -d '|'` 961 echo ${PORTSDIR}/${FILE} 962 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 963 echo "files/${HASH}.gz not found -- snapshot corrupt." 964 return 1 965 fi 966 case ${FILE} in 967 */) 968 mkdir -p ${PORTSDIR}/${FILE} 969 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 970 -C ${PORTSDIR}/${FILE} 971 ;; 972 *) 973 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 974 -C ${PORTSDIR} ${FILE} 975 ;; 976 esac 977 done; then 978 return 1 979 fi 980 981 extract_metadata 982 extract_indices 983} 984 985#### Main functions -- call parameter-handling and core functions 986 987# Using the command line, configuration file, and defaults, 988# set all the parameters which are needed later. 989get_params() { 990 init_params 991 parse_cmdline $@ 992 sanity_conffile 993 default_conffile 994 parse_conffile 995 default_params 996} 997 998# Fetch command. Make sure that we're being called 999# interactively, then run fetch_check_params and fetch_run 1000cmd_fetch() { 1001 if [ ! -t 0 ]; then 1002 echo -n "`basename $0` fetch should not " 1003 echo "be run non-interactively." 1004 echo "Run `basename $0` cron instead." 1005 exit 1 1006 fi 1007 fetch_check_params 1008 fetch_run || exit 1 1009} 1010 1011# Cron command. Make sure the parameters are sensible; wait 1012# rand(3600) seconds; then fetch updates. While fetching updates, 1013# send output to a temporary file; only print that file if the 1014# fetching failed. 1015cmd_cron() { 1016 fetch_check_params 1017 sleep `jot -r 1 0 3600` 1018 1019 TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1 1020 if ! fetch_run >> ${TMPFILE}; then 1021 cat ${TMPFILE} 1022 rm ${TMPFILE} 1023 exit 1 1024 fi 1025 1026 rm ${TMPFILE} 1027} 1028 1029# Extract command. Make sure the parameters are sensible, 1030# then extract the ports tree (or part thereof). 1031cmd_extract() { 1032 extract_check_params 1033 extract_run || exit 1 1034} 1035 1036# Update command. Make sure the parameters are sensible, 1037# then update the ports tree. 1038cmd_update() { 1039 update_check_params 1040 update_run || exit 1 1041} 1042 1043# Alfred command. Run 'fetch' or 'cron' depending on 1044# whether stdin is a terminal; then run 'update' or 1045# 'extract' depending on whether ${PORTSDIR} exists. 1046cmd_alfred() { 1047 if [ -t 0 ]; then 1048 cmd_fetch 1049 else 1050 cmd_cron 1051 fi 1052 if [ -d ${PORTSDIR} ]; then 1053 cmd_update 1054 else 1055 cmd_extract 1056 fi 1057} 1058 1059#### Entry point 1060 1061# Make sure we find utilities from the base system 1062export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH} 1063 1064# Set LC_ALL in order to avoid problems with character ranges like [A-Z]. 1065export LC_ALL=C 1066 1067get_params $@ 1068for COMMAND in ${COMMANDS}; do 1069 cmd_${COMMAND} 1070done 1071