portsnap.sh revision 150159
1#!/bin/sh 2 3#- 4# Copyright 2004-2005 Colin Percival 5# All rights reserved 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted providing that the following conditions 9# are met: 10# 1. Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# 2. Redistributions in binary form must reproduce the above copyright 13# notice, this list of conditions and the following disclaimer in the 14# documentation and/or other materials provided with the distribution. 15# 16# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 20# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26# POSSIBILITY OF SUCH DAMAGE. 27 28# $FreeBSD: head/usr.sbin/portsnap/portsnap/portsnap.sh 150159 2005-09-15 13:29:10Z cperciva $ 29 30#### Usage function -- called from command-line handling code. 31 32# Usage instructions. Options not listed: 33# --debug -- don't filter output from utilities 34# --no-stats -- don't show progress statistics while fetching files 35usage() { 36 cat <<EOF 37usage: `basename $0` [options] command ... [path] 38 39Options: 40 -d workdir -- Store working files in workdir 41 (default: /var/db/portsnap/) 42 -f conffile -- Read configuration options from conffile 43 (default: /etc/portsnap.conf) 44 -I -- Update INDEX only. (update command only) 45 -k KEY -- Trust an RSA key with SHA256 hash of KEY 46 -p portsdir -- Location of uncompressed ports tree 47 (default: /usr/ports/) 48 -s server -- Server from which to fetch updates. 49 (default: portsnap.FreeBSD.org) 50 path -- Extract only parts of the tree starting with the given 51 string. (extract command only) 52Commands: 53 fetch -- Fetch a compressed snapshot of the ports tree, 54 or update an existing snapshot. 55 cron -- Sleep rand(3600) seconds, and then fetch updates. 56 extract -- Extract snapshot of ports tree, replacing existing 57 files and directories. 58 update -- Update ports tree to match current snapshot, replacing 59 files and directories which have changed. 60EOF 61 exit 0 62} 63 64#### Parameter handling functions. 65 66# Initialize parameters to null, just in case they're 67# set in the environment. 68init_params() { 69 KEYPRINT="" 70 EXTRACTPATH="" 71 WORKDIR="" 72 PORTSDIR="" 73 CONFFILE="" 74 COMMAND="" 75 COMMANDS="" 76 QUIETREDIR="" 77 QUIETFLAG="" 78 STATSREDIR="" 79 XARGST="" 80 NDEBUG="" 81 DDSTATS="" 82 INDEXONLY="" 83 SERVERNAME="" 84 REFUSE="" 85} 86 87# Parse the command line 88parse_cmdline() { 89 while [ $# -gt 0 ]; do 90 case "$1" in 91 -d) 92 if [ $# -eq 1 ]; then usage; fi 93 if [ ! -z "${WORKDIR}" ]; then usage; fi 94 shift; WORKDIR="$1" 95 ;; 96 --debug) 97 QUIETREDIR="/dev/stderr" 98 STATSREDIR="/dev/stderr" 99 QUIETFLAG=" " 100 NDEBUG=" " 101 XARGST="-t" 102 DDSTATS=".." 103 ;; 104 -f) 105 if [ $# -eq 1 ]; then usage; fi 106 if [ ! -z "${CONFFILE}" ]; then usage; fi 107 shift; CONFFILE="$1" 108 ;; 109 -h | --help | help) 110 usage 111 ;; 112 -I) 113 INDEXONLY="YES" 114 ;; 115 -k) 116 if [ $# -eq 1 ]; then usage; fi 117 if [ ! -z "${KEYPRINT}" ]; then usage; fi 118 shift; KEYPRINT="$1" 119 ;; 120 --no-stats) 121 if [ -z "${STATSREDIR}" ]; then 122 STATSREDIR="/dev/null" 123 DDSTATS=".. " 124 fi 125 ;; 126 -p) 127 if [ $# -eq 1 ]; then usage; fi 128 if [ ! -z "${PORTSDIR}" ]; then usage; fi 129 shift; PORTSDIR="$1" 130 ;; 131 -s) 132 if [ $# -eq 1 ]; then usage; fi 133 if [ ! -z "${SERVERNAME}" ]; then usage; fi 134 shift; SERVERNAME="$1" 135 ;; 136 cron | extract | fetch | update) 137 COMMANDS="${COMMANDS} $1" 138 ;; 139 *) 140 if [ $# -gt 1 ]; then usage; fi 141 if echo ${COMMANDS} | grep -vq extract; then 142 usage 143 fi 144 EXTRACTPATH="$1" 145 ;; 146 esac 147 shift 148 done 149 150 if [ -z "${COMMANDS}" ]; then 151 usage 152 fi 153} 154 155# If CONFFILE was specified at the command-line, make 156# sure that it exists and is readable. 157sanity_conffile() { 158 if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then 159 echo -n "File does not exist " 160 echo -n "or is not readable: " 161 echo ${CONFFILE} 162 exit 1 163 fi 164} 165 166# If a configuration file hasn't been specified, use 167# the default value (/etc/portsnap.conf) 168default_conffile() { 169 if [ -z "${CONFFILE}" ]; then 170 CONFFILE="/etc/portsnap.conf" 171 fi 172} 173 174# Read {KEYPRINT, SERVERNAME, WORKDIR, PORTSDIR} from the configuration 175# file if they haven't already been set. If the configuration 176# file doesn't exist, do nothing. 177# Also read REFUSE (which cannot be set via the command line) if it is 178# present in the configuration file. 179parse_conffile() { 180 if [ -r "${CONFFILE}" ]; then 181 for X in KEYPRINT WORKDIR PORTSDIR SERVERNAME; do 182 eval _=\$${X} 183 if [ -z "${_}" ]; then 184 eval ${X}=`grep "^${X}=" "${CONFFILE}" | 185 cut -f 2- -d '=' | tail -1` 186 fi 187 done 188 189 if grep -qE "^REFUSE[[:space:]]" ${CONFFILE}; then 190 REFUSE="^(` 191 grep -E "^REFUSE[[:space:]]" "${CONFFILE}" | 192 cut -c 7- | xargs echo | tr ' ' '|' 193 `)" 194 fi 195 fi 196} 197 198# If parameters have not been set, use default values 199default_params() { 200 _QUIETREDIR="/dev/null" 201 _QUIETFLAG="-q" 202 _STATSREDIR="/dev/stdout" 203 _WORKDIR="/var/db/portsnap" 204 _PORTSDIR="/usr/ports" 205 _NDEBUG="-n" 206 for X in QUIETREDIR QUIETFLAG STATSREDIR WORKDIR PORTSDIR NDEBUG; do 207 eval _=\$${X} 208 eval __=\$_${X} 209 if [ -z "${_}" ]; then 210 eval ${X}=${__} 211 fi 212 done 213} 214 215# Perform sanity checks and set some final parameters 216# in preparation for fetching files. Also chdir into 217# the working directory. 218fetch_check_params() { 219 export HTTP_USER_AGENT="portsnap (${COMMAND}, `uname -r`)" 220 221 _SERVERNAME_z=\ 222"SERVERNAME must be given via command line or configuration file." 223 _KEYPRINT_z="Key must be given via -k option or configuration file." 224 _KEYPRINT_bad="Invalid key fingerprint: " 225 _WORKDIR_bad="Directory does not exist or is not writable: " 226 227 if [ -z "${SERVERNAME}" ]; then 228 echo -n "`basename $0`: " 229 echo "${_SERVERNAME_z}" 230 exit 1 231 fi 232 if [ -z "${KEYPRINT}" ]; then 233 echo -n "`basename $0`: " 234 echo "${_KEYPRINT_z}" 235 exit 1 236 fi 237 if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then 238 echo -n "`basename $0`: " 239 echo -n "${_KEYPRINT_bad}" 240 echo ${KEYPRINT} 241 exit 1 242 fi 243 if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then 244 echo -n "`basename $0`: " 245 echo -n "${_WORKDIR_bad}" 246 echo ${WORKDIR} 247 exit 1 248 fi 249 cd ${WORKDIR} || exit 1 250 251 BSPATCH=/usr/bin/bspatch 252 SHA256=/sbin/sha256 253 PHTTPGET=/usr/libexec/phttpget 254} 255 256# Perform sanity checks and set some final parameters 257# in preparation for extracting or updating ${PORTSDIR} 258extract_check_params() { 259 _WORKDIR_bad="Directory does not exist: " 260 _PORTSDIR_bad="Directory does not exist or is not writable: " 261 262 if ! [ -d "${WORKDIR}" ]; then 263 echo -n "`basename $0`: " 264 echo -n "${_WORKDIR_bad}" 265 echo ${WORKDIR} 266 exit 1 267 fi 268 if ! [ -d "${PORTSDIR}" -a -w "${PORTSDIR}" ]; then 269 echo -n "`basename $0`: " 270 echo -n "${_PORTSDIR_bad}" 271 echo ${PORTSDIR} 272 exit 1 273 fi 274 275 if ! [ -d "${WORKDIR}/files" -a -r "${WORKDIR}/tag" \ 276 -a -r "${WORKDIR}/INDEX" -a -r "${WORKDIR}/tINDEX" ]; then 277 echo "No snapshot available. Try running" 278 echo "# `basename $0` fetch" 279 exit 1 280 fi 281 282 MKINDEX=/usr/libexec/make_index 283} 284 285# Perform sanity checks and set some final parameters 286# in preparation for updating ${PORTSDIR} 287update_check_params() { 288 extract_check_params 289 290 if ! [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 291 echo "${PORTSDIR} was not created by portsnap." 292 echo -n "You must run '`basename $0` extract' before " 293 echo "running '`basename $0` update'." 294 exit 1 295 fi 296 297} 298 299#### Core functionality -- the actual work gets done here 300 301# Use an SRV query to pick a server. If the SRV query doesn't provide 302# a useful answer, use the server name specified by the user. 303# Put another way... look up _http._tcp.${SERVERNAME} and pick a server 304# from that; or if no servers are returned, use ${SERVERNAME}. 305# This allows a user to specify "portsnap.freebsd.org" (in which case 306# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org" 307# (in which case portsnap will use that particular server, since there 308# won't be an SRV entry for that name). 309# 310# We don't implement the recommendations from RFC 2782 completely, since 311# we are only looking to pick a single server -- the recommendations are 312# targetted at applications which obtain a list of servers and then try 313# each in turn, but we are instead just going to pick one server and let 314# the user re-run portsnap if a broken server was selected. 315# 316# We also ignore the Port field, since we are always going to use port 80. 317fetch_pick_server() { 318# Check that host(1) exists (i.e., that the system wasn't built with the 319# NO_BIND flag set) and don't try to find a mirror if it doesn't exist. 320 if ! which -s host; then 321 return 322 fi 323 324 echo -n "Looking up ${SERVERNAME} mirrors..." 325 326# Issue the SRV query and pull out the Priority, Weight, and Target fields. 327 host -t srv "_http._tcp.${SERVERNAME}" | 328 grep -E "^_http._tcp.${SERVERNAME} has SRV record" | 329 cut -f 5,6,8 -d ' ' > serverlist 330 331# If no records, give up -- we'll just use the server name we were given. 332 if [ `wc -l < serverlist` -eq 0 ]; then 333 echo " none found." 334 return 335 fi 336 337# Find the highest priority level (lowest numeric value). 338 SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1` 339 340# Add up the weights of the response lines at that priority level. 341 SRV_WSUM=0; 342 while read X; do 343 case "$X" in 344 ${SRV_PRIORITY}\ *) 345 SRV_W=`echo $X | cut -f 2 -d ' '` 346 SRV_WSUM=$(($SRV_WSUM + $SRV_W)) 347 ;; 348 esac 349 done < serverlist 350 351# If all the weights are 0, pretend that they are all 1 instead. 352 if [ ${SRV_WSUM} -eq 0 ]; then 353 SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l` 354 SRV_W_ADD=1 355 else 356 SRV_W_ADD=0 357 fi 358 359# Pick a random value between 1 and the sum of the weights 360 SRV_RND=`jot -r 1 1 ${SRV_WSUM}` 361 362# Read through the list of mirrors and set SERVERNAME 363 while read X; do 364 case "$X" in 365 ${SRV_PRIORITY}\ *) 366 SRV_W=`echo $X | cut -f 2 -d ' '` 367 SRV_W=$(($SRV_W + $SRV_W_ADD)) 368 if [ $SRV_RND -le $SRV_W ]; then 369 SERVERNAME=`echo $X | cut -f 3 -d ' '` 370 break 371 else 372 SRV_RND=$(($SRV_RND - $SRV_W)) 373 fi 374 ;; 375 esac 376 done < serverlist 377 378 echo " using ${SERVERNAME}" 379} 380 381# Check that we have a public key with an appropriate hash, or 382# fetch the key if it doesn't exist. 383fetch_key() { 384 if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 385 return 386 fi 387 388 echo -n "Fetching public key... " 389 rm -f pub.ssl 390 fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \ 391 2>${QUIETREDIR} || true 392 if ! [ -r pub.ssl ]; then 393 echo "failed." 394 return 1 395 fi 396 if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 397 echo "key has incorrect hash." 398 rm -f pub.ssl 399 return 1 400 fi 401 echo "done." 402} 403 404# Fetch a snapshot tag 405fetch_tag() { 406 rm -f snapshot.ssl tag.new 407 408 echo ${NDEBUG} "Fetching snapshot tag... " 409 fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl 410 2>${QUIETREDIR} || true 411 if ! [ -r $1.ssl ]; then 412 echo "failed." 413 return 1 414 fi 415 416 openssl rsautl -pubin -inkey pub.ssl -verify \ 417 < $1.ssl > tag.new 2>${QUIETREDIR} || true 418 rm $1.ssl 419 420 if ! [ `wc -l < tag.new` = 1 ] || 421 ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then 422 echo "invalid snapshot tag." 423 return 1 424 fi 425 426 echo "done." 427 428 SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new` 429 SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new` 430} 431 432# Sanity-check the date on a snapshot tag 433fetch_snapshot_tagsanity() { 434 if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then 435 echo "Snapshot appears to be more than a year old!" 436 echo "(Is the system clock correct?)" 437 echo "Cowardly refusing to proceed any further." 438 return 1 439 fi 440 if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then 441 echo -n "Snapshot appears to have been created more than " 442 echo "one day into the future!" 443 echo "(Is the system clock correct?)" 444 echo "Cowardly refusing to proceed any further." 445 return 1 446 fi 447} 448 449# Sanity-check the date on a snapshot update tag 450fetch_update_tagsanity() { 451 fetch_snapshot_tagsanity || return 1 452 453 if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then 454 echo -n "Latest snapshot on server is " 455 echo "older than what we already have!" 456 echo -n "Cowardly refusing to downgrade from " 457 date -r ${OLDSNAPSHOTDATE} 458 echo "to `date -r ${SNAPSHOTDATE}`." 459 return 1 460 fi 461} 462 463# Compare old and new tags; return 1 if update is unnecessary 464fetch_update_neededp() { 465 if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then 466 echo -n "Latest snapshot on server matches " 467 echo "what we already have." 468 echo "No updates needed." 469 rm tag.new 470 return 1 471 fi 472 if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then 473 echo -n "Ports tree hasn't changed since " 474 echo "last snapshot." 475 echo "No updates needed." 476 rm tag.new 477 return 1 478 fi 479 480 return 0 481} 482 483# Fetch snapshot metadata file 484fetch_metadata() { 485 rm -f ${SNAPSHOTHASH} tINDEX.new 486 487 echo ${NDEBUG} "Fetching snapshot metadata... " 488 fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} 489 2>${QUIETREDIR} || return 490 if [ `${SHA256} -q ${SNAPSHOTHASH}` != ${SNAPSHOTHASH} ]; then 491 echo "snapshot metadata corrupt." 492 return 1 493 fi 494 mv ${SNAPSHOTHASH} tINDEX.new 495 echo "done." 496} 497 498# Warn user about bogus metadata 499fetch_metadata_freakout() { 500 echo 501 echo "Portsnap metadata is correctly signed, but contains" 502 echo "at least one line which appears bogus." 503 echo "Cowardly refusing to proceed any further." 504} 505 506# Sanity-check a snapshot metadata file 507fetch_metadata_sanity() { 508 if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then 509 fetch_metadata_freakout 510 return 1 511 fi 512 if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then 513 echo 514 echo "Portsnap metadata appears bogus." 515 echo "Cowardly refusing to proceed any further." 516 return 1 517 fi 518} 519 520# Take a list of ${oldhash}|${newhash} and output a list of needed patches 521fetch_make_patchlist() { 522 grep -vE "^([0-9a-f]{64})\|\1$" | 523 while read LINE; do 524 X=`echo ${LINE} | cut -f 1 -d '|'` 525 Y=`echo ${LINE} | cut -f 2 -d '|'` 526 if [ -f "files/${Y}.gz" ]; then continue; fi 527 if [ ! -f "files/${X}.gz" ]; then continue; fi 528 echo "${LINE}" 529 done 530} 531 532# Print user-friendly progress statistics 533fetch_progress() { 534 LNC=0 535 while read x; do 536 LNC=$(($LNC + 1)) 537 if [ $(($LNC % 10)) = 0 ]; then 538 echo -n $LNC 539 elif [ $(($LNC % 2)) = 0 ]; then 540 echo -n . 541 fi 542 done 543 echo -n " " 544} 545 546# Sanity-check an index file 547fetch_index_sanity() { 548 if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new || 549 fgrep -q "./" INDEX.new; then 550 fetch_metadata_freakout 551 return 1 552 fi 553} 554 555# Verify a list of files 556fetch_snapshot_verify() { 557 while read F; do 558 if [ `gunzip -c snap/${F} | ${SHA256} -q` != ${F} ]; then 559 echo "snapshot corrupt." 560 return 1 561 fi 562 done 563 return 0 564} 565 566# Fetch a snapshot tarball, extract, and verify. 567fetch_snapshot() { 568 fetch_tag snapshot || return 1 569 fetch_snapshot_tagsanity || return 1 570 fetch_metadata || return 1 571 fetch_metadata_sanity || return 1 572 573 rm -f ${SNAPSHOTHASH}.tgz 574 rm -rf snap/ 575 576# Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will 577# probably take a while, so the progrees reports that fetch(1) generates 578# will be useful for keeping the users' attention from drifting. 579 echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:" 580 fetch http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1 581 582 echo -n "Extracting snapshot... " 583 tar -xzf ${SNAPSHOTHASH}.tgz snap/ || return 1 584 rm ${SNAPSHOTHASH}.tgz 585 echo "done." 586 587 echo -n "Verifying snapshot integrity... " 588# Verify the metadata files 589 cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1 590# Extract the index 591 rm -f INDEX.new 592 gunzip -c snap/`look INDEX tINDEX.new | 593 cut -f 2 -d '|'`.gz > INDEX.new 594 fetch_index_sanity || return 1 595# Verify the snapshot contents 596 cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1 597 echo "done." 598 599# Move files into their proper locations 600 rm -f tag INDEX tINDEX 601 rm -rf files 602 mv tag.new tag 603 mv tINDEX.new tINDEX 604 mv INDEX.new INDEX 605 mv snap/ files/ 606 607 return 0 608} 609 610# Update a compressed snapshot 611fetch_update() { 612 rm -f patchlist diff OLD NEW filelist INDEX.new 613 614 OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag` 615 OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag` 616 617 fetch_tag latest || return 1 618 fetch_update_tagsanity || return 1 619 fetch_update_neededp || return 0 620 fetch_metadata || return 1 621 fetch_metadata_sanity || return 1 622 623 echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` " 624 echo "to `date -r ${SNAPSHOTDATE}`." 625 626# Generate a list of wanted metadata patches 627 join -t '|' -o 1.2,2.2 tINDEX tINDEX.new | 628 fetch_make_patchlist > patchlist 629 630# Attempt to fetch metadata patches 631 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 632 echo ${NDEBUG} "metadata patches.${DDSTATS}" 633 tr '|' '-' < patchlist | 634 lam -s "tp/" - -s ".gz" | 635 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 636 2>${STATSREDIR} | fetch_progress 637 echo "done." 638 639# Attempt to apply metadata patches 640 echo -n "Applying metadata patches... " 641 while read LINE; do 642 X=`echo ${LINE} | cut -f 1 -d '|'` 643 Y=`echo ${LINE} | cut -f 2 -d '|'` 644 if [ ! -f "${X}-${Y}.gz" ]; then continue; fi 645 gunzip -c < ${X}-${Y}.gz > diff 646 gunzip -c < files/${X}.gz > OLD 647 cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp 648 grep '^\+' diff | cut -c 2- | 649 sort -k 1,1 -t '|' -m - ptmp > NEW 650 if [ `${SHA256} -q NEW` = ${Y} ]; then 651 mv NEW files/${Y} 652 gzip -n files/${Y} 653 fi 654 rm -f diff OLD NEW ${X}-${Y}.gz ptmp 655 done < patchlist 2>${QUIETREDIR} 656 echo "done." 657 658# Update metadata without patches 659 join -t '|' -v 2 tINDEX tINDEX.new | 660 cut -f 2 -d '|' /dev/stdin patchlist | 661 while read Y; do 662 if [ ! -f "files/${Y}.gz" ]; then 663 echo ${Y}; 664 fi 665 done > filelist 666 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 667 echo ${NDEBUG} "metadata files... " 668 lam -s "f/" - -s ".gz" < filelist | 669 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 670 2>${QUIETREDIR} 671 672 while read Y; do 673 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 674 mv ${Y}.gz files/${Y}.gz 675 else 676 echo "metadata is corrupt." 677 return 1 678 fi 679 done < filelist 680 echo "done." 681 682# Extract the index 683 gunzip -c files/`look INDEX tINDEX.new | 684 cut -f 2 -d '|'`.gz > INDEX.new 685 fetch_index_sanity || return 1 686 687# If we have decided to refuse certain updates, construct a hybrid index which 688# is equal to the old index for parts of the tree which we don't want to 689# update, and equal to the new index for parts of the tree which gets updates. 690# This means that we should always have a "complete snapshot" of the ports 691# tree -- with the caveat that it isn't actually a snapshot. 692 if [ ! -z "${REFUSE}" ]; then 693 echo "Refusing to download updates for ${REFUSE}" \ 694 >${QUIETREDIR} 695 696 grep -Ev "${REFUSE}" INDEX.new > INDEX.tmp 697 grep -E "${REFUSE}" INDEX | 698 sort -m -k 1,1 -t '|' - INDEX.tmp > INDEX.new 699 rm -f INDEX.tmp 700 fi 701 702# Generate a list of wanted ports patches 703 join -t '|' -o 1.2,2.2 INDEX INDEX.new | 704 fetch_make_patchlist > patchlist 705 706# Attempt to fetch ports patches 707 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 708 echo ${NDEBUG} "patches.${DDSTATS}" 709 tr '|' '-' < patchlist | lam -s "bp/" - | 710 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 711 2>${STATSREDIR} | fetch_progress 712 echo "done." 713 714# Attempt to apply ports patches 715 echo -n "Applying patches... " 716 while read LINE; do 717 X=`echo ${LINE} | cut -f 1 -d '|'` 718 Y=`echo ${LINE} | cut -f 2 -d '|'` 719 if [ ! -f "${X}-${Y}" ]; then continue; fi 720 gunzip -c < files/${X}.gz > OLD 721 ${BSPATCH} OLD NEW ${X}-${Y} 722 if [ `${SHA256} -q NEW` = ${Y} ]; then 723 mv NEW files/${Y} 724 gzip -n files/${Y} 725 fi 726 rm -f diff OLD NEW ${X}-${Y} 727 done < patchlist 2>${QUIETREDIR} 728 echo "done." 729 730# Update ports without patches 731 join -t '|' -v 2 INDEX INDEX.new | 732 cut -f 2 -d '|' /dev/stdin patchlist | 733 while read Y; do 734 if [ ! -f "files/${Y}.gz" ]; then 735 echo ${Y}; 736 fi 737 done > filelist 738 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 739 echo ${NDEBUG} "new ports or files... " 740 lam -s "f/" - -s ".gz" < filelist | 741 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 742 2>${QUIETREDIR} 743 744 while read Y; do 745 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 746 mv ${Y}.gz files/${Y}.gz 747 else 748 echo "snapshot is corrupt." 749 return 1 750 fi 751 done < filelist 752 echo "done." 753 754# Remove files which are no longer needed 755 cut -f 2 -d '|' tINDEX INDEX | sort > oldfiles 756 cut -f 2 -d '|' tINDEX.new INDEX.new | sort | comm -13 - oldfiles | 757 lam -s "files/" - -s ".gz" | xargs rm -f 758 rm patchlist filelist oldfiles 759 760# We're done! 761 mv INDEX.new INDEX 762 mv tINDEX.new tINDEX 763 mv tag.new tag 764 765 return 0 766} 767 768# Do the actual work involved in "fetch" / "cron". 769fetch_run() { 770 fetch_pick_server 771 772 fetch_key || return 1 773 774 if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then 775 fetch_snapshot || return 1 776 fi 777 fetch_update || return 1 778} 779 780# Build a ports INDEX file 781extract_make_index() { 782 gunzip -c "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX | 783 cut -f 2 -d '|'`.gz" | ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2 784} 785 786# Create INDEX, INDEX-5, INDEX-6 787extract_indices() { 788 echo -n "Building new INDEX files... " 789 extract_make_index DESCRIBE.4 INDEX || return 1 790 extract_make_index DESCRIBE.5 INDEX-5 || return 1 791 extract_make_index DESCRIBE.6 INDEX-6 || return 1 792 echo "done." 793} 794 795# Create .portsnap.INDEX; if we are REFUSEing to touch certain directories, 796# merge the values from any exiting .portsnap.INDEX file. 797extract_metadata() { 798 if [ -z "${REFUSE}" ]; then 799 sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX 800 elif [ -f ${PORTSDIR}/.portsnap.INDEX ]; then 801 grep -E "${REFUSE}" ${PORTSDIR}/.portsnap.INDEX \ 802 > ${PORTSDIR}/.portsnap.INDEX.tmp 803 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort | 804 sort -m - ${PORTSDIR}/.portsnap.INDEX.tmp \ 805 > ${PORTSDIR}/.portsnap.INDEX 806 rm -f ${PORTSDIR}/.portsnap.INDEX.tmp 807 else 808 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort \ 809 > ${PORTSDIR}/.portsnap.INDEX 810 fi 811} 812 813# Do the actual work involved in "extract" 814extract_run() { 815 if ! 816 if ! [ -z "${EXTRACTPATH}" ]; then 817 grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX 818 elif ! [ -z "${REFUSE}" ]; then 819 grep -vE "${REFUSE}" ${WORKDIR}/INDEX 820 else 821 cat ${WORKDIR}/INDEX 822 fi | while read LINE; do 823 FILE=`echo ${LINE} | cut -f 1 -d '|'` 824 HASH=`echo ${LINE} | cut -f 2 -d '|'` 825 echo ${PORTSDIR}/${FILE} 826 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 827 echo "files/${HASH}.gz not found -- snapshot corrupt." 828 return 1 829 fi 830 case ${FILE} in 831 */) 832 rm -rf ${PORTSDIR}/${FILE} 833 mkdir -p ${PORTSDIR}/${FILE} 834 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 835 -C ${PORTSDIR}/${FILE} 836 ;; 837 *) 838 rm -f ${PORTSDIR}/${FILE} 839 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 840 -C ${PORTSDIR} ${FILE} 841 ;; 842 esac 843 done; then 844 return 1 845 fi 846 if [ ! -z "${EXTRACTPATH}" ]; then 847 return 0; 848 fi 849 850 extract_metadata 851 extract_indices 852} 853 854# Do the actual work involved in "update" 855update_run() { 856 if ! [ -z "${INDEXONLY}" ]; then 857 extract_indices >/dev/null || return 1 858 return 0 859 fi 860 861 if sort ${WORKDIR}/INDEX | 862 cmp -s ${PORTSDIR}/.portsnap.INDEX -; then 863 echo "Ports tree is already up to date." 864 return 0 865 fi 866 867# If we are REFUSEing to touch certain directories, don't remove files 868# from those directories (even if they are out of date) 869 echo -n "Removing old files and directories... " 870 if ! [ -z "${REFUSE}" ]; then 871 sort ${WORKDIR}/INDEX | 872 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 873 grep -vE "${REFUSE}" | 874 lam -s "${PORTSDIR}/" - | xargs rm -rf 875 else 876 sort ${WORKDIR}/INDEX | 877 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 878 lam -s "${PORTSDIR}/" - | xargs rm -rf 879 fi 880 echo "done." 881 882# Install new files 883 echo "Extracting new files:" 884 if ! 885 if ! [ -z "${REFUSE}" ]; then 886 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort 887 else 888 sort ${WORKDIR}/INDEX 889 fi | 890 comm -13 ${PORTSDIR}/.portsnap.INDEX - | 891 while read LINE; do 892 FILE=`echo ${LINE} | cut -f 1 -d '|'` 893 HASH=`echo ${LINE} | cut -f 2 -d '|'` 894 echo ${PORTSDIR}/${FILE} 895 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 896 echo "files/${HASH}.gz not found -- snapshot corrupt." 897 return 1 898 fi 899 case ${FILE} in 900 */) 901 mkdir -p ${PORTSDIR}/${FILE} 902 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 903 -C ${PORTSDIR}/${FILE} 904 ;; 905 *) 906 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 907 -C ${PORTSDIR} ${FILE} 908 ;; 909 esac 910 done; then 911 return 1 912 fi 913 914 extract_metadata 915 extract_indices 916} 917 918#### Main functions -- call parameter-handling and core functions 919 920# Using the command line, configuration file, and defaults, 921# set all the parameters which are needed later. 922get_params() { 923 init_params 924 parse_cmdline $@ 925 sanity_conffile 926 default_conffile 927 parse_conffile 928 default_params 929} 930 931# Fetch command. Make sure that we're being called 932# interactively, then run fetch_check_params and fetch_run 933cmd_fetch() { 934 if [ ! -t 0 ]; then 935 echo -n "`basename $0` fetch should not " 936 echo "be run non-interactively." 937 echo "Run `basename $0` cron instead." 938 exit 1 939 fi 940 fetch_check_params 941 fetch_run || exit 1 942} 943 944# Cron command. Make sure the parameters are sensible; wait 945# rand(3600) seconds; then fetch updates. While fetching updates, 946# send output to a temporary file; only print that file if the 947# fetching failed. 948cmd_cron() { 949 fetch_check_params 950 sleep `jot -r 1 0 3600` 951 952 TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1 953 if ! fetch_run >> ${TMPFILE}; then 954 cat ${TMPFILE} 955 rm ${TMPFILE} 956 exit 1 957 fi 958 959 rm ${TMPFILE} 960} 961 962# Extract command. Make sure the parameters are sensible, 963# then extract the ports tree (or part thereof). 964cmd_extract() { 965 extract_check_params 966 extract_run || exit 1 967} 968 969# Update command. Make sure the parameters are sensible, 970# then update the ports tree. 971cmd_update() { 972 update_check_params 973 update_run || exit 1 974} 975 976#### Entry point 977 978# Make sure we find utilities from the base system 979export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH} 980 981get_params $@ 982for COMMAND in ${COMMANDS}; do 983 cmd_${COMMAND} 984done 985