portsnap.sh revision 149041
1#!/bin/sh 2 3#- 4# Copyright 2004-2005 Colin Percival 5# All rights reserved 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted providing that the following conditions 9# are met: 10# 1. Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# 2. Redistributions in binary form must reproduce the above copyright 13# notice, this list of conditions and the following disclaimer in the 14# documentation and/or other materials provided with the distribution. 15# 16# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 20# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26# POSSIBILITY OF SUCH DAMAGE. 27 28# $FreeBSD: head/usr.sbin/portsnap/portsnap/portsnap.sh 149041 2005-08-13 21:28:43Z cperciva $ 29 30#### Usage function -- called from command-line handling code. 31 32# Usage instructions. Options not listed: 33# --debug -- don't filter output from utilities 34# --no-stats -- don't show progress statistics while fetching files 35usage() { 36 cat <<EOF 37usage: `basename $0` [options] command ... [path] 38 39Options: 40 -d workdir -- Store working files in workdir 41 (default: /var/db/portsnap/) 42 -f conffile -- Read configuration options from conffile 43 (default: /etc/portsnap.conf) 44 -I -- Update INDEX only. (update command only) 45 -k KEY -- Trust an RSA key with SHA256 hash of KEY 46 -p portsdir -- Location of uncompressed ports tree 47 (default: /usr/ports/) 48 -s server -- Server from which to fetch updates. 49 (default: portsnap.FreeBSD.org) 50 path -- Extract only parts of the tree starting with the given 51 string. (extract command only) 52Commands: 53 fetch -- Fetch a compressed snapshot of the ports tree, 54 or update an existing snapshot. 55 cron -- Sleep rand(3600) seconds, and then fetch updates. 56 extract -- Extract snapshot of ports tree, replacing existing 57 files and directories. 58 update -- Update ports tree to match current snapshot, replacing 59 files and directories which have changed. 60EOF 61 exit 0 62} 63 64#### Parameter handling functions. 65 66# Initialize parameters to null, just in case they're 67# set in the environment. 68init_params() { 69 KEYPRINT="" 70 EXTRACTPATH="" 71 WORKDIR="" 72 PORTSDIR="" 73 CONFFILE="" 74 COMMAND="" 75 COMMANDS="" 76 QUIETREDIR="" 77 QUIETFLAG="" 78 STATSREDIR="" 79 XARGST="" 80 NDEBUG="" 81 DDSTATS="" 82 INDEXONLY="" 83 SERVERNAME="" 84} 85 86# Parse the command line 87parse_cmdline() { 88 while [ $# -gt 0 ]; do 89 case "$1" in 90 -d) 91 if [ $# -eq 1 ]; then usage; fi 92 if [ ! -z "${WORKDIR}" ]; then usage; fi 93 shift; WORKDIR="$1" 94 ;; 95 --debug) 96 QUIETREDIR="/dev/stderr" 97 STATSREDIR="/dev/stderr" 98 QUIETFLAG=" " 99 NDEBUG=" " 100 XARGST="-t" 101 DDSTATS=".." 102 ;; 103 -f) 104 if [ $# -eq 1 ]; then usage; fi 105 if [ ! -z "${CONFFILE}" ]; then usage; fi 106 shift; CONFFILE="$1" 107 ;; 108 -h | --help | help) 109 usage 110 ;; 111 -I) 112 INDEXONLY="YES" 113 ;; 114 -k) 115 if [ $# -eq 1 ]; then usage; fi 116 if [ ! -z "${KEYPRINT}" ]; then usage; fi 117 shift; KEYPRINT="$1" 118 ;; 119 --no-stats) 120 if [ -z "${STATSREDIR}" ]; then 121 STATSREDIR="/dev/null" 122 DDSTATS=".. " 123 fi 124 ;; 125 -p) 126 if [ $# -eq 1 ]; then usage; fi 127 if [ ! -z "${PORTSDIR}" ]; then usage; fi 128 shift; PORTSDIR="$1" 129 ;; 130 -s) 131 if [ $# -eq 1 ]; then usage; fi 132 if [ ! -z "${SERVERNAME}" ]; then usage; fi 133 shift; SERVERNAME="$1" 134 ;; 135 cron | extract | fetch | update) 136 COMMANDS="${COMMANDS} $1" 137 ;; 138 *) 139 if [ $# -gt 1 ]; then usage; fi 140 if echo ${COMMANDS} | grep -vq extract; then 141 usage 142 fi 143 EXTRACTPATH="$1" 144 ;; 145 esac 146 shift 147 done 148 149 if [ -z "${COMMANDS}" ]; then 150 usage 151 fi 152} 153 154# If CONFFILE was specified at the command-line, make 155# sure that it exists and is readable. 156sanity_conffile() { 157 if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then 158 echo -n "File does not exist " 159 echo -n "or is not readable: " 160 echo ${CONFFILE} 161 exit 1 162 fi 163} 164 165# If a configuration file hasn't been specified, use 166# the default value (/etc/portsnap.conf) 167default_conffile() { 168 if [ -z "${CONFFILE}" ]; then 169 CONFFILE="/etc/portsnap.conf" 170 fi 171} 172 173# Read {KEYPRINT, SERVERNAME, WORKDIR, PORTSDIR} from the configuration 174# file if they haven't already been set. If the configuration 175# file doesn't exist, do nothing. 176parse_conffile() { 177 if [ -r "${CONFFILE}" ]; then 178 for X in KEYPRINT WORKDIR PORTSDIR SERVERNAME; do 179 eval _=\$${X} 180 if [ -z "${_}" ]; then 181 eval ${X}=`grep "^${X}=" "${CONFFILE}" | 182 cut -f 2- -d '=' | tail -1` 183 fi 184 done 185 fi 186} 187 188# If parameters have not been set, use default values 189default_params() { 190 _QUIETREDIR="/dev/null" 191 _QUIETFLAG="-q" 192 _STATSREDIR="/dev/stdout" 193 _WORKDIR="/var/db/portsnap" 194 _PORTSDIR="/usr/ports" 195 _NDEBUG="-n" 196 for X in QUIETREDIR QUIETFLAG STATSREDIR WORKDIR PORTSDIR NDEBUG; do 197 eval _=\$${X} 198 eval __=\$_${X} 199 if [ -z "${_}" ]; then 200 eval ${X}=${__} 201 fi 202 done 203} 204 205# Perform sanity checks and set some final parameters 206# in preparation for fetching files. Also chdir into 207# the working directory. 208fetch_check_params() { 209 export HTTP_USER_AGENT="portsnap (${COMMAND}, `uname -r`)" 210 211 _SERVERNAME_z=\ 212"SERVERNAME must be given via command line or configuration file." 213 _KEYPRINT_z="Key must be given via -k option or configuration file." 214 _KEYPRINT_bad="Invalid key fingerprint: " 215 _WORKDIR_bad="Directory does not exist or is not writable: " 216 217 if [ -z "${SERVERNAME}" ]; then 218 echo -n "`basename $0`: " 219 echo "${_SERVERNAME_z}" 220 exit 1 221 fi 222 if [ -z "${KEYPRINT}" ]; then 223 echo -n "`basename $0`: " 224 echo "${_KEYPRINT_z}" 225 exit 1 226 fi 227 if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then 228 echo -n "`basename $0`: " 229 echo -n "${_KEYPRINT_bad}" 230 echo ${KEYPRINT} 231 exit 1 232 fi 233 if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then 234 echo -n "`basename $0`: " 235 echo -n "${_WORKDIR_bad}" 236 echo ${WORKDIR} 237 exit 1 238 fi 239 cd ${WORKDIR} || exit 1 240 241 BSPATCH=/usr/bin/bspatch 242 SHA256=/sbin/sha256 243 PHTTPGET=/usr/libexec/phttpget 244} 245 246# Perform sanity checks and set some final parameters 247# in preparation for extracting or updating ${PORTSDIR} 248extract_check_params() { 249 _WORKDIR_bad="Directory does not exist: " 250 _PORTSDIR_bad="Directory does not exist or is not writable: " 251 252 if ! [ -d "${WORKDIR}" ]; then 253 echo -n "`basename $0`: " 254 echo -n "${_WORKDIR_bad}" 255 echo ${WORKDIR} 256 exit 1 257 fi 258 if ! [ -d "${PORTSDIR}" -a -w "${PORTSDIR}" ]; then 259 echo -n "`basename $0`: " 260 echo -n "${_PORTSDIR_bad}" 261 echo ${PORTSDIR} 262 exit 1 263 fi 264 265 if ! [ -d "${WORKDIR}/files" -a -r "${WORKDIR}/tag" \ 266 -a -r "${WORKDIR}/INDEX" -a -r "${WORKDIR}/tINDEX" ]; then 267 echo "No snapshot available. Try running" 268 echo "# `basename $0` fetch" 269 exit 1 270 fi 271 272 MKINDEX=/usr/libexec/make_index 273} 274 275# Perform sanity checks and set some final parameters 276# in preparation for updating ${PORTSDIR} 277update_check_params() { 278 extract_check_params 279 280 if ! [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 281 echo "${PORTSDIR} was not created by portsnap." 282 echo -n "You must run '`basename $0` extract' before " 283 echo "running '`basename $0` update'." 284 exit 1 285 fi 286 287} 288 289#### Core functionality -- the actual work gets done here 290 291# Use an SRV query to pick a server. If the SRV query doesn't provide 292# a useful answer, use the server name specified by the user. 293# Put another way... look up _http._tcp.${SERVERNAME} and pick a server 294# from that; or if no servers are returned, use ${SERVERNAME}. 295# This allows a user to specify "portsnap.freebsd.org" (in which case 296# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org" 297# (in which case portsnap will use that particular server, since there 298# won't be an SRV entry for that name). 299# 300# We don't implement the recommendations from RFC 2782 completely, since 301# we are only looking to pick a single server -- the recommendations are 302# targetted at applications which obtain a list of servers and then try 303# each in turn, but we are instead just going to pick one server and let 304# the user re-run portsnap if a broken server was selected. 305# 306# We also ignore the Port field, since we are always going to use port 80. 307fetch_pick_server() { 308 echo -n "Looking up ${SERVERNAME} mirrors..." 309 310# Issue the SRV query and pull out the Priority, Weight, and Target fields. 311 host -t srv "_http._tcp.${SERVERNAME}" | 312 grep -E "^_http._tcp.${SERVERNAME} has SRV record" | 313 cut -f 5,6,8 -d ' ' > serverlist 314 315# If no records, give up -- we'll just use the server name we were given. 316 if [ `wc -l < serverlist` -eq 0 ]; then 317 echo " none found." 318 return 319 fi 320 321# Find the highest priority level (lowest numeric value). 322 SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1` 323 324# Add up the weights of the response lines at that priority level. 325 SRV_WSUM=0; 326 while read X; do 327 case "$X" in 328 ${SRV_PRIORITY}\ *) 329 SRV_W=`echo $X | cut -f 2 -d ' '` 330 SRV_WSUM=$(($SRV_WSUM + $SRV_W)) 331 ;; 332 esac 333 done < serverlist 334 335# If all the weights are 0, pretend that they are all 1 instead. 336 if [ ${SRV_WSUM} -eq 0 ]; then 337 SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l` 338 SRV_W_ADD=1 339 else 340 SRV_W_ADD=0 341 fi 342 343# Pick a random value between 1 and the sum of the weights 344 SRV_RND=`jot -r 1 1 ${SRV_WSUM}` 345 346# Read through the list of mirrors and set SERVERNAME 347 while read X; do 348 case "$X" in 349 ${SRV_PRIORITY}\ *) 350 SRV_W=`echo $X | cut -f 2 -d ' '` 351 SRV_W=$(($SRV_W + $SRV_W_ADD)) 352 if [ $SRV_RND -le $SRV_W ]; then 353 SERVERNAME=`echo $X | cut -f 3 -d ' '` 354 break 355 else 356 SRV_RND=$(($SRV_RND - $SRV_W)) 357 fi 358 ;; 359 esac 360 done < serverlist 361 362 echo " using ${SERVERNAME}" 363} 364 365# Check that we have a public key with an appropriate hash, or 366# fetch the key if it doesn't exist. 367fetch_key() { 368 if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 369 return 370 fi 371 372 echo -n "Fetching public key... " 373 rm -f pub.ssl 374 fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \ 375 2>${QUIETREDIR} || true 376 if ! [ -r pub.ssl ]; then 377 echo "failed." 378 return 1 379 fi 380 if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 381 echo "key has incorrect hash." 382 rm -f pub.ssl 383 return 1 384 fi 385 echo "done." 386} 387 388# Fetch a snapshot tag 389fetch_tag() { 390 rm -f snapshot.ssl tag.new 391 392 echo ${NDEBUG} "Fetching snapshot tag... " 393 fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl 394 2>${QUIETREDIR} || true 395 if ! [ -r $1.ssl ]; then 396 echo "failed." 397 return 1 398 fi 399 400 openssl rsautl -pubin -inkey pub.ssl -verify \ 401 < $1.ssl > tag.new 2>${QUIETREDIR} || true 402 rm $1.ssl 403 404 if ! [ `wc -l < tag.new` = 1 ] || 405 ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then 406 echo "invalid snapshot tag." 407 return 1 408 fi 409 410 echo "done." 411 412 SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new` 413 SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new` 414} 415 416# Sanity-check the date on a snapshot tag 417fetch_snapshot_tagsanity() { 418 if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then 419 echo "Snapshot appears to be more than a year old!" 420 echo "(Is the system clock correct?)" 421 echo "Cowarly refusing to proceed any further." 422 return 1 423 fi 424 if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then 425 echo -n "Snapshot appears to have been created more than " 426 echo "one day into the future!" 427 echo "(Is the system clock correct?)" 428 echo "Cowardly refusing to proceed any further." 429 return 1 430 fi 431} 432 433# Sanity-check the date on a snapshot update tag 434fetch_update_tagsanity() { 435 fetch_snapshot_tagsanity || return 1 436 437 if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then 438 echo -n "Latest snapshot on server is " 439 echo "older than what we already have!" 440 echo -n "Cowardly refusing to downgrade from " 441 date -r ${OLDSNAPSHOTDATE} 442 echo "to `date -r ${SNAPSHOTDATE}`." 443 return 1 444 fi 445} 446 447# Compare old and new tags; return 1 if update is unnecessary 448fetch_update_neededp() { 449 if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then 450 echo -n "Latest snapshot on server matches " 451 echo "what we already have." 452 echo "No updates needed." 453 rm tag.new 454 return 1 455 fi 456 if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then 457 echo -n "Ports tree hasn't changed since " 458 echo "last snapshot." 459 echo "No updates needed." 460 rm tag.new 461 return 1 462 fi 463 464 return 0 465} 466 467# Fetch snapshot metadata file 468fetch_metadata() { 469 rm -f ${SNAPSHOTHASH} tINDEX.new 470 471 echo ${NDEBUG} "Fetching snapshot metadata... " 472 fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} 473 2>${QUIETREDIR} || return 474 if [ `${SHA256} -q ${SNAPSHOTHASH}` != ${SNAPSHOTHASH} ]; then 475 echo "snapshot metadata corrupt." 476 return 1 477 fi 478 mv ${SNAPSHOTHASH} tINDEX.new 479 echo "done." 480} 481 482# Warn user about bogus metadata 483fetch_metadata_freakout() { 484 echo 485 echo "Portsnap metadata is correctly signed, but contains" 486 echo "at least one line which appears bogus." 487 echo "Cowardly refusing to proceed any further." 488} 489 490# Sanity-check a snapshot metadata file 491fetch_metadata_sanity() { 492 if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then 493 fetch_metadata_freakout 494 return 1 495 fi 496 if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then 497 echo 498 echo "Portsnap metadata appears bogus." 499 echo "Cowardly refusing to proceed any further." 500 return 1 501 fi 502} 503 504# Take a list of ${oldhash}|${newhash} and output a list of needed patches 505fetch_make_patchlist() { 506 grep -vE "^([0-9a-f]{64})\|\1$" | 507 while read LINE; do 508 X=`echo ${LINE} | cut -f 1 -d '|'` 509 Y=`echo ${LINE} | cut -f 2 -d '|'` 510 if [ -f "files/${Y}.gz" ]; then continue; fi 511 if [ ! -f "files/${X}.gz" ]; then continue; fi 512 echo "${LINE}" 513 done 514} 515 516# Print user-friendly progress statistics 517fetch_progress() { 518 LNC=0 519 while read x; do 520 LNC=$(($LNC + 1)) 521 if [ $(($LNC % 10)) = 0 ]; then 522 echo -n $LNC 523 elif [ $(($LNC % 2)) = 0 ]; then 524 echo -n . 525 fi 526 done 527 echo -n " " 528} 529 530# Sanity-check an index file 531fetch_index_sanity() { 532 if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new || 533 fgrep -q "./" INDEX.new; then 534 fetch_metadata_freakout 535 return 1 536 fi 537} 538 539# Verify a list of files 540fetch_snapshot_verify() { 541 while read F; do 542 if [ `gunzip -c snap/${F} | ${SHA256} -q` != ${F} ]; then 543 echo "snapshot corrupt." 544 return 1 545 fi 546 done 547 return 0 548} 549 550# Fetch a snapshot tarball, extract, and verify. 551fetch_snapshot() { 552 fetch_tag snapshot || return 1 553 fetch_snapshot_tagsanity || return 1 554 fetch_metadata || return 1 555 fetch_metadata_sanity || return 1 556 557 rm -f ${SNAPSHOTHASH}.tgz 558 rm -rf snap/ 559 560# Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will 561# probably take a while, so the progrees reports that fetch(1) generates 562# will be useful for keeping the users' attention from drifting. 563 echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:" 564 fetch http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1 565 566 echo -n "Extracting snapshot... " 567 tar -xzf ${SNAPSHOTHASH}.tgz snap/ || return 1 568 rm ${SNAPSHOTHASH}.tgz 569 echo "done." 570 571 echo -n "Verifying snapshot integrity... " 572# Verify the metadata files 573 cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1 574# Extract the index 575 rm -f INDEX.new 576 gunzip -c snap/`look INDEX tINDEX.new | 577 cut -f 2 -d '|'`.gz > INDEX.new 578 fetch_index_sanity || return 1 579# Verify the snapshot contents 580 cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1 581 echo "done." 582 583# Move files into their proper locations 584 rm -f tag INDEX tINDEX 585 rm -rf files 586 mv tag.new tag 587 mv tINDEX.new tINDEX 588 mv INDEX.new INDEX 589 mv snap/ files/ 590 591 return 0 592} 593 594# Update a compressed snapshot 595fetch_update() { 596 rm -f patchlist diff OLD NEW filelist INDEX.new 597 598 OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag` 599 OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag` 600 601 fetch_tag latest || return 1 602 fetch_update_tagsanity || return 1 603 fetch_update_neededp || return 0 604 fetch_metadata || return 1 605 fetch_metadata_sanity || return 1 606 607 echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` " 608 echo "to `date -r ${SNAPSHOTDATE}`." 609 610# Generate a list of wanted metadata patches 611 join -t '|' -o 1.2,2.2 tINDEX tINDEX.new | 612 fetch_make_patchlist > patchlist 613 614# Attempt to fetch metadata patches 615 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 616 echo ${NDEBUG} "metadata patches.${DDSTATS}" 617 tr '|' '-' < patchlist | 618 lam -s "tp/" - -s ".gz" | 619 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 620 2>${STATSREDIR} | fetch_progress 621 echo "done." 622 623# Attempt to apply metadata patches 624 echo -n "Applying metadata patches... " 625 while read LINE; do 626 X=`echo ${LINE} | cut -f 1 -d '|'` 627 Y=`echo ${LINE} | cut -f 2 -d '|'` 628 if [ ! -f "${X}-${Y}.gz" ]; then continue; fi 629 gunzip -c < ${X}-${Y}.gz > diff 630 gunzip -c < files/${X}.gz > OLD 631 cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp 632 grep '^\+' diff | cut -c 2- | 633 sort -k 1,1 -t '|' -m - ptmp > NEW 634 if [ `${SHA256} -q NEW` = ${Y} ]; then 635 mv NEW files/${Y} 636 gzip -n files/${Y} 637 fi 638 rm -f diff OLD NEW ${X}-${Y}.gz ptmp 639 done < patchlist 2>${QUIETREDIR} 640 echo "done." 641 642# Update metadata without patches 643 join -t '|' -v 2 tINDEX tINDEX.new | 644 cut -f 2 -d '|' /dev/stdin patchlist | 645 while read Y; do 646 if [ ! -f "files/${Y}.gz" ]; then 647 echo ${Y}; 648 fi 649 done > filelist 650 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 651 echo ${NDEBUG} "metadata files... " 652 lam -s "f/" - -s ".gz" < filelist | 653 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 654 2>${QUIETREDIR} 655 656 while read Y; do 657 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 658 mv ${Y}.gz files/${Y}.gz 659 else 660 echo "metadata is corrupt." 661 return 1 662 fi 663 done < filelist 664 echo "done." 665 666# Extract the index 667 gunzip -c files/`look INDEX tINDEX.new | 668 cut -f 2 -d '|'`.gz > INDEX.new 669 fetch_index_sanity || return 1 670 671# Generate a list of wanted ports patches 672 join -t '|' -o 1.2,2.2 INDEX INDEX.new | 673 fetch_make_patchlist > patchlist 674 675# Attempt to fetch ports patches 676 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 677 echo ${NDEBUG} "patches.${DDSTATS}" 678 tr '|' '-' < patchlist | lam -s "bp/" - | 679 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 680 2>${STATSREDIR} | fetch_progress 681 echo "done." 682 683# Attempt to apply ports patches 684 echo -n "Applying patches... " 685 while read LINE; do 686 X=`echo ${LINE} | cut -f 1 -d '|'` 687 Y=`echo ${LINE} | cut -f 2 -d '|'` 688 if [ ! -f "${X}-${Y}" ]; then continue; fi 689 gunzip -c < files/${X}.gz > OLD 690 ${BSPATCH} OLD NEW ${X}-${Y} 691 if [ `${SHA256} -q NEW` = ${Y} ]; then 692 mv NEW files/${Y} 693 gzip -n files/${Y} 694 fi 695 rm -f diff OLD NEW ${X}-${Y} 696 done < patchlist 2>${QUIETREDIR} 697 echo "done." 698 699# Update ports without patches 700 join -t '|' -v 2 INDEX INDEX.new | 701 cut -f 2 -d '|' /dev/stdin patchlist | 702 while read Y; do 703 if [ ! -f "files/${Y}.gz" ]; then 704 echo ${Y}; 705 fi 706 done > filelist 707 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 708 echo ${NDEBUG} "new ports or files... " 709 lam -s "f/" - -s ".gz" < filelist | 710 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 711 2>${QUIETREDIR} 712 713 while read Y; do 714 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 715 mv ${Y}.gz files/${Y}.gz 716 else 717 echo "snapshot is corrupt." 718 return 1 719 fi 720 done < filelist 721 echo "done." 722 723# Remove files which are no longer needed 724 cut -f 2 -d '|' tINDEX INDEX | sort > oldfiles 725 cut -f 2 -d '|' tINDEX.new INDEX.new | sort | comm -13 - oldfiles | 726 lam -s "files/" - -s ".gz" | xargs rm -f 727 rm patchlist filelist oldfiles 728 729# We're done! 730 mv INDEX.new INDEX 731 mv tINDEX.new tINDEX 732 mv tag.new tag 733 734 return 0 735} 736 737# Do the actual work involved in "fetch" / "cron". 738fetch_run() { 739 fetch_pick_server 740 741 fetch_key || return 1 742 743 if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then 744 fetch_snapshot || return 1 745 fi 746 fetch_update || return 1 747} 748 749# Build a ports INDEX file 750extract_make_index() { 751 gunzip -c "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX | 752 cut -f 2 -d '|'`.gz" | ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2 753} 754 755# Create INDEX, INDEX-5, INDEX-6 756extract_indices() { 757 echo -n "Building new INDEX files... " 758 extract_make_index DESCRIBE.4 INDEX || return 1 759 extract_make_index DESCRIBE.5 INDEX-5 || return 1 760 extract_make_index DESCRIBE.6 INDEX-6 || return 1 761 echo "done." 762} 763 764# Create .portsnap.INDEX 765extract_metadata() { 766 sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX 767} 768 769# Do the actual work involved in "extract" 770extract_run() { 771 if ! grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX | while read LINE; do 772 FILE=`echo ${LINE} | cut -f 1 -d '|'` 773 HASH=`echo ${LINE} | cut -f 2 -d '|'` 774 echo ${PORTSDIR}/${FILE} 775 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 776 echo "files/${HASH}.gz not found -- snapshot corrupt." 777 return 1 778 fi 779 case ${FILE} in 780 */) 781 rm -rf ${PORTSDIR}/${FILE} 782 mkdir -p ${PORTSDIR}/${FILE} 783 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 784 -C ${PORTSDIR}/${FILE} 785 ;; 786 *) 787 rm -f ${PORTSDIR}/${FILE} 788 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 789 -C ${PORTSDIR} ${FILE} 790 ;; 791 esac 792 done; then 793 return 1 794 fi 795 if [ ! -z "${EXTRACTPATH}" ]; then 796 return 0; 797 fi 798 799 extract_metadata 800 extract_indices 801} 802 803# Do the actual work involved in "update" 804update_run() { 805 if ! [ -z "${INDEXONLY}" ]; then 806 extract_indices >/dev/null || return 1 807 return 0 808 fi 809 810 if sort ${WORKDIR}/INDEX | 811 cmp ${PORTSDIR}/.portsnap.INDEX - >/dev/null; then 812 echo "Ports tree is already up to date." 813 return 0 814 fi 815 816 echo -n "Removing old files and directories... " 817 sort ${WORKDIR}/INDEX | comm -23 ${PORTSDIR}/.portsnap.INDEX - | 818 cut -f 1 -d '|' | lam -s "${PORTSDIR}/" - | xargs rm -rf 819 echo "done." 820 821# Install new files 822 echo "Extracting new files:" 823 if ! sort ${WORKDIR}/INDEX | 824 comm -13 ${PORTSDIR}/.portsnap.INDEX - | 825 while read LINE; do 826 FILE=`echo ${LINE} | cut -f 1 -d '|'` 827 HASH=`echo ${LINE} | cut -f 2 -d '|'` 828 echo ${PORTSDIR}/${FILE} 829 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 830 echo "files/${HASH}.gz not found -- snapshot corrupt." 831 return 1 832 fi 833 case ${FILE} in 834 */) 835 mkdir -p ${PORTSDIR}/${FILE} 836 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 837 -C ${PORTSDIR}/${FILE} 838 ;; 839 *) 840 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 841 -C ${PORTSDIR} ${FILE} 842 ;; 843 esac 844 done; then 845 return 1 846 fi 847 848 extract_metadata 849 extract_indices 850} 851 852#### Main functions -- call parameter-handling and core functions 853 854# Using the command line, configuration file, and defaults, 855# set all the parameters which are needed later. 856get_params() { 857 init_params 858 parse_cmdline $@ 859 sanity_conffile 860 default_conffile 861 parse_conffile 862 default_params 863} 864 865# Fetch command. Make sure that we're being called 866# interactively, then run fetch_check_params and fetch_run 867cmd_fetch() { 868 if [ ! -t 0 ]; then 869 echo -n "`basename $0` fetch should not " 870 echo "be run non-interactively." 871 echo "Run `basename $0` cron instead." 872 exit 1 873 fi 874 fetch_check_params 875 fetch_run || exit 1 876} 877 878# Cron command. Make sure the parameters are sensible; wait 879# rand(3600) seconds; then fetch updates. While fetching updates, 880# send output to a temporary file; only print that file if the 881# fetching failed. 882cmd_cron() { 883 fetch_check_params 884 sleep `jot -r 1 0 3600` 885 886 TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1 887 if ! fetch_run >> ${TMPFILE}; then 888 cat ${TMPFILE} 889 rm ${TMPFILE} 890 exit 1 891 fi 892 893 rm ${TMPFILE} 894} 895 896# Extract command. Make sure the parameters are sensible, 897# then extract the ports tree (or part thereof). 898cmd_extract() { 899 extract_check_params 900 extract_run || exit 1 901} 902 903# Update command. Make sure the parameters are sensible, 904# then update the ports tree. 905cmd_update() { 906 update_check_params 907 update_run || exit 1 908} 909 910#### Entry point 911 912# Make sure we find utilities from the base system 913export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH} 914 915get_params $@ 916for COMMAND in ${COMMANDS}; do 917 cmd_${COMMAND} 918done 919