portsnap.sh revision 158523
1#!/bin/sh 2 3#- 4# Copyright 2004-2005 Colin Percival 5# All rights reserved 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted providing that the following conditions 9# are met: 10# 1. Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# 2. Redistributions in binary form must reproduce the above copyright 13# notice, this list of conditions and the following disclaimer in the 14# documentation and/or other materials provided with the distribution. 15# 16# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 20# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26# POSSIBILITY OF SUCH DAMAGE. 27 28# $FreeBSD: head/usr.sbin/portsnap/portsnap/portsnap.sh 158523 2006-05-13 15:56:35Z cperciva $ 29 30#### Usage function -- called from command-line handling code. 31 32# Usage instructions. Options not listed: 33# --debug -- don't filter output from utilities 34# --no-stats -- don't show progress statistics while fetching files 35usage() { 36 cat <<EOF 37usage: `basename $0` [options] command ... [path] 38 39Options: 40 -d workdir -- Store working files in workdir 41 (default: /var/db/portsnap/) 42 -f conffile -- Read configuration options from conffile 43 (default: /etc/portsnap.conf) 44 -I -- Update INDEX only. (update command only) 45 -k KEY -- Trust an RSA key with SHA256 hash of KEY 46 -l descfile -- Merge the specified local describes file into the INDEX. 47 -p portsdir -- Location of uncompressed ports tree 48 (default: /usr/ports/) 49 -s server -- Server from which to fetch updates. 50 (default: portsnap.FreeBSD.org) 51 path -- Extract only parts of the tree starting with the given 52 string. (extract command only) 53Commands: 54 fetch -- Fetch a compressed snapshot of the ports tree, 55 or update an existing snapshot. 56 cron -- Sleep rand(3600) seconds, and then fetch updates. 57 extract -- Extract snapshot of ports tree, replacing existing 58 files and directories. 59 update -- Update ports tree to match current snapshot, replacing 60 files and directories which have changed. 61EOF 62 exit 0 63} 64 65#### Parameter handling functions. 66 67# Initialize parameters to null, just in case they're 68# set in the environment. 69init_params() { 70 KEYPRINT="" 71 EXTRACTPATH="" 72 WORKDIR="" 73 PORTSDIR="" 74 CONFFILE="" 75 COMMAND="" 76 COMMANDS="" 77 QUIETREDIR="" 78 QUIETFLAG="" 79 STATSREDIR="" 80 XARGST="" 81 NDEBUG="" 82 DDSTATS="" 83 INDEXONLY="" 84 SERVERNAME="" 85 REFUSE="" 86 LOCALDESC="" 87} 88 89# Parse the command line 90parse_cmdline() { 91 while [ $# -gt 0 ]; do 92 case "$1" in 93 -d) 94 if [ $# -eq 1 ]; then usage; fi 95 if [ ! -z "${WORKDIR}" ]; then usage; fi 96 shift; WORKDIR="$1" 97 ;; 98 --debug) 99 QUIETREDIR="/dev/stderr" 100 STATSREDIR="/dev/stderr" 101 QUIETFLAG=" " 102 NDEBUG=" " 103 XARGST="-t" 104 DDSTATS=".." 105 ;; 106 -f) 107 if [ $# -eq 1 ]; then usage; fi 108 if [ ! -z "${CONFFILE}" ]; then usage; fi 109 shift; CONFFILE="$1" 110 ;; 111 -h | --help | help) 112 usage 113 ;; 114 -I) 115 INDEXONLY="YES" 116 ;; 117 -k) 118 if [ $# -eq 1 ]; then usage; fi 119 if [ ! -z "${KEYPRINT}" ]; then usage; fi 120 shift; KEYPRINT="$1" 121 ;; 122 -l) 123 if [ $# -eq 1 ]; then usage; fi 124 if [ ! -z "${LOCALDESC}" ]; then usage; fi 125 shift; LOCALDESC="$1" 126 ;; 127 --no-stats) 128 if [ -z "${STATSREDIR}" ]; then 129 STATSREDIR="/dev/null" 130 DDSTATS=".. " 131 fi 132 ;; 133 -p) 134 if [ $# -eq 1 ]; then usage; fi 135 if [ ! -z "${PORTSDIR}" ]; then usage; fi 136 shift; PORTSDIR="$1" 137 ;; 138 -s) 139 if [ $# -eq 1 ]; then usage; fi 140 if [ ! -z "${SERVERNAME}" ]; then usage; fi 141 shift; SERVERNAME="$1" 142 ;; 143 cron | extract | fetch | update) 144 COMMANDS="${COMMANDS} $1" 145 ;; 146 *) 147 if [ $# -gt 1 ]; then usage; fi 148 if echo ${COMMANDS} | grep -vq extract; then 149 usage 150 fi 151 EXTRACTPATH="$1" 152 ;; 153 esac 154 shift 155 done 156 157 if [ -z "${COMMANDS}" ]; then 158 usage 159 fi 160} 161 162# If CONFFILE was specified at the command-line, make 163# sure that it exists and is readable. 164sanity_conffile() { 165 if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then 166 echo -n "File does not exist " 167 echo -n "or is not readable: " 168 echo ${CONFFILE} 169 exit 1 170 fi 171} 172 173# If a configuration file hasn't been specified, use 174# the default value (/etc/portsnap.conf) 175default_conffile() { 176 if [ -z "${CONFFILE}" ]; then 177 CONFFILE="/etc/portsnap.conf" 178 fi 179} 180 181# Read {KEYPRINT, SERVERNAME, WORKDIR, PORTSDIR} from the configuration 182# file if they haven't already been set. If the configuration 183# file doesn't exist, do nothing. 184# Also read REFUSE (which cannot be set via the command line) if it is 185# present in the configuration file. 186parse_conffile() { 187 if [ -r "${CONFFILE}" ]; then 188 for X in KEYPRINT WORKDIR PORTSDIR SERVERNAME; do 189 eval _=\$${X} 190 if [ -z "${_}" ]; then 191 eval ${X}=`grep "^${X}=" "${CONFFILE}" | 192 cut -f 2- -d '=' | tail -1` 193 fi 194 done 195 196 if grep -qE "^REFUSE[[:space:]]" ${CONFFILE}; then 197 REFUSE="^(` 198 grep -E "^REFUSE[[:space:]]" "${CONFFILE}" | 199 cut -c 7- | xargs echo | tr ' ' '|' 200 `)" 201 fi 202 fi 203} 204 205# If parameters have not been set, use default values 206default_params() { 207 _QUIETREDIR="/dev/null" 208 _QUIETFLAG="-q" 209 _STATSREDIR="/dev/stdout" 210 _WORKDIR="/var/db/portsnap" 211 _PORTSDIR="/usr/ports" 212 _NDEBUG="-n" 213 _LOCALDESC="/dev/null" 214 for X in QUIETREDIR QUIETFLAG STATSREDIR WORKDIR PORTSDIR \ 215 NDEBUG LOCALDESC; do 216 eval _=\$${X} 217 eval __=\$_${X} 218 if [ -z "${_}" ]; then 219 eval ${X}=${__} 220 fi 221 done 222} 223 224# Perform sanity checks and set some final parameters 225# in preparation for fetching files. Also chdir into 226# the working directory. 227fetch_check_params() { 228 export HTTP_USER_AGENT="portsnap (${COMMAND}, `uname -r`)" 229 230 _SERVERNAME_z=\ 231"SERVERNAME must be given via command line or configuration file." 232 _KEYPRINT_z="Key must be given via -k option or configuration file." 233 _KEYPRINT_bad="Invalid key fingerprint: " 234 _WORKDIR_bad="Directory does not exist or is not writable: " 235 236 if [ -z "${SERVERNAME}" ]; then 237 echo -n "`basename $0`: " 238 echo "${_SERVERNAME_z}" 239 exit 1 240 fi 241 if [ -z "${KEYPRINT}" ]; then 242 echo -n "`basename $0`: " 243 echo "${_KEYPRINT_z}" 244 exit 1 245 fi 246 if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then 247 echo -n "`basename $0`: " 248 echo -n "${_KEYPRINT_bad}" 249 echo ${KEYPRINT} 250 exit 1 251 fi 252 if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then 253 echo -n "`basename $0`: " 254 echo -n "${_WORKDIR_bad}" 255 echo ${WORKDIR} 256 exit 1 257 fi 258 cd ${WORKDIR} || exit 1 259 260 BSPATCH=/usr/bin/bspatch 261 SHA256=/sbin/sha256 262 PHTTPGET=/usr/libexec/phttpget 263} 264 265# Perform sanity checks and set some final parameters 266# in preparation for extracting or updating ${PORTSDIR} 267# Complain if ${PORTSDIR} exists but is not writable, 268# but don't complain if ${PORTSDIR} doesn't exist. 269extract_check_params() { 270 _WORKDIR_bad="Directory does not exist: " 271 _PORTSDIR_bad="Directory is not writable: " 272 273 if ! [ -d "${WORKDIR}" ]; then 274 echo -n "`basename $0`: " 275 echo -n "${_WORKDIR_bad}" 276 echo ${WORKDIR} 277 exit 1 278 fi 279 if [ -d "${PORTSDIR}" ] && ! [ -w "${PORTSDIR}" ]; then 280 echo -n "`basename $0`: " 281 echo -n "${_PORTSDIR_bad}" 282 echo ${PORTSDIR} 283 exit 1 284 fi 285 286 if ! [ -d "${WORKDIR}/files" -a -r "${WORKDIR}/tag" \ 287 -a -r "${WORKDIR}/INDEX" -a -r "${WORKDIR}/tINDEX" ]; then 288 echo "No snapshot available. Try running" 289 echo "# `basename $0` fetch" 290 exit 1 291 fi 292 293 MKINDEX=/usr/libexec/make_index 294} 295 296# Perform sanity checks and set some final parameters 297# in preparation for updating ${PORTSDIR} 298update_check_params() { 299 extract_check_params 300 301 if ! [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 302 echo "${PORTSDIR} was not created by portsnap." 303 echo -n "You must run '`basename $0` extract' before " 304 echo "running '`basename $0` update'." 305 exit 1 306 fi 307 308} 309 310#### Core functionality -- the actual work gets done here 311 312# Use an SRV query to pick a server. If the SRV query doesn't provide 313# a useful answer, use the server name specified by the user. 314# Put another way... look up _http._tcp.${SERVERNAME} and pick a server 315# from that; or if no servers are returned, use ${SERVERNAME}. 316# This allows a user to specify "portsnap.freebsd.org" (in which case 317# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org" 318# (in which case portsnap will use that particular server, since there 319# won't be an SRV entry for that name). 320# 321# We ignore the Port field, since we are always going to use port 80. 322 323# Fetch the mirror list, but do not pick a mirror yet. Returns 1 if 324# no mirrors are available for any reason. 325fetch_pick_server_init() { 326 : > serverlist_tried 327 328# Check that host(1) exists (i.e., that the system wasn't built with the 329# WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist. 330 if ! which -s host; then 331 : > serverlist_full 332 return 1 333 fi 334 335 echo -n "Looking up ${SERVERNAME} mirrors... " 336 337# Issue the SRV query and pull out the Priority, Weight, and Target fields. 338# BIND 9 prints "$name has SRV record ..." while BIND 8 prints 339# "$name server selection ..."; we allow either format. 340 MLIST="_http._tcp.${SERVERNAME}" 341 host -t srv "${MLIST}" | 342 sed -nE "s/${MLIST} (has SRV record|server selection) //p" | 343 cut -f 1,2,4 -d ' ' | 344 sed -e 's/\.$//' | 345 sort > serverlist_full 346 347# If no records, give up -- we'll just use the server name we were given. 348 if [ `wc -l < serverlist_full` -eq 0 ]; then 349 echo "none found." 350 return 1 351 fi 352 353# Report how many mirrors we found. 354 echo `wc -l < serverlist_full` "mirrors found." 355 356# Generate a random seed for use in picking mirrors. If HTTP_PROXY 357# is set, this will be used to generate the seed; otherwise, the seed 358# will be random. 359 if [ -n "${HTTP_PROXY}${http_proxy}" ]; then 360 RANDVALUE=`sha256 -qs "${HTTP_PROXY}${http_proxy}" | 361 tr -d 'a-f' | 362 cut -c 1-9` 363 else 364 RANDVALUE=`jot -r 1 0 999999999` 365 fi 366} 367 368# Pick a mirror. Returns 1 if we have run out of mirrors to try. 369fetch_pick_server() { 370# Generate a list of not-yet-tried mirrors 371 sort serverlist_tried | 372 comm -23 serverlist_full - > serverlist 373 374# Have we run out of mirrors? 375 if [ `wc -l < serverlist` -eq 0 ]; then 376 echo "No mirrors remaining, giving up." 377 return 1 378 fi 379 380# Find the highest priority level (lowest numeric value). 381 SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1` 382 383# Add up the weights of the response lines at that priority level. 384 SRV_WSUM=0; 385 while read X; do 386 case "$X" in 387 ${SRV_PRIORITY}\ *) 388 SRV_W=`echo $X | cut -f 2 -d ' '` 389 SRV_WSUM=$(($SRV_WSUM + $SRV_W)) 390 ;; 391 esac 392 done < serverlist 393 394# If all the weights are 0, pretend that they are all 1 instead. 395 if [ ${SRV_WSUM} -eq 0 ]; then 396 SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l` 397 SRV_W_ADD=1 398 else 399 SRV_W_ADD=0 400 fi 401 402# Pick a value between 0 and the sum of the weights - 1 403 SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}` 404 405# Read through the list of mirrors and set SERVERNAME. Write the line 406# corresponding to the mirror we selected into serverlist_tried so that 407# we won't try it again. 408 while read X; do 409 case "$X" in 410 ${SRV_PRIORITY}\ *) 411 SRV_W=`echo $X | cut -f 2 -d ' '` 412 SRV_W=$(($SRV_W + $SRV_W_ADD)) 413 if [ $SRV_RND -lt $SRV_W ]; then 414 SERVERNAME=`echo $X | cut -f 3 -d ' '` 415 echo "$X" >> serverlist_tried 416 break 417 else 418 SRV_RND=$(($SRV_RND - $SRV_W)) 419 fi 420 ;; 421 esac 422 done < serverlist 423} 424 425# Check that we have a public key with an appropriate hash, or 426# fetch the key if it doesn't exist. Returns 1 if the key has 427# not yet been fetched. 428fetch_key() { 429 if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 430 return 0 431 fi 432 433 echo -n "Fetching public key from ${SERVERNAME}... " 434 rm -f pub.ssl 435 fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \ 436 2>${QUIETREDIR} || true 437 if ! [ -r pub.ssl ]; then 438 echo "failed." 439 return 1 440 fi 441 if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 442 echo "key has incorrect hash." 443 rm -f pub.ssl 444 return 1 445 fi 446 echo "done." 447} 448 449# Fetch a snapshot tag 450fetch_tag() { 451 rm -f snapshot.ssl tag.new 452 453 echo ${NDEBUG} "Fetching snapshot tag from ${SERVERNAME}... " 454 fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl \ 455 2>${QUIETREDIR} || true 456 if ! [ -r $1.ssl ]; then 457 echo "failed." 458 return 1 459 fi 460 461 openssl rsautl -pubin -inkey pub.ssl -verify \ 462 < $1.ssl > tag.new 2>${QUIETREDIR} || true 463 rm $1.ssl 464 465 if ! [ `wc -l < tag.new` = 1 ] || 466 ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then 467 echo "invalid snapshot tag." 468 return 1 469 fi 470 471 echo "done." 472 473 SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new` 474 SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new` 475} 476 477# Sanity-check the date on a snapshot tag 478fetch_snapshot_tagsanity() { 479 if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then 480 echo "Snapshot appears to be more than a year old!" 481 echo "(Is the system clock correct?)" 482 echo "Cowardly refusing to proceed any further." 483 return 1 484 fi 485 if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then 486 echo -n "Snapshot appears to have been created more than " 487 echo "one day into the future!" 488 echo "(Is the system clock correct?)" 489 echo "Cowardly refusing to proceed any further." 490 return 1 491 fi 492} 493 494# Sanity-check the date on a snapshot update tag 495fetch_update_tagsanity() { 496 fetch_snapshot_tagsanity || return 1 497 498 if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then 499 echo -n "Latest snapshot on server is " 500 echo "older than what we already have!" 501 echo -n "Cowardly refusing to downgrade from " 502 date -r ${OLDSNAPSHOTDATE} 503 echo "to `date -r ${SNAPSHOTDATE}`." 504 return 1 505 fi 506} 507 508# Compare old and new tags; return 1 if update is unnecessary 509fetch_update_neededp() { 510 if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then 511 echo -n "Latest snapshot on server matches " 512 echo "what we already have." 513 echo "No updates needed." 514 rm tag.new 515 return 1 516 fi 517 if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then 518 echo -n "Ports tree hasn't changed since " 519 echo "last snapshot." 520 echo "No updates needed." 521 rm tag.new 522 return 1 523 fi 524 525 return 0 526} 527 528# Fetch snapshot metadata file 529fetch_metadata() { 530 rm -f ${SNAPSHOTHASH} tINDEX.new 531 532 echo ${NDEBUG} "Fetching snapshot metadata... " 533 fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} 534 2>${QUIETREDIR} || return 535 if [ `${SHA256} -q ${SNAPSHOTHASH}` != ${SNAPSHOTHASH} ]; then 536 echo "snapshot metadata corrupt." 537 return 1 538 fi 539 mv ${SNAPSHOTHASH} tINDEX.new 540 echo "done." 541} 542 543# Warn user about bogus metadata 544fetch_metadata_freakout() { 545 echo 546 echo "Portsnap metadata is correctly signed, but contains" 547 echo "at least one line which appears bogus." 548 echo "Cowardly refusing to proceed any further." 549} 550 551# Sanity-check a snapshot metadata file 552fetch_metadata_sanity() { 553 if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then 554 fetch_metadata_freakout 555 return 1 556 fi 557 if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then 558 echo 559 echo "Portsnap metadata appears bogus." 560 echo "Cowardly refusing to proceed any further." 561 return 1 562 fi 563} 564 565# Take a list of ${oldhash}|${newhash} and output a list of needed patches 566fetch_make_patchlist() { 567 grep -vE "^([0-9a-f]{64})\|\1$" | 568 while read LINE; do 569 X=`echo ${LINE} | cut -f 1 -d '|'` 570 Y=`echo ${LINE} | cut -f 2 -d '|'` 571 if [ -f "files/${Y}.gz" ]; then continue; fi 572 if [ ! -f "files/${X}.gz" ]; then continue; fi 573 echo "${LINE}" 574 done 575} 576 577# Print user-friendly progress statistics 578fetch_progress() { 579 LNC=0 580 while read x; do 581 LNC=$(($LNC + 1)) 582 if [ $(($LNC % 10)) = 0 ]; then 583 echo -n $LNC 584 elif [ $(($LNC % 2)) = 0 ]; then 585 echo -n . 586 fi 587 done 588 echo -n " " 589} 590 591# Sanity-check an index file 592fetch_index_sanity() { 593 if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new || 594 fgrep -q "./" INDEX.new; then 595 fetch_metadata_freakout 596 return 1 597 fi 598} 599 600# Verify a list of files 601fetch_snapshot_verify() { 602 while read F; do 603 if [ `gunzip -c snap/${F} | ${SHA256} -q` != ${F} ]; then 604 echo "snapshot corrupt." 605 return 1 606 fi 607 done 608 return 0 609} 610 611# Fetch a snapshot tarball, extract, and verify. 612fetch_snapshot() { 613 while ! fetch_tag snapshot; do 614 fetch_pick_server || return 1 615 done 616 fetch_snapshot_tagsanity || return 1 617 fetch_metadata || return 1 618 fetch_metadata_sanity || return 1 619 620 rm -rf snap/ 621 622# Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will 623# probably take a while, so the progrees reports that fetch(1) generates 624# will be useful for keeping the users' attention from drifting. 625 echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:" 626 fetch -r http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1 627 628 echo -n "Extracting snapshot... " 629 tar -xzf ${SNAPSHOTHASH}.tgz snap/ || return 1 630 rm ${SNAPSHOTHASH}.tgz 631 echo "done." 632 633 echo -n "Verifying snapshot integrity... " 634# Verify the metadata files 635 cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1 636# Extract the index 637 rm -f INDEX.new 638 gunzip -c snap/`look INDEX tINDEX.new | 639 cut -f 2 -d '|'`.gz > INDEX.new 640 fetch_index_sanity || return 1 641# Verify the snapshot contents 642 cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1 643 echo "done." 644 645# Move files into their proper locations 646 rm -f tag INDEX tINDEX 647 rm -rf files 648 mv tag.new tag 649 mv tINDEX.new tINDEX 650 mv INDEX.new INDEX 651 mv snap/ files/ 652 653 return 0 654} 655 656# Update a compressed snapshot 657fetch_update() { 658 rm -f patchlist diff OLD NEW filelist INDEX.new 659 660 OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag` 661 OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag` 662 663 while ! fetch_tag latest; do 664 fetch_pick_server || return 1 665 done 666 fetch_update_tagsanity || return 1 667 fetch_update_neededp || return 0 668 fetch_metadata || return 1 669 fetch_metadata_sanity || return 1 670 671 echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` " 672 echo "to `date -r ${SNAPSHOTDATE}`." 673 674# Generate a list of wanted metadata patches 675 join -t '|' -o 1.2,2.2 tINDEX tINDEX.new | 676 fetch_make_patchlist > patchlist 677 678# Attempt to fetch metadata patches 679 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 680 echo ${NDEBUG} "metadata patches.${DDSTATS}" 681 tr '|' '-' < patchlist | 682 lam -s "tp/" - -s ".gz" | 683 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 684 2>${STATSREDIR} | fetch_progress 685 echo "done." 686 687# Attempt to apply metadata patches 688 echo -n "Applying metadata patches... " 689 while read LINE; do 690 X=`echo ${LINE} | cut -f 1 -d '|'` 691 Y=`echo ${LINE} | cut -f 2 -d '|'` 692 if [ ! -f "${X}-${Y}.gz" ]; then continue; fi 693 gunzip -c < ${X}-${Y}.gz > diff 694 gunzip -c < files/${X}.gz > OLD 695 cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp 696 grep '^\+' diff | cut -c 2- | 697 sort -k 1,1 -t '|' -m - ptmp > NEW 698 if [ `${SHA256} -q NEW` = ${Y} ]; then 699 mv NEW files/${Y} 700 gzip -n files/${Y} 701 fi 702 rm -f diff OLD NEW ${X}-${Y}.gz ptmp 703 done < patchlist 2>${QUIETREDIR} 704 echo "done." 705 706# Update metadata without patches 707 join -t '|' -v 2 tINDEX tINDEX.new | 708 cut -f 2 -d '|' /dev/stdin patchlist | 709 while read Y; do 710 if [ ! -f "files/${Y}.gz" ]; then 711 echo ${Y}; 712 fi 713 done > filelist 714 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 715 echo ${NDEBUG} "metadata files... " 716 lam -s "f/" - -s ".gz" < filelist | 717 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 718 2>${QUIETREDIR} 719 720 while read Y; do 721 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 722 mv ${Y}.gz files/${Y}.gz 723 else 724 echo "metadata is corrupt." 725 return 1 726 fi 727 done < filelist 728 echo "done." 729 730# Extract the index 731 gunzip -c files/`look INDEX tINDEX.new | 732 cut -f 2 -d '|'`.gz > INDEX.new 733 fetch_index_sanity || return 1 734 735# If we have decided to refuse certain updates, construct a hybrid index which 736# is equal to the old index for parts of the tree which we don't want to 737# update, and equal to the new index for parts of the tree which gets updates. 738# This means that we should always have a "complete snapshot" of the ports 739# tree -- with the caveat that it isn't actually a snapshot. 740 if [ ! -z "${REFUSE}" ]; then 741 echo "Refusing to download updates for ${REFUSE}" \ 742 >${QUIETREDIR} 743 744 grep -Ev "${REFUSE}" INDEX.new > INDEX.tmp 745 grep -E "${REFUSE}" INDEX | 746 sort -m -k 1,1 -t '|' - INDEX.tmp > INDEX.new 747 rm -f INDEX.tmp 748 fi 749 750# Generate a list of wanted ports patches 751 join -t '|' -o 1.2,2.2 INDEX INDEX.new | 752 fetch_make_patchlist > patchlist 753 754# Attempt to fetch ports patches 755 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 756 echo ${NDEBUG} "patches.${DDSTATS}" 757 tr '|' '-' < patchlist | lam -s "bp/" - | 758 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 759 2>${STATSREDIR} | fetch_progress 760 echo "done." 761 762# Attempt to apply ports patches 763 echo -n "Applying patches... " 764 while read LINE; do 765 X=`echo ${LINE} | cut -f 1 -d '|'` 766 Y=`echo ${LINE} | cut -f 2 -d '|'` 767 if [ ! -f "${X}-${Y}" ]; then continue; fi 768 gunzip -c < files/${X}.gz > OLD 769 ${BSPATCH} OLD NEW ${X}-${Y} 770 if [ `${SHA256} -q NEW` = ${Y} ]; then 771 mv NEW files/${Y} 772 gzip -n files/${Y} 773 fi 774 rm -f diff OLD NEW ${X}-${Y} 775 done < patchlist 2>${QUIETREDIR} 776 echo "done." 777 778# Update ports without patches 779 join -t '|' -v 2 INDEX INDEX.new | 780 cut -f 2 -d '|' /dev/stdin patchlist | 781 while read Y; do 782 if [ ! -f "files/${Y}.gz" ]; then 783 echo ${Y}; 784 fi 785 done > filelist 786 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 787 echo ${NDEBUG} "new ports or files... " 788 lam -s "f/" - -s ".gz" < filelist | 789 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 790 2>${QUIETREDIR} 791 792 while read Y; do 793 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 794 mv ${Y}.gz files/${Y}.gz 795 else 796 echo "snapshot is corrupt." 797 return 1 798 fi 799 done < filelist 800 echo "done." 801 802# Remove files which are no longer needed 803 cut -f 2 -d '|' tINDEX INDEX | sort > oldfiles 804 cut -f 2 -d '|' tINDEX.new INDEX.new | sort | comm -13 - oldfiles | 805 lam -s "files/" - -s ".gz" | xargs rm -f 806 rm patchlist filelist oldfiles 807 808# We're done! 809 mv INDEX.new INDEX 810 mv tINDEX.new tINDEX 811 mv tag.new tag 812 813 return 0 814} 815 816# Do the actual work involved in "fetch" / "cron". 817fetch_run() { 818 fetch_pick_server_init && fetch_pick_server 819 820 while ! fetch_key; do 821 fetch_pick_server || return 1 822 done 823 824 if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then 825 fetch_snapshot || return 1 826 fi 827 fetch_update || return 1 828} 829 830# Build a ports INDEX file 831extract_make_index() { 832 gunzip -c "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX | 833 cut -f 2 -d '|'`.gz" | 834 cat - ${LOCALDESC} | 835 ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2 836} 837 838# Create INDEX, INDEX-5, INDEX-6 839extract_indices() { 840 echo -n "Building new INDEX files... " 841 extract_make_index DESCRIBE.4 INDEX || return 1 842 extract_make_index DESCRIBE.5 INDEX-5 || return 1 843 extract_make_index DESCRIBE.6 INDEX-6 || return 1 844 echo "done." 845} 846 847# Create .portsnap.INDEX; if we are REFUSEing to touch certain directories, 848# merge the values from any exiting .portsnap.INDEX file. 849extract_metadata() { 850 if [ -z "${REFUSE}" ]; then 851 sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX 852 elif [ -f ${PORTSDIR}/.portsnap.INDEX ]; then 853 grep -E "${REFUSE}" ${PORTSDIR}/.portsnap.INDEX \ 854 > ${PORTSDIR}/.portsnap.INDEX.tmp 855 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort | 856 sort -m - ${PORTSDIR}/.portsnap.INDEX.tmp \ 857 > ${PORTSDIR}/.portsnap.INDEX 858 rm -f ${PORTSDIR}/.portsnap.INDEX.tmp 859 else 860 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort \ 861 > ${PORTSDIR}/.portsnap.INDEX 862 fi 863} 864 865# Do the actual work involved in "extract" 866extract_run() { 867 mkdir -p ${PORTSDIR} || return 1 868 869 if ! 870 if ! [ -z "${EXTRACTPATH}" ]; then 871 grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX 872 elif ! [ -z "${REFUSE}" ]; then 873 grep -vE "${REFUSE}" ${WORKDIR}/INDEX 874 else 875 cat ${WORKDIR}/INDEX 876 fi | while read LINE; do 877 FILE=`echo ${LINE} | cut -f 1 -d '|'` 878 HASH=`echo ${LINE} | cut -f 2 -d '|'` 879 echo ${PORTSDIR}/${FILE} 880 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 881 echo "files/${HASH}.gz not found -- snapshot corrupt." 882 return 1 883 fi 884 case ${FILE} in 885 */) 886 DIR=`echo ${FILE} | sed -e 's|/$||'` 887 rm -rf ${PORTSDIR}/${DIR} 888 mkdir -p ${PORTSDIR}/${FILE} 889 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 890 -C ${PORTSDIR}/${FILE} 891 ;; 892 *) 893 rm -f ${PORTSDIR}/${FILE} 894 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 895 -C ${PORTSDIR} ${FILE} 896 ;; 897 esac 898 done; then 899 return 1 900 fi 901 if [ ! -z "${EXTRACTPATH}" ]; then 902 return 0; 903 fi 904 905 extract_metadata 906 extract_indices 907} 908 909# Do the actual work involved in "update" 910update_run() { 911 if ! [ -z "${INDEXONLY}" ]; then 912 extract_indices >/dev/null || return 1 913 return 0 914 fi 915 916 if sort ${WORKDIR}/INDEX | 917 cmp -s ${PORTSDIR}/.portsnap.INDEX -; then 918 echo "Ports tree is already up to date." 919 return 0 920 fi 921 922# If we are REFUSEing to touch certain directories, don't remove files 923# from those directories (even if they are out of date) 924 echo -n "Removing old files and directories... " 925 if ! [ -z "${REFUSE}" ]; then 926 sort ${WORKDIR}/INDEX | 927 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 928 grep -vE "${REFUSE}" | 929 lam -s "${PORTSDIR}/" - | 930 sed -e 's|/$||' | xargs rm -rf 931 else 932 sort ${WORKDIR}/INDEX | 933 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 934 lam -s "${PORTSDIR}/" - | 935 sed -e 's|/$||' | xargs rm -rf 936 fi 937 echo "done." 938 939# Install new files 940 echo "Extracting new files:" 941 if ! 942 if ! [ -z "${REFUSE}" ]; then 943 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort 944 else 945 sort ${WORKDIR}/INDEX 946 fi | 947 comm -13 ${PORTSDIR}/.portsnap.INDEX - | 948 while read LINE; do 949 FILE=`echo ${LINE} | cut -f 1 -d '|'` 950 HASH=`echo ${LINE} | cut -f 2 -d '|'` 951 echo ${PORTSDIR}/${FILE} 952 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 953 echo "files/${HASH}.gz not found -- snapshot corrupt." 954 return 1 955 fi 956 case ${FILE} in 957 */) 958 mkdir -p ${PORTSDIR}/${FILE} 959 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 960 -C ${PORTSDIR}/${FILE} 961 ;; 962 *) 963 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 964 -C ${PORTSDIR} ${FILE} 965 ;; 966 esac 967 done; then 968 return 1 969 fi 970 971 extract_metadata 972 extract_indices 973} 974 975#### Main functions -- call parameter-handling and core functions 976 977# Using the command line, configuration file, and defaults, 978# set all the parameters which are needed later. 979get_params() { 980 init_params 981 parse_cmdline $@ 982 sanity_conffile 983 default_conffile 984 parse_conffile 985 default_params 986} 987 988# Fetch command. Make sure that we're being called 989# interactively, then run fetch_check_params and fetch_run 990cmd_fetch() { 991 if [ ! -t 0 ]; then 992 echo -n "`basename $0` fetch should not " 993 echo "be run non-interactively." 994 echo "Run `basename $0` cron instead." 995 exit 1 996 fi 997 fetch_check_params 998 fetch_run || exit 1 999} 1000 1001# Cron command. Make sure the parameters are sensible; wait 1002# rand(3600) seconds; then fetch updates. While fetching updates, 1003# send output to a temporary file; only print that file if the 1004# fetching failed. 1005cmd_cron() { 1006 fetch_check_params 1007 sleep `jot -r 1 0 3600` 1008 1009 TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1 1010 if ! fetch_run >> ${TMPFILE}; then 1011 cat ${TMPFILE} 1012 rm ${TMPFILE} 1013 exit 1 1014 fi 1015 1016 rm ${TMPFILE} 1017} 1018 1019# Extract command. Make sure the parameters are sensible, 1020# then extract the ports tree (or part thereof). 1021cmd_extract() { 1022 extract_check_params 1023 extract_run || exit 1 1024} 1025 1026# Update command. Make sure the parameters are sensible, 1027# then update the ports tree. 1028cmd_update() { 1029 update_check_params 1030 update_run || exit 1 1031} 1032 1033#### Entry point 1034 1035# Make sure we find utilities from the base system 1036export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH} 1037 1038get_params $@ 1039for COMMAND in ${COMMANDS}; do 1040 cmd_${COMMAND} 1041done 1042