portsnap.sh revision 158301
1#!/bin/sh 2 3#- 4# Copyright 2004-2005 Colin Percival 5# All rights reserved 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted providing that the following conditions 9# are met: 10# 1. Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# 2. Redistributions in binary form must reproduce the above copyright 13# notice, this list of conditions and the following disclaimer in the 14# documentation and/or other materials provided with the distribution. 15# 16# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 20# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26# POSSIBILITY OF SUCH DAMAGE. 27 28# $FreeBSD: head/usr.sbin/portsnap/portsnap/portsnap.sh 158301 2006-05-05 04:47:00Z cperciva $ 29 30#### Usage function -- called from command-line handling code. 31 32# Usage instructions. Options not listed: 33# --debug -- don't filter output from utilities 34# --no-stats -- don't show progress statistics while fetching files 35usage() { 36 cat <<EOF 37usage: `basename $0` [options] command ... [path] 38 39Options: 40 -d workdir -- Store working files in workdir 41 (default: /var/db/portsnap/) 42 -f conffile -- Read configuration options from conffile 43 (default: /etc/portsnap.conf) 44 -I -- Update INDEX only. (update command only) 45 -k KEY -- Trust an RSA key with SHA256 hash of KEY 46 -p portsdir -- Location of uncompressed ports tree 47 (default: /usr/ports/) 48 -s server -- Server from which to fetch updates. 49 (default: portsnap.FreeBSD.org) 50 path -- Extract only parts of the tree starting with the given 51 string. (extract command only) 52Commands: 53 fetch -- Fetch a compressed snapshot of the ports tree, 54 or update an existing snapshot. 55 cron -- Sleep rand(3600) seconds, and then fetch updates. 56 extract -- Extract snapshot of ports tree, replacing existing 57 files and directories. 58 update -- Update ports tree to match current snapshot, replacing 59 files and directories which have changed. 60EOF 61 exit 0 62} 63 64#### Parameter handling functions. 65 66# Initialize parameters to null, just in case they're 67# set in the environment. 68init_params() { 69 KEYPRINT="" 70 EXTRACTPATH="" 71 WORKDIR="" 72 PORTSDIR="" 73 CONFFILE="" 74 COMMAND="" 75 COMMANDS="" 76 QUIETREDIR="" 77 QUIETFLAG="" 78 STATSREDIR="" 79 XARGST="" 80 NDEBUG="" 81 DDSTATS="" 82 INDEXONLY="" 83 SERVERNAME="" 84 REFUSE="" 85} 86 87# Parse the command line 88parse_cmdline() { 89 while [ $# -gt 0 ]; do 90 case "$1" in 91 -d) 92 if [ $# -eq 1 ]; then usage; fi 93 if [ ! -z "${WORKDIR}" ]; then usage; fi 94 shift; WORKDIR="$1" 95 ;; 96 --debug) 97 QUIETREDIR="/dev/stderr" 98 STATSREDIR="/dev/stderr" 99 QUIETFLAG=" " 100 NDEBUG=" " 101 XARGST="-t" 102 DDSTATS=".." 103 ;; 104 -f) 105 if [ $# -eq 1 ]; then usage; fi 106 if [ ! -z "${CONFFILE}" ]; then usage; fi 107 shift; CONFFILE="$1" 108 ;; 109 -h | --help | help) 110 usage 111 ;; 112 -I) 113 INDEXONLY="YES" 114 ;; 115 -k) 116 if [ $# -eq 1 ]; then usage; fi 117 if [ ! -z "${KEYPRINT}" ]; then usage; fi 118 shift; KEYPRINT="$1" 119 ;; 120 --no-stats) 121 if [ -z "${STATSREDIR}" ]; then 122 STATSREDIR="/dev/null" 123 DDSTATS=".. " 124 fi 125 ;; 126 -p) 127 if [ $# -eq 1 ]; then usage; fi 128 if [ ! -z "${PORTSDIR}" ]; then usage; fi 129 shift; PORTSDIR="$1" 130 ;; 131 -s) 132 if [ $# -eq 1 ]; then usage; fi 133 if [ ! -z "${SERVERNAME}" ]; then usage; fi 134 shift; SERVERNAME="$1" 135 ;; 136 cron | extract | fetch | update) 137 COMMANDS="${COMMANDS} $1" 138 ;; 139 *) 140 if [ $# -gt 1 ]; then usage; fi 141 if echo ${COMMANDS} | grep -vq extract; then 142 usage 143 fi 144 EXTRACTPATH="$1" 145 ;; 146 esac 147 shift 148 done 149 150 if [ -z "${COMMANDS}" ]; then 151 usage 152 fi 153} 154 155# If CONFFILE was specified at the command-line, make 156# sure that it exists and is readable. 157sanity_conffile() { 158 if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then 159 echo -n "File does not exist " 160 echo -n "or is not readable: " 161 echo ${CONFFILE} 162 exit 1 163 fi 164} 165 166# If a configuration file hasn't been specified, use 167# the default value (/etc/portsnap.conf) 168default_conffile() { 169 if [ -z "${CONFFILE}" ]; then 170 CONFFILE="/etc/portsnap.conf" 171 fi 172} 173 174# Read {KEYPRINT, SERVERNAME, WORKDIR, PORTSDIR} from the configuration 175# file if they haven't already been set. If the configuration 176# file doesn't exist, do nothing. 177# Also read REFUSE (which cannot be set via the command line) if it is 178# present in the configuration file. 179parse_conffile() { 180 if [ -r "${CONFFILE}" ]; then 181 for X in KEYPRINT WORKDIR PORTSDIR SERVERNAME; do 182 eval _=\$${X} 183 if [ -z "${_}" ]; then 184 eval ${X}=`grep "^${X}=" "${CONFFILE}" | 185 cut -f 2- -d '=' | tail -1` 186 fi 187 done 188 189 if grep -qE "^REFUSE[[:space:]]" ${CONFFILE}; then 190 REFUSE="^(` 191 grep -E "^REFUSE[[:space:]]" "${CONFFILE}" | 192 cut -c 7- | xargs echo | tr ' ' '|' 193 `)" 194 fi 195 fi 196} 197 198# If parameters have not been set, use default values 199default_params() { 200 _QUIETREDIR="/dev/null" 201 _QUIETFLAG="-q" 202 _STATSREDIR="/dev/stdout" 203 _WORKDIR="/var/db/portsnap" 204 _PORTSDIR="/usr/ports" 205 _NDEBUG="-n" 206 for X in QUIETREDIR QUIETFLAG STATSREDIR WORKDIR PORTSDIR NDEBUG; do 207 eval _=\$${X} 208 eval __=\$_${X} 209 if [ -z "${_}" ]; then 210 eval ${X}=${__} 211 fi 212 done 213} 214 215# Perform sanity checks and set some final parameters 216# in preparation for fetching files. Also chdir into 217# the working directory. 218fetch_check_params() { 219 export HTTP_USER_AGENT="portsnap (${COMMAND}, `uname -r`)" 220 221 _SERVERNAME_z=\ 222"SERVERNAME must be given via command line or configuration file." 223 _KEYPRINT_z="Key must be given via -k option or configuration file." 224 _KEYPRINT_bad="Invalid key fingerprint: " 225 _WORKDIR_bad="Directory does not exist or is not writable: " 226 227 if [ -z "${SERVERNAME}" ]; then 228 echo -n "`basename $0`: " 229 echo "${_SERVERNAME_z}" 230 exit 1 231 fi 232 if [ -z "${KEYPRINT}" ]; then 233 echo -n "`basename $0`: " 234 echo "${_KEYPRINT_z}" 235 exit 1 236 fi 237 if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then 238 echo -n "`basename $0`: " 239 echo -n "${_KEYPRINT_bad}" 240 echo ${KEYPRINT} 241 exit 1 242 fi 243 if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then 244 echo -n "`basename $0`: " 245 echo -n "${_WORKDIR_bad}" 246 echo ${WORKDIR} 247 exit 1 248 fi 249 cd ${WORKDIR} || exit 1 250 251 BSPATCH=/usr/bin/bspatch 252 SHA256=/sbin/sha256 253 PHTTPGET=/usr/libexec/phttpget 254} 255 256# Perform sanity checks and set some final parameters 257# in preparation for extracting or updating ${PORTSDIR} 258# Complain if ${PORTSDIR} exists but is not writable, 259# but don't complain if ${PORTSDIR} doesn't exist. 260extract_check_params() { 261 _WORKDIR_bad="Directory does not exist: " 262 _PORTSDIR_bad="Directory is not writable: " 263 264 if ! [ -d "${WORKDIR}" ]; then 265 echo -n "`basename $0`: " 266 echo -n "${_WORKDIR_bad}" 267 echo ${WORKDIR} 268 exit 1 269 fi 270 if [ -d "${PORTSDIR}" ] && ! [ -w "${PORTSDIR}" ]; then 271 echo -n "`basename $0`: " 272 echo -n "${_PORTSDIR_bad}" 273 echo ${PORTSDIR} 274 exit 1 275 fi 276 277 if ! [ -d "${WORKDIR}/files" -a -r "${WORKDIR}/tag" \ 278 -a -r "${WORKDIR}/INDEX" -a -r "${WORKDIR}/tINDEX" ]; then 279 echo "No snapshot available. Try running" 280 echo "# `basename $0` fetch" 281 exit 1 282 fi 283 284 MKINDEX=/usr/libexec/make_index 285} 286 287# Perform sanity checks and set some final parameters 288# in preparation for updating ${PORTSDIR} 289update_check_params() { 290 extract_check_params 291 292 if ! [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 293 echo "${PORTSDIR} was not created by portsnap." 294 echo -n "You must run '`basename $0` extract' before " 295 echo "running '`basename $0` update'." 296 exit 1 297 fi 298 299} 300 301#### Core functionality -- the actual work gets done here 302 303# Use an SRV query to pick a server. If the SRV query doesn't provide 304# a useful answer, use the server name specified by the user. 305# Put another way... look up _http._tcp.${SERVERNAME} and pick a server 306# from that; or if no servers are returned, use ${SERVERNAME}. 307# This allows a user to specify "portsnap.freebsd.org" (in which case 308# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org" 309# (in which case portsnap will use that particular server, since there 310# won't be an SRV entry for that name). 311# 312# We ignore the Port field, since we are always going to use port 80. 313 314# Fetch the mirror list, but do not pick a mirror yet. Returns 1 if 315# no mirrors are available for any reason. 316fetch_pick_server_init() { 317 : > serverlist_tried 318 319# Check that host(1) exists (i.e., that the system wasn't built with the 320# WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist. 321 if ! which -s host; then 322 : > serverlist_full 323 return 1 324 fi 325 326 echo -n "Looking up ${SERVERNAME} mirrors... " 327 328# Issue the SRV query and pull out the Priority, Weight, and Target fields. 329# BIND 9 prints "$name has SRV record ..." while BIND 8 prints 330# "$name server selection ..."; we allow either format. 331 MLIST="_http._tcp.${SERVERNAME}" 332 host -t srv "${MLIST}" | 333 sed -nE "s/${MLIST} (has SRV record|server selection) //p" | 334 cut -f 1,2,4 -d ' ' | 335 sed -e 's/\.$//' | 336 sort > serverlist_full 337 338# If no records, give up -- we'll just use the server name we were given. 339 if [ `wc -l < serverlist_full` -eq 0 ]; then 340 echo "none found." 341 return 1 342 fi 343 344# Report how many mirrors we found. 345 echo `wc -l < serverlist_full` "mirrors found." 346 347# Generate a random seed for use in picking mirrors. If HTTP_PROXY 348# is set, this will be used to generate the seed; otherwise, the seed 349# will be random. 350 if [ -n "${HTTP_PROXY}${http_proxy}" ]; then 351 RANDVALUE=`sha256 -qs "${HTTP_PROXY}${http_proxy}" | 352 tr -d 'a-f' | 353 cut -c 1-9` 354 else 355 RANDVALUE=`jot -r 1 0 999999999` 356 fi 357} 358 359# Pick a mirror. Returns 1 if we have run out of mirrors to try. 360fetch_pick_server() { 361# Generate a list of not-yet-tried mirrors 362 sort serverlist_tried | 363 comm -23 serverlist_full - > serverlist 364 365# Have we run out of mirrors? 366 if [ `wc -l < serverlist` -eq 0 ]; then 367 echo "No mirrors remaining, giving up." 368 return 1 369 fi 370 371# Find the highest priority level (lowest numeric value). 372 SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1` 373 374# Add up the weights of the response lines at that priority level. 375 SRV_WSUM=0; 376 while read X; do 377 case "$X" in 378 ${SRV_PRIORITY}\ *) 379 SRV_W=`echo $X | cut -f 2 -d ' '` 380 SRV_WSUM=$(($SRV_WSUM + $SRV_W)) 381 ;; 382 esac 383 done < serverlist 384 385# If all the weights are 0, pretend that they are all 1 instead. 386 if [ ${SRV_WSUM} -eq 0 ]; then 387 SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l` 388 SRV_W_ADD=1 389 else 390 SRV_W_ADD=0 391 fi 392 393# Pick a value between 0 and the sum of the weights - 1 394 SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}` 395 396# Read through the list of mirrors and set SERVERNAME. Write the line 397# corresponding to the mirror we selected into serverlist_tried so that 398# we won't try it again. 399 while read X; do 400 case "$X" in 401 ${SRV_PRIORITY}\ *) 402 SRV_W=`echo $X | cut -f 2 -d ' '` 403 SRV_W=$(($SRV_W + $SRV_W_ADD)) 404 if [ $SRV_RND -lt $SRV_W ]; then 405 SERVERNAME=`echo $X | cut -f 3 -d ' '` 406 echo "$X" >> serverlist_tried 407 break 408 else 409 SRV_RND=$(($SRV_RND - $SRV_W)) 410 fi 411 ;; 412 esac 413 done < serverlist 414} 415 416# Check that we have a public key with an appropriate hash, or 417# fetch the key if it doesn't exist. Returns 1 if the key has 418# not yet been fetched. 419fetch_key() { 420 if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 421 return 0 422 fi 423 424 echo -n "Fetching public key from ${SERVERNAME}... " 425 rm -f pub.ssl 426 fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \ 427 2>${QUIETREDIR} || true 428 if ! [ -r pub.ssl ]; then 429 echo "failed." 430 return 1 431 fi 432 if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 433 echo "key has incorrect hash." 434 rm -f pub.ssl 435 return 1 436 fi 437 echo "done." 438} 439 440# Fetch a snapshot tag 441fetch_tag() { 442 rm -f snapshot.ssl tag.new 443 444 echo ${NDEBUG} "Fetching snapshot tag from ${SERVERNAME}... " 445 fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl \ 446 2>${QUIETREDIR} || true 447 if ! [ -r $1.ssl ]; then 448 echo "failed." 449 return 1 450 fi 451 452 openssl rsautl -pubin -inkey pub.ssl -verify \ 453 < $1.ssl > tag.new 2>${QUIETREDIR} || true 454 rm $1.ssl 455 456 if ! [ `wc -l < tag.new` = 1 ] || 457 ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then 458 echo "invalid snapshot tag." 459 return 1 460 fi 461 462 echo "done." 463 464 SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new` 465 SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new` 466} 467 468# Sanity-check the date on a snapshot tag 469fetch_snapshot_tagsanity() { 470 if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then 471 echo "Snapshot appears to be more than a year old!" 472 echo "(Is the system clock correct?)" 473 echo "Cowardly refusing to proceed any further." 474 return 1 475 fi 476 if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then 477 echo -n "Snapshot appears to have been created more than " 478 echo "one day into the future!" 479 echo "(Is the system clock correct?)" 480 echo "Cowardly refusing to proceed any further." 481 return 1 482 fi 483} 484 485# Sanity-check the date on a snapshot update tag 486fetch_update_tagsanity() { 487 fetch_snapshot_tagsanity || return 1 488 489 if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then 490 echo -n "Latest snapshot on server is " 491 echo "older than what we already have!" 492 echo -n "Cowardly refusing to downgrade from " 493 date -r ${OLDSNAPSHOTDATE} 494 echo "to `date -r ${SNAPSHOTDATE}`." 495 return 1 496 fi 497} 498 499# Compare old and new tags; return 1 if update is unnecessary 500fetch_update_neededp() { 501 if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then 502 echo -n "Latest snapshot on server matches " 503 echo "what we already have." 504 echo "No updates needed." 505 rm tag.new 506 return 1 507 fi 508 if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then 509 echo -n "Ports tree hasn't changed since " 510 echo "last snapshot." 511 echo "No updates needed." 512 rm tag.new 513 return 1 514 fi 515 516 return 0 517} 518 519# Fetch snapshot metadata file 520fetch_metadata() { 521 rm -f ${SNAPSHOTHASH} tINDEX.new 522 523 echo ${NDEBUG} "Fetching snapshot metadata... " 524 fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} 525 2>${QUIETREDIR} || return 526 if [ `${SHA256} -q ${SNAPSHOTHASH}` != ${SNAPSHOTHASH} ]; then 527 echo "snapshot metadata corrupt." 528 return 1 529 fi 530 mv ${SNAPSHOTHASH} tINDEX.new 531 echo "done." 532} 533 534# Warn user about bogus metadata 535fetch_metadata_freakout() { 536 echo 537 echo "Portsnap metadata is correctly signed, but contains" 538 echo "at least one line which appears bogus." 539 echo "Cowardly refusing to proceed any further." 540} 541 542# Sanity-check a snapshot metadata file 543fetch_metadata_sanity() { 544 if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then 545 fetch_metadata_freakout 546 return 1 547 fi 548 if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then 549 echo 550 echo "Portsnap metadata appears bogus." 551 echo "Cowardly refusing to proceed any further." 552 return 1 553 fi 554} 555 556# Take a list of ${oldhash}|${newhash} and output a list of needed patches 557fetch_make_patchlist() { 558 grep -vE "^([0-9a-f]{64})\|\1$" | 559 while read LINE; do 560 X=`echo ${LINE} | cut -f 1 -d '|'` 561 Y=`echo ${LINE} | cut -f 2 -d '|'` 562 if [ -f "files/${Y}.gz" ]; then continue; fi 563 if [ ! -f "files/${X}.gz" ]; then continue; fi 564 echo "${LINE}" 565 done 566} 567 568# Print user-friendly progress statistics 569fetch_progress() { 570 LNC=0 571 while read x; do 572 LNC=$(($LNC + 1)) 573 if [ $(($LNC % 10)) = 0 ]; then 574 echo -n $LNC 575 elif [ $(($LNC % 2)) = 0 ]; then 576 echo -n . 577 fi 578 done 579 echo -n " " 580} 581 582# Sanity-check an index file 583fetch_index_sanity() { 584 if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new || 585 fgrep -q "./" INDEX.new; then 586 fetch_metadata_freakout 587 return 1 588 fi 589} 590 591# Verify a list of files 592fetch_snapshot_verify() { 593 while read F; do 594 if [ `gunzip -c snap/${F} | ${SHA256} -q` != ${F} ]; then 595 echo "snapshot corrupt." 596 return 1 597 fi 598 done 599 return 0 600} 601 602# Fetch a snapshot tarball, extract, and verify. 603fetch_snapshot() { 604 while ! fetch_tag snapshot; do 605 fetch_pick_server || return 1 606 done 607 fetch_snapshot_tagsanity || return 1 608 fetch_metadata || return 1 609 fetch_metadata_sanity || return 1 610 611 rm -rf snap/ 612 613# Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will 614# probably take a while, so the progrees reports that fetch(1) generates 615# will be useful for keeping the users' attention from drifting. 616 echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:" 617 fetch -r http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1 618 619 echo -n "Extracting snapshot... " 620 tar -xzf ${SNAPSHOTHASH}.tgz snap/ || return 1 621 rm ${SNAPSHOTHASH}.tgz 622 echo "done." 623 624 echo -n "Verifying snapshot integrity... " 625# Verify the metadata files 626 cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1 627# Extract the index 628 rm -f INDEX.new 629 gunzip -c snap/`look INDEX tINDEX.new | 630 cut -f 2 -d '|'`.gz > INDEX.new 631 fetch_index_sanity || return 1 632# Verify the snapshot contents 633 cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1 634 echo "done." 635 636# Move files into their proper locations 637 rm -f tag INDEX tINDEX 638 rm -rf files 639 mv tag.new tag 640 mv tINDEX.new tINDEX 641 mv INDEX.new INDEX 642 mv snap/ files/ 643 644 return 0 645} 646 647# Update a compressed snapshot 648fetch_update() { 649 rm -f patchlist diff OLD NEW filelist INDEX.new 650 651 OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag` 652 OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag` 653 654 while ! fetch_tag latest; do 655 fetch_pick_server || return 1 656 done 657 fetch_update_tagsanity || return 1 658 fetch_update_neededp || return 0 659 fetch_metadata || return 1 660 fetch_metadata_sanity || return 1 661 662 echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` " 663 echo "to `date -r ${SNAPSHOTDATE}`." 664 665# Generate a list of wanted metadata patches 666 join -t '|' -o 1.2,2.2 tINDEX tINDEX.new | 667 fetch_make_patchlist > patchlist 668 669# Attempt to fetch metadata patches 670 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 671 echo ${NDEBUG} "metadata patches.${DDSTATS}" 672 tr '|' '-' < patchlist | 673 lam -s "tp/" - -s ".gz" | 674 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 675 2>${STATSREDIR} | fetch_progress 676 echo "done." 677 678# Attempt to apply metadata patches 679 echo -n "Applying metadata patches... " 680 while read LINE; do 681 X=`echo ${LINE} | cut -f 1 -d '|'` 682 Y=`echo ${LINE} | cut -f 2 -d '|'` 683 if [ ! -f "${X}-${Y}.gz" ]; then continue; fi 684 gunzip -c < ${X}-${Y}.gz > diff 685 gunzip -c < files/${X}.gz > OLD 686 cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp 687 grep '^\+' diff | cut -c 2- | 688 sort -k 1,1 -t '|' -m - ptmp > NEW 689 if [ `${SHA256} -q NEW` = ${Y} ]; then 690 mv NEW files/${Y} 691 gzip -n files/${Y} 692 fi 693 rm -f diff OLD NEW ${X}-${Y}.gz ptmp 694 done < patchlist 2>${QUIETREDIR} 695 echo "done." 696 697# Update metadata without patches 698 join -t '|' -v 2 tINDEX tINDEX.new | 699 cut -f 2 -d '|' /dev/stdin patchlist | 700 while read Y; do 701 if [ ! -f "files/${Y}.gz" ]; then 702 echo ${Y}; 703 fi 704 done > filelist 705 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 706 echo ${NDEBUG} "metadata files... " 707 lam -s "f/" - -s ".gz" < filelist | 708 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 709 2>${QUIETREDIR} 710 711 while read Y; do 712 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 713 mv ${Y}.gz files/${Y}.gz 714 else 715 echo "metadata is corrupt." 716 return 1 717 fi 718 done < filelist 719 echo "done." 720 721# Extract the index 722 gunzip -c files/`look INDEX tINDEX.new | 723 cut -f 2 -d '|'`.gz > INDEX.new 724 fetch_index_sanity || return 1 725 726# If we have decided to refuse certain updates, construct a hybrid index which 727# is equal to the old index for parts of the tree which we don't want to 728# update, and equal to the new index for parts of the tree which gets updates. 729# This means that we should always have a "complete snapshot" of the ports 730# tree -- with the caveat that it isn't actually a snapshot. 731 if [ ! -z "${REFUSE}" ]; then 732 echo "Refusing to download updates for ${REFUSE}" \ 733 >${QUIETREDIR} 734 735 grep -Ev "${REFUSE}" INDEX.new > INDEX.tmp 736 grep -E "${REFUSE}" INDEX | 737 sort -m -k 1,1 -t '|' - INDEX.tmp > INDEX.new 738 rm -f INDEX.tmp 739 fi 740 741# Generate a list of wanted ports patches 742 join -t '|' -o 1.2,2.2 INDEX INDEX.new | 743 fetch_make_patchlist > patchlist 744 745# Attempt to fetch ports patches 746 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 747 echo ${NDEBUG} "patches.${DDSTATS}" 748 tr '|' '-' < patchlist | lam -s "bp/" - | 749 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 750 2>${STATSREDIR} | fetch_progress 751 echo "done." 752 753# Attempt to apply ports patches 754 echo -n "Applying patches... " 755 while read LINE; do 756 X=`echo ${LINE} | cut -f 1 -d '|'` 757 Y=`echo ${LINE} | cut -f 2 -d '|'` 758 if [ ! -f "${X}-${Y}" ]; then continue; fi 759 gunzip -c < files/${X}.gz > OLD 760 ${BSPATCH} OLD NEW ${X}-${Y} 761 if [ `${SHA256} -q NEW` = ${Y} ]; then 762 mv NEW files/${Y} 763 gzip -n files/${Y} 764 fi 765 rm -f diff OLD NEW ${X}-${Y} 766 done < patchlist 2>${QUIETREDIR} 767 echo "done." 768 769# Update ports without patches 770 join -t '|' -v 2 INDEX INDEX.new | 771 cut -f 2 -d '|' /dev/stdin patchlist | 772 while read Y; do 773 if [ ! -f "files/${Y}.gz" ]; then 774 echo ${Y}; 775 fi 776 done > filelist 777 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 778 echo ${NDEBUG} "new ports or files... " 779 lam -s "f/" - -s ".gz" < filelist | 780 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 781 2>${QUIETREDIR} 782 783 while read Y; do 784 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 785 mv ${Y}.gz files/${Y}.gz 786 else 787 echo "snapshot is corrupt." 788 return 1 789 fi 790 done < filelist 791 echo "done." 792 793# Remove files which are no longer needed 794 cut -f 2 -d '|' tINDEX INDEX | sort > oldfiles 795 cut -f 2 -d '|' tINDEX.new INDEX.new | sort | comm -13 - oldfiles | 796 lam -s "files/" - -s ".gz" | xargs rm -f 797 rm patchlist filelist oldfiles 798 799# We're done! 800 mv INDEX.new INDEX 801 mv tINDEX.new tINDEX 802 mv tag.new tag 803 804 return 0 805} 806 807# Do the actual work involved in "fetch" / "cron". 808fetch_run() { 809 fetch_pick_server_init && fetch_pick_server 810 811 while ! fetch_key; do 812 fetch_pick_server || return 1 813 done 814 815 if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then 816 fetch_snapshot || return 1 817 fi 818 fetch_update || return 1 819} 820 821# Build a ports INDEX file 822extract_make_index() { 823 gunzip -c "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX | 824 cut -f 2 -d '|'`.gz" | ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2 825} 826 827# Create INDEX, INDEX-5, INDEX-6 828extract_indices() { 829 echo -n "Building new INDEX files... " 830 extract_make_index DESCRIBE.4 INDEX || return 1 831 extract_make_index DESCRIBE.5 INDEX-5 || return 1 832 extract_make_index DESCRIBE.6 INDEX-6 || return 1 833 echo "done." 834} 835 836# Create .portsnap.INDEX; if we are REFUSEing to touch certain directories, 837# merge the values from any exiting .portsnap.INDEX file. 838extract_metadata() { 839 if [ -z "${REFUSE}" ]; then 840 sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX 841 elif [ -f ${PORTSDIR}/.portsnap.INDEX ]; then 842 grep -E "${REFUSE}" ${PORTSDIR}/.portsnap.INDEX \ 843 > ${PORTSDIR}/.portsnap.INDEX.tmp 844 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort | 845 sort -m - ${PORTSDIR}/.portsnap.INDEX.tmp \ 846 > ${PORTSDIR}/.portsnap.INDEX 847 rm -f ${PORTSDIR}/.portsnap.INDEX.tmp 848 else 849 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort \ 850 > ${PORTSDIR}/.portsnap.INDEX 851 fi 852} 853 854# Do the actual work involved in "extract" 855extract_run() { 856 mkdir -p ${PORTSDIR} || return 1 857 858 if ! 859 if ! [ -z "${EXTRACTPATH}" ]; then 860 grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX 861 elif ! [ -z "${REFUSE}" ]; then 862 grep -vE "${REFUSE}" ${WORKDIR}/INDEX 863 else 864 cat ${WORKDIR}/INDEX 865 fi | while read LINE; do 866 FILE=`echo ${LINE} | cut -f 1 -d '|'` 867 HASH=`echo ${LINE} | cut -f 2 -d '|'` 868 echo ${PORTSDIR}/${FILE} 869 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 870 echo "files/${HASH}.gz not found -- snapshot corrupt." 871 return 1 872 fi 873 case ${FILE} in 874 */) 875 rm -rf ${PORTSDIR}/${FILE} 876 mkdir -p ${PORTSDIR}/${FILE} 877 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 878 -C ${PORTSDIR}/${FILE} 879 ;; 880 *) 881 rm -f ${PORTSDIR}/${FILE} 882 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 883 -C ${PORTSDIR} ${FILE} 884 ;; 885 esac 886 done; then 887 return 1 888 fi 889 if [ ! -z "${EXTRACTPATH}" ]; then 890 return 0; 891 fi 892 893 extract_metadata 894 extract_indices 895} 896 897# Do the actual work involved in "update" 898update_run() { 899 if ! [ -z "${INDEXONLY}" ]; then 900 extract_indices >/dev/null || return 1 901 return 0 902 fi 903 904 if sort ${WORKDIR}/INDEX | 905 cmp -s ${PORTSDIR}/.portsnap.INDEX -; then 906 echo "Ports tree is already up to date." 907 return 0 908 fi 909 910# If we are REFUSEing to touch certain directories, don't remove files 911# from those directories (even if they are out of date) 912 echo -n "Removing old files and directories... " 913 if ! [ -z "${REFUSE}" ]; then 914 sort ${WORKDIR}/INDEX | 915 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 916 grep -vE "${REFUSE}" | 917 lam -s "${PORTSDIR}/" - | xargs rm -rf 918 else 919 sort ${WORKDIR}/INDEX | 920 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 921 lam -s "${PORTSDIR}/" - | xargs rm -rf 922 fi 923 echo "done." 924 925# Install new files 926 echo "Extracting new files:" 927 if ! 928 if ! [ -z "${REFUSE}" ]; then 929 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort 930 else 931 sort ${WORKDIR}/INDEX 932 fi | 933 comm -13 ${PORTSDIR}/.portsnap.INDEX - | 934 while read LINE; do 935 FILE=`echo ${LINE} | cut -f 1 -d '|'` 936 HASH=`echo ${LINE} | cut -f 2 -d '|'` 937 echo ${PORTSDIR}/${FILE} 938 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 939 echo "files/${HASH}.gz not found -- snapshot corrupt." 940 return 1 941 fi 942 case ${FILE} in 943 */) 944 mkdir -p ${PORTSDIR}/${FILE} 945 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 946 -C ${PORTSDIR}/${FILE} 947 ;; 948 *) 949 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 950 -C ${PORTSDIR} ${FILE} 951 ;; 952 esac 953 done; then 954 return 1 955 fi 956 957 extract_metadata 958 extract_indices 959} 960 961#### Main functions -- call parameter-handling and core functions 962 963# Using the command line, configuration file, and defaults, 964# set all the parameters which are needed later. 965get_params() { 966 init_params 967 parse_cmdline $@ 968 sanity_conffile 969 default_conffile 970 parse_conffile 971 default_params 972} 973 974# Fetch command. Make sure that we're being called 975# interactively, then run fetch_check_params and fetch_run 976cmd_fetch() { 977 if [ ! -t 0 ]; then 978 echo -n "`basename $0` fetch should not " 979 echo "be run non-interactively." 980 echo "Run `basename $0` cron instead." 981 exit 1 982 fi 983 fetch_check_params 984 fetch_run || exit 1 985} 986 987# Cron command. Make sure the parameters are sensible; wait 988# rand(3600) seconds; then fetch updates. While fetching updates, 989# send output to a temporary file; only print that file if the 990# fetching failed. 991cmd_cron() { 992 fetch_check_params 993 sleep `jot -r 1 0 3600` 994 995 TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1 996 if ! fetch_run >> ${TMPFILE}; then 997 cat ${TMPFILE} 998 rm ${TMPFILE} 999 exit 1 1000 fi 1001 1002 rm ${TMPFILE} 1003} 1004 1005# Extract command. Make sure the parameters are sensible, 1006# then extract the ports tree (or part thereof). 1007cmd_extract() { 1008 extract_check_params 1009 extract_run || exit 1 1010} 1011 1012# Update command. Make sure the parameters are sensible, 1013# then update the ports tree. 1014cmd_update() { 1015 update_check_params 1016 update_run || exit 1 1017} 1018 1019#### Entry point 1020 1021# Make sure we find utilities from the base system 1022export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH} 1023 1024get_params $@ 1025for COMMAND in ${COMMANDS}; do 1026 cmd_${COMMAND} 1027done 1028