portsnap.sh revision 148958
1#!/bin/sh 2 3#- 4# Copyright 2004-2005 Colin Percival 5# All rights reserved 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted providing that the following conditions 9# are met: 10# 1. Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# 2. Redistributions in binary form must reproduce the above copyright 13# notice, this list of conditions and the following disclaimer in the 14# documentation and/or other materials provided with the distribution. 15# 16# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 20# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26# POSSIBILITY OF SUCH DAMAGE. 27 28# $FreeBSD: head/usr.sbin/portsnap/portsnap/portsnap.sh 148958 2005-08-11 13:48:13Z cperciva $ 29 30#### Usage function -- called from command-line handling code. 31 32# Usage instructions. Options not listed: 33# --debug -- don't filter output from utilities 34# --no-stats -- don't show progress statistics while fetching files 35usage() { 36 cat <<EOF 37usage: `basename $0` [options] command [path] 38 39Options: 40 -d workdir -- Store working files in workdir 41 (default: /var/db/portsnap/) 42 -f conffile -- Read configuration options from conffile 43 (default: /etc/portsnap.conf) 44 -I -- Update INDEX only. (update command only) 45 -k KEY -- Trust an RSA key with SHA256 hash of KEY 46 -p portsdir -- Location of uncompressed ports tree 47 (default: /usr/ports/) 48 -s server -- Server from which to fetch updates. 49 (default: portsnap.FreeBSD.org) 50 path -- Extract only parts of the tree starting with the given 51 string. (extract command only) 52Commands: 53 fetch -- Fetch a compressed snapshot of the ports tree, 54 or update an existing snapshot. 55 cron -- Sleep rand(3600) seconds, and then fetch updates. 56 extract -- Extract snapshot of ports tree, replacing existing 57 files and directories. 58 update -- Update ports tree to match current snapshot, replacing 59 files and directories which have changed. 60EOF 61 exit 0 62} 63 64#### Parameter handling functions. 65 66# Initialize parameters to null, just in case they're 67# set in the environment. 68init_params() { 69 KEYPRINT="" 70 EXTRACTPATH="" 71 WORKDIR="" 72 PORTSDIR="" 73 CONFFILE="" 74 COMMAND="" 75 QUIETREDIR="" 76 QUIETFLAG="" 77 STATSREDIR="" 78 XARGST="" 79 NDEBUG="" 80 DDSTATS="" 81 INDEXONLY="" 82 SERVERNAME="" 83} 84 85# Parse the command line 86parse_cmdline() { 87 while [ $# -gt 0 ]; do 88 case "$1" in 89 -d) 90 if [ $# -eq 1 ]; then usage; fi 91 if [ ! -z "${WORKDIR}" ]; then usage; fi 92 shift; WORKDIR="$1" 93 ;; 94 --debug) 95 QUIETREDIR="/dev/stderr" 96 STATSREDIR="/dev/stderr" 97 QUIETFLAG=" " 98 NDEBUG=" " 99 XARGST="-t" 100 DDSTATS=".." 101 ;; 102 -f) 103 if [ $# -eq 1 ]; then usage; fi 104 if [ ! -z "${CONFFILE}" ]; then usage; fi 105 shift; CONFFILE="$1" 106 ;; 107 -h | --help | help) 108 usage 109 ;; 110 -I) 111 INDEXONLY="YES" 112 ;; 113 -k) 114 if [ $# -eq 1 ]; then usage; fi 115 if [ ! -z "${KEYPRINT}" ]; then usage; fi 116 shift; KEYPRINT="$1" 117 ;; 118 --no-stats) 119 if [ -z "${STATSREDIR}" ]; then 120 STATSREDIR="/dev/null" 121 DDSTATS=".. " 122 fi 123 ;; 124 -p) 125 if [ $# -eq 1 ]; then usage; fi 126 if [ ! -z "${PORTSDIR}" ]; then usage; fi 127 shift; PORTSDIR="$1" 128 ;; 129 -s) 130 if [ $# -eq 1 ]; then usage; fi 131 if [ ! -z "${SERVERNAME}" ]; then usage; fi 132 shift; SERVERNAME="$1" 133 ;; 134 cron | extract | fetch | update) 135 if [ ! -z "${COMMAND}" ]; then usage; fi 136 COMMAND="$1" 137 ;; 138 *) 139 if [ $# -gt 1 ]; then usage; fi 140 if [ "${COMMAND}" = "extract" ]; then usage; fi 141 EXTRACTPATH="$1" 142 ;; 143 esac 144 shift 145 done 146 147 if [ -z "${COMMAND}" ]; then 148 usage 149 fi 150} 151 152# If CONFFILE was specified at the command-line, make 153# sure that it exists and is readable. 154sanity_conffile() { 155 if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then 156 echo -n "File does not exist " 157 echo -n "or is not readable: " 158 echo ${CONFFILE} 159 exit 1 160 fi 161} 162 163# If a configuration file hasn't been specified, use 164# the default value (/etc/portsnap.conf) 165default_conffile() { 166 if [ -z "${CONFFILE}" ]; then 167 CONFFILE="/etc/portsnap.conf" 168 fi 169} 170 171# Read {KEYPRINT, SERVERNAME, WORKDIR, PORTSDIR} from the configuration 172# file if they haven't already been set. If the configuration 173# file doesn't exist, do nothing. 174parse_conffile() { 175 if [ -r "${CONFFILE}" ]; then 176 for X in KEYPRINT WORKDIR PORTSDIR SERVERNAME; do 177 eval _=\$${X} 178 if [ -z "${_}" ]; then 179 eval ${X}=`grep "^${X}=" "${CONFFILE}" | 180 cut -f 2- -d '=' | tail -1` 181 fi 182 done 183 fi 184} 185 186# If parameters have not been set, use default values 187default_params() { 188 _QUIETREDIR="/dev/null" 189 _QUIETFLAG="-q" 190 _STATSREDIR="/dev/stdout" 191 _WORKDIR="/var/db/portsnap" 192 _PORTSDIR="/usr/ports" 193 _NDEBUG="-n" 194 for X in QUIETREDIR QUIETFLAG STATSREDIR WORKDIR PORTSDIR NDEBUG; do 195 eval _=\$${X} 196 eval __=\$_${X} 197 if [ -z "${_}" ]; then 198 eval ${X}=${__} 199 fi 200 done 201} 202 203# Perform sanity checks and set some final parameters 204# in preparation for fetching files. Also chdir into 205# the working directory. 206fetch_check_params() { 207 export HTTP_USER_AGENT="portsnap (${COMMAND}, `uname -r`)" 208 209 _SERVERNAME_z=\ 210"SERVERNAME must be given via command line or configuration file." 211 _KEYPRINT_z="Key must be given via -k option or configuration file." 212 _KEYPRINT_bad="Invalid key fingerprint: " 213 _WORKDIR_bad="Directory does not exist or is not writable: " 214 215 if [ -z "${SERVERNAME}" ]; then 216 echo -n "`basename $0`: " 217 echo "${_SERVERNAME_z}" 218 exit 1 219 fi 220 if [ -z "${KEYPRINT}" ]; then 221 echo -n "`basename $0`: " 222 echo "${_KEYPRINT_z}" 223 exit 1 224 fi 225 if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then 226 echo -n "`basename $0`: " 227 echo -n "${_KEYPRINT_bad}" 228 echo ${KEYPRINT} 229 exit 1 230 fi 231 if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then 232 echo -n "`basename $0`: " 233 echo -n "${_WORKDIR_bad}" 234 echo ${WORKDIR} 235 exit 1 236 fi 237 cd ${WORKDIR} || exit 1 238 239 BSPATCH=/usr/bin/bspatch 240 SHA256=/sbin/sha256 241 PHTTPGET=/usr/libexec/phttpget 242} 243 244# Perform sanity checks and set some final parameters 245# in preparation for extracting or updating ${PORTSDIR} 246extract_check_params() { 247 _WORKDIR_bad="Directory does not exist: " 248 _PORTSDIR_bad="Directory does not exist or is not writable: " 249 250 if ! [ -d "${WORKDIR}" ]; then 251 echo -n "`basename $0`: " 252 echo -n "${_WORKDIR_bad}" 253 echo ${WORKDIR} 254 exit 1 255 fi 256 if ! [ -d "${PORTSDIR}" -a -w "${PORTSDIR}" ]; then 257 echo -n "`basename $0`: " 258 echo -n "${_PORTSDIR_bad}" 259 echo ${PORTSDIR} 260 exit 1 261 fi 262 263 if ! [ -d "${WORKDIR}/files" -a -r "${WORKDIR}/tag" \ 264 -a -r "${WORKDIR}/INDEX" -a -r "${WORKDIR}/tINDEX" ]; then 265 echo "No snapshot available. Try running" 266 echo "# `basename $0` fetch" 267 exit 1 268 fi 269 270 MKINDEX=/usr/libexec/make_index 271} 272 273# Perform sanity checks and set some final parameters 274# in preparation for updating ${PORTSDIR} 275update_check_params() { 276 extract_check_params 277 278 if ! [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 279 echo "${PORTSDIR} was not created by portsnap." 280 echo -n "You must run '`basename $0` extract' before " 281 echo "running '`basename $0` update'." 282 exit 1 283 fi 284 285} 286 287#### Core functionality -- the actual work gets done here 288 289# Use an SRV query to pick a server. If the SRV query doesn't provide 290# a useful answer, use the server name specified by the user. 291# Put another way... look up _http._tcp.${SERVERNAME} and pick a server 292# from that; or if no servers are returned, use ${SERVERNAME}. 293# This allows a user to specify "portsnap.freebsd.org" (in which case 294# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org" 295# (in which case portsnap will use that particular server, since there 296# won't be an SRV entry for that name). 297# 298# We don't implement the recommendations from RFC 2782 completely, since 299# we are only looking to pick a single server -- the recommendations are 300# targetted at applications which obtain a list of servers and then try 301# each in turn, but we are instead just going to pick one server and let 302# the user re-run portsnap if a broken server was selected. 303# 304# We also ignore the Port field, since we are always going to use port 80. 305fetch_pick_server() { 306 echo -n "Looking up ${SERVERNAME} mirrors..." 307 308# Issue the SRV query and pull out the Priority, Weight, and Target fields. 309 host -t srv "_http._tcp.${SERVERNAME}" | 310 grep -E "^_http._tcp.${SERVERNAME} has SRV record" | 311 cut -f 5,6,8 -d ' ' > serverlist 312 313# If no records, give up -- we'll just use the server name we were given. 314 if [ `wc -l < serverlist` -eq 0 ]; then 315 echo " none found." 316 return 317 fi 318 319# Find the highest priority level (lowest numeric value). 320 SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1` 321 322# Add up the weights of the response lines at that priority level. 323 SRV_WSUM=0; 324 while read X; do 325 case "$X" in 326 ${SRV_PRIORITY}\ *) 327 SRV_W=`echo $X | cut -f 2 -d ' '` 328 SRV_WSUM=$(($SRV_WSUM + $SRV_W)) 329 ;; 330 esac 331 done < serverlist 332 333# If all the weights are 0, pretend that they are all 1 instead. 334 if [ ${SRV_WSUM} -eq 0 ]; then 335 SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l` 336 SRV_W_ADD=1 337 else 338 SRV_W_ADD=0 339 fi 340 341# Pick a random value between 1 and the sum of the weights 342 SRV_RND=`jot -r 1 1 ${SRV_WSUM}` 343 344# Read through the list of mirrors and set SERVERNAME 345 while read X; do 346 case "$X" in 347 ${SRV_PRIORITY}\ *) 348 SRV_W=`echo $X | cut -f 2 -d ' '` 349 SRV_W=$(($SRV_W + $SRV_W_ADD)) 350 if [ $SRV_RND -le $SRV_W ]; then 351 SERVERNAME=`echo $X | cut -f 3 -d ' '` 352 break 353 else 354 SRV_RND=$(($SRV_RND - $SRV_W)) 355 fi 356 ;; 357 esac 358 done < serverlist 359 360 echo " using ${SERVERNAME}" 361} 362 363# Check that we have a public key with an appropriate hash, or 364# fetch the key if it doesn't exist. 365fetch_key() { 366 if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 367 return 368 fi 369 370 echo -n "Fetching public key... " 371 rm -f pub.ssl 372 fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \ 373 2>${QUIETREDIR} || true 374 if ! [ -r pub.ssl ]; then 375 echo "failed." 376 return 1 377 fi 378 if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 379 echo "key has incorrect hash." 380 rm -f pub.ssl 381 return 1 382 fi 383 echo "done." 384} 385 386# Fetch a snapshot tag 387fetch_tag() { 388 rm -f snapshot.ssl tag.new 389 390 echo ${NDEBUG} "Fetching snapshot tag... " 391 fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl 392 2>${QUIETREDIR} || true 393 if ! [ -r $1.ssl ]; then 394 echo "failed." 395 return 1 396 fi 397 398 openssl rsautl -pubin -inkey pub.ssl -verify \ 399 < $1.ssl > tag.new 2>${QUIETREDIR} || true 400 rm $1.ssl 401 402 if ! [ `wc -l < tag.new` = 1 ] || 403 ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then 404 echo "invalid snapshot tag." 405 return 1 406 fi 407 408 echo "done." 409 410 SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new` 411 SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new` 412} 413 414# Sanity-check the date on a snapshot tag 415fetch_snapshot_tagsanity() { 416 if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then 417 echo "Snapshot appears to be more than a year old!" 418 echo "(Is the system clock correct?)" 419 echo "Cowarly refusing to proceed any further." 420 return 1 421 fi 422 if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then 423 echo -n "Snapshot appears to have been created more than " 424 echo "one day into the future!" 425 echo "(Is the system clock correct?)" 426 echo "Cowardly refusing to proceed any further." 427 return 1 428 fi 429} 430 431# Sanity-check the date on a snapshot update tag 432fetch_update_tagsanity() { 433 fetch_snapshot_tagsanity || return 1 434 435 if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then 436 echo -n "Latest snapshot on server is " 437 echo "older than what we already have!" 438 echo -n "Cowardly refusing to downgrade from " 439 date -r ${OLDSNAPSHOTDATE} 440 echo "to `date -r ${SNAPSHOTDATE}`." 441 return 1 442 fi 443} 444 445# Compare old and new tags; return 1 if update is unnecessary 446fetch_update_neededp() { 447 if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then 448 echo -n "Latest snapshot on server matches " 449 echo "what we already have." 450 echo "No updates needed." 451 rm tag.new 452 return 1 453 fi 454 if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then 455 echo -n "Ports tree hasn't changed since " 456 echo "last snapshot." 457 echo "No updates needed." 458 rm tag.new 459 return 1 460 fi 461 462 return 0 463} 464 465# Fetch snapshot metadata file 466fetch_metadata() { 467 rm -f ${SNAPSHOTHASH} tINDEX.new 468 469 echo ${NDEBUG} "Fetching snapshot metadata... " 470 fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} 471 2>${QUIETREDIR} || return 472 if [ `${SHA256} -q ${SNAPSHOTHASH}` != ${SNAPSHOTHASH} ]; then 473 echo "snapshot metadata corrupt." 474 return 1 475 fi 476 mv ${SNAPSHOTHASH} tINDEX.new 477 echo "done." 478} 479 480# Warn user about bogus metadata 481fetch_metadata_freakout() { 482 echo 483 echo "Portsnap metadata is correctly signed, but contains" 484 echo "at least one line which appears bogus." 485 echo "Cowardly refusing to proceed any further." 486} 487 488# Sanity-check a snapshot metadata file 489fetch_metadata_sanity() { 490 if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then 491 fetch_metadata_freakout 492 return 1 493 fi 494 if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then 495 echo 496 echo "Portsnap metadata appears bogus." 497 echo "Cowardly refusing to proceed any further." 498 return 1 499 fi 500} 501 502# Take a list of ${oldhash}|${newhash} and output a list of needed patches 503fetch_make_patchlist() { 504 grep -vE "^([0-9a-f]{64})\|\1$" | 505 while read LINE; do 506 X=`echo ${LINE} | cut -f 1 -d '|'` 507 Y=`echo ${LINE} | cut -f 2 -d '|'` 508 if [ -f "files/${Y}.gz" ]; then continue; fi 509 if [ ! -f "files/${X}.gz" ]; then continue; fi 510 echo "${LINE}" 511 done 512} 513 514# Print user-friendly progress statistics 515fetch_progress() { 516 LNC=0 517 while read x; do 518 LNC=$(($LNC + 1)) 519 if [ $(($LNC % 10)) = 0 ]; then 520 echo -n $FN2 521 elif [ $(($LNC % 2)) = 0 ]; then 522 echo -n . 523 fi 524 done 525 echo -n " " 526} 527 528# Sanity-check an index file 529fetch_index_sanity() { 530 if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new || 531 fgrep -q "./" INDEX.new; then 532 fetch_metadata_freakout 533 return 1 534 fi 535} 536 537# Verify a list of files 538fetch_snapshot_verify() { 539 while read F; do 540 if [ `gunzip -c snap/${F} | ${SHA256} -q` != ${F} ]; then 541 echo "snapshot corrupt." 542 return 1 543 fi 544 done 545 return 0 546} 547 548# Fetch a snapshot tarball, extract, and verify. 549fetch_snapshot() { 550 fetch_tag snapshot || return 1 551 fetch_snapshot_tagsanity || return 1 552 fetch_metadata || return 1 553 fetch_metadata_sanity || return 1 554 555 rm -f ${SNAPSHOTHASH}.tgz 556 rm -rf snap/ 557 558# Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will 559# probably take a while, so the progrees reports that fetch(1) generates 560# will be useful for keeping the users' attention from drifting. 561 echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:" 562 fetch http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1 563 564 echo -n "Extracting snapshot... " 565 tar -xzf ${SNAPSHOTHASH}.tgz snap/ || return 1 566 rm ${SNAPSHOTHASH}.tgz 567 echo "done." 568 569 echo -n "Verifying snapshot integrity... " 570# Verify the metadata files 571 cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1 572# Extract the index 573 rm -f INDEX.new 574 gunzip -c snap/`look INDEX tINDEX.new | 575 cut -f 2 -d '|'`.gz > INDEX.new 576 fetch_index_sanity || return 1 577# Verify the snapshot contents 578 cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1 579 echo "done." 580 581# Move files into their proper locations 582 rm -f tag INDEX tINDEX 583 rm -rf files 584 mv tag.new tag 585 mv tINDEX.new tINDEX 586 mv INDEX.new INDEX 587 mv snap/ files/ 588 589 return 0 590} 591 592# Update a compressed snapshot 593fetch_update() { 594 rm -f patchlist diff OLD NEW filelist INDEX.new 595 596 OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag` 597 OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag` 598 599 fetch_tag latest || return 1 600 fetch_update_tagsanity || return 1 601 fetch_update_neededp || return 0 602 fetch_metadata || return 1 603 fetch_metadata_sanity || return 1 604 605 echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` " 606 echo "to `date -r ${SNAPSHOTDATE}`." 607 608# Generate a list of wanted metadata patches 609 join -t '|' -o 1.2,2.2 tINDEX tINDEX.new | 610 fetch_make_patchlist > patchlist 611 612# Attempt to fetch metadata patches 613 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 614 echo ${NDEBUG} "metadata patches.${DDSTATS}" 615 tr '|' '-' < patchlist | 616 lam -s "tp/" - -s ".gz" | 617 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 618 2>${STATSREDIR} | fetch_progress 619 echo "done." 620 621# Attempt to apply metadata patches 622 echo -n "Applying metadata patches... " 623 while read LINE; do 624 X=`echo ${LINE} | cut -f 1 -d '|'` 625 Y=`echo ${LINE} | cut -f 2 -d '|'` 626 if [ ! -f "${X}-${Y}.gz" ]; then continue; fi 627 gunzip -c < ${X}-${Y}.gz > diff 628 gunzip -c < files/${X}.gz > OLD 629 cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp 630 grep '^\+' diff | cut -c 2- | 631 sort -k 1,1 -t '|' -m - ptmp > NEW 632 if [ `${SHA256} -q NEW` = ${Y} ]; then 633 mv NEW files/${Y} 634 gzip -n files/${Y} 635 fi 636 rm -f diff OLD NEW ${X}-${Y}.gz ptmp 637 done < patchlist 2>${QUIETREDIR} 638 echo "done." 639 640# Update metadata without patches 641 join -t '|' -v 2 tINDEX tINDEX.new | 642 cut -f 2 -d '|' /dev/stdin patchlist | 643 while read Y; do 644 if [ ! -f "files/${Y}.gz" ]; then 645 echo ${Y}; 646 fi 647 done > filelist 648 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 649 echo ${NDEBUG} "metadata files... " 650 lam -s "f/" - -s ".gz" < filelist | 651 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 652 2>${QUIETREDIR} 653 654 while read Y; do 655 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 656 mv ${Y}.gz files/${Y}.gz 657 else 658 echo "metadata is corrupt." 659 return 1 660 fi 661 done < filelist 662 echo "done." 663 664# Extract the index 665 gunzip -c files/`look INDEX tINDEX.new | 666 cut -f 2 -d '|'`.gz > INDEX.new 667 fetch_index_sanity || return 1 668 669# Generate a list of wanted ports patches 670 join -t '|' -o 1.2,2.2 INDEX INDEX.new | 671 fetch_make_patchlist > patchlist 672 673# Attempt to fetch ports patches 674 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 675 echo ${NDEBUG} "patches.${DDSTATS}" 676 tr '|' '-' < patchlist | lam -s "bp/" - | 677 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 678 2>${STATSREDIR} | fetch_progress 679 echo "done." 680 681# Attempt to apply ports patches 682 echo -n "Applying patches... " 683 while read LINE; do 684 X=`echo ${LINE} | cut -f 1 -d '|'` 685 Y=`echo ${LINE} | cut -f 2 -d '|'` 686 if [ ! -f "${X}-${Y}" ]; then continue; fi 687 gunzip -c < files/${X}.gz > OLD 688 ${BSPATCH} OLD NEW ${X}-${Y} 689 if [ `${SHA256} -q NEW` = ${Y} ]; then 690 mv NEW files/${Y} 691 gzip -n files/${Y} 692 fi 693 rm -f diff OLD NEW ${X}-${Y} 694 done < patchlist 2>${QUIETREDIR} 695 echo "done." 696 697# Update ports without patches 698 join -t '|' -v 2 INDEX INDEX.new | 699 cut -f 2 -d '|' /dev/stdin patchlist | 700 while read Y; do 701 if [ ! -f "files/${Y}.gz" ]; then 702 echo ${Y}; 703 fi 704 done > filelist 705 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 706 echo ${NDEBUG} "new ports or files... " 707 lam -s "f/" - -s ".gz" < filelist | 708 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 709 2>${QUIETREDIR} 710 711 while read Y; do 712 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 713 mv ${Y}.gz files/${Y}.gz 714 else 715 echo "snapshot is corrupt." 716 return 1 717 fi 718 done < filelist 719 echo "done." 720 721# Remove files which are no longer needed 722 cut -f 2 -d '|' tINDEX INDEX | sort > oldfiles 723 cut -f 2 -d '|' tINDEX.new INDEX.new | sort | comm -13 - oldfiles | 724 lam -s "files/" - -s ".gz" | xargs rm -f 725 rm patchlist filelist oldfiles 726 727# We're done! 728 mv INDEX.new INDEX 729 mv tINDEX.new tINDEX 730 mv tag.new tag 731 732 return 0 733} 734 735# Do the actual work involved in "fetch" / "cron". 736fetch_run() { 737 fetch_pick_server 738 739 fetch_key || return 1 740 741 if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then 742 fetch_snapshot || return 1 743 fi 744 fetch_update || return 1 745} 746 747# Build a ports INDEX file 748extract_make_index() { 749 gunzip -c "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX | 750 cut -f 2 -d '|'`.gz" | ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2 751} 752 753# Create INDEX, INDEX-5, INDEX-6 754extract_indices() { 755 echo -n "Building new INDEX files... " 756 extract_make_index DESCRIBE.4 INDEX || return 1 757 extract_make_index DESCRIBE.5 INDEX-5 || return 1 758 extract_make_index DESCRIBE.6 INDEX-6 || return 1 759 echo "done." 760} 761 762# Create .portsnap.INDEX 763extract_metadata() { 764 sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX 765} 766 767# Do the actual work involved in "extract" 768extract_run() { 769 grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX | while read LINE; do 770 FILE=`echo ${LINE} | cut -f 1 -d '|'` 771 HASH=`echo ${LINE} | cut -f 2 -d '|'` 772 echo ${PORTSDIR}/${FILE} 773 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 774 echo "files/${HASH}.gz not found -- snapshot corrupt." 775 return 1 776 fi 777 case ${FILE} in 778 */) 779 rm -rf ${PORTSDIR}/${FILE} 780 mkdir -p ${PORTSDIR}/${FILE} 781 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 782 -C ${PORTSDIR}/${FILE} 783 ;; 784 *) 785 rm -f ${PORTSDIR}/${FILE} 786 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 787 -C ${PORTSDIR} ${FILE} 788 ;; 789 esac 790 done 791 if [ ! -z "${EXTRACTPATH}" ]; then 792 return 0; 793 fi 794 795 extract_metadata 796 extract_indices 797} 798 799# Do the actual work involved in "update" 800update_run() { 801 if ! [ -z "${INDEXONLY}" ]; then 802 extract_indices >/dev/null || return 1 803 return 0 804 fi 805 806 if sort ${WORKDIR}/INDEX | cmp ${PORTSDIR}/.portsnap.INDEX -; then 807 echo "Ports tree is already up to date." 808 return 0 809 fi 810 811 echo -n "Removing old files and directories... " 812 sort ${WORKDIR}/INDEX | comm -23 ${PORTSDIR}/.portsnap.INDEX - | 813 cut -f 1 -d '|' | lam -s "${PORTSDIR}/" - | xargs rm -rf 814 echo "done." 815 816# Install new files 817 echo "Extracting new files:" 818 sort ${WORKDIR}/INDEX | comm -13 ${PORTSDIR}/.portsnap.INDEX - | 819 while read LINE; do 820 FILE=`echo ${LINE} | cut -f 1 -d '|'` 821 HASH=`echo ${LINE} | cut -f 2 -d '|'` 822 echo ${PORTSDIR}/${FILE} 823 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 824 echo "files/${HASH}.gz not found -- snapshot corrupt." 825 return 1 826 fi 827 case ${FILE} in 828 */) 829 mkdir -p ${PORTSDIR}/${FILE} 830 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 831 -C ${PORTSDIR}/${FILE} 832 ;; 833 *) 834 tar -xzf ${WORKDIR}/files/${HASH}.gz \ 835 -C ${PORTSDIR} ${FILE} 836 ;; 837 esac 838 done 839 840 extract_metadata 841 extract_indices 842} 843 844#### Main functions -- call parameter-handling and core functions 845 846# Using the command line, configuration file, and defaults, 847# set all the parameters which are needed later. 848get_params() { 849 init_params 850 parse_cmdline $@ 851 sanity_conffile 852 default_conffile 853 parse_conffile 854 default_params 855} 856 857# Fetch command. Make sure that we're being called 858# interactively, then run fetch_check_params and fetch_run 859cmd_fetch() { 860 if [ ! -t 0 ]; then 861 echo -n "`basename $0` fetch should not " 862 echo "be run non-interactively." 863 echo "Run `basename $0` cron instead." 864 exit 1 865 fi 866 fetch_check_params 867 fetch_run || exit 1 868} 869 870# Cron command. Make sure the parameters are sensible; wait 871# rand(3600) seconds; then fetch updates. While fetching updates, 872# send output to a temporary file; only print that file if the 873# fetching failed. 874cmd_cron() { 875 fetch_check_params 876 sleep `jot -r 1 0 3600` 877 878 TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1 879 if ! fetch_run >> ${TMPFILE}; then 880 cat ${TMPFILE} 881 rm ${TMPFILE} 882 exit 1 883 fi 884 885 rm ${TMPFILE} 886} 887 888# Extract command. Make sure the parameters are sensible, 889# then extract the ports tree (or part thereof). 890cmd_extract() { 891 extract_check_params 892 extract_run || exit 1 893} 894 895# Update command. Make sure the parameters are sensible, 896# then update the ports tree. 897cmd_update() { 898 update_check_params 899 update_run || exit 1 900} 901 902#### Entry point 903 904# Make sure we find utilities from the base system 905export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH} 906 907get_params $@ 908cmd_${COMMAND} 909