1#!/bin/sh 2 3#- 4# Copyright 2004-2005 Colin Percival 5# All rights reserved 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted providing that the following conditions 9# are met: 10# 1. Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# 2. Redistributions in binary form must reproduce the above copyright 13# notice, this list of conditions and the following disclaimer in the 14# documentation and/or other materials provided with the distribution. 15# 16# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 20# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26# POSSIBILITY OF SUCH DAMAGE. 27 28# $FreeBSD: releng/9.3/usr.sbin/portsnap/portsnap/portsnap.sh 306942 2016-10-10 07:19:16Z delphij $ 29 30#### Usage function -- called from command-line handling code. 31 32# Usage instructions. Options not listed: 33# --debug -- don't filter output from utilities 34# --no-stats -- don't show progress statistics while fetching files 35usage() { 36 cat <<EOF 37usage: `basename $0` [options] command ... [path] 38 39Options: 40 -d workdir -- Store working files in workdir 41 (default: /var/db/portsnap/) 42 -f conffile -- Read configuration options from conffile 43 (default: /etc/portsnap.conf) 44 -I -- Update INDEX only. (update command only) 45 -k KEY -- Trust an RSA key with SHA256 hash of KEY 46 -l descfile -- Merge the specified local describes file into the INDEX. 47 -p portsdir -- Location of uncompressed ports tree 48 (default: /usr/ports/) 49 -s server -- Server from which to fetch updates. 50 (default: portsnap.FreeBSD.org) 51 path -- Extract only parts of the tree starting with the given 52 string. (extract command only) 53Commands: 54 fetch -- Fetch a compressed snapshot of the ports tree, 55 or update an existing snapshot. 56 cron -- Sleep rand(3600) seconds, and then fetch updates. 57 extract -- Extract snapshot of ports tree, replacing existing 58 files and directories. 59 update -- Update ports tree to match current snapshot, replacing 60 files and directories which have changed. 61EOF 62 exit 0 63} 64 65#### Parameter handling functions. 66 67# Initialize parameters to null, just in case they're 68# set in the environment. 69init_params() { 70 KEYPRINT="" 71 EXTRACTPATH="" 72 WORKDIR="" 73 PORTSDIR="" 74 CONFFILE="" 75 COMMAND="" 76 COMMANDS="" 77 QUIETREDIR="" 78 QUIETFLAG="" 79 STATSREDIR="" 80 XARGST="" 81 NDEBUG="" 82 DDSTATS="" 83 INDEXONLY="" 84 SERVERNAME="" 85 REFUSE="" 86 LOCALDESC="" 87} 88 89# Parse the command line 90parse_cmdline() { 91 while [ $# -gt 0 ]; do 92 case "$1" in 93 -d) 94 if [ $# -eq 1 ]; then usage; fi 95 if [ ! -z "${WORKDIR}" ]; then usage; fi 96 shift; WORKDIR="$1" 97 ;; 98 --debug) 99 QUIETREDIR="/dev/stderr" 100 STATSREDIR="/dev/stderr" 101 QUIETFLAG=" " 102 NDEBUG=" " 103 XARGST="-t" 104 DDSTATS=".." 105 ;; 106 -f) 107 if [ $# -eq 1 ]; then usage; fi 108 if [ ! -z "${CONFFILE}" ]; then usage; fi 109 shift; CONFFILE="$1" 110 ;; 111 -h | --help | help) 112 usage 113 ;; 114 -I) 115 INDEXONLY="YES" 116 ;; 117 -k) 118 if [ $# -eq 1 ]; then usage; fi 119 if [ ! -z "${KEYPRINT}" ]; then usage; fi 120 shift; KEYPRINT="$1" 121 ;; 122 -l) 123 if [ $# -eq 1 ]; then usage; fi 124 if [ ! -z "${LOCALDESC}" ]; then usage; fi 125 shift; LOCALDESC="$1" 126 ;; 127 --no-stats) 128 if [ -z "${STATSREDIR}" ]; then 129 STATSREDIR="/dev/null" 130 DDSTATS=".. " 131 fi 132 ;; 133 -p) 134 if [ $# -eq 1 ]; then usage; fi 135 if [ ! -z "${PORTSDIR}" ]; then usage; fi 136 shift; PORTSDIR="$1" 137 ;; 138 -s) 139 if [ $# -eq 1 ]; then usage; fi 140 if [ ! -z "${SERVERNAME}" ]; then usage; fi 141 shift; SERVERNAME="$1" 142 ;; 143 cron | extract | fetch | update | alfred) 144 COMMANDS="${COMMANDS} $1" 145 ;; 146 up) 147 COMMANDS="${COMMANDS} update" 148 ;; 149 *) 150 if [ $# -gt 1 ]; then usage; fi 151 if echo ${COMMANDS} | grep -vq extract; then 152 usage 153 fi 154 EXTRACTPATH="$1" 155 ;; 156 esac 157 shift 158 done 159 160 if [ -z "${COMMANDS}" ]; then 161 usage 162 fi 163} 164 165# If CONFFILE was specified at the command-line, make 166# sure that it exists and is readable. 167sanity_conffile() { 168 if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then 169 echo -n "File does not exist " 170 echo -n "or is not readable: " 171 echo ${CONFFILE} 172 exit 1 173 fi 174} 175 176# If a configuration file hasn't been specified, use 177# the default value (/etc/portsnap.conf) 178default_conffile() { 179 if [ -z "${CONFFILE}" ]; then 180 CONFFILE="/etc/portsnap.conf" 181 fi 182} 183 184# Read {KEYPRINT, SERVERNAME, WORKDIR, PORTSDIR} from the configuration 185# file if they haven't already been set. If the configuration 186# file doesn't exist, do nothing. 187# Also read REFUSE (which cannot be set via the command line) if it is 188# present in the configuration file. 189parse_conffile() { 190 if [ -r "${CONFFILE}" ]; then 191 for X in KEYPRINT WORKDIR PORTSDIR SERVERNAME; do 192 eval _=\$${X} 193 if [ -z "${_}" ]; then 194 eval ${X}=`grep "^${X}=" "${CONFFILE}" | 195 cut -f 2- -d '=' | tail -1` 196 fi 197 done 198 199 if grep -qE "^REFUSE[[:space:]]" ${CONFFILE}; then 200 REFUSE="^(` 201 grep -E "^REFUSE[[:space:]]" "${CONFFILE}" | 202 cut -c 7- | xargs echo | tr ' ' '|' 203 `)" 204 fi 205 206 if grep -qE "^INDEX[[:space:]]" ${CONFFILE}; then 207 INDEXPAIRS="` 208 grep -E "^INDEX[[:space:]]" "${CONFFILE}" | 209 cut -c 7- | tr ' ' '|' | xargs echo`" 210 fi 211 fi 212} 213 214# If parameters have not been set, use default values 215default_params() { 216 _QUIETREDIR="/dev/null" 217 _QUIETFLAG="-q" 218 _STATSREDIR="/dev/stdout" 219 _WORKDIR="/var/db/portsnap" 220 _PORTSDIR="/usr/ports" 221 _NDEBUG="-n" 222 _LOCALDESC="/dev/null" 223 for X in QUIETREDIR QUIETFLAG STATSREDIR WORKDIR PORTSDIR \ 224 NDEBUG LOCALDESC; do 225 eval _=\$${X} 226 eval __=\$_${X} 227 if [ -z "${_}" ]; then 228 eval ${X}=${__} 229 fi 230 done 231} 232 233# Perform sanity checks and set some final parameters 234# in preparation for fetching files. Also chdir into 235# the working directory. 236fetch_check_params() { 237 export HTTP_USER_AGENT="portsnap (${COMMAND}, `uname -r`)" 238 239 _SERVERNAME_z=\ 240"SERVERNAME must be given via command line or configuration file." 241 _KEYPRINT_z="Key must be given via -k option or configuration file." 242 _KEYPRINT_bad="Invalid key fingerprint: " 243 _WORKDIR_bad="Directory does not exist or is not writable: " 244 245 if [ -z "${SERVERNAME}" ]; then 246 echo -n "`basename $0`: " 247 echo "${_SERVERNAME_z}" 248 exit 1 249 fi 250 if [ -z "${KEYPRINT}" ]; then 251 echo -n "`basename $0`: " 252 echo "${_KEYPRINT_z}" 253 exit 1 254 fi 255 if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then 256 echo -n "`basename $0`: " 257 echo -n "${_KEYPRINT_bad}" 258 echo ${KEYPRINT} 259 exit 1 260 fi 261 if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then 262 echo -n "`basename $0`: " 263 echo -n "${_WORKDIR_bad}" 264 echo ${WORKDIR} 265 exit 1 266 fi 267 cd ${WORKDIR} || exit 1 268 269 BSPATCH=/usr/bin/bspatch 270 SHA256=/sbin/sha256 271 PHTTPGET=/usr/libexec/phttpget 272} 273 274# Perform sanity checks and set some final parameters 275# in preparation for extracting or updating ${PORTSDIR} 276# Complain if ${PORTSDIR} exists but is not writable, 277# but don't complain if ${PORTSDIR} doesn't exist. 278extract_check_params() { 279 _WORKDIR_bad="Directory does not exist: " 280 _PORTSDIR_bad="Directory is not writable: " 281 282 if ! [ -d "${WORKDIR}" ]; then 283 echo -n "`basename $0`: " 284 echo -n "${_WORKDIR_bad}" 285 echo ${WORKDIR} 286 exit 1 287 fi 288 if [ -d "${PORTSDIR}" ] && ! [ -w "${PORTSDIR}" ]; then 289 echo -n "`basename $0`: " 290 echo -n "${_PORTSDIR_bad}" 291 echo ${PORTSDIR} 292 exit 1 293 fi 294 295 if ! [ -d "${WORKDIR}/files" -a -r "${WORKDIR}/tag" \ 296 -a -r "${WORKDIR}/INDEX" -a -r "${WORKDIR}/tINDEX" ]; then 297 echo "No snapshot available. Try running" 298 echo "# `basename $0` fetch" 299 exit 1 300 fi 301 302 MKINDEX=/usr/libexec/make_index 303} 304 305# Perform sanity checks and set some final parameters 306# in preparation for updating ${PORTSDIR} 307update_check_params() { 308 extract_check_params 309 310 if ! [ -r ${PORTSDIR}/.portsnap.INDEX ]; then 311 echo "${PORTSDIR} was not created by portsnap." 312 echo -n "You must run '`basename $0` extract' before " 313 echo "running '`basename $0` update'." 314 exit 1 315 fi 316 317} 318 319#### Core functionality -- the actual work gets done here 320 321# Use an SRV query to pick a server. If the SRV query doesn't provide 322# a useful answer, use the server name specified by the user. 323# Put another way... look up _http._tcp.${SERVERNAME} and pick a server 324# from that; or if no servers are returned, use ${SERVERNAME}. 325# This allows a user to specify "portsnap.freebsd.org" (in which case 326# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org" 327# (in which case portsnap will use that particular server, since there 328# won't be an SRV entry for that name). 329# 330# We ignore the Port field, since we are always going to use port 80. 331 332# Fetch the mirror list, but do not pick a mirror yet. Returns 1 if 333# no mirrors are available for any reason. 334fetch_pick_server_init() { 335 : > serverlist_tried 336 337# Check that host(1) exists (i.e., that the system wasn't built with the 338# WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist. 339 if ! which -s host; then 340 : > serverlist_full 341 return 1 342 fi 343 344 echo -n "Looking up ${SERVERNAME} mirrors... " 345 346# Issue the SRV query and pull out the Priority, Weight, and Target fields. 347# BIND 9 prints "$name has SRV record ..." while BIND 8 prints 348# "$name server selection ..."; we allow either format. 349 MLIST="_http._tcp.${SERVERNAME}" 350 host -t srv "${MLIST}" | 351 sed -nE "s/${MLIST} (has SRV record|server selection) //Ip" | 352 cut -f 1,2,4 -d ' ' | 353 sed -e 's/\.$//' | 354 sort > serverlist_full 355 356# If no records, give up -- we'll just use the server name we were given. 357 if [ `wc -l < serverlist_full` -eq 0 ]; then 358 echo "none found." 359 return 1 360 fi 361 362# Report how many mirrors we found. 363 echo `wc -l < serverlist_full` "mirrors found." 364 365# Generate a random seed for use in picking mirrors. If HTTP_PROXY 366# is set, this will be used to generate the seed; otherwise, the seed 367# will be random. 368 if [ -n "${HTTP_PROXY}${http_proxy}" ]; then 369 RANDVALUE=`sha256 -qs "${HTTP_PROXY}${http_proxy}" | 370 tr -d 'a-f' | 371 cut -c 1-9` 372 else 373 RANDVALUE=`jot -r 1 0 999999999` 374 fi 375} 376 377# Pick a mirror. Returns 1 if we have run out of mirrors to try. 378fetch_pick_server() { 379# Generate a list of not-yet-tried mirrors 380 sort serverlist_tried | 381 comm -23 serverlist_full - > serverlist 382 383# Have we run out of mirrors? 384 if [ `wc -l < serverlist` -eq 0 ]; then 385 echo "No mirrors remaining, giving up." 386 return 1 387 fi 388 389# Find the highest priority level (lowest numeric value). 390 SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1` 391 392# Add up the weights of the response lines at that priority level. 393 SRV_WSUM=0; 394 while read X; do 395 case "$X" in 396 ${SRV_PRIORITY}\ *) 397 SRV_W=`echo $X | cut -f 2 -d ' '` 398 SRV_WSUM=$(($SRV_WSUM + $SRV_W)) 399 ;; 400 esac 401 done < serverlist 402 403# If all the weights are 0, pretend that they are all 1 instead. 404 if [ ${SRV_WSUM} -eq 0 ]; then 405 SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l` 406 SRV_W_ADD=1 407 else 408 SRV_W_ADD=0 409 fi 410 411# Pick a value between 0 and the sum of the weights - 1 412 SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}` 413 414# Read through the list of mirrors and set SERVERNAME. Write the line 415# corresponding to the mirror we selected into serverlist_tried so that 416# we won't try it again. 417 while read X; do 418 case "$X" in 419 ${SRV_PRIORITY}\ *) 420 SRV_W=`echo $X | cut -f 2 -d ' '` 421 SRV_W=$(($SRV_W + $SRV_W_ADD)) 422 if [ $SRV_RND -lt $SRV_W ]; then 423 SERVERNAME=`echo $X | cut -f 3 -d ' '` 424 echo "$X" >> serverlist_tried 425 break 426 else 427 SRV_RND=$(($SRV_RND - $SRV_W)) 428 fi 429 ;; 430 esac 431 done < serverlist 432} 433 434# Check that we have a public key with an appropriate hash, or 435# fetch the key if it doesn't exist. Returns 1 if the key has 436# not yet been fetched. 437fetch_key() { 438 if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 439 return 0 440 fi 441 442 echo -n "Fetching public key from ${SERVERNAME}... " 443 rm -f pub.ssl 444 fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \ 445 2>${QUIETREDIR} || true 446 if ! [ -r pub.ssl ]; then 447 echo "failed." 448 return 1 449 fi 450 if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then 451 echo "key has incorrect hash." 452 rm -f pub.ssl 453 return 1 454 fi 455 echo "done." 456} 457 458# Fetch a snapshot tag 459fetch_tag() { 460 rm -f snapshot.ssl tag.new 461 462 echo ${NDEBUG} "Fetching snapshot tag from ${SERVERNAME}... " 463 fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl \ 464 2>${QUIETREDIR} || true 465 if ! [ -r $1.ssl ]; then 466 echo "failed." 467 return 1 468 fi 469 470 openssl rsautl -pubin -inkey pub.ssl -verify \ 471 < $1.ssl > tag.new 2>${QUIETREDIR} || true 472 rm $1.ssl 473 474 if ! [ `wc -l < tag.new` = 1 ] || 475 ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then 476 echo "invalid snapshot tag." 477 return 1 478 fi 479 480 echo "done." 481 482 SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new` 483 SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new` 484} 485 486# Sanity-check the date on a snapshot tag 487fetch_snapshot_tagsanity() { 488 if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then 489 echo "Snapshot appears to be more than a year old!" 490 echo "(Is the system clock correct?)" 491 echo "Cowardly refusing to proceed any further." 492 return 1 493 fi 494 if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then 495 echo -n "Snapshot appears to have been created more than " 496 echo "one day into the future!" 497 echo "(Is the system clock correct?)" 498 echo "Cowardly refusing to proceed any further." 499 return 1 500 fi 501} 502 503# Sanity-check the date on a snapshot update tag 504fetch_update_tagsanity() { 505 fetch_snapshot_tagsanity || return 1 506 507 if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then 508 echo -n "Latest snapshot on server is " 509 echo "older than what we already have!" 510 echo -n "Cowardly refusing to downgrade from " 511 date -r ${OLDSNAPSHOTDATE} 512 echo "to `date -r ${SNAPSHOTDATE}`." 513 return 1 514 fi 515} 516 517# Compare old and new tags; return 1 if update is unnecessary 518fetch_update_neededp() { 519 if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then 520 echo -n "Latest snapshot on server matches " 521 echo "what we already have." 522 echo "No updates needed." 523 rm tag.new 524 return 1 525 fi 526 if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then 527 echo -n "Ports tree hasn't changed since " 528 echo "last snapshot." 529 echo "No updates needed." 530 rm tag.new 531 return 1 532 fi 533 534 return 0 535} 536 537# Fetch snapshot metadata file 538fetch_metadata() { 539 rm -f ${SNAPSHOTHASH} tINDEX.new 540 541 echo ${NDEBUG} "Fetching snapshot metadata... " 542 fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} \ 543 2>${QUIETREDIR} || return 544 if [ "`${SHA256} -q ${SNAPSHOTHASH}`" != ${SNAPSHOTHASH} ]; then 545 echo "snapshot metadata corrupt." 546 return 1 547 fi 548 mv ${SNAPSHOTHASH} tINDEX.new 549 echo "done." 550} 551 552# Warn user about bogus metadata 553fetch_metadata_freakout() { 554 echo 555 echo "Portsnap metadata is correctly signed, but contains" 556 echo "at least one line which appears bogus." 557 echo "Cowardly refusing to proceed any further." 558} 559 560# Sanity-check a snapshot metadata file 561fetch_metadata_sanity() { 562 if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then 563 fetch_metadata_freakout 564 return 1 565 fi 566 if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then 567 echo 568 echo "Portsnap metadata appears bogus." 569 echo "Cowardly refusing to proceed any further." 570 return 1 571 fi 572} 573 574# Take a list of ${oldhash}|${newhash} and output a list of needed patches 575fetch_make_patchlist() { 576 grep -vE "^([0-9a-f]{64})\|\1$" | 577 while read LINE; do 578 X=`echo ${LINE} | cut -f 1 -d '|'` 579 Y=`echo ${LINE} | cut -f 2 -d '|'` 580 if [ -f "files/${Y}.gz" ]; then continue; fi 581 if [ ! -f "files/${X}.gz" ]; then continue; fi 582 echo "${LINE}" 583 done 584} 585 586# Print user-friendly progress statistics 587fetch_progress() { 588 LNC=0 589 while read x; do 590 LNC=$(($LNC + 1)) 591 if [ $(($LNC % 10)) = 0 ]; then 592 echo -n $LNC 593 elif [ $(($LNC % 2)) = 0 ]; then 594 echo -n . 595 fi 596 done 597 echo -n " " 598} 599 600# Sanity-check an index file 601fetch_index_sanity() { 602 if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new || 603 fgrep -q "./" INDEX.new; then 604 fetch_metadata_freakout 605 return 1 606 fi 607} 608 609# Verify a list of files 610fetch_snapshot_verify() { 611 while read F; do 612 if [ "`gunzip -c < snap/${F}.gz | ${SHA256} -q`" != ${F} ]; then 613 echo "snapshot corrupt." 614 return 1 615 fi 616 done 617 return 0 618} 619 620# Fetch a snapshot tarball, extract, and verify. 621fetch_snapshot() { 622 while ! fetch_tag snapshot; do 623 fetch_pick_server || return 1 624 done 625 fetch_snapshot_tagsanity || return 1 626 fetch_metadata || return 1 627 fetch_metadata_sanity || return 1 628 629 rm -rf snap/ 630 631# Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will 632# probably take a while, so the progrees reports that fetch(1) generates 633# will be useful for keeping the users' attention from drifting. 634 echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:" 635 fetch -r http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1 636 637 echo -n "Extracting snapshot... " 638 tar -xz --numeric-owner -f ${SNAPSHOTHASH}.tgz snap/ || return 1 639 rm ${SNAPSHOTHASH}.tgz 640 echo "done." 641 642 echo -n "Verifying snapshot integrity... " 643# Verify the metadata files 644 cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1 645# Extract the index 646 rm -f INDEX.new 647 gunzip -c < snap/`look INDEX tINDEX.new | 648 cut -f 2 -d '|'`.gz > INDEX.new 649 fetch_index_sanity || return 1 650# Verify the snapshot contents 651 cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1 652 cut -f 2 -d '|' tINDEX.new INDEX.new | sort -u > files.expected 653 find snap -mindepth 1 | sed -E 's^snap/(.*)\.gz^\1^' | sort > files.snap 654 if ! cmp -s files.expected files.snap; then 655 echo "unexpected files in snapshot." 656 return 1 657 fi 658 rm files.expected files.snap 659 echo "done." 660 661# Move files into their proper locations 662 rm -f tag INDEX tINDEX 663 rm -rf files 664 mv tag.new tag 665 mv tINDEX.new tINDEX 666 mv INDEX.new INDEX 667 mv snap/ files/ 668 669 return 0 670} 671 672# Update a compressed snapshot 673fetch_update() { 674 rm -f patchlist diff OLD NEW filelist INDEX.new 675 676 OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag` 677 OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag` 678 679 while ! fetch_tag latest; do 680 fetch_pick_server || return 1 681 done 682 fetch_update_tagsanity || return 1 683 fetch_update_neededp || return 0 684 fetch_metadata || return 1 685 fetch_metadata_sanity || return 1 686 687 echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` " 688 echo "to `date -r ${SNAPSHOTDATE}`." 689 690# Generate a list of wanted metadata patches 691 join -t '|' -o 1.2,2.2 tINDEX tINDEX.new | 692 fetch_make_patchlist > patchlist 693 694# Attempt to fetch metadata patches 695 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 696 echo ${NDEBUG} "metadata patches.${DDSTATS}" 697 tr '|' '-' < patchlist | 698 lam -s "tp/" - -s ".gz" | 699 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 700 2>${STATSREDIR} | fetch_progress 701 echo "done." 702 703# Attempt to apply metadata patches 704 echo -n "Applying metadata patches... " 705 while read LINE; do 706 X=`echo ${LINE} | cut -f 1 -d '|'` 707 Y=`echo ${LINE} | cut -f 2 -d '|'` 708 if [ ! -f "${X}-${Y}.gz" ]; then continue; fi 709 gunzip -c < ${X}-${Y}.gz > diff 710 gunzip -c < files/${X}.gz > OLD 711 cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp 712 grep '^\+' diff | cut -c 2- | 713 sort -k 1,1 -t '|' -m - ptmp > NEW 714 if [ `${SHA256} -q NEW` = ${Y} ]; then 715 mv NEW files/${Y} 716 gzip -n files/${Y} 717 fi 718 rm -f diff OLD NEW ${X}-${Y}.gz ptmp 719 done < patchlist 2>${QUIETREDIR} 720 echo "done." 721 722# Update metadata without patches 723 join -t '|' -v 2 tINDEX tINDEX.new | 724 cut -f 2 -d '|' /dev/stdin patchlist | 725 while read Y; do 726 if [ ! -f "files/${Y}.gz" ]; then 727 echo ${Y}; 728 fi 729 done > filelist 730 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 731 echo ${NDEBUG} "metadata files... " 732 lam -s "f/" - -s ".gz" < filelist | 733 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 734 2>${QUIETREDIR} 735 736 while read Y; do 737 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 738 mv ${Y}.gz files/${Y}.gz 739 else 740 echo "metadata is corrupt." 741 return 1 742 fi 743 done < filelist 744 echo "done." 745 746# Extract the index 747 gunzip -c < files/`look INDEX tINDEX.new | 748 cut -f 2 -d '|'`.gz > INDEX.new 749 fetch_index_sanity || return 1 750 751# If we have decided to refuse certain updates, construct a hybrid index which 752# is equal to the old index for parts of the tree which we don't want to 753# update, and equal to the new index for parts of the tree which gets updates. 754# This means that we should always have a "complete snapshot" of the ports 755# tree -- with the caveat that it isn't actually a snapshot. 756 if [ ! -z "${REFUSE}" ]; then 757 echo "Refusing to download updates for ${REFUSE}" \ 758 >${QUIETREDIR} 759 760 grep -Ev "${REFUSE}" INDEX.new > INDEX.tmp 761 grep -E "${REFUSE}" INDEX | 762 sort -m -k 1,1 -t '|' - INDEX.tmp > INDEX.new 763 rm -f INDEX.tmp 764 fi 765 766# Generate a list of wanted ports patches 767 join -t '|' -o 1.2,2.2 INDEX INDEX.new | 768 fetch_make_patchlist > patchlist 769 770# Attempt to fetch ports patches 771 echo -n "Fetching `wc -l < patchlist | tr -d ' '` " 772 echo ${NDEBUG} "patches.${DDSTATS}" 773 tr '|' '-' < patchlist | lam -s "bp/" - | 774 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 775 2>${STATSREDIR} | fetch_progress 776 echo "done." 777 778# Attempt to apply ports patches 779 echo -n "Applying patches... " 780 while read LINE; do 781 X=`echo ${LINE} | cut -f 1 -d '|'` 782 Y=`echo ${LINE} | cut -f 2 -d '|'` 783 if [ ! -f "${X}-${Y}" ]; then continue; fi 784 gunzip -c < files/${X}.gz > OLD 785 ${BSPATCH} OLD NEW ${X}-${Y} 786 if [ `${SHA256} -q NEW` = ${Y} ]; then 787 mv NEW files/${Y} 788 gzip -n files/${Y} 789 fi 790 rm -f diff OLD NEW ${X}-${Y} 791 done < patchlist 2>${QUIETREDIR} 792 echo "done." 793 794# Update ports without patches 795 join -t '|' -v 2 INDEX INDEX.new | 796 cut -f 2 -d '|' /dev/stdin patchlist | 797 while read Y; do 798 if [ ! -f "files/${Y}.gz" ]; then 799 echo ${Y}; 800 fi 801 done > filelist 802 echo -n "Fetching `wc -l < filelist | tr -d ' '` " 803 echo ${NDEBUG} "new ports or files... " 804 lam -s "f/" - -s ".gz" < filelist | 805 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ 806 2>${QUIETREDIR} 807 808 while read Y; do 809 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then 810 mv ${Y}.gz files/${Y}.gz 811 else 812 echo "snapshot is corrupt." 813 return 1 814 fi 815 done < filelist 816 echo "done." 817 818# Remove files which are no longer needed 819 cut -f 2 -d '|' tINDEX INDEX | sort -u > oldfiles 820 cut -f 2 -d '|' tINDEX.new INDEX.new | sort -u | comm -13 - oldfiles | 821 lam -s "files/" - -s ".gz" | xargs rm -f 822 rm patchlist filelist oldfiles 823 824# We're done! 825 mv INDEX.new INDEX 826 mv tINDEX.new tINDEX 827 mv tag.new tag 828 829 return 0 830} 831 832# Do the actual work involved in "fetch" / "cron". 833fetch_run() { 834 fetch_pick_server_init && fetch_pick_server 835 836 while ! fetch_key; do 837 fetch_pick_server || return 1 838 done 839 840 if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then 841 fetch_snapshot || return 1 842 fi 843 fetch_update || return 1 844} 845 846# Build a ports INDEX file 847extract_make_index() { 848 if ! look $1 ${WORKDIR}/tINDEX > /dev/null; then 849 echo -n "$1 not provided by portsnap server; " 850 echo "$2 not being generated." 851 else 852 gunzip -c < "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX | 853 cut -f 2 -d '|'`.gz" | 854 cat - ${LOCALDESC} | 855 ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2 856 fi 857} 858 859# Create INDEX, INDEX-5, INDEX-6 860extract_indices() { 861 echo -n "Building new INDEX files... " 862 for PAIR in ${INDEXPAIRS}; do 863 INDEXFILE=`echo ${PAIR} | cut -f 1 -d '|'` 864 DESCRIBEFILE=`echo ${PAIR} | cut -f 2 -d '|'` 865 extract_make_index ${DESCRIBEFILE} ${INDEXFILE} || return 1 866 done 867 echo "done." 868} 869 870# Create .portsnap.INDEX; if we are REFUSEing to touch certain directories, 871# merge the values from any exiting .portsnap.INDEX file. 872extract_metadata() { 873 if [ -z "${REFUSE}" ]; then 874 sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX 875 elif [ -f ${PORTSDIR}/.portsnap.INDEX ]; then 876 grep -E "${REFUSE}" ${PORTSDIR}/.portsnap.INDEX \ 877 > ${PORTSDIR}/.portsnap.INDEX.tmp 878 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort | 879 sort -m - ${PORTSDIR}/.portsnap.INDEX.tmp \ 880 > ${PORTSDIR}/.portsnap.INDEX 881 rm -f ${PORTSDIR}/.portsnap.INDEX.tmp 882 else 883 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort \ 884 > ${PORTSDIR}/.portsnap.INDEX 885 fi 886} 887 888# Do the actual work involved in "extract" 889extract_run() { 890 mkdir -p ${PORTSDIR} || return 1 891 892 if ! 893 if ! [ -z "${EXTRACTPATH}" ]; then 894 grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX 895 elif ! [ -z "${REFUSE}" ]; then 896 grep -vE "${REFUSE}" ${WORKDIR}/INDEX 897 else 898 cat ${WORKDIR}/INDEX 899 fi | tr '|' ' ' | while read FILE HASH; do 900 echo ${PORTSDIR}/${FILE} 901 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 902 echo "files/${HASH}.gz not found -- snapshot corrupt." 903 return 1 904 fi 905 case ${FILE} in 906 */) 907 rm -rf ${PORTSDIR}/${FILE%/} 908 mkdir -p ${PORTSDIR}/${FILE} 909 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 910 -C ${PORTSDIR}/${FILE} 911 ;; 912 *) 913 rm -f ${PORTSDIR}/${FILE} 914 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 915 -C ${PORTSDIR} ${FILE} 916 ;; 917 esac 918 done; then 919 return 1 920 fi 921 if [ ! -z "${EXTRACTPATH}" ]; then 922 return 0; 923 fi 924 925 extract_metadata 926 extract_indices 927} 928 929# Do the actual work involved in "update" 930update_run() { 931 if ! [ -z "${INDEXONLY}" ]; then 932 extract_indices >/dev/null || return 1 933 return 0 934 fi 935 936 if sort ${WORKDIR}/INDEX | 937 cmp -s ${PORTSDIR}/.portsnap.INDEX -; then 938 echo "Ports tree is already up to date." 939 return 0 940 fi 941 942# If we are REFUSEing to touch certain directories, don't remove files 943# from those directories (even if they are out of date) 944 echo -n "Removing old files and directories... " 945 if ! [ -z "${REFUSE}" ]; then 946 sort ${WORKDIR}/INDEX | 947 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 948 grep -vE "${REFUSE}" | 949 lam -s "${PORTSDIR}/" - | 950 sed -e 's|/$||' | xargs rm -rf 951 else 952 sort ${WORKDIR}/INDEX | 953 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | 954 lam -s "${PORTSDIR}/" - | 955 sed -e 's|/$||' | xargs rm -rf 956 fi 957 echo "done." 958 959# Install new files 960 echo "Extracting new files:" 961 if ! 962 if ! [ -z "${REFUSE}" ]; then 963 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort 964 else 965 sort ${WORKDIR}/INDEX 966 fi | 967 comm -13 ${PORTSDIR}/.portsnap.INDEX - | 968 while read LINE; do 969 FILE=`echo ${LINE} | cut -f 1 -d '|'` 970 HASH=`echo ${LINE} | cut -f 2 -d '|'` 971 echo ${PORTSDIR}/${FILE} 972 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then 973 echo "files/${HASH}.gz not found -- snapshot corrupt." 974 return 1 975 fi 976 case ${FILE} in 977 */) 978 mkdir -p ${PORTSDIR}/${FILE} 979 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 980 -C ${PORTSDIR}/${FILE} 981 ;; 982 *) 983 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ 984 -C ${PORTSDIR} ${FILE} 985 ;; 986 esac 987 done; then 988 return 1 989 fi 990 991 extract_metadata 992 extract_indices 993} 994 995#### Main functions -- call parameter-handling and core functions 996 997# Using the command line, configuration file, and defaults, 998# set all the parameters which are needed later. 999get_params() { 1000 init_params 1001 parse_cmdline $@ 1002 sanity_conffile 1003 default_conffile 1004 parse_conffile 1005 default_params 1006} 1007 1008# Fetch command. Make sure that we're being called 1009# interactively, then run fetch_check_params and fetch_run 1010cmd_fetch() { 1011 if [ ! -t 0 ]; then 1012 echo -n "`basename $0` fetch should not " 1013 echo "be run non-interactively." 1014 echo "Run `basename $0` cron instead." 1015 exit 1 1016 fi 1017 fetch_check_params 1018 fetch_run || exit 1 1019} 1020 1021# Cron command. Make sure the parameters are sensible; wait 1022# rand(3600) seconds; then fetch updates. While fetching updates, 1023# send output to a temporary file; only print that file if the 1024# fetching failed. 1025cmd_cron() { 1026 fetch_check_params 1027 sleep `jot -r 1 0 3600` 1028 1029 TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1 1030 if ! fetch_run >> ${TMPFILE}; then 1031 cat ${TMPFILE} 1032 rm ${TMPFILE} 1033 exit 1 1034 fi 1035 1036 rm ${TMPFILE} 1037} 1038 1039# Extract command. Make sure the parameters are sensible, 1040# then extract the ports tree (or part thereof). 1041cmd_extract() { 1042 extract_check_params 1043 extract_run || exit 1 1044} 1045 1046# Update command. Make sure the parameters are sensible, 1047# then update the ports tree. 1048cmd_update() { 1049 update_check_params 1050 update_run || exit 1 1051} 1052 1053# Alfred command. Run 'fetch' or 'cron' depending on 1054# whether stdin is a terminal; then run 'update' or 1055# 'extract' depending on whether ${PORTSDIR} exists. 1056cmd_alfred() { 1057 if [ -t 0 ]; then 1058 cmd_fetch 1059 else 1060 cmd_cron 1061 fi 1062 if [ -d ${PORTSDIR} ]; then 1063 cmd_update 1064 else 1065 cmd_extract 1066 fi 1067} 1068 1069#### Entry point 1070 1071# Make sure we find utilities from the base system 1072export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH} 1073 1074# Set LC_ALL in order to avoid problems with character ranges like [A-Z]. 1075export LC_ALL=C 1076 1077get_params $@ 1078for COMMAND in ${COMMANDS}; do 1079 cmd_${COMMAND} 1080done 1081