index.subr revision 259054
1if [ ! "$_PACKAGES_INDEX_SUBR" ]; then _PACKAGES_INDEX_SUBR=1 2# 3# Copyright (c) 2013 Devin Teske 4# All rights reserved. 5# 6# Redistribution and use in source and binary forms, with or without 7# modification, are permitted provided that the following conditions 8# are met: 9# 1. Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# 2. Redistributions in binary form must reproduce the above copyright 12# notice, this list of conditions and the following disclaimer in the 13# documentation and/or other materials provided with the distribution. 14# 15# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25# SUCH DAMAGE. 26# 27# $FreeBSD: head/usr.sbin/bsdconfig/share/packages/index.subr 259054 2013-12-07 00:31:01Z dteske $ 28# 29############################################################ INCLUDES 30 31BSDCFG_SHARE="/usr/share/bsdconfig" 32. $BSDCFG_SHARE/common.subr || exit 1 33f_dprintf "%s: loading includes..." packages/index.subr 34f_include $BSDCFG_SHARE/device.subr 35f_include $BSDCFG_SHARE/media/common.subr 36f_include $BSDCFG_SHARE/strings.subr 37 38BSDCFG_LIBE="/usr/libexec/bsdconfig" 39f_include_lang $BSDCFG_LIBE/include/messages.subr 40 41############################################################ GLOBALS 42 43PACKAGE_INDEX= 44_INDEX_INITTED= 45 46# 47# Default path to pkg(8) repo-packagesite.sqlite database 48# 49SQLITE_REPO="/var/db/pkg/repo-packagesite.sqlite" 50 51# 52# Default path to on-disk cache INDEX file 53# 54PACKAGES_INDEX_CACHEFILE="/var/run/bsdconfig/packages_INDEX.cache" 55 56# 57# INDEX format for FreeBSD-6.0 or higher: 58# 59# package|port-origin|install-prefix|comment|port-desc-file|maintainer| 60# categories|build-deps|run-deps|www-site|reserve|reserve|reserve|disc 61# 62INDEX_FORMAT="%n-%v" # package 63INDEX_FORMAT="$INDEX_FORMAT|/usr/ports/%o" # port-origin 64INDEX_FORMAT="$INDEX_FORMAT|%p" # install-prefix 65INDEX_FORMAT="$INDEX_FORMAT|%c" # comment 66INDEX_FORMAT="$INDEX_FORMAT|/usr/ports/%o/pkg-descr" # port-desc-file 67INDEX_FORMAT="$INDEX_FORMAT|%m" # maintainer 68INDEX_FORMAT="$INDEX_FORMAT|@CATEGORIES@" # place-holder 69INDEX_FORMAT="$INDEX_FORMAT|" # build-deps 70INDEX_FORMAT="$INDEX_FORMAT|@RUNDEPS@" # place-holder 71INDEX_FORMAT="$INDEX_FORMAT|%w" # www-site 72INDEX_FORMAT="$INDEX_FORMAT|" # reserved 73INDEX_FORMAT="$INDEX_FORMAT|" # reserved 74INDEX_FORMAT="$INDEX_FORMAT|" # reserved 75INDEX_FORMAT="$INDEX_FORMAT|" # disc 76 77############################################################ FUNCTIONS 78 79# f_index_initialize [$var_to_set] 80# 81# Read and initialize the global index. Returns success unless media cannot be 82# initialized for any reason (e.g. user cancels media selection dialog or an 83# error occurs). The index is sorted before being loaded into $var_to_set. 84# 85# NOTE: The index is processed with f_index_read() [below] after being loaded. 86# 87f_index_initialize() 88{ 89 local __funcname=f_index_initialize 90 local __var_to_set="${2:-PACKAGE_INDEX}" 91 92 [ "$_INDEX_INITTED" ] && return $SUCCESS 93 94 # Got any media? 95 f_media_verify || return $FAILURE 96 97 # Does it move when you kick it? 98 f_device_init media || return $FAILURE 99 100 f_show_info "$msg_attempting_to_update_repository_catalogue" 101 102 # 103 # Generate $PACKAGESITE variable for pkg(8) based on media type 104 # 105 local __type __data __site 106 device_media get type __type 107 device_media get private __data 108 case "$__type" in 109 $DEVICE_TYPE_DIRECTORY) 110 __site="file://$__data/packages/$PKG_ABI" ;; 111 $DEVICE_TYPE_FLOPPY) 112 __site="file://${__data:-$MOUNTPOINT}/packages/$PKG_ABI" ;; 113 $DEVICE_TYPE_FTP) 114 f_getvar $VAR_FTP_PATH __site 115 __site="$__site/packages/$PKG_ABI" ;; 116 $DEVICE_TYPE_HTTP) 117 f_getvar $VAR_HTTP_PATH __site 118 __site="$__site/$PKG_ABI/latest" ;; 119 $DEVICE_TYPE_HTTP_PROXY) 120 f_getvar $VAR_HTTP_PROXY_PATH __site 121 __site="$__site/packages/$PKG_ABI" ;; 122 *) # UFS, DISK, CDROM, USB, DOS, NFS, etc. 123 __site="file://$MOUNTPOINT/packages/$PKG_ABI" 124 esac 125 126 export PACKAGESITE="$__site" 127 f_dprintf "PACKAGESITE=[%s]" "$PACKAGESITE" 128 if ! f_eval_catch $__funcname pkg "pkg update"; then 129 f_show_err "$msg_unable_to_update_pkg_from_selected_media" 130 f_device_shutdown media 131 return $FAILURE 132 fi 133 134 # 135 # Try to get contents from validated on-disk cache 136 # 137 138 # 139 # Calculate digest used to determine if the on-disk persistant cache 140 # INDEX (containing this digest on the first line) is valid and can be 141 # used to quickly populate the environment. 142 # 143 local __sqlite_digest 144 if ! __sqlite_digest=$( md5 < "$SQLITE_REPO" 2> /dev/null ); then 145 f_show_err "$msg_no_pkg_database_found" 146 f_device_shutdown media 147 return $FAILURE 148 fi 149 150 # 151 # Check to see if the persistant cache INDEX file exists 152 # 153 if [ -f "$PACKAGES_INDEX_CACHEFILE" ]; then 154 # 155 # Attempt to populate the environment with the (soon to be) 156 # validated on-disk cache. If validation fails, fall-back to 157 # generating a fresh cache. 158 # 159 if eval $__var_to_set='$( 160 ( # Get digest as the first word on first line 161 read digest rest_ignored 162 163 # 164 # If the stored digest matches the calculated- 165 # one populate the environment from the on-disk 166 # cache and provide success exit status. 167 # 168 if [ "$digest" = "$__sqlite_digest" ]; then 169 cat 170 exit $SUCCESS 171 else 172 # Otherwise, return the current value 173 eval echo \"\$__var_to_set\" 174 exit $FAILURE 175 fi 176 ) < "$PACKAGES_INDEX_CACHEFILE" 2> /dev/null 177 )'; then 178 f_show_info \ 179 "$msg_located_index_now_reading_package_data_from_it" 180 if ! f_index_read "$__var_to_set"; then 181 f_show_err \ 182 "$msg_io_or_format_error_on_index_file" 183 return $FAILURE 184 fi 185 _INDEX_INITTED=1 186 return $SUCCESS 187 fi 188 # Otherwise, fall-thru to create a fresh cache from scratch 189 fi 190 191 # 192 # If we reach this point, we need to generate the data from scratch 193 # 194 195 f_show_info "$msg_getting_package_categories_via_pkg_rquery" 196 if ! eval "$( pkg rquery "%n-%v %C" | awk ' 197 { categories[$1] = categories[$1] " " $2 } 198 END { 199 for (package in categories) 200 { 201 cats = categories[package] 202 sub(/^ /, "", cats) 203 gsub(/[^[:alnum:]_]/, "_", package) 204 printf "local _%s_categories=\"%s\";\n", package, cats 205 } 206 }' )"; then 207 f_show_err "$msg_unable_to_pkg_rquery_package_dependencies" 208 f_device_shutdown media 209 return $FAILURE 210 fi 211 212 f_show_info "$msg_getting_package_dependencies_via_pkg_rquery" 213 if ! eval "$( pkg rquery "%n-%v %dn-%dv" | awk ' 214 { rundeps[$1] = rundeps[$1] " " $2 } 215 END { 216 for (package in rundeps) 217 { 218 deps = rundeps[package] 219 sub(/^ /, "", deps) 220 gsub(/[^[:alnum:]_]/, "_", package) 221 printf "local _%s_rundeps=\"%s\";\n", package, deps 222 } 223 }' )"; then 224 f_show_err "$msg_unable_to_pkg_rquery_package_dependencies" 225 f_device_shutdown media 226 return $FAILURE 227 fi 228 229 f_show_info "$msg_generating_index_from_pkg_database" 230 eval "$__var_to_set"='$( pkg rquery "$INDEX_FORMAT" | 231 while read LINE; do 232 package="${LINE%%|*}"; 233 f_str2varname "$package" varpkg; 234 eval f_replaceall \"\$LINE\" \"\|@CATEGORIES@\|\" \ 235 \"\|\$_${varpkg}_categories\|\" LINE 236 eval f_replaceall \"\$LINE\" \"\|@RUNDEPS@\|\" \ 237 \"\|\$_${varpkg}_rundeps\|\" LINE 238 echo "$LINE" 239 done 240 )' # always returns true (status of last item in pipe-chain) 241 eval "$__var_to_set"='$( debug= f_getvar "$__var_to_set" | sort )' 242 243 # 244 # Attempt to create the persistant on-disk cache 245 # 246 247 # Create a new temporary file to write to 248 local __tmpfile 249 if f_eval_catch -dk __tmpfile $__funcname mktemp \ 250 'mktemp -t "%s"' "$pgm" 251 then 252 # Write the temporary file contents 253 echo "$__sqlite_digest" > "$__tmpfile" 254 debug= f_getvar "$__var_to_set" >> "$__tmpfile" 255 256 # Finally, move the temporary file into place 257 case "$PACKAGES_INDEX_CACHEFILE" in 258 */*) f_eval_catch -d $funcname mkdir \ 259 'mkdir -p "%s"' "${PACKAGES_INDEX_CACHEFILE%/*}" 260 esac 261 f_eval_catch -d $__funcname mv 'mv -f "%s" "%s"' \ 262 "$__tmpfile" "$PACKAGES_INDEX_CACHEFILE" 263 fi 264 265 f_show_info "$msg_located_index_now_reading_package_data_from_it" 266 if ! f_index_read "$__var_to_set"; then 267 f_show_err "$msg_io_or_format_error_on_index_file" 268 return $FAILURE 269 fi 270 271 _INDEX_INITTED=1 272 return $SUCCESS 273} 274 275# f_index_read [$var_to_get] 276# 277# Process the INDEX file (contents contained in $var_to_get) and... 278# 279# 1. create a list ($CATEGORY_MENU_LIST) of categories with package counts 280# 2. For convenience, create $_npkgs holding the total number of all packages 281# 3. extract associative categories for each package into $_categories_$varpkg 282# 4. extract runtime dependencies for each package into $_rundeps_$varpkg 283# 5. extract a [sorted] list of categories into $PACKAGE_CATEGORIES 284# 6. create $_npkgs_$varcat holding the total number of packages in category 285# 286# NOTE: $varpkg is the product of f_str2varname $package varpkg 287# NOTE: $package is the name as it appears in the INDEX (no archive suffix) 288# NOTE: We only show categories for which there are at least one package. 289# NOTE: $varcat is the product of f_str2varname $category varcat 290# 291f_index_read() 292{ 293 local var_to_get="${1:-PACKAGE_INDEX}" 294 295 # Export variables required by awk(1) below 296 export msg_no_description_provided 297 export msg_all msg_all_desc 298 export VALID_VARNAME_CHARS 299 export msg_packages 300 301 eval "$( debug= f_getvar "$var_to_get" | awk -F'|' ' 302 function asorti(src, dest) 303 { 304 # Copy src indices to dest and calculate array length 305 nitems = 0; for (i in src) dest[++nitems] = i 306 307 # Sort the array of indices (dest) using insertion sort method 308 for (i = 1; i <= nitems; k = i++) 309 { 310 idx = dest[i] 311 while ((k > 0) && (dest[k] > idx)) 312 { 313 dest[k+1] = dest[k] 314 k-- 315 } 316 dest[k+1] = idx 317 } 318 319 return nitems 320 } 321 function print_category(category, npkgs, desc) 322 { 323 cat = category 324 # Accent the category if the first page has been 325 # cached (also acting as a visitation indicator) 326 if ( ENVIRON["_index_page_" varcat "_1"] ) 327 cat = cat "*" 328 printf "'\''%s'\'' '\''%s " packages "'\'' '\''%s'\''\n", 329 cat, npkgs, desc 330 } 331 BEGIN { 332 valid_chars = ENVIRON["VALID_VARNAME_CHARS"] 333 default_desc = ENVIRON["msg_no_description_provided"] 334 packages = ENVIRON["msg_packages"] 335 tpkgs = 0 336 prefix = "" 337 } 338 { 339 tpkgs++ 340 varpkg = $1 341 gsub("[^" valid_chars "]", "_", varpkg) 342 print "_categories_" varpkg "=\"" $7 "\"" 343 split($7, pkg_categories, /[[:space:]]+/) 344 for (pkg_category in pkg_categories) 345 categories[pkg_categories[pkg_category]]++ 346 print "_rundeps_" varpkg "=\"" $9 "\"" 347 } 348 END { 349 print "_npkgs=" tpkgs # For convenience, total package count 350 351 n = asorti(categories, categories_sorted) 352 353 # Produce package counts for each category 354 for (i = 1; i <= n; i++) 355 { 356 cat = varcat = categories_sorted[i] 357 npkgs = categories[cat] 358 gsub("[^" valid_chars "]", "_", varcat) 359 print "_npkgs_" varcat "=\"" npkgs "\"" 360 } 361 362 # Create menu list and generate list of categories at same time 363 print "CATEGORY_MENU_LIST=\"" 364 print_category(ENVIRON["msg_all"], tpkgs, 365 ENVIRON["msg_all_desc"]) 366 category_list = "" 367 for (i = 1; i <= n; i++) 368 { 369 cat = varcat = categories_sorted[i] 370 npkgs = categories[cat] 371 cur_prefix = tolower(substr(cat, 1, 1)) 372 if ( prefix != cur_prefix ) 373 prefix = cur_prefix 374 else 375 cat = " " cat 376 gsub("[^" valid_chars "]", "_", varcat) 377 desc = ENVIRON["_category_" varcat] 378 if ( ! desc ) desc = default_desc 379 print_category(cat, npkgs, desc) 380 category_list = category_list " " cat 381 } 382 print "\"" 383 384 # Produce the list of categories (calculated in above block) 385 sub(/^ /, "", category_list) 386 print "PACKAGE_CATEGORIES=\"" category_list "\"" 387 388 }' )" # End-Quote 389} 390 391# f_index_extract_pages $var_to_get $var_basename $pagesize [$category] 392# 393# Extracts the package INDEX ($PACKAGE_INDEX by default if/when $var_to_get is 394# NULL; but should not be missing) into a series of sequential variables 395# corresponding to "pages" containing up to $pagesize packages. The package 396# INDEX data must be contained in the variable $var_to_get. The extracted pages 397# are stored in variables ${var_basename}_# -- where "#" is a the page number. 398# If $category is set, only packages for that category are extracted. 399# Otherwise, if $category is "All", missing, or NULL, all packages are 400# extracted and no filtering is done. 401# 402f_index_extract_pages() 403{ 404 local var_to_get="${1:-PACKAGE_INDEX}" var_basename="$2" pagesize="$3" 405 local category="$4" # Optional 406 407 eval "$( 408 debug= f_getvar "$var_to_get" | awk -F'|' \ 409 -v cat="$category" \ 410 -v pagesize="$pagesize" \ 411 -v var_basename="$var_basename" \ 412 -v i18n_all="$msg_all" ' 413 BEGIN { n = page = 0 } 414 /'\''/{ gsub(/'\''/, "'\''\\'\'\''") } 415 { 416 if ( cat !~ "(^$|^" i18n_all "$)" && $7 !~ \ 417 "(^|[[:space:]])" cat "([[:space:]]|$)" ) next 418 starting_new_page = (n++ == (pagesize * page)) 419 if ( starting_new_page ) 420 printf "%s%s", ( n > 1 ? "'\''\n" : "" ), 421 var_basename "_" ++page "='\''" 422 printf "%s%s", ( starting_new_page ? "" : "\n" ), $0 423 } 424 END { if ( n > 0 ) print "'\''" }' 425 )" 426} 427 428# f_index_search $var_to_get $name [$var_to_set] 429# 430# Search the package INDEX ($PACKAGE_INDEX by default if/when $var_to_get is 431# NULL; but should not be missing) for $name, returning the first match. 432# Matches are strict (not regular expressions) and must match the beginning 433# portion of the package name to be considered a match. If $var_to_set is 434# missing or NULL, output is sent to standard output. If a match is found, 435# returns success; otherwise failure. 436# 437f_index_search() 438{ 439 local __var_to_get="${1:-PACKAGE_INDEX}" __pkg_basename="$2" 440 local __var_to_set="$3" 441 442 f_dprintf "f_index_search: Searching package data (in %s) for %s" \ 443 "$__var_to_get" "$__pkg_basename" 444 445 local __pkg= 446 __pkg=$( debug= f_getvar "$__var_to_get" | 447 awk -F'|' -v basename="$__pkg_basename" ' 448 BEGIN { n = length(basename) } 449 substr($1, 0, n) == basename { print $1; exit } 450 ' ) 451 if [ ! "$__pkg" ]; then 452 f_dprintf "f_index_search: No packages matching %s found" \ 453 "$__pkg_basename" 454 return $FAILURE 455 fi 456 457 f_dprintf "f_index_search: Found package %s" "$__pkg" 458 if [ "$__var_to_set" ]; then 459 setvar "$__var_to_set" "$__pkg" 460 else 461 echo "$__pkg" 462 fi 463 return $SUCCESS 464} 465 466############################################################ MAIN 467 468f_dprintf "%s: Successfully loaded." packages/index.subr 469 470fi # ! $_PACKAGES_INDEX_SUBR 471