1#!/bin/sh 2 3 4usage() 5{ 6 cat << EOF 7Usage: $0 <options> 8 9Options - in order of intended workflow: 10 11--help 12 13--list List the languages of the local set of catkeys. 14 15--download Download tarballs of the locally known languages. 16 (The ones from --list. Ignoring all other.) 17 18--download <lang> Download tarball of language <lang>. 19 20 E.g retries of 0k file. 21 Or a language still unknown locally. 22 23--unpack Unpack all tarballs in a big pile. 24 25--weed Remove catkey files with less than two lines in them. 26 27--copy Copy catkeys from the scripts temp folder to trunk/data/calogs. 28 29--test Test building catalogs from catkeys files. (jam -q catalogs") 30 31--stat Show pre-commit subversion statistics. 32 33--add Add new catkey files to subversion source control. (svn add) 34 35 36--diff Show diffs for all languages. Paged. Divided by language. 37--diff <lang> Show diff for language <lang>. Paged. 38 39--commit Commit all languages, as separate commits. 40 A standard message will be used for each. 41 ("Catalog update for language xx.") 42 You will be asked for svn password once per language. 43 44 CAVEAT! :: hta_committer currently fails to commit a langauge 45 if one of the catkey files resides in a newly added folder, 46 which is not part of the arguments. (Due to the split up of 47 languages, by filename 'xx.catkeys'-part, in separate commits.) 48 49(--commit <lang>) Not implemented. 50 51--delete Delete tarballs and tempfolder. 52 53--fix_fingerprints Force expected fingerprints on catkeys that fail to build. 54 55 Don't use this unless the fingerprints broke due to your 56 editing of catalog entries, and you need the fingerprints 57 to be updated. (It's a brute-force approach, including two 58 full "jam clean". Caveat emptor.) 59 60 61'********************************************' 62'* This script is probably not safe. *' 63'* *' 64'* Use at your own peril! *' 65'********************************************' 66 67EOF 68} 69 70 71 72subversionRootDirectory="$(pwd)" 73 74tempDirName=hta_committer_tempfolder 75 76tempDirectory="$subversionRootDirectory/$tempDirName" 77 78listOfLanguagesFoundInSubversion="none found" 79 80 81 82AssertCurrentDirIsSubversionTrunk() 83{ 84 for entry in data/catalogs src headers ; do 85 if ! [ -e "${entry}" ] ; then 86 echo "Need to be in SVN trunk." 87 exit 1 88 fi 89 done 90} 91 92 93CreateTempDir() 94{ 95 mkdir -p "$tempDirectory" 96} 97 98 99PopulateLanguageList() 100{ 101 listOfLanguagesFoundInSubversion="$( \ 102 find data/catalogs -name '*.catkeys' -exec basename '{}' ';' \ 103 | sort -u | awk '-F.' '{print$1}' | xargs echo)" 104} 105 106 107DownloadAllLanguageTarballs() 108{ 109 echo "Downloading languages:" $listOfLanguagesFoundInSubversion 110 for language in $listOfLanguagesFoundInSubversion ; do 111 DownloadLanguageTarball "$language" 112 done 113} 114 115 116DownloadLanguageTarball() 117{ 118 LANGUAGE=$1 119 # The package for zh_hans catalogs is actually called zh-hans 120 if [ "$LANGUAGE" = "zh_hans" ] ; then 121 LANGUAGE="zh-hans" 122 fi 123 124 cd "$tempDirectory" 125 echo "Downloading language: $LANGUAGE" 126 127 if [ -e "$LANGUAGE.tgz" ] ; then 128 rm "$LANGUAGE.tgz" 129 fi 130 131 attempt=0 132 local tarballURL="http://hta.polytect.org/catalogs/tarball" 133 134 while [ $(wget -nv -U hta_committer -O "$LANGUAGE.tgz" "$tarballURL/$LANGUAGE" ; echo $?) -ne 0 ]; do 135 if [ $attempt -eq 5 ]; then 136 break 137 fi 138 (( attempt++ )) 139 echo "Download attempt #$attempt failed. Retrying ..." 140 if [ -e "$LANGUAGE" ]; then 141 rm "$LANGUAGE" 142 fi 143 done 144 if [ $attempt -ge 5 ]; then 145 if [ -e "$LANGUAGE" ]; then 146 rm "$LANGUAGE" 147 fi 148 echo "WARNING: Max download retries exceeded for language $LANGUAGE." 149 fi 150 151 cd "$subversionRootDirectory" 152} 153 154 155PopulateTarballList() 156{ 157 cd "$tempDirectory" 158 TARBALL_LIST=$(ls | grep .tgz | sort) 159 echo "$TARBALL_LIST" 160 cd "$subversionRootDirectory" 161} 162 163 164UnpackAllTarballs() 165{ 166 cd "$tempDirectory" 167 echo "Unpacking tarballs:" $TARBALL_LIST 168 for tarball in $TARBALL_LIST ; do 169 UnpackTarball "$tarball" 170 done 171 cd "$subversionRootDirectory" 172} 173 174 175UnpackTarball() 176{ 177 cd "$tempDirectory" 178 echo "Unpacking tarball:" $1 179 tar -zxf "$1" 180# mv data "$(echo $1 | sed 's/.tgz//g')" #in separate folders 181 cd "$subversionRootDirectory" 182} 183 184 185DeleteEmptyCatkeyFiles() 186{ 187 numberOfEmptyFiles=0 188 189 unpackedCatkeyFiles=$(find "$tempDirectory" -name '*.catkeys' -exec echo '{}' ';') 190 for file in $unpackedCatkeyFiles ; do 191 LINES_IN_FILE=$(cat $file | wc | awk '{print$1}') 192 if [ $LINES_IN_FILE -lt 2 ] ; then 193 #echo Deleting empty file "$file" 194 numberOfEmptyFiles=$(($numberOfEmptyFiles + 1)) 195 rm $file 196 fi 197 done 198 echo Deleted "$numberOfEmptyFiles" empty files 199} 200 201 202CopyCatkeyFilesInPlace() 203{ 204 cp -r "${tempDirectory}/data/catalogs/." "${subversionRootDirectory}/data/catalogs/" 205} 206 207 208TestBuildingCatalogsFromCatkeyFiles() 209{ 210 cd "$subversionRootDirectory" 211 jam catalogs 212 jam catalogs 213} 214 215 216ShowSubversionStatistics() 217{ 218 cd "$subversionRootDirectory" 219 220 echo 221 echo Languages: "$listOfLanguagesToCommit" 222 echo 223 echo Total: "$(svn stat data/catalogs | grep catkeys | wc | awk '{print$1}')" 224 echo Modified: "$(svn stat data/catalogs | grep catkeys | grep ^M | wc | awk '{print$1}')" 225 echo Added: "$(svn stat data/catalogs | grep catkeys | grep ^A | wc | awk '{print$1}')" 226 echo 227 echo Other: 228 echo "$(svn stat data/catalogs | grep ^?)" 229} 230 231 232AddUnversionedToSubversion() 233{ 234 cd "$subversionRootDirectory" 235 svn stat data/catalogs | grep ^? | awk '{print$2}' | xargs svn add 236} 237 238 239PopulateCommitLanguageList() 240{ 241 listOfLanguagesToCommit="$(svn stat data/catalogs | grep catkeys | awk '{print$2}' \ 242 | awk -F / '{print$NF}' | awk '-F.' '{print$1}' | sort -u | xargs echo)" 243} 244 245 246DiffAllLanguages() 247{ 248 echo Diffing languages: "$listOfLanguagesToCommit" 249 250 for language in $listOfLanguagesToCommit ; do 251 DiffLanguage "$language" 252 done 253} 254 255 256DiffLanguage() 257{ 258 echo Diffing language "$1" 259 svn stat data/catalogs/ | grep [^A\|^M] | grep "$1.catkeys" \ 260 | awk '{print$2}' | xargs svn diff | less 261} 262 263 264CommitAllLanguages() 265{ 266 echo Committing languages: "$listOfLanguagesToCommit" 267 268 for language in $listOfLanguagesToCommit ; do 269 CommitLanguage "$language" 270 done 271} 272 273 274CommitLanguage() 275{ 276 echo Committing language "$1" 277 svn stat data/catalogs/ | grep [^A\|^M] | grep "$1.catkeys" \ 278 | awk '{print$2}' | xargs svn commit -m "Catalog update for language $1." 279 280 echo 281} 282 283 284DeleteTarballs() 285{ 286 find "$subversionRootDirectory/$tempDirName" -name '*.tgz' -exec rm '{}' ';' 287} 288 289 290DeleteTempDirectory() 291{ 292 cd "$subversionRootDirectory" 293 rm -rf "$tempDirectory" 294} 295 296 297RemoveTempDirectoryIfEmpty() 298{ 299 rmdir --ignore-fail-on-non-empty hta_committer_tempfolder/ 300} 301 302 303FixFingerprintsOfFailingCatalogs() 304{ 305 # Find the catalogs that won't build. 306 bad_catalogs=$( \ 307 echo $( \ 308 jam catalogs 2>&1 \ 309 | awk '-Fsource-catalog ' '{print$2}' \ 310 | awk '-F - error' '{print$1}' \ 311 | sort -u)); 312 313 # Find the present and the expected fingerprints. 314 # Produce a script to replace present fingerprints with the expected ones. 315 jam catalogs 2>&1 \ 316 | grep 'instead of' \ 317 | awk '-Fafter load ' '{print$2}' \ 318 | awk '-F. The catalog data' '{print$1}' \ 319 | sed 's/ instead of /\t/g' | sed 's/[()]//g' \ 320 | awk '{print "sed -i s/" $2 "/" $1 "/g"}' \ 321 | awk "{ print \$0\" $bad_catalogs\" }" \ 322 > hta_fingerprint_correction_script.sh; 323 324 chmod u+x hta_fingerprint_correction_script.sh; 325 $(hta_fingerprint_correction_script.sh); 326 rm hta_fingerprint_correction_script.sh 327} 328 329 330if [ $# -eq 1 ] ; then 331 case "$1" in 332 --help) 333 usage 334 exit 335 ;; 336 --list) 337 AssertCurrentDirIsSubversionTrunk 338 PopulateLanguageList 339 echo $listOfLanguagesFoundInSubversion 340 exit 341 ;; 342 --download) 343 AssertCurrentDirIsSubversionTrunk 344 CreateTempDir 345 PopulateLanguageList 346 DownloadAllLanguageTarballs 347 exit 348 ;; 349 --unpack) 350 AssertCurrentDirIsSubversionTrunk 351 CreateTempDir 352 PopulateTarballList 353 UnpackAllTarballs 354 exit 355 ;; 356 --weed) 357 AssertCurrentDirIsSubversionTrunk 358 CreateTempDir 359 DeleteEmptyCatkeyFiles 360 exit 361 ;; 362 --delete) 363 AssertCurrentDirIsSubversionTrunk 364 CreateTempDir 365 DeleteTarballs 366 DeleteTempDirectory 367 exit 368 ;; 369 --copy) 370 AssertCurrentDirIsSubversionTrunk 371 CopyCatkeyFilesInPlace 372 exit 373 ;; 374 --test) 375 AssertCurrentDirIsSubversionTrunk 376 TestBuildingCatalogsFromCatkeyFiles 377 exit 378 ;; 379 --stat) 380 AssertCurrentDirIsSubversionTrunk 381 PopulateCommitLanguageList 382 ShowSubversionStatistics 383 exit 384 ;; 385 --add) 386 AssertCurrentDirIsSubversionTrunk 387 AddUnversionedToSubversion 388 exit 389 ;; 390 --diff) 391 AssertCurrentDirIsSubversionTrunk 392 PopulateCommitLanguageList 393 DiffAllLanguages 394 exit 395 ;; 396 --commit) 397 AssertCurrentDirIsSubversionTrunk 398 PopulateCommitLanguageList 399 CommitAllLanguages 400 exit 401 ;; 402 --fix_fingerprints) 403 AssertCurrentDirIsSubversionTrunk 404 FixFingerprintsOfFailingCatalogs 405 exit 406 ;; 407 *) 408 usage 409 exit 1 410 ;; 411 esac 412fi 413 414 415if [ $# -eq 2 ] ; then 416 case "$1" in 417 --help) 418 usage 419 exit 420 ;; 421 --download) 422 AssertCurrentDirIsSubversionTrunk 423 CreateTempDir 424 DownloadLanguageTarball "$2" 425 exit; 426 ;; 427 --diff) 428 AssertCurrentDirIsSubversionTrunk 429 DiffLanguage "$2" 430 exit 431 ;; 432 *) 433 usage 434 exit 1 435 ;; 436 esac 437fi 438 439 440usage 441exit 1 442 443