hgforest.sh revision 966:76533a3f52ce
1139826Simp#!/bin/sh 256723Sshin 356723Sshin# 456723Sshin# Copyright (c) 2009, 2014, Oracle and/or its affiliates. All rights reserved. 556723Sshin# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 656723Sshin# 756723Sshin# This code is free software; you can redistribute it and/or modify it 856723Sshin# under the terms of the GNU General Public License version 2 only, as 956723Sshin# published by the Free Software Foundation. 1056723Sshin# 1156723Sshin# This code is distributed in the hope that it will be useful, but WITHOUT 1256723Sshin# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1356723Sshin# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 1456723Sshin# version 2 for more details (a copy is included in the LICENSE file that 1556723Sshin# accompanied this code). 1656723Sshin# 1756723Sshin# You should have received a copy of the GNU General Public License version 1856723Sshin# 2 along with this work; if not, write to the Free Software Foundation, 1956723Sshin# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 2056723Sshin# 2156723Sshin# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 2256723Sshin# or visit www.oracle.com if you need additional information or have any 2356723Sshin# questions. 2456723Sshin# 2556723Sshin 2656723Sshin# Shell script for a fast parallel forest command 2756723Sshincommand="$1" 28174510Sobrienpull_extra_base="$2" 29174510Sobrien 30174510Sobrienif [ "" = "$command" ] ; then 3156723Sshin echo No command to hg supplied! 3256723Sshin exit 1 3356723Sshinfi 3456723Sshin 3556723Sshin# Clean out the temporary directory that stores the pid files. 3656723Sshintmp=/tmp/forest.$$ 3756723Sshinrm -f -r ${tmp} 3856723Sshinmkdir -p ${tmp} 3956723Sshin 4056723Sshinsafe_interrupt () { 4156723Sshin if [ -d ${tmp} ]; then 4256723Sshin if [ "`ls ${tmp}/*.pid`" != "" ]; then 4356723Sshin echo "Waiting for processes ( `cat ${tmp}/*.pid | tr '\n' ' '`) to terminate nicely!" 4456723Sshin sleep 1 4556723Sshin # Pipe stderr to dev/null to silence kill, that complains when trying to kill 4656723Sshin # a subprocess that has already exited. 4756723Sshin kill -TERM `cat ${tmp}/*.pid | tr '\n' ' '` 2> /dev/null 4862587Sitojun wait 4956723Sshin echo Interrupt complete! 5056723Sshin fi 5156723Sshin fi 5256723Sshin rm -f -r ${tmp} 5362587Sitojun exit 1 5462587Sitojun} 5556723Sshin 5656723Sshinnice_exit () { 5756723Sshin if [ -d ${tmp} ]; then 5856723Sshin if [ "`ls ${tmp}`" != "" ]; then 5956723Sshin wait 6056723Sshin fi 6156723Sshin fi 6262587Sitojun rm -f -r ${tmp} 6356723Sshin} 6462587Sitojun 6556723Sshintrap 'safe_interrupt' INT QUIT 6656723Sshintrap 'nice_exit' EXIT 6756723Sshin 6856723Sshin# Only look in specific locations for possible forests (avoids long searches) 6956723Sshinpull_default="" 7056723Sshinrepos="" 7156723Sshinrepos_extra="" 7256723Sshinif [ "${command}" = "clone" -o "${command}" = "fclone" ] ; then 7356723Sshin subrepos="corba jaxp jaxws langtools jdk hotspot nashorn" 7456723Sshin if [ -f .hg/hgrc ] ; then 7556723Sshin pull_default=`hg paths default` 7656723Sshin if [ "${pull_default}" = "" ] ; then 7756723Sshin echo "ERROR: Need initial clone with 'hg paths default' defined" 7856723Sshin exit 1 7956723Sshin fi 8056723Sshin fi 8156723Sshin if [ "${pull_default}" = "" ] ; then 8278064Sume echo "ERROR: Need initial repository to use this script" 8356723Sshin exit 1 8456723Sshin fi 8556723Sshin for i in ${subrepos} ; do 8656723Sshin if [ ! -f ${i}/.hg/hgrc ] ; then 8756723Sshin repos="${repos} ${i}" 8856723Sshin fi 8956723Sshin done 9078064Sume if [ "${pull_extra_base}" != "" ] ; then 9156723Sshin subrepos_extra="closed jdk/src/closed jdk/make/closed jdk/test/closed hotspot/make/closed hotspot/src/closed hotspot/test/closed deploy install sponsors pubs" 9256723Sshin pull_default_tail=`echo ${pull_default} | sed -e 's@^.*://[^/]*/\(.*\)@\1@'` 9356723Sshin pull_extra="${pull_extra_base}/${pull_default_tail}" 9456723Sshin for i in ${subrepos_extra} ; do 9556723Sshin if [ ! -f ${i}/.hg/hgrc ] ; then 9656723Sshin repos_extra="${repos_extra} ${i}" 9756723Sshin fi 9856723Sshin done 9956723Sshin fi 10056723Sshin at_a_time=2 10156723Sshin # Any repos to deal with? 10256723Sshin if [ "${repos}" = "" -a "${repos_extra}" = "" ] ; then 103171260Sdelphij exit 104171260Sdelphij fi 10556723Sshinelse 10656723Sshin hgdirs=`ls -d ./.hg ./*/.hg ./*/*/.hg ./*/*/*/.hg ./*/*/*/*/.hg 2>/dev/null` 10756723Sshin # Derive repository names from the .hg directory locations 10856723Sshin for i in ${hgdirs} ; do 10956723Sshin repos="${repos} `echo ${i} | sed -e 's@/.hg$@@'`" 11056723Sshin done 11156723Sshin for i in ${repos} ; do 11256723Sshin if [ -h ${i}/.hg/store/lock -o -f ${i}/.hg/store/lock ] ; then 11356723Sshin locked="${i} ${locked}" 11456723Sshin fi 11556723Sshin done 11656723Sshin at_a_time=8 11756723Sshin # Any repos to deal with? 11856723Sshin if [ "${repos}" = "" ] ; then 11956723Sshin echo "No repositories to process." 12056723Sshin exit 12156723Sshin fi 12256723Sshin if [ "${locked}" != "" ] ; then 12356723Sshin echo "These repositories are locked: ${locked}" 124253081Sae exit 125253081Sae fi 126253081Saefi 127253081Sae 128253081Sae# Echo out what repositories we do a command on. 129253081Saeecho "# Repositories: ${repos} ${repos_extra}" 130253081Saeecho 131253081Sae 132253081Sae# Run the supplied command on all repos in parallel. 133253081Saen=0 134253081Saefor i in ${repos} ${repos_extra} ; do 135253081Sae n=`expr ${n} '+' 1` 136253081Sae repopidfile=`echo ${i} | sed -e 's@./@@' -e 's@/@_@g'` 13756723Sshin reponame=`echo ${i} | sed -e :a -e 's/^.\{1,20\}$/ &/;ta'` 13856723Sshin pull_base="${pull_default}" 13962587Sitojun for j in $repos_extra ; do 14056723Sshin if [ "$i" = "$j" ] ; then 14156723Sshin pull_base="${pull_extra}" 14256723Sshin fi 14362587Sitojun done 14456723Sshin ( 14562587Sitojun ( 14656723Sshin if [ "${command}" = "clone" -o "${command}" = "fclone" ] ; then 14756723Sshin pull_newrepo="`echo ${pull_base}/${i} | sed -e 's@\([^:]/\)//*@\1@g'`" 14856723Sshin echo hg clone ${pull_newrepo} ${i} 14956723Sshin path="`dirname ${i}`" 15056723Sshin if [ "${path}" != "." ] ; then 15156723Sshin times=0 15256723Sshin while [ ! -d "${path}" ] ## nested repo, ensure containing dir exists 15362587Sitojun do 15456723Sshin times=`expr ${times} '+' 1` 15556723Sshin if [ `expr ${times} '%' 10` -eq 0 ] ; then 15662587Sitojun echo ${path} still not created, waiting... 15762587Sitojun fi 15862587Sitojun sleep 5 15962587Sitojun done 16062587Sitojun fi 161148169Sume (PYTHONUNBUFFERED=true hg clone ${pull_newrepo} ${i}; echo "$?" > ${tmp}/${repopidfile}.pid.rc )& 16262587Sitojun else 16362587Sitojun echo "cd ${i} && hg $*" 16462587Sitojun cd ${i} && (PYTHONUNBUFFERED=true hg "$@"; echo "$?" > ${tmp}/${repopidfile}.pid.rc )& 16562587Sitojun fi 16662587Sitojun echo $! > ${tmp}/${repopidfile}.pid 16762587Sitojun ) 2>&1 | sed -e "s@^@${reponame}: @") & 16862587Sitojun 16962587Sitojun if [ `expr ${n} '%' ${at_a_time}` -eq 0 ] ; then 17062587Sitojun sleep 2 17162587Sitojun echo Waiting 5 secs before spawning next background command. 17262587Sitojun sleep 3 17362587Sitojun fi 17462587Sitojundone 17556723Sshin# Wait for all hg commands to complete 17656723Sshinwait 17756723Sshin 17856723Sshin# Terminate with exit 0 only if all subprocesses were successful 17956723Sshinec=0 18056723Sshinif [ -d ${tmp} ]; then 18156723Sshin for rc in ${tmp}/*.pid.rc ; do 18256723Sshin exit_code=`cat ${rc} | tr -d ' \n\r'` 18356723Sshin if [ "${exit_code}" != "0" ] ; then 18456723Sshin echo "WARNING: ${rc} exited abnormally." 18556723Sshin ec=1 18656723Sshin fi 18756723Sshin done 18856723Sshinfi 18956723Sshinexit ${ec} 19056723Sshin