hgforest.sh revision 881:e877cb3eb4d6
1#!/bin/sh 2 3# 4# Copyright (c) 2009, 2012, Oracle and/or its affiliates. All rights reserved. 5# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6# 7# This code is free software; you can redistribute it and/or modify it 8# under the terms of the GNU General Public License version 2 only, as 9# published by the Free Software Foundation. 10# 11# This code is distributed in the hope that it will be useful, but WITHOUT 12# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14# version 2 for more details (a copy is included in the LICENSE file that 15# accompanied this code). 16# 17# You should have received a copy of the GNU General Public License version 18# 2 along with this work; if not, write to the Free Software Foundation, 19# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20# 21# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22# or visit www.oracle.com if you need additional information or have any 23# questions. 24# 25 26# Shell script for a fast parallel forest command 27command="$1" 28pull_extra_base="$2" 29 30# Python always buffers stdout significantly, thus we will not see any output from hg clone jdk, 31# until a lot of time has passed! By passing -u to python, we get incremental updates 32# on stdout. Much nicer. 33whichhg="`which hg`" 34 35if [ "${whichhg}" = "" ] ; then 36 echo Cannot find hg! 37 exit 1 38fi 39 40if [ "" = "$command" ] ; then 41 echo No command to hg supplied! 42 exit 1 43fi 44 45has_hash_bang="`head -n 1 "${whichhg}" | cut -b 1-2`" 46python="" 47bpython="" 48 49if [ "#!" = "$has_hash_bang" ] ; then 50 python="`head -n 1 ${whichhg} | cut -b 3-`" 51 bpython="`basename "$python"`" 52fi 53 54if [ "python" = "$bpython" -a -x "$python" ] ; then 55 hg="${python} -u ${whichhg}" 56else 57 echo Cannot find python from hg launcher. Running plain hg, which probably has buffered stdout. 58 hg="hg" 59fi 60 61# Clean out the temporary directory that stores the pid files. 62tmp=/tmp/forest.$$ 63rm -f -r ${tmp} 64mkdir -p ${tmp} 65 66safe_interrupt () { 67 if [ -d ${tmp} ]; then 68 if [ "`ls ${tmp}/*.pid`" != "" ]; then 69 echo "Waiting for processes ( `cat ${tmp}/*.pid | tr '\n' ' '`) to terminate nicely!" 70 sleep 1 71 # Pipe stderr to dev/null to silence kill, that complains when trying to kill 72 # a subprocess that has already exited. 73 kill -TERM `cat ${tmp}/*.pid | tr '\n' ' '` 2> /dev/null 74 wait 75 echo Interrupt complete! 76 fi 77 fi 78 rm -f -r ${tmp} 79 exit 1 80} 81 82nice_exit () { 83 if [ -d ${tmp} ]; then 84 if [ "`ls ${tmp}`" != "" ]; then 85 wait 86 fi 87 fi 88 rm -f -r ${tmp} 89} 90 91trap 'safe_interrupt' INT QUIT 92trap 'nice_exit' EXIT 93 94# Only look in specific locations for possible forests (avoids long searches) 95pull_default="" 96repos="" 97repos_extra="" 98if [ "${command}" = "clone" -o "${command}" = "fclone" ] ; then 99 subrepos="corba jaxp jaxws langtools jdk hotspot nashorn" 100 if [ -f .hg/hgrc ] ; then 101 pull_default=`hg paths default` 102 if [ "${pull_default}" = "" ] ; then 103 echo "ERROR: Need initial clone with 'hg paths default' defined" 104 exit 1 105 fi 106 fi 107 if [ "${pull_default}" = "" ] ; then 108 echo "ERROR: Need initial repository to use this script" 109 exit 1 110 fi 111 for i in ${subrepos} ; do 112 if [ ! -f ${i}/.hg/hgrc ] ; then 113 repos="${repos} ${i}" 114 fi 115 done 116 if [ "${pull_extra_base}" != "" ] ; then 117 subrepos_extra="jdk/src/closed jdk/make/closed jdk/test/closed hotspot/make/closed hotspot/src/closed hotspot/test/closed deploy install sponsors pubs" 118 pull_default_tail=`echo ${pull_default} | sed -e 's@^.*://[^/]*/\(.*\)@\1@'` 119 pull_extra="${pull_extra_base}/${pull_default_tail}" 120 for i in ${subrepos_extra} ; do 121 if [ ! -f ${i}/.hg/hgrc ] ; then 122 repos_extra="${repos_extra} ${i}" 123 fi 124 done 125 fi 126 at_a_time=2 127 # Any repos to deal with? 128 if [ "${repos}" = "" -a "${repos_extra}" = "" ] ; then 129 exit 130 fi 131else 132 hgdirs=`ls -d ./.hg ./*/.hg ./*/*/.hg ./*/*/*/.hg ./*/*/*/*/.hg 2>/dev/null` 133 # Derive repository names from the .hg directory locations 134 for i in ${hgdirs} ; do 135 repos="${repos} `echo ${i} | sed -e 's@/.hg$@@'`" 136 done 137 for i in ${repos} ; do 138 if [ -h ${i}/.hg/store/lock -o -f ${i}/.hg/store/lock ] ; then 139 locked="${i} ${locked}" 140 fi 141 done 142 at_a_time=8 143 # Any repos to deal with? 144 if [ "${repos}" = "" ] ; then 145 echo "No repositories to process." 146 exit 147 fi 148 if [ "${locked}" != "" ] ; then 149 echo "These repositories are locked: ${locked}" 150 exit 151 fi 152fi 153 154# Echo out what repositories we do a command on. 155echo "# Repositories: ${repos} ${repos_extra}" 156echo 157 158# Run the supplied command on all repos in parallel. 159n=0 160for i in ${repos} ${repos_extra} ; do 161 n=`expr ${n} '+' 1` 162 repopidfile=`echo ${i} | sed -e 's@./@@' -e 's@/@_@g'` 163 reponame=`echo ${i} | sed -e :a -e 's/^.\{1,20\}$/ &/;ta'` 164 pull_base="${pull_default}" 165 for j in $repos_extra ; do 166 if [ "$i" = "$j" ] ; then 167 pull_base="${pull_extra}" 168 fi 169 done 170 ( 171 ( 172 if [ "${command}" = "clone" -o "${command}" = "fclone" ] ; then 173 pull_newrepo="`echo ${pull_base}/${i} | sed -e 's@\([^:]/\)//*@\1@g'`" 174 echo ${hg} clone ${pull_newrepo} ${i} 175 path="`dirname ${i}`" 176 if [ "${path}" != "." ] ; then 177 times=0 178 while [ ! -d "${path}" ] ## nested repo, ensure containing dir exists 179 do 180 times=`expr ${times} '+' 1` 181 if [ `expr ${times} '%' 10` -eq 0 ] ; then 182 echo ${path} still not created, waiting... 183 fi 184 sleep 5 185 done 186 fi 187 (${hg} clone ${pull_newrepo} ${i}; echo "$?" > ${tmp}/${repopidfile}.pid.rc )& 188 else 189 echo "cd ${i} && ${hg} $*" 190 cd ${i} && (${hg} "$@"; echo "$?" > ${tmp}/${repopidfile}.pid.rc )& 191 fi 192 echo $! > ${tmp}/${repopidfile}.pid 193 ) 2>&1 | sed -e "s@^@${reponame}: @") & 194 195 if [ `expr ${n} '%' ${at_a_time}` -eq 0 ] ; then 196 sleep 2 197 echo Waiting 5 secs before spawning next background command. 198 sleep 3 199 fi 200done 201# Wait for all hg commands to complete 202wait 203 204# Terminate with exit 0 only if all subprocesses were successful 205ec=0 206if [ -d ${tmp} ]; then 207 for rc in ${tmp}/*.pid.rc ; do 208 exit_code=`cat ${rc} | tr -d ' \n\r'` 209 if [ "${exit_code}" != "0" ] ; then 210 echo "WARNING: ${rc} exited abnormally." 211 ec=1 212 fi 213 done 214fi 215exit ${ec} 216