hgforest.sh revision 1005:233d0452e038 
1#!/bin/bash
2
3#
4# Copyright (c) 2009, 2014, Oracle and/or its affiliates. All rights reserved.
5# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6#
7# This code is free software; you can redistribute it and/or modify it
8# under the terms of the GNU General Public License version 2 only, as
9# published by the Free Software Foundation.
10#
11# This code is distributed in the hope that it will be useful, but WITHOUT
12# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14# version 2 for more details (a copy is included in the LICENSE file that
15# accompanied this code).
16#
17# You should have received a copy of the GNU General Public License version
18# 2 along with this work; if not, write to the Free Software Foundation,
19# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20#
21# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22# or visit www.oracle.com if you need additional information or have any
23# questions.
24#
25
26# Shell script for a fast parallel forest command
27
28global_opts=""
29status_output="/dev/stdout"
30qflag="false"
31vflag="false"
32while [ $# -gt 0 ]
33do
34  case $1 in
35    -q | --quiet )
36      qflag="true"
37      global_opts="${global_opts} -q"
38      status_output="/dev/null"
39      ;;
40
41    -v | --verbose )
42      vflag="true"
43      global_opts="${global_opts} -v"
44      ;;
45
46    '--' ) # no more options
47      shift; break
48      ;;
49
50    -*)  # bad option
51      usage
52      ;;
53
54     * )  # non option
55      break
56      ;;
57  esac
58  shift
59done
60
61
62command="$1"; shift
63repo_base="$@"
64
65usage() {
66      echo "usage: $0 [-q|--quiet] [-v|--verbose] [--] <command> [repo_base_path]" > ${status_output}
67      exit 1
68}
69
70
71if [ "x" = "x$command" ] ; then
72  echo "ERROR: No command to hg supplied!"
73  usage
74fi
75
76# Clean out the temporary directory that stores the pid files.
77tmp=/tmp/forest.$$
78rm -f -r ${tmp}
79mkdir -p ${tmp}
80
81safe_interrupt () {
82  if [ -d ${tmp} ]; then
83    if [ "`ls ${tmp}/*.pid`" != "" ]; then
84      echo "Waiting for processes ( `cat ${tmp}/*.pid | tr '\n' ' '`) to terminate nicely!" > ${status_output}
85      sleep 1
86      # Pipe stderr to dev/null to silence kill, that complains when trying to kill
87      # a subprocess that has already exited.
88      kill -TERM `cat ${tmp}/*.pid | tr '\n' ' '` 2> /dev/null
89      wait
90      echo "Interrupt complete!" > ${status_output}
91    fi
92    rm -f -r ${tmp}
93  fi
94  exit 130
95}
96
97nice_exit () {
98  if [ -d ${tmp} ]; then
99    if [ "`ls ${tmp}`" != "" ]; then
100      wait
101    fi
102    rm -f -r ${tmp}
103  fi
104}
105
106trap 'safe_interrupt' INT QUIT
107trap 'nice_exit' EXIT
108
109subrepos="corba jaxp jaxws langtools jdk hotspot nashorn"
110subrepos_extra="closed jdk/src/closed jdk/make/closed jdk/test/closed hotspot/make/closed hotspot/src/closed hotspot/test/closed deploy install sponsors pubs"
111
112# Only look in specific locations for possible forests (avoids long searches)
113pull_default=""
114repos=""
115repos_extra=""
116if [ "${command}" = "clone" -o "${command}" = "fclone" ] ; then
117  if [ -f .hg/hgrc ] ; then
118    pull_default=`hg paths default`
119    if [ "${pull_default}" = "" ] ; then
120      echo "ERROR: Need initial clone with 'hg paths default' defined" > ${status_output}
121      exit 1
122    fi
123  fi
124  if [ "${pull_default}" = "" ] ; then
125    echo "ERROR: Need initial repository to use this script" > ${status_output}
126    exit 1
127  fi
128  for i in ${subrepos} ; do
129    if [ ! -f ${i}/.hg/hgrc ] ; then
130      repos="${repos} ${i}"
131    fi
132  done
133  if [ "${repo_base}" != "" ] ; then
134    pull_default_tail=`echo ${pull_default} | sed -e 's@^.*://[^/]*/\(.*\)@\1@'`
135    pull_extra="${repo_base}/${pull_default_tail}"
136    for i in ${subrepos_extra} ; do
137      if [ ! -f ${i}/.hg/hgrc ] ; then
138        repos_extra="${repos_extra} ${i}"
139      fi
140    done
141  fi
142  at_a_time=2
143  # Any repos to deal with?
144  if [ "${repos}" = "" -a "${repos_extra}" = "" ] ; then
145    echo "No repositories to process." > ${status_output}
146    exit
147  fi
148else
149  for i in . ${subrepos} ${subrepos_extra} ; do
150    if [ -d ${i}/.hg ] ; then
151      repos="${repos} ${i}"
152    fi
153  done
154
155  # Any repos to deal with?
156  if [ "${repos}" = "" ] ; then
157    echo "No repositories to process." > ${status_output}
158    exit
159  fi
160
161  # any of the repos locked?
162  for i in ${repos} ; do
163    if [ -h ${i}/.hg/store/lock -o -f ${i}/.hg/store/lock ] ; then
164      locked="${i} ${locked}"
165    fi
166  done
167  if [ "${locked}" != "" ] ; then
168    echo "ERROR: These repositories are locked: ${locked}" > ${status_output}
169    exit 1
170  fi
171  at_a_time=8
172fi
173
174# Echo out what repositories we do a command on.
175echo "# Repositories: ${repos} ${repos_extra}" > ${status_output}
176
177if [ "${command}" = "serve" ] ; then
178  # "serve" is run for all the repos.
179  (
180    (
181      (
182        echo "[web]"
183        echo "description = $(basename $(pwd))"
184        echo "allow_push = *"
185        echo "push_ssl = False"
186
187        echo "[paths]"
188        for i in ${repos} ${repos_extra} ; do
189          if [ "${i}" != "." ] ; then
190            echo "/$(basename $(pwd))/${i} = ${i}"
191          else
192            echo "/$(basename $(pwd)) = $(pwd)"
193          fi
194        done
195      ) > ${tmp}/serve.web-conf
196
197      echo "serving root repo $(basename $(pwd))"
198
199      (PYTHONUNBUFFERED=true hg${global_opts} serve -A ${status_output} -E ${status_output} --pid-file ${tmp}/serve.pid --web-conf ${tmp}/serve.web-conf; echo "$?" > ${tmp}/serve.pid.rc ) 2>&1 &
200    ) 2>&1 | sed -e "s@^@serve:   @" > ${status_output}
201  ) &
202else
203  # Run the supplied command on all repos in parallel.
204  n=0
205  for i in ${repos} ${repos_extra} ; do
206    n=`expr ${n} '+' 1`
207    repopidfile=`echo ${i} | sed -e 's@./@@' -e 's@/@_@g'`
208    reponame=`echo ${i} | sed -e :a -e 's/^.\{1,20\}$/ &/;ta'`
209    pull_base="${pull_default}"
210    for j in $repos_extra ; do
211      if [ "$i" = "$j" ] ; then
212          pull_base="${pull_extra}"
213      fi
214    done
215    (
216      (
217        if [ "${command}" = "clone" -o "${command}" = "fclone" ] ; then
218          pull_newrepo="`echo ${pull_base}/${i} | sed -e 's@\([^:]/\)//*@\1@g'`"
219          echo "hg clone ${pull_newrepo} ${i}" > ${status_output}
220          path="`dirname ${i}`"
221          if [ "${path}" != "." ] ; then
222            times=0
223            while [ ! -d "${path}" ]   ## nested repo, ensure containing dir exists
224            do
225              times=`expr ${times} '+' 1`
226              if [ `expr ${times} '%' 10` -eq 0 ] ; then
227                echo "${path} still not created, waiting..." > ${status_output}
228              fi
229              sleep 5
230            done
231          fi
232          (PYTHONUNBUFFERED=true hg${global_opts} clone ${pull_newrepo} ${i}; echo "$?" > ${tmp}/${repopidfile}.pid.rc ) 2>&1 &
233        else
234          echo "cd ${i} && hg${global_opts} ${command} ${repo_base}" > ${status_output}
235          cd ${i} && (PYTHONUNBUFFERED=true hg${global_opts} ${command} ${command_repo}; echo "$?" > ${tmp}/${repopidfile}.pid.rc ) 2>&1 &
236        fi
237
238        echo $! > ${tmp}/${repopidfile}.pid
239      ) 2>&1 | sed -e "s@^@${reponame}:   @" > ${status_output}
240    ) &
241
242    if [ `expr ${n} '%' ${at_a_time}` -eq 0 ] ; then
243      sleep 2
244      echo "Waiting 5 secs before spawning next background command." > ${status_output}
245      sleep 3
246    fi
247  done
248fi
249
250# Wait for all hg commands to complete
251wait
252
253# Terminate with exit 0 only if all subprocesses were successful
254ec=0
255if [ -d ${tmp} ]; then
256  for rc in ${tmp}/*.pid.rc ; do
257    exit_code=`cat ${rc} | tr -d ' \n\r'`
258    if [ "${exit_code}" != "0" ] ; then
259      repo="`echo ${rc} | sed -e s@^${tmp}@@ -e 's@/*\([^/]*\)\.pid\.rc$@\1@' -e 's@_@/@g'`"
260      echo "WARNING: ${repo} exited abnormally." > ${status_output}
261      ec=1
262    fi
263  done
264fi
265exit ${ec}
266