1#
2#  This program is free software; you can redistribute it and/or modify
3#  it under the terms of the GNU General Public License version 2
4#  as published by the Free Software Foundation.
5#
6#  This program is distributed in the hope that it will be useful,
7#  but WITHOUT ANY WARRANTY; without even the implied warranty of
8#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
9#  GNU General Public License for more details.
10#
11#  You should have received a copy of the GNU General Public License
12#  along with this program; if not, write to the Free Software
13#  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
14#
15
16#
17# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
18# Use is subject to license terms.
19#
20
21'''
22Workspace backup
23
24Backup format is:
25   backupdir/
26      wsname/
27         generation#/
28            dirstate (handled by CdmUncommittedBackup)
29                File containing dirstate nodeid (the changeset we need
30                to update the workspace to after applying the bundle).
31                This is the node to which the working copy changes
32                (see 'diff', below) will be applied if applicable.
33
34            bundle (handled by CdmCommittedBackup)
35                An Hg bundle containing outgoing committed changes.
36
37            nodes (handled by CdmCommittedBackup)
38                A text file listing the full (hex) nodeid of all nodes in
39                bundle, used by need_backup.
40
41            diff (handled by CdmUncommittedBackup)
42                A Git-formatted diff containing uncommitted changes.
43
44            renames (handled by CdmUncommittedBackup)
45                A list of renames in the working copy that have to be
46                applied manually, rather than by the diff.
47
48            metadata.tar.gz (handled by CdmMetadataBackup)
49                $CODEMGR_WS/.hg/hgrc
50                $CODEMGR_WS/.hg/localtags
51                $CODEMGR_WS/.hg/patches (Mq data)
52
53         latest -> generation#
54            Newest backup generation.
55
56All files in a given backup generation, with the exception of
57dirstate, are optional.
58'''
59
60import os, pwd, shutil, tarfile, time, traceback
61from mercurial import changegroup, node, patch, util
62
63from onbld.Scm.WorkSpace import HgLookupError
64import onbld.Scm.Version as Version
65
66
67class CdmNodeMissing(util.Abort):
68    '''a required node is not present in the destination workspace.
69
70    This may occur both in the case where the bundle contains a
71    changeset which is a child of a node not present in the
72    destination workspace (because the destination workspace is not as
73    up-to-date as the source), or because the source and destination
74    workspace are not related.
75
76    It may also happen in cases where the uncommitted changes need to
77    be applied onto a node that the workspace does not possess even
78    after application of the bundle (on a branch not present
79    in the bundle or destination workspace, for instance)'''
80
81    def __init__(self, msg, name):
82        #
83        # If e.name is a string 20 characters long, it is
84        # assumed to be a node.  (Mercurial makes this
85        # same assumption, when creating a LookupError)
86        #
87        if isinstance(name, str) and len(name) == 20:
88            n = node.short(name)
89        else:
90            n = name
91
92        util.Abort.__init__(self, "%s: changeset '%s' is missing\n"
93                            "Your workspace is either not "
94                            "sufficiently up to date,\n"
95                            "or is unrelated to the workspace from "
96                            "which the backup was taken.\n" % (msg, n))
97
98
99class CdmCommittedBackup(object):
100    '''Backup of committed changes'''
101
102    def __init__(self, backup, ws):
103        self.ws = ws
104        self.bu = backup
105        self.files = ('bundle', 'nodes')
106
107    def _outgoing_nodes(self, parent):
108        '''Return a list of all outgoing nodes in hex format'''
109
110        if parent:
111            outgoing = self.ws.findoutgoing(parent)
112            nodes = self.ws.repo.changelog.nodesbetween(outgoing)[0]
113            return map(node.hex, nodes)
114        else:
115            return []
116
117    def backup(self):
118        '''Backup committed changes'''
119        parent = self.ws.parent()
120
121        if not parent:
122            self.ws.ui.warn('Workspace has no parent, committed changes will '
123                            'not be backed up\n')
124            return
125
126        out = self.ws.findoutgoing(parent)
127        if not out:
128            return
129
130        cg = self.ws.repo.changegroup(out, 'bundle')
131        changegroup.writebundle(cg, self.bu.backupfile('bundle'), 'HG10BZ')
132
133        outnodes = self._outgoing_nodes(parent)
134        if outnodes:
135            fp = None
136            try:
137                try:
138                    fp = open(self.bu.backupfile('nodes'), 'w')
139                    fp.write('%s\n' % '\n'.join(outnodes))
140                except EnvironmentError, e:
141                    raise util.Abort("couldn't store outgoing nodes: %s" % e)
142            finally:
143                if fp and not fp.closed:
144                    fp.close()
145
146    def restore(self):
147        '''Restore committed changes from backup'''
148        bfile = self.bu.backupfile('bundle')
149
150        if os.path.exists(bfile):
151            f = None
152            try:
153                try:
154                    f = open(bfile, 'r')
155                    bundle = changegroup.readbundle(f, bfile)
156                    self.ws.repo.addchangegroup(bundle, 'strip',
157                                                'bundle:%s' % bfile)
158                except EnvironmentError, e:
159                    raise util.Abort("couldn't restore committed changes: %s\n"
160                                     "   %s" % (bfile, e))
161                except HgLookupError, e:
162                    raise CdmNodeMissing("couldn't restore committed changes",
163                                                     e.name)
164            finally:
165                if f and not f.closed:
166                    f.close()
167
168    def need_backup(self):
169        '''Compare backup of committed changes to workspace'''
170
171        if os.path.exists(self.bu.backupfile('nodes')):
172            f = None
173            try:
174                try:
175                    f = open(self.bu.backupfile('nodes'))
176                    bnodes = set([line.rstrip('\r\n')
177                                  for line in f.readlines()])
178                    f.close()
179                except EnvironmentError, e:
180                    raise util.Abort("couldn't open backup node list: %s" % e)
181            finally:
182                if f and not f.closed:
183                    f.close()
184        else:
185            bnodes = set()
186
187        outnodes = set(self._outgoing_nodes(self.ws.parent()))
188        if outnodes != bnodes:
189            return True
190
191        return False
192
193    def cleanup(self):
194        '''Remove backed up committed changes'''
195
196        for fname in self.files:
197            if os.path.exists(self.bu.backupfile(fname)):
198                os.unlink(self.bu.backupfile(fname))
199
200
201class CdmUncommittedBackup(object):
202    '''Backup of uncommitted changes'''
203
204    def __init__(self, backup, ws):
205        self.ws = ws
206        self.bu = backup
207
208    def _clobbering_renames(self):
209        '''Return a list of pairs of files representing renames/copies
210        that clobber already versioned files.  [(oldname newname)...]'''
211
212        #
213        # Note that this doesn't handle uncommitted merges
214        # as CdmUncommittedBackup itself doesn't.
215        #
216        wctx = self.ws.workingctx()
217        parent = wctx.parents()[0]
218
219        ret = []
220        for fname in wctx.added() + wctx.modified():
221            rn = wctx.filectx(fname).renamed()
222            if rn and fname in parent:
223                ret.append((rn[0], fname))
224        return ret
225
226    def backup(self):
227        '''Backup uncommitted changes'''
228
229        if self.ws.merged():
230            raise util.Abort("Unable to backup an uncommitted merge.\n"
231                             "Please complete your merge and commit")
232
233        dirstate = node.hex(self.ws.workingctx().parents()[0].node())
234
235        fp = None
236        try:
237            try:
238                fp = open(self.bu.backupfile('dirstate'), 'w')
239                fp.write(dirstate + '\n')
240            except EnvironmentError, e:
241                raise util.Abort("couldn't save working copy parent: %s" % e)
242        finally:
243            if fp and not fp.closed:
244                fp.close()
245
246        try:
247            try:
248                fp = open(self.bu.backupfile('renames'), 'w')
249                for cons in self._clobbering_renames():
250                    fp.write("%s %s\n" % cons)
251            except EnvironmentError, e:
252                raise util.Abort("couldn't save clobbering copies: %s" % e)
253        finally:
254            if fp and not fp.closed:
255                fp.close()
256
257        try:
258            try:
259                fp = open(self.bu.backupfile('diff'), 'w')
260                opts = patch.diffopts(self.ws.ui, opts={'git': True})
261                fp.write(self.ws.diff(opts=opts))
262            except EnvironmentError, e:
263                raise util.Abort("couldn't save working copy diff: %s" % e)
264        finally:
265            if fp and not fp.closed:
266                fp.close()
267
268    def _dirstate(self):
269        '''Return the desired working copy node from the backup'''
270        fp = None
271        try:
272            try:
273                fp = open(self.bu.backupfile('dirstate'))
274                dirstate = fp.readline().strip()
275                return dirstate
276            except EnvironmentError, e:
277                raise util.Abort("couldn't read saved parent: %s" % e)
278        finally:
279            if fp and not fp.closed:
280                fp.close()
281
282    def restore(self):
283        '''Restore uncommitted changes'''
284        diff = self.bu.backupfile('diff')
285        dirstate = self._dirstate()
286
287        #
288        # Check that the patch's parent changeset exists.
289        #
290        try:
291            n = node.bin(dirstate)
292            self.ws.repo.changelog.lookup(n)
293        except HgLookupError, e:
294            raise CdmNodeMissing("couldn't restore uncommitted changes",
295                                 e.name)
296
297        try:
298            self.ws.clean(rev=dirstate)
299        except util.Abort, e:
300            raise util.Abort("couldn't update to saved node: %s" % e)
301
302        if not os.path.exists(diff):
303            return
304
305        #
306        # There's a race here whereby if the patch (or part thereof)
307        # is applied within the same second as the clean above (such
308        # that mtime doesn't change) and if the size of that file
309        # does not change, Hg may not see the change.
310        #
311        # We sleep a full second to avoid this, as sleeping merely
312        # until the next second begins would require very close clock
313        # synchronization on network filesystems.
314        #
315        time.sleep(1)
316
317        files = {}
318        try:
319            try:
320                fuzz = patch.patch(diff, self.ws.ui, strip=1,
321                                   cwd=self.ws.repo.root, files=files)
322                if fuzz:
323                    raise util.Abort('working copy diff applied with fuzz')
324            except Exception, e:
325                raise util.Abort("couldn't apply working copy diff: %s\n"
326                                 "   %s" % (diff, e))
327        finally:
328            patch.updatedir(self.ws.ui, self.ws.repo, files)
329
330        if not os.path.exists(self.bu.backupfile('renames')):
331            return
332
333        #
334        # We need to re-apply name changes where the new name
335        # (rename/copy destination) is an already versioned file, as
336        # Hg would otherwise ignore them.
337        #
338        try:
339            fp = open(self.bu.backupfile('renames'))
340            for line in fp:
341                source, dest = line.strip().split()
342                self.ws.repo.copy(source, dest)
343        except EnvironmentError, e:
344            raise util.Abort('unable to open renames file: %s' % e)
345        except ValueError:
346            raise util.Abort('corrupt renames file: %s' %
347                             self.bu.backupfile('renames'))
348
349    def need_backup(self):
350        '''Compare backup of uncommitted changes to workspace'''
351        cnode = self.ws.workingctx().parents()[0].node()
352        if self._dirstate() != node.hex(cnode):
353            return True
354
355        opts = patch.diffopts(self.ws.ui, opts={'git': True})
356        curdiff = self.ws.diff(opts=opts)
357
358        diff = self.bu.backupfile('diff')
359        if os.path.exists(diff):
360            try:
361                try:
362                    fd = open(diff)
363                    backdiff = fd.read()
364                except EnvironmentError, e:
365                    raise util.Abort("couldn't open backup diff %s\n"
366                                     "   %s" % (diff, e))
367            finally:
368                if fd and not fd.closed:
369                    fd.close()
370        else:
371            backdiff = ''
372
373        if backdiff != curdiff:
374            return True
375
376
377        currrenamed = self._clobbering_renames()
378        bakrenamed = None
379
380        if os.path.exists(self.bu.backupfile('renames')):
381            try:
382                try:
383                    fd = open(self.bu.backupfile('renames'))
384                    bakrenamed = [line.strip().split(' ') for line in fd]
385                except EnvironmentError, e:
386                    raise util.Abort("couldn't open renames file %s: %s\n" %
387                                     (self.bu.backupfile('renames'), e))
388            finally:
389                if fd and not fd.closed:
390                    fd.close()
391
392            if currrenamed != bakrenamed:
393                return True
394
395        return False
396
397    def cleanup(self):
398        '''Remove backed up uncommitted changes'''
399        for fname in ('dirstate', 'diff', 'renames'):
400            if os.path.exists(self.bu.backupfile(fname)):
401                os.unlink(self.bu.backupfile(fname))
402
403
404class CdmMetadataBackup(object):
405    '''Backup of workspace metadata'''
406
407    def __init__(self, backup, ws):
408        self.bu = backup
409        self.ws = ws
410        self.files = ('hgrc', 'localtags', 'patches', 'cdm')
411
412    def backup(self):
413        '''Backup workspace metadata'''
414
415        tar = None
416
417        try:
418            try:
419                tar = tarfile.open(self.bu.backupfile('metadata.tar.gz'),
420                                   'w:gz')
421                tar.errorlevel = 2
422            except (EnvironmentError, tarfile.TarError), e:
423                raise util.Abort("couldn't open %s for writing: %s" %
424                                 (self.bu.backupfile('metadata.tar.gz'), e))
425
426            try:
427                for elt in self.files:
428                    fpath = self.ws.repo.join(elt)
429                    if os.path.exists(fpath):
430                        tar.add(fpath, elt)
431            except (EnvironmentError, tarfile.TarError), e:
432                #
433                # tarfile.TarError doesn't include the tar member or file
434                # in question, so we have to do so ourselves.
435                #
436                if isinstance(e, tarfile.TarError):
437                    error = "%s: %s" % (elt, e)
438                else:
439                    error = str(e)
440
441                raise util.Abort("couldn't backup metadata to %s:\n"
442                                 "  %s" %
443                                 (self.bu.backupfile('metadata.tar.gz'),
444                                  error))
445        finally:
446            if tar and not tar.closed:
447                tar.close()
448
449    def old_restore(self):
450        '''Restore workspace metadata from an pre-tar backup'''
451
452        for fname in self.files:
453            bfile = self.bu.backupfile(fname)
454            wfile = self.ws.repo.join(fname)
455
456            if os.path.exists(bfile):
457                try:
458                    shutil.copy2(bfile, wfile)
459                except EnvironmentError, e:
460                    raise util.Abort("couldn't restore metadata from %s:\n"
461                                     "   %s" % (bfile, e))
462
463    def tar_restore(self):
464        '''Restore workspace metadata (from a tar-style backup)'''
465
466        if os.path.exists(self.bu.backupfile('metadata.tar.gz')):
467            tar = None
468
469            try:
470                try:
471                    tar = tarfile.open(self.bu.backupfile('metadata.tar.gz'))
472                    tar.errorlevel = 2
473                except (EnvironmentError, tarfile.TarError), e:
474                    raise util.Abort("couldn't open %s: %s" %
475                                 (self.bu.backupfile('metadata.tar.gz'), e))
476
477                try:
478                    for elt in tar:
479                        tar.extract(elt, path=self.ws.repo.path)
480                except (EnvironmentError, tarfile.TarError), e:
481                    # Make sure the member name is in the exception message.
482                    if isinstance(e, tarfile.TarError):
483                        error = "%s: %s" % (elt.name, e)
484                    else:
485                        error = str(e)
486
487                    raise util.Abort("couldn't restore metadata from %s:\n"
488                                     "   %s" %
489                                     (self.bu.backupfile('metadata.tar.gz'),
490                                      error))
491            finally:
492                if tar and not tar.closed:
493                    tar.close()
494
495    def restore(self):
496        '''Restore workspace metadata'''
497
498        if os.path.exists(self.bu.backupfile('hgrc')):
499            self.old_restore()
500        else:
501            self.tar_restore()
502
503    def need_backup(self):
504        '''Compare backed up workspace metadata to workspace'''
505
506        if os.path.exists(self.bu.backupfile('metadata.tar.gz')):
507            try:
508                tar = tarfile.open(self.bu.backupfile('metadata.tar.gz'))
509                tar.errorlevel = 2
510            except (EnvironmentError, tarfile.TarError), e:
511                raise util.Abort("couldn't open metadata tarball: %s\n"
512                                 "   %s" %
513                                 (self.bu.backupfile('metadata.tar.gz'), e))
514
515            for elt in tar:
516                fpath = self.ws.repo.join(elt.name)
517                if not os.path.exists(fpath):
518                    return True     # File in tar, not workspace
519
520                if elt.isdir():     # Don't care about directories
521                    continue
522
523                #
524                # The filesystem can give us mtime with fractional seconds
525                # (as a float), whereas tar files only keep it to the second.
526                #
527                # Always compare to the integer (second-granularity) mtime.
528                #
529                if (elt.mtime != int(os.path.getmtime(fpath)) or
530                    elt.size != os.path.getsize(fpath)):
531                    return True
532
533            tarnames = tar.getnames()
534            tar.close()
535        else:
536            tarnames = []
537
538        for mfile in self.files:
539            fpath = self.ws.repo.join(mfile)
540
541            if os.path.isdir(fpath):
542                # Directories in tarfile always end with a '/'
543                if not mfile.endswith('/'):
544                    mfile += '/'
545
546                if mfile not in tarnames:
547                    return True
548
549                for root, dirs, files in os.walk(fpath, topdown=True):
550                    for elt in files:
551                        path = os.path.join(root, elt)
552
553                        rpath = self.ws.repo.path
554                        if not rpath.endswith('/'):
555                            rpath += '/'
556
557                        path = path.replace(rpath, '', 1)
558                        if path not in tarnames:
559                            return True # In workspace not tar
560            else:
561                if os.path.exists(fpath) and mfile not in tarnames:
562                    return True
563
564        return False
565
566    def cleanup(self):
567        '''Remove backed up workspace metadata'''
568        if os.path.exists(self.bu.backupfile('metadata.tar.gz')):
569            os.unlink(self.bu.backupfile('metadata.tar.gz'))
570
571
572class CdmBackup(object):
573    '''A backup of a given workspace'''
574
575    def __init__(self, ui, ws, name):
576        self.ws = ws
577        self.ui = ui
578        self.backupdir = self._find_backup_dir(name)
579
580        #
581        # The order of instances here controls the order the various operations
582        # are run.
583        #
584        # There's some inherent dependence, in that on restore we need
585        # to restore committed changes prior to uncommitted changes
586        # (as the parent revision of any uncommitted changes is quite
587        # likely to not exist until committed changes are restored).
588        # Metadata restore can happen at any point, but happens last
589        # as a matter of convention.
590        #
591        self.modules = [x(self, ws) for x in [CdmCommittedBackup,
592                                              CdmUncommittedBackup,
593                                              CdmMetadataBackup]]
594
595
596        if os.path.exists(os.path.join(self.backupdir, 'latest')):
597            generation = os.readlink(os.path.join(self.backupdir, 'latest'))
598            self.generation = int(os.path.split(generation)[1])
599        else:
600            self.generation = 0
601
602    def _find_backup_dir(self, name):
603        '''Find the path to an appropriate backup directory based on NAME'''
604        backupdir = None
605        backupbase = None
606
607        if os.path.isabs(name):
608            return name
609
610        if self.ui.config('cdm', 'backupdir'):
611            backupbase = os.path.expanduser(self.ui.config('cdm', 'backupdir'))
612        else:
613            home = None
614
615            try:
616                home = os.getenv('HOME') or pwd.getpwuid(os.getuid()).pw_dir
617            except KeyError:
618                pass                    # Handled anyway
619
620            if not home:
621                raise util.Abort('Could not determine your HOME directory to '
622                                 'find backup path')
623
624            backupbase = os.path.join(home, 'cdm.backup')
625
626        backupdir = os.path.join(backupbase, name)
627
628        # If backupdir exists, it must be a directory.
629        if (os.path.exists(backupdir) and not os.path.isdir(backupdir)):
630            raise util.Abort('%s exists but is not a directory' % backupdir)
631
632        return backupdir
633
634    def backupfile(self, path):
635        '''return full path to backup file FILE at GEN'''
636        return os.path.join(self.backupdir, str(self.generation), path)
637
638    def update_latest(self, gen):
639        '''Update latest symlink to point to the current generation'''
640        linkpath = os.path.join(self.backupdir, 'latest')
641
642        if os.path.lexists(linkpath):
643            os.unlink(linkpath)
644
645        os.symlink(str(gen), linkpath)
646
647    def create_gen(self, gen):
648        '''Create a new backup generation'''
649        try:
650            os.makedirs(os.path.join(self.backupdir, str(gen)))
651            self.update_latest(gen)
652        except EnvironmentError, e:
653            raise util.Abort("Couldn't create backup generation %s: %s" %
654                             (os.path.join(self.backupdir, str(gen)), e))
655
656    def need_backup(self):
657        '''Compare backed up changes to workspace'''
658        #
659        # If there's no current backup generation, or the last backup was
660        # invalid (lacking the dirstate file), we need a backup regardless
661        # of anything else.
662        #
663        if (not self.generation or
664            not os.path.exists(self.backupfile('dirstate'))):
665            return True
666
667        for x in self.modules:
668            if x.need_backup():
669                return True
670
671        return False
672
673    def backup(self):
674        '''Take a backup of the current workspace
675
676        Calling code is expected to hold both the working copy lock
677        and repository lock.'''
678
679        if not os.path.exists(self.backupdir):
680            try:
681                os.makedirs(self.backupdir)
682            except EnvironmentError, e:
683                raise util.Abort('Could not create backup directory %s: %s' %
684                                 (self.backupdir, e))
685
686        self.generation += 1
687        self.create_gen(self.generation)
688
689        try:
690            for x in self.modules:
691                x.backup()
692        except Exception, e:
693            if isinstance(e, KeyboardInterrupt):
694                self.ws.ui.warn("Interrupted\n")
695            else:
696                self.ws.ui.warn("Error: %s\n" % e)
697                if Version.at_least("1.3.0"):
698                    show_traceback = self.ws.ui.configbool('ui', 'traceback',
699                                                   False)
700                else:
701                    show_traceback = self.ws.ui.traceback
702
703                #
704                # If it's not a 'normal' error, we want to print a stack
705                # trace now in case the attempt to remove the partial
706                # backup also fails, and raises a second exception.
707                #
708                if (not isinstance(e, (EnvironmentError, util.Abort))
709                    or show_traceback):
710                    traceback.print_exc()
711
712            for x in self.modules:
713                x.cleanup()
714
715            os.rmdir(os.path.join(self.backupdir, str(self.generation)))
716            self.generation -= 1
717
718            if self.generation != 0:
719                self.update_latest(self.generation)
720            else:
721                os.unlink(os.path.join(self.backupdir, 'latest'))
722
723            raise util.Abort('Backup failed')
724
725    def restore(self, gen=None):
726        '''Restore workspace from backup
727
728        Restores from backup generation GEN (defaulting to the latest)
729        into workspace WS.
730
731        Calling code is expected to hold both the working copy lock
732        and repository lock of the destination workspace.'''
733
734        if not os.path.exists(self.backupdir):
735            raise util.Abort('Backup directory does not exist: %s' %
736                             (self.backupdir))
737
738        if gen:
739            if not os.path.exists(os.path.join(self.backupdir, str(gen))):
740                raise util.Abort('Backup generation does not exist: %s' %
741                                 (os.path.join(self.backupdir, str(gen))))
742            self.generation = int(gen)
743
744        if not self.generation: # This is ok, 0 is not a valid generation
745            raise util.Abort('Backup has no generations: %s' % self.backupdir)
746
747        if not os.path.exists(self.backupfile('dirstate')):
748            raise util.Abort('Backup %s/%s is incomplete (dirstate missing)' %
749                             (self.backupdir, self.generation))
750
751        try:
752            for x in self.modules:
753                x.restore()
754        except util.Abort, e:
755            raise util.Abort('Error restoring workspace:\n'
756                             '%s\n'
757                             'Workspace may be partially restored' % e)
758