1289177Speter/* hotcopys.c --- FS hotcopy functionality for FSX
2289177Speter *
3289177Speter * ====================================================================
4289177Speter *    Licensed to the Apache Software Foundation (ASF) under one
5289177Speter *    or more contributor license agreements.  See the NOTICE file
6289177Speter *    distributed with this work for additional information
7289177Speter *    regarding copyright ownership.  The ASF licenses this file
8289177Speter *    to you under the Apache License, Version 2.0 (the
9289177Speter *    "License"); you may not use this file except in compliance
10289177Speter *    with the License.  You may obtain a copy of the License at
11289177Speter *
12289177Speter *      http://www.apache.org/licenses/LICENSE-2.0
13289177Speter *
14289177Speter *    Unless required by applicable law or agreed to in writing,
15289177Speter *    software distributed under the License is distributed on an
16289177Speter *    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17289177Speter *    KIND, either express or implied.  See the License for the
18289177Speter *    specific language governing permissions and limitations
19289177Speter *    under the License.
20289177Speter * ====================================================================
21289177Speter */
22289177Speter#include "svn_pools.h"
23289177Speter#include "svn_path.h"
24289177Speter#include "svn_dirent_uri.h"
25289177Speter
26289177Speter#include "fs_x.h"
27289177Speter#include "hotcopy.h"
28289177Speter#include "util.h"
29289177Speter#include "revprops.h"
30289177Speter#include "rep-cache.h"
31289177Speter#include "transaction.h"
32289177Speter#include "recovery.h"
33289177Speter
34289177Speter#include "../libsvn_fs/fs-loader.h"
35289177Speter
36289177Speter#include "svn_private_config.h"
37289177Speter
38289177Speter/* Like svn_io_dir_file_copy(), but doesn't copy files that exist at
39289177Speter * the destination and do not differ in terms of kind, size, and mtime.
40289177Speter * Set *SKIPPED_P to FALSE only if the file was copied, do not change
41289177Speter * the value in *SKIPPED_P otherwise. SKIPPED_P may be NULL if not
42289177Speter * required. */
43289177Speterstatic svn_error_t *
44289177Speterhotcopy_io_dir_file_copy(svn_boolean_t *skipped_p,
45289177Speter                         const char *src_path,
46289177Speter                         const char *dst_path,
47289177Speter                         const char *file,
48289177Speter                         apr_pool_t *scratch_pool)
49289177Speter{
50289177Speter  const svn_io_dirent2_t *src_dirent;
51289177Speter  const svn_io_dirent2_t *dst_dirent;
52289177Speter  const char *src_target;
53289177Speter  const char *dst_target;
54289177Speter
55289177Speter  /* Does the destination already exist? If not, we must copy it. */
56289177Speter  dst_target = svn_dirent_join(dst_path, file, scratch_pool);
57289177Speter  SVN_ERR(svn_io_stat_dirent2(&dst_dirent, dst_target, FALSE, TRUE,
58289177Speter                              scratch_pool, scratch_pool));
59289177Speter  if (dst_dirent->kind != svn_node_none)
60289177Speter    {
61289177Speter      /* If the destination's stat information indicates that the file
62289177Speter       * is equal to the source, don't bother copying the file again. */
63289177Speter      src_target = svn_dirent_join(src_path, file, scratch_pool);
64289177Speter      SVN_ERR(svn_io_stat_dirent2(&src_dirent, src_target, FALSE, FALSE,
65289177Speter                                  scratch_pool, scratch_pool));
66289177Speter      if (src_dirent->kind == dst_dirent->kind &&
67289177Speter          src_dirent->special == dst_dirent->special &&
68289177Speter          src_dirent->filesize == dst_dirent->filesize &&
69289177Speter          src_dirent->mtime <= dst_dirent->mtime)
70289177Speter        return SVN_NO_ERROR;
71289177Speter    }
72289177Speter
73289177Speter  if (skipped_p)
74289177Speter    *skipped_p = FALSE;
75289177Speter
76289177Speter  return svn_error_trace(svn_io_dir_file_copy(src_path, dst_path, file,
77289177Speter                                              scratch_pool));
78289177Speter}
79289177Speter
80289177Speter/* Set *NAME_P to the UTF-8 representation of directory entry NAME.
81289177Speter * NAME is in the internal encoding used by APR; PARENT is in
82289177Speter * UTF-8 and in internal (not local) style.
83289177Speter *
84289177Speter * Use PARENT only for generating an error string if the conversion
85289177Speter * fails because NAME could not be represented in UTF-8.  In that
86289177Speter * case, return a two-level error in which the outer error's message
87289177Speter * mentions PARENT, but the inner error's message does not mention
88289177Speter * NAME (except possibly in hex) since NAME may not be printable.
89289177Speter * Such a compound error at least allows the user to go looking in the
90289177Speter * right directory for the problem.
91289177Speter *
92289177Speter * If there is any other error, just return that error directly.
93289177Speter *
94289177Speter * If there is any error, the effect on *NAME_P is undefined.
95289177Speter *
96289177Speter * *NAME_P and NAME may refer to the same storage.
97289177Speter */
98289177Speterstatic svn_error_t *
99289177Speterentry_name_to_utf8(const char **name_p,
100289177Speter                   const char *name,
101289177Speter                   const char *parent,
102289177Speter                   apr_pool_t *result_pool)
103289177Speter{
104289177Speter  svn_error_t *err = svn_path_cstring_to_utf8(name_p, name, result_pool);
105289177Speter  if (err && err->apr_err == APR_EINVAL)
106289177Speter    {
107289177Speter      return svn_error_createf(err->apr_err, err,
108289177Speter                               _("Error converting entry "
109289177Speter                                 "in directory '%s' to UTF-8"),
110289177Speter                               svn_dirent_local_style(parent, result_pool));
111289177Speter    }
112289177Speter  return err;
113289177Speter}
114289177Speter
115289177Speter/* Like svn_io_copy_dir_recursively() but doesn't copy regular files that
116289177Speter * exist in the destination and do not differ from the source in terms of
117289177Speter * kind, size, and mtime. Set *SKIPPED_P to FALSE only if at least one
118289177Speter * file was copied, do not change the value in *SKIPPED_P otherwise.
119289177Speter * SKIPPED_P may be NULL if not required. */
120289177Speterstatic svn_error_t *
121289177Speterhotcopy_io_copy_dir_recursively(svn_boolean_t *skipped_p,
122289177Speter                                const char *src,
123289177Speter                                const char *dst_parent,
124289177Speter                                const char *dst_basename,
125289177Speter                                svn_boolean_t copy_perms,
126289177Speter                                svn_cancel_func_t cancel_func,
127289177Speter                                void *cancel_baton,
128289177Speter                                apr_pool_t *scratch_pool)
129289177Speter{
130289177Speter  svn_node_kind_t kind;
131289177Speter  apr_status_t status;
132289177Speter  const char *dst_path;
133289177Speter  apr_dir_t *this_dir;
134289177Speter  apr_finfo_t this_entry;
135289177Speter  apr_int32_t flags = APR_FINFO_TYPE | APR_FINFO_NAME;
136289177Speter
137289177Speter  /* Make a subpool for recursion */
138289177Speter  apr_pool_t *subpool = svn_pool_create(scratch_pool);
139289177Speter
140289177Speter  /* The 'dst_path' is simply dst_parent/dst_basename */
141289177Speter  dst_path = svn_dirent_join(dst_parent, dst_basename, scratch_pool);
142289177Speter
143289177Speter  /* Sanity checks:  SRC and DST_PARENT are directories, and
144289177Speter     DST_BASENAME doesn't already exist in DST_PARENT. */
145289177Speter  SVN_ERR(svn_io_check_path(src, &kind, subpool));
146289177Speter  if (kind != svn_node_dir)
147289177Speter    return svn_error_createf(SVN_ERR_NODE_UNEXPECTED_KIND, NULL,
148289177Speter                             _("Source '%s' is not a directory"),
149289177Speter                             svn_dirent_local_style(src, scratch_pool));
150289177Speter
151289177Speter  SVN_ERR(svn_io_check_path(dst_parent, &kind, subpool));
152289177Speter  if (kind != svn_node_dir)
153289177Speter    return svn_error_createf(SVN_ERR_NODE_UNEXPECTED_KIND, NULL,
154289177Speter                             _("Destination '%s' is not a directory"),
155289177Speter                             svn_dirent_local_style(dst_parent,
156289177Speter                                                    scratch_pool));
157289177Speter
158289177Speter  SVN_ERR(svn_io_check_path(dst_path, &kind, subpool));
159289177Speter
160289177Speter  /* Create the new directory. */
161289177Speter  /* ### TODO: copy permissions (needs apr_file_attrs_get()) */
162289177Speter  SVN_ERR(svn_io_make_dir_recursively(dst_path, scratch_pool));
163289177Speter
164289177Speter  /* Loop over the dirents in SRC.  ('.' and '..' are auto-excluded) */
165289177Speter  SVN_ERR(svn_io_dir_open(&this_dir, src, subpool));
166289177Speter
167289177Speter  for (status = apr_dir_read(&this_entry, flags, this_dir);
168289177Speter       status == APR_SUCCESS;
169289177Speter       status = apr_dir_read(&this_entry, flags, this_dir))
170289177Speter    {
171289177Speter      if ((this_entry.name[0] == '.')
172289177Speter          && ((this_entry.name[1] == '\0')
173289177Speter              || ((this_entry.name[1] == '.')
174289177Speter                  && (this_entry.name[2] == '\0'))))
175289177Speter        {
176289177Speter          continue;
177289177Speter        }
178289177Speter      else
179289177Speter        {
180289177Speter          const char *entryname_utf8;
181289177Speter
182289177Speter          if (cancel_func)
183289177Speter            SVN_ERR(cancel_func(cancel_baton));
184289177Speter
185289177Speter          SVN_ERR(entry_name_to_utf8(&entryname_utf8, this_entry.name,
186289177Speter                                     src, subpool));
187289177Speter          if (this_entry.filetype == APR_REG) /* regular file */
188289177Speter            {
189289177Speter              SVN_ERR(hotcopy_io_dir_file_copy(skipped_p, src, dst_path,
190289177Speter                                               entryname_utf8, subpool));
191289177Speter            }
192289177Speter          else if (this_entry.filetype == APR_LNK) /* symlink */
193289177Speter            {
194289177Speter              const char *src_target = svn_dirent_join(src, entryname_utf8,
195289177Speter                                                       subpool);
196289177Speter              const char *dst_target = svn_dirent_join(dst_path,
197289177Speter                                                       entryname_utf8,
198289177Speter                                                       subpool);
199289177Speter              SVN_ERR(svn_io_copy_link(src_target, dst_target,
200289177Speter                                       subpool));
201289177Speter            }
202289177Speter          else if (this_entry.filetype == APR_DIR) /* recurse */
203289177Speter            {
204289177Speter              const char *src_target;
205289177Speter
206289177Speter              /* Prevent infinite recursion by filtering off our
207289177Speter                 newly created destination path. */
208289177Speter              if (strcmp(src, dst_parent) == 0
209289177Speter                  && strcmp(entryname_utf8, dst_basename) == 0)
210289177Speter                continue;
211289177Speter
212289177Speter              src_target = svn_dirent_join(src, entryname_utf8, subpool);
213289177Speter              SVN_ERR(hotcopy_io_copy_dir_recursively(skipped_p,
214289177Speter                                                      src_target,
215289177Speter                                                      dst_path,
216289177Speter                                                      entryname_utf8,
217289177Speter                                                      copy_perms,
218289177Speter                                                      cancel_func,
219289177Speter                                                      cancel_baton,
220289177Speter                                                      subpool));
221289177Speter            }
222289177Speter          /* ### support other APR node types someday?? */
223289177Speter
224289177Speter        }
225289177Speter    }
226289177Speter
227289177Speter  if (! (APR_STATUS_IS_ENOENT(status)))
228289177Speter    return svn_error_wrap_apr(status, _("Can't read directory '%s'"),
229289177Speter                              svn_dirent_local_style(src, scratch_pool));
230289177Speter
231289177Speter  status = apr_dir_close(this_dir);
232289177Speter  if (status)
233289177Speter    return svn_error_wrap_apr(status, _("Error closing directory '%s'"),
234289177Speter                              svn_dirent_local_style(src, scratch_pool));
235289177Speter
236289177Speter  /* Free any memory used by recursion */
237289177Speter  svn_pool_destroy(subpool);
238289177Speter
239289177Speter  return SVN_NO_ERROR;
240289177Speter}
241289177Speter
242289177Speter/* Copy an un-packed revision or revprop file for revision REV from SRC_SUBDIR
243289177Speter * to DST_SUBDIR. Assume a sharding layout based on MAX_FILES_PER_DIR.
244289177Speter * Set *SKIPPED_P to FALSE only if the file was copied, do not change the
245289177Speter * value in *SKIPPED_P otherwise. SKIPPED_P may be NULL if not required.
246362181Sdim * If PROPS is set, copy the revprops file, otherwise copy the rev data file.
247289177Speter * Use SCRATCH_POOL for temporary allocations. */
248289177Speterstatic svn_error_t *
249289177Speterhotcopy_copy_shard_file(svn_boolean_t *skipped_p,
250289177Speter                        const char *src_subdir,
251289177Speter                        const char *dst_subdir,
252289177Speter                        svn_revnum_t rev,
253289177Speter                        int max_files_per_dir,
254362181Sdim                        svn_boolean_t props,
255289177Speter                        apr_pool_t *scratch_pool)
256289177Speter{
257289177Speter  const char *src_subdir_shard = src_subdir,
258289177Speter             *dst_subdir_shard = dst_subdir;
259289177Speter
260289177Speter  const char *shard = apr_psprintf(scratch_pool, "%ld",
261289177Speter                                   rev / max_files_per_dir);
262289177Speter  src_subdir_shard = svn_dirent_join(src_subdir, shard, scratch_pool);
263289177Speter  dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool);
264289177Speter
265289177Speter  if (rev % max_files_per_dir == 0)
266289177Speter    {
267289177Speter      SVN_ERR(svn_io_make_dir_recursively(dst_subdir_shard, scratch_pool));
268289177Speter      SVN_ERR(svn_io_copy_perms(dst_subdir, dst_subdir_shard,
269289177Speter                                scratch_pool));
270289177Speter    }
271289177Speter
272289177Speter  SVN_ERR(hotcopy_io_dir_file_copy(skipped_p,
273289177Speter                                   src_subdir_shard, dst_subdir_shard,
274362181Sdim                                   apr_psprintf(scratch_pool, "%c%ld",
275362181Sdim                                                props ? 'p' : 'r',
276362181Sdim                                                rev),
277289177Speter                                   scratch_pool));
278289177Speter  return SVN_NO_ERROR;
279289177Speter}
280289177Speter
281289177Speter
282289177Speter/* Copy a packed shard containing revision REV, and which contains
283289177Speter * MAX_FILES_PER_DIR revisions, from SRC_FS to DST_FS.
284289177Speter * Update *DST_MIN_UNPACKED_REV in case the shard is new in DST_FS.
285289177Speter * Do not re-copy data which already exists in DST_FS.
286289177Speter * Set *SKIPPED_P to FALSE only if at least one part of the shard
287289177Speter * was copied, do not change the value in *SKIPPED_P otherwise.
288289177Speter * SKIPPED_P may be NULL if not required.
289289177Speter * Use SCRATCH_POOL for temporary allocations. */
290289177Speterstatic svn_error_t *
291289177Speterhotcopy_copy_packed_shard(svn_boolean_t *skipped_p,
292289177Speter                          svn_revnum_t *dst_min_unpacked_rev,
293289177Speter                          svn_fs_t *src_fs,
294289177Speter                          svn_fs_t *dst_fs,
295289177Speter                          svn_revnum_t rev,
296289177Speter                          int max_files_per_dir,
297289177Speter                          apr_pool_t *scratch_pool)
298289177Speter{
299289177Speter  const char *src_subdir;
300289177Speter  const char *dst_subdir;
301289177Speter  const char *packed_shard;
302289177Speter  const char *src_subdir_packed_shard;
303289177Speter
304289177Speter  /* Copy the packed shard. */
305289177Speter  src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, scratch_pool);
306289177Speter  dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, scratch_pool);
307289177Speter  packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD,
308289177Speter                              rev / max_files_per_dir);
309289177Speter  src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard,
310289177Speter                                            scratch_pool);
311289177Speter  SVN_ERR(hotcopy_io_copy_dir_recursively(skipped_p, src_subdir_packed_shard,
312289177Speter                                          dst_subdir, packed_shard,
313289177Speter                                          TRUE /* copy_perms */,
314289177Speter                                          NULL /* cancel_func */, NULL,
315289177Speter                                          scratch_pool));
316289177Speter
317289177Speter  /* If necessary, update the min-unpacked rev file in the hotcopy. */
318289177Speter  if (*dst_min_unpacked_rev < rev + max_files_per_dir)
319289177Speter    {
320289177Speter      *dst_min_unpacked_rev = rev + max_files_per_dir;
321289177Speter      SVN_ERR(svn_fs_x__write_min_unpacked_rev(dst_fs,
322289177Speter                                               *dst_min_unpacked_rev,
323289177Speter                                               scratch_pool));
324289177Speter    }
325289177Speter
326289177Speter  return SVN_NO_ERROR;
327289177Speter}
328289177Speter
329289177Speter/* Remove file PATH, if it exists - even if it is read-only.
330289177Speter * Use SCRATCH_POOL for temporary allocations. */
331289177Speterstatic svn_error_t *
332289177Speterhotcopy_remove_file(const char *path,
333289177Speter                    apr_pool_t *scratch_pool)
334289177Speter{
335289177Speter  /* Make the rev file writable and remove it. */
336289177Speter  SVN_ERR(svn_io_set_file_read_write(path, TRUE, scratch_pool));
337289177Speter  SVN_ERR(svn_io_remove_file2(path, TRUE, scratch_pool));
338289177Speter
339289177Speter  return SVN_NO_ERROR;
340289177Speter}
341289177Speter
342289177Speter/* Verify that DST_FS is a suitable destination for an incremental
343289177Speter * hotcopy from SRC_FS. */
344289177Speterstatic svn_error_t *
345289177Speterhotcopy_incremental_check_preconditions(svn_fs_t *src_fs,
346289177Speter                                        svn_fs_t *dst_fs)
347289177Speter{
348289177Speter  svn_fs_x__data_t *src_ffd = src_fs->fsap_data;
349289177Speter  svn_fs_x__data_t *dst_ffd = dst_fs->fsap_data;
350289177Speter
351289177Speter  /* We only support incremental hotcopy between the same format. */
352289177Speter  if (src_ffd->format != dst_ffd->format)
353289177Speter    return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
354289177Speter      _("The FSX format (%d) of the hotcopy source does not match the "
355289177Speter        "FSX format (%d) of the hotcopy destination; please upgrade "
356289177Speter        "both repositories to the same format"),
357289177Speter      src_ffd->format, dst_ffd->format);
358289177Speter
359289177Speter  /* Make sure the UUID of source and destination match up.
360289177Speter   * We don't want to copy over a different repository. */
361289177Speter  if (strcmp(src_fs->uuid, dst_fs->uuid) != 0)
362289177Speter    return svn_error_create(SVN_ERR_RA_UUID_MISMATCH, NULL,
363289177Speter                            _("The UUID of the hotcopy source does "
364289177Speter                              "not match the UUID of the hotcopy "
365289177Speter                              "destination"));
366289177Speter
367289177Speter  /* Also require same shard size. */
368289177Speter  if (src_ffd->max_files_per_dir != dst_ffd->max_files_per_dir)
369289177Speter    return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
370289177Speter                            _("The sharding layout configuration "
371289177Speter                              "of the hotcopy source does not match "
372289177Speter                              "the sharding layout configuration of "
373289177Speter                              "the hotcopy destination"));
374289177Speter  return SVN_NO_ERROR;
375289177Speter}
376289177Speter
377289177Speter/* Copy the revision and revprop files (possibly sharded / packed) from
378289177Speter * SRC_FS to DST_FS.  Do not re-copy data which already exists in DST_FS.
379289177Speter * When copying packed or unpacked shards, checkpoint the result in DST_FS
380289177Speter * for every shard by updating the 'current' file if necessary.  Assume
381289177Speter * the >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT filesystem format without
382289177Speter * global next-ID counters.  Indicate progress via the optional NOTIFY_FUNC
383289177Speter * callback using NOTIFY_BATON.  Use SCRATCH_POOL for temporary allocations.
384289177Speter */
385289177Speterstatic svn_error_t *
386289177Speterhotcopy_revisions(svn_fs_t *src_fs,
387289177Speter                  svn_fs_t *dst_fs,
388289177Speter                  svn_revnum_t src_youngest,
389289177Speter                  svn_revnum_t dst_youngest,
390289177Speter                  svn_boolean_t incremental,
391289177Speter                  const char *src_revs_dir,
392289177Speter                  const char *dst_revs_dir,
393289177Speter                  svn_fs_hotcopy_notify_t notify_func,
394289177Speter                  void* notify_baton,
395289177Speter                  svn_cancel_func_t cancel_func,
396289177Speter                  void* cancel_baton,
397289177Speter                  apr_pool_t *scratch_pool)
398289177Speter{
399289177Speter  svn_fs_x__data_t *src_ffd = src_fs->fsap_data;
400289177Speter  int max_files_per_dir = src_ffd->max_files_per_dir;
401289177Speter  svn_revnum_t src_min_unpacked_rev;
402289177Speter  svn_revnum_t dst_min_unpacked_rev;
403289177Speter  svn_revnum_t rev;
404289177Speter  apr_pool_t *iterpool;
405289177Speter
406289177Speter  /* Copy the min unpacked rev, and read its value. */
407289177Speter  SVN_ERR(svn_fs_x__read_min_unpacked_rev(&src_min_unpacked_rev, src_fs,
408289177Speter                                          scratch_pool));
409289177Speter  SVN_ERR(svn_fs_x__read_min_unpacked_rev(&dst_min_unpacked_rev, dst_fs,
410289177Speter                                          scratch_pool));
411289177Speter
412289177Speter  /* We only support packs coming from the hotcopy source.
413289177Speter    * The destination should not be packed independently from
414289177Speter    * the source. This also catches the case where users accidentally
415289177Speter    * swap the source and destination arguments. */
416289177Speter  if (src_min_unpacked_rev < dst_min_unpacked_rev)
417289177Speter    return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
418289177Speter                             _("The hotcopy destination already contains "
419289177Speter                               "more packed revisions (%lu) than the "
420289177Speter                               "hotcopy source contains (%lu)"),
421289177Speter                             dst_min_unpacked_rev - 1,
422289177Speter                             src_min_unpacked_rev - 1);
423289177Speter
424289177Speter  SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path,
425289177Speter                               PATH_MIN_UNPACKED_REV, scratch_pool));
426289177Speter
427289177Speter  if (cancel_func)
428289177Speter    SVN_ERR(cancel_func(cancel_baton));
429289177Speter
430289177Speter  /*
431289177Speter   * Copy the necessary rev files.
432289177Speter   */
433289177Speter
434289177Speter  iterpool = svn_pool_create(scratch_pool);
435289177Speter  /* First, copy packed shards. */
436289177Speter  for (rev = 0; rev < src_min_unpacked_rev; rev += max_files_per_dir)
437289177Speter    {
438289177Speter      svn_boolean_t skipped = TRUE;
439289177Speter      svn_revnum_t pack_end_rev;
440289177Speter
441289177Speter      svn_pool_clear(iterpool);
442289177Speter
443289177Speter      if (cancel_func)
444289177Speter        SVN_ERR(cancel_func(cancel_baton));
445289177Speter
446289177Speter      /* Copy the packed shard. */
447289177Speter      SVN_ERR(hotcopy_copy_packed_shard(&skipped, &dst_min_unpacked_rev,
448289177Speter                                        src_fs, dst_fs,
449289177Speter                                        rev, max_files_per_dir,
450289177Speter                                        iterpool));
451289177Speter
452289177Speter      pack_end_rev = rev + max_files_per_dir - 1;
453289177Speter
454289177Speter      /* Whenever this pack did not previously exist in the destination,
455289177Speter       * update 'current' to the most recent packed rev (so readers can see
456289177Speter       * new revisions which arrived in this pack). */
457289177Speter      if (pack_end_rev > dst_youngest)
458289177Speter        {
459289177Speter          SVN_ERR(svn_fs_x__write_current(dst_fs, pack_end_rev, iterpool));
460289177Speter        }
461289177Speter
462289177Speter      /* When notifying about packed shards, make things simpler by either
463289177Speter       * reporting a full revision range, i.e [pack start, pack end] or
464289177Speter       * reporting nothing. There is one case when this approach might not
465289177Speter       * be exact (incremental hotcopy with a pack replacing last unpacked
466289177Speter       * revisions), but generally this is good enough. */
467289177Speter      if (notify_func && !skipped)
468289177Speter        notify_func(notify_baton, rev, pack_end_rev, iterpool);
469289177Speter
470289177Speter      /* Now that all revisions have moved into the pack, the original
471289177Speter       * rev dir can be removed. */
472362181Sdim      SVN_ERR(svn_io_remove_dir2(svn_fs_x__path_shard(dst_fs, rev, iterpool),
473362181Sdim                                 TRUE, cancel_func, cancel_baton, iterpool));
474289177Speter    }
475289177Speter
476289177Speter  if (cancel_func)
477289177Speter    SVN_ERR(cancel_func(cancel_baton));
478289177Speter
479289177Speter  SVN_ERR_ASSERT(rev == src_min_unpacked_rev);
480289177Speter  SVN_ERR_ASSERT(src_min_unpacked_rev == dst_min_unpacked_rev);
481289177Speter
482289177Speter  /* Now, copy pairs of non-packed revisions and revprop files.
483289177Speter   * If necessary, update 'current' after copying all files from a shard. */
484289177Speter  for (; rev <= src_youngest; rev++)
485289177Speter    {
486289177Speter      svn_boolean_t skipped = TRUE;
487289177Speter
488289177Speter      svn_pool_clear(iterpool);
489289177Speter
490289177Speter      if (cancel_func)
491289177Speter        SVN_ERR(cancel_func(cancel_baton));
492289177Speter
493289177Speter      /* Copying non-packed revisions is racy in case the source repository is
494289177Speter       * being packed concurrently with this hotcopy operation.  With the pack
495289177Speter       * lock, however, the race is impossible, because hotcopy and pack
496289177Speter       * operations block each other.
497289177Speter       *
498289177Speter       * We assume that all revisions coming after 'min-unpacked-rev' really
499289177Speter       * are unpacked and that's not necessarily true with concurrent packing.
500289177Speter       * Don't try to be smart in this edge case, because handling it properly
501289177Speter       * might require copying *everything* from the start. Just abort the
502289177Speter       * hotcopy with an ENOENT (revision file moved to a pack, so it is no
503289177Speter       * longer where we expect it to be). */
504289177Speter
505289177Speter      /* Copy the rev file. */
506289177Speter      SVN_ERR(hotcopy_copy_shard_file(&skipped, src_revs_dir, dst_revs_dir,
507362181Sdim                                      rev, max_files_per_dir, FALSE,
508289177Speter                                      iterpool));
509289177Speter
510289177Speter      /* Copy the revprop file. */
511362181Sdim      SVN_ERR(hotcopy_copy_shard_file(&skipped, src_revs_dir, dst_revs_dir,
512362181Sdim                                      rev, max_files_per_dir, TRUE,
513289177Speter                                      iterpool));
514289177Speter
515289177Speter      /* Whenever this revision did not previously exist in the destination,
516289177Speter       * checkpoint the progress via 'current' (do that once per full shard
517289177Speter       * in order not to slow things down). */
518289177Speter      if (rev > dst_youngest)
519289177Speter        {
520289177Speter          if (max_files_per_dir && (rev % max_files_per_dir == 0))
521289177Speter            {
522289177Speter              SVN_ERR(svn_fs_x__write_current(dst_fs, rev, iterpool));
523289177Speter            }
524289177Speter        }
525289177Speter
526289177Speter      if (notify_func && !skipped)
527289177Speter        notify_func(notify_baton, rev, rev, iterpool);
528289177Speter    }
529289177Speter  svn_pool_destroy(iterpool);
530289177Speter
531289177Speter  /* We assume that all revisions were copied now, i.e. we didn't exit the
532289177Speter   * above loop early. 'rev' was last incremented during exit of the loop. */
533289177Speter  SVN_ERR_ASSERT(rev == src_youngest + 1);
534289177Speter
535289177Speter  return SVN_NO_ERROR;
536289177Speter}
537289177Speter
538289177Speter/* Baton for hotcopy_body(). */
539289177Spetertypedef struct hotcopy_body_baton_t {
540289177Speter  svn_fs_t *src_fs;
541289177Speter  svn_fs_t *dst_fs;
542289177Speter  svn_boolean_t incremental;
543289177Speter  svn_fs_hotcopy_notify_t notify_func;
544289177Speter  void *notify_baton;
545289177Speter  svn_cancel_func_t cancel_func;
546289177Speter  void *cancel_baton;
547289177Speter} hotcopy_body_baton_t;
548289177Speter
549289177Speter/* Perform a hotcopy, either normal or incremental.
550289177Speter *
551289177Speter * Normal hotcopy assumes that the destination exists as an empty
552289177Speter * directory. It behaves like an incremental hotcopy except that
553289177Speter * none of the copied files already exist in the destination.
554289177Speter *
555289177Speter * An incremental hotcopy copies only changed or new files to the destination,
556289177Speter * and removes files from the destination no longer present in the source.
557289177Speter * While the incremental hotcopy is running, readers should still be able
558362181Sdim * to access the destination repository without error and should not see
559289177Speter * revisions currently in progress of being copied. Readers are able to see
560289177Speter * new fully copied revisions even if the entire incremental hotcopy procedure
561289177Speter * has not yet completed.
562289177Speter *
563289177Speter * Writers are blocked out completely during the entire incremental hotcopy
564289177Speter * process to ensure consistency. This function assumes that the repository
565289177Speter * write-lock is held.
566289177Speter */
567289177Speterstatic svn_error_t *
568289177Speterhotcopy_body(void *baton,
569289177Speter             apr_pool_t *scratch_pool)
570289177Speter{
571289177Speter  hotcopy_body_baton_t *hbb = baton;
572289177Speter  svn_fs_t *src_fs = hbb->src_fs;
573289177Speter  svn_fs_t *dst_fs = hbb->dst_fs;
574289177Speter  svn_boolean_t incremental = hbb->incremental;
575289177Speter  svn_fs_hotcopy_notify_t notify_func = hbb->notify_func;
576289177Speter  void* notify_baton = hbb->notify_baton;
577289177Speter  svn_cancel_func_t cancel_func = hbb->cancel_func;
578289177Speter  void* cancel_baton = hbb->cancel_baton;
579289177Speter  svn_revnum_t src_youngest;
580289177Speter  svn_revnum_t dst_youngest;
581289177Speter  const char *src_revs_dir;
582289177Speter  const char *dst_revs_dir;
583289177Speter  const char *src_subdir;
584289177Speter  const char *dst_subdir;
585289177Speter  svn_node_kind_t kind;
586289177Speter
587289177Speter  /* Try to copy the config.
588289177Speter   *
589289177Speter   * ### We try copying the config file before doing anything else,
590289177Speter   * ### because higher layers will abort the hotcopy if we throw
591289177Speter   * ### an error from this function, and that renders the hotcopy
592289177Speter   * ### unusable anyway. */
593289177Speter  SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path, PATH_CONFIG,
594289177Speter                               scratch_pool));
595289177Speter
596289177Speter  if (cancel_func)
597289177Speter    SVN_ERR(cancel_func(cancel_baton));
598289177Speter
599289177Speter  /* Find the youngest revision in the source and destination.
600289177Speter   * We only support hotcopies from sources with an equal or greater amount
601289177Speter   * of revisions than the destination.
602289177Speter   * This also catches the case where users accidentally swap the
603289177Speter   * source and destination arguments. */
604289177Speter  SVN_ERR(svn_fs_x__read_current(&src_youngest, src_fs, scratch_pool));
605289177Speter  if (incremental)
606289177Speter    {
607289177Speter      SVN_ERR(svn_fs_x__youngest_rev(&dst_youngest, dst_fs, scratch_pool));
608289177Speter      if (src_youngest < dst_youngest)
609289177Speter        return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
610289177Speter                 _("The hotcopy destination already contains more revisions "
611289177Speter                   "(%lu) than the hotcopy source contains (%lu); are source "
612289177Speter                   "and destination swapped?"),
613289177Speter                   dst_youngest, src_youngest);
614289177Speter    }
615289177Speter  else
616289177Speter    dst_youngest = 0;
617289177Speter
618289177Speter  src_revs_dir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, scratch_pool);
619289177Speter  dst_revs_dir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, scratch_pool);
620289177Speter
621289177Speter  /* Ensure that the required folders exist in the destination
622289177Speter   * before actually copying the revisions and revprops. */
623289177Speter  SVN_ERR(svn_io_make_dir_recursively(dst_revs_dir, scratch_pool));
624289177Speter  if (cancel_func)
625289177Speter    SVN_ERR(cancel_func(cancel_baton));
626289177Speter
627289177Speter  /* Split the logic for new and old FS formats. The latter is much simpler
628289177Speter   * due to the absense of sharding and packing. However, it requires special
629289177Speter   * care when updating the 'current' file (which contains not just the
630289177Speter   * revision number, but also the next-ID counters). */
631289177Speter  SVN_ERR(hotcopy_revisions(src_fs, dst_fs, src_youngest, dst_youngest,
632289177Speter                            incremental, src_revs_dir, dst_revs_dir,
633289177Speter                            notify_func, notify_baton,
634289177Speter                            cancel_func, cancel_baton, scratch_pool));
635289177Speter  SVN_ERR(svn_fs_x__write_current(dst_fs, src_youngest, scratch_pool));
636289177Speter
637289177Speter  /* Replace the locks tree.
638289177Speter   * This is racy in case readers are currently trying to list locks in
639289177Speter   * the destination. However, we need to get rid of stale locks.
640289177Speter   * This is the simplest way of doing this, so we accept this small race. */
641289177Speter  dst_subdir = svn_dirent_join(dst_fs->path, PATH_LOCKS_DIR, scratch_pool);
642289177Speter  SVN_ERR(svn_io_remove_dir2(dst_subdir, TRUE, cancel_func, cancel_baton,
643289177Speter                             scratch_pool));
644289177Speter  src_subdir = svn_dirent_join(src_fs->path, PATH_LOCKS_DIR, scratch_pool);
645289177Speter  SVN_ERR(svn_io_check_path(src_subdir, &kind, scratch_pool));
646289177Speter  if (kind == svn_node_dir)
647289177Speter    SVN_ERR(svn_io_copy_dir_recursively(src_subdir, dst_fs->path,
648289177Speter                                        PATH_LOCKS_DIR, TRUE,
649289177Speter                                        cancel_func, cancel_baton,
650289177Speter                                        scratch_pool));
651289177Speter
652289177Speter  /*
653289177Speter   * NB: Data copied below is only read by writers, not readers.
654289177Speter   *     Writers are still locked out at this point.
655289177Speter   */
656289177Speter
657289177Speter  /* Copy the rep cache and then remove entries for revisions
658289177Speter   * younger than the destination's youngest revision. */
659289177Speter  src_subdir = svn_dirent_join(src_fs->path, REP_CACHE_DB_NAME, scratch_pool);
660289177Speter  dst_subdir = svn_dirent_join(dst_fs->path, REP_CACHE_DB_NAME, scratch_pool);
661289177Speter  SVN_ERR(svn_io_check_path(src_subdir, &kind, scratch_pool));
662289177Speter  if (kind == svn_node_file)
663289177Speter    {
664289177Speter      /* Copy the rep cache and then remove entries for revisions
665289177Speter       * that did not make it into the destination. */
666289177Speter      SVN_ERR(svn_sqlite__hotcopy(src_subdir, dst_subdir, scratch_pool));
667362181Sdim
668362181Sdim      /* The source might have r/o flags set on it - which would be
669362181Sdim         carried over to the copy. */
670362181Sdim      SVN_ERR(svn_io_set_file_read_write(dst_subdir, FALSE, scratch_pool));
671289177Speter      SVN_ERR(svn_fs_x__del_rep_reference(dst_fs, src_youngest,
672289177Speter                                          scratch_pool));
673289177Speter    }
674289177Speter
675289177Speter  /* Copy the txn-current file. */
676289177Speter  SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path,
677289177Speter                                PATH_TXN_CURRENT, scratch_pool));
678289177Speter
679289177Speter  /* If a revprop generation file exists in the source filesystem,
680289177Speter   * reset it to zero (since this is on a different path, it will not
681289177Speter   * overlap with data already in cache).  Also, clean up stale files
682289177Speter   * used for the named atomics implementation. */
683289177Speter  SVN_ERR(svn_fs_x__reset_revprop_generation_file(dst_fs, scratch_pool));
684289177Speter
685362181Sdim  /* Hotcopied FS is complete. Stamp it with a format file. */
686362181Sdim  SVN_ERR(svn_fs_x__write_format(dst_fs, TRUE, scratch_pool));
687362181Sdim
688289177Speter  return SVN_NO_ERROR;
689289177Speter}
690289177Speter
691362181Sdimsvn_error_t *
692362181Sdimsvn_fs_x__hotcopy(svn_fs_t *src_fs,
693362181Sdim                  svn_fs_t *dst_fs,
694362181Sdim                  const char *src_path,
695362181Sdim                  const char *dst_path,
696362181Sdim                  svn_boolean_t incremental,
697362181Sdim                  svn_fs_hotcopy_notify_t notify_func,
698362181Sdim                  void *notify_baton,
699362181Sdim                  svn_cancel_func_t cancel_func,
700362181Sdim                  void *cancel_baton,
701362181Sdim                  svn_mutex__t *common_pool_lock,
702362181Sdim                  apr_pool_t *scratch_pool,
703362181Sdim                  apr_pool_t *common_pool)
704289177Speter{
705362181Sdim  hotcopy_body_baton_t hbb;
706289177Speter
707362181Sdim  if (cancel_func)
708362181Sdim    SVN_ERR(cancel_func(cancel_baton));
709289177Speter
710362181Sdim  SVN_ERR(svn_fs_x__open(src_fs, src_path, scratch_pool));
711289177Speter
712289177Speter  if (incremental)
713289177Speter    {
714289177Speter      const char *dst_format_abspath;
715289177Speter      svn_node_kind_t dst_format_kind;
716289177Speter
717289177Speter      /* Check destination format to be sure we know how to incrementally
718289177Speter       * hotcopy to the destination FS. */
719289177Speter      dst_format_abspath = svn_dirent_join(dst_path, PATH_FORMAT,
720289177Speter                                           scratch_pool);
721289177Speter      SVN_ERR(svn_io_check_path(dst_format_abspath, &dst_format_kind,
722289177Speter                                scratch_pool));
723289177Speter      if (dst_format_kind == svn_node_none)
724289177Speter        {
725362181Sdim          /* No destination?  Fallback to a non-incremental hotcopy. */
726362181Sdim          incremental = FALSE;
727289177Speter        }
728289177Speter    }
729362181Sdim
730362181Sdim  if (incremental)
731362181Sdim    {
732362181Sdim      /* Check the existing repository. */
733362181Sdim      SVN_ERR(svn_fs_x__open(dst_fs, dst_path, scratch_pool));
734362181Sdim      SVN_ERR(hotcopy_incremental_check_preconditions(src_fs, dst_fs));
735362181Sdim
736362181Sdim      SVN_ERR(svn_fs_x__initialize_shared_data(dst_fs, common_pool_lock,
737362181Sdim                                               scratch_pool, common_pool));
738362181Sdim      SVN_ERR(svn_fs_x__initialize_caches(dst_fs, scratch_pool));
739362181Sdim    }
740289177Speter  else
741289177Speter    {
742289177Speter      /* Start out with an empty destination using the same configuration
743289177Speter       * as the source. */
744362181Sdim      svn_fs_x__data_t *src_ffd = src_fs->fsap_data;
745362181Sdim
746362181Sdim      /* Create the DST_FS repository with the same layout as SRC_FS. */
747362181Sdim      SVN_ERR(svn_fs_x__create_file_tree(dst_fs, dst_path, src_ffd->format,
748362181Sdim                                         src_ffd->max_files_per_dir,
749362181Sdim                                         scratch_pool));
750362181Sdim
751362181Sdim      /* Copy the UUID.  Hotcopy destination receives a new instance ID, but
752362181Sdim       * has the same filesystem UUID as the source. */
753362181Sdim      SVN_ERR(svn_fs_x__set_uuid(dst_fs, src_fs->uuid, NULL, TRUE,
754362181Sdim                                 scratch_pool));
755362181Sdim
756362181Sdim      /* Remove revision 0 contents.  Otherwise, it may not get overwritten
757362181Sdim       * due to having a newer timestamp. */
758362181Sdim      SVN_ERR(hotcopy_remove_file(svn_fs_x__path_rev(dst_fs, 0,
759362181Sdim                                                     scratch_pool),
760362181Sdim                                  scratch_pool));
761362181Sdim      SVN_ERR(hotcopy_remove_file(svn_fs_x__path_revprops(dst_fs, 0,
762362181Sdim                                                          scratch_pool),
763362181Sdim                                  scratch_pool));
764362181Sdim
765362181Sdim      SVN_ERR(svn_fs_x__initialize_shared_data(dst_fs, common_pool_lock,
766362181Sdim                                               scratch_pool, common_pool));
767362181Sdim      SVN_ERR(svn_fs_x__initialize_caches(dst_fs, scratch_pool));
768289177Speter    }
769289177Speter
770362181Sdim  if (cancel_func)
771362181Sdim    SVN_ERR(cancel_func(cancel_baton));
772289177Speter
773289177Speter  hbb.src_fs = src_fs;
774289177Speter  hbb.dst_fs = dst_fs;
775289177Speter  hbb.incremental = incremental;
776289177Speter  hbb.notify_func = notify_func;
777289177Speter  hbb.notify_baton = notify_baton;
778289177Speter  hbb.cancel_func = cancel_func;
779289177Speter  hbb.cancel_baton = cancel_baton;
780289177Speter
781362181Sdim  /* Lock the destination in the incremental mode.  For a non-incremental
782362181Sdim   * hotcopy, don't take any locks.  In that case the destination cannot be
783362181Sdim   * opened until the hotcopy finishes, and we don't have to worry about
784362181Sdim   * concurrency. */
785362181Sdim  if (incremental)
786362181Sdim    SVN_ERR(svn_fs_x__with_all_locks(dst_fs, hotcopy_body, &hbb,
787362181Sdim                                     scratch_pool));
788362181Sdim  else
789362181Sdim    SVN_ERR(hotcopy_body(&hbb, scratch_pool));
790362181Sdim
791289177Speter  return SVN_NO_ERROR;
792289177Speter}
793