1/* hotcopy.c --- FS hotcopy functionality for FSFS 2 * 3 * ==================================================================== 4 * Licensed to the Apache Software Foundation (ASF) under one 5 * or more contributor license agreements. See the NOTICE file 6 * distributed with this work for additional information 7 * regarding copyright ownership. The ASF licenses this file 8 * to you under the Apache License, Version 2.0 (the 9 * "License"); you may not use this file except in compliance 10 * with the License. You may obtain a copy of the License at 11 * 12 * http://www.apache.org/licenses/LICENSE-2.0 13 * 14 * Unless required by applicable law or agreed to in writing, 15 * software distributed under the License is distributed on an 16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 * KIND, either express or implied. See the License for the 18 * specific language governing permissions and limitations 19 * under the License. 20 * ==================================================================== 21 */ 22#include "svn_pools.h" 23#include "svn_path.h" 24#include "svn_dirent_uri.h" 25 26#include "fs_fs.h" 27#include "hotcopy.h" 28#include "util.h" 29#include "recovery.h" 30#include "revprops.h" 31#include "rep-cache.h" 32 33#include "../libsvn_fs/fs-loader.h" 34 35#include "svn_private_config.h" 36 37/* Like svn_io_dir_file_copy(), but doesn't copy files that exist at 38 * the destination and do not differ in terms of kind, size, and mtime. 39 * Set *SKIPPED_P to FALSE only if the file was copied, do not change 40 * the value in *SKIPPED_P otherwise. SKIPPED_P may be NULL if not 41 * required. */ 42static svn_error_t * 43hotcopy_io_dir_file_copy(svn_boolean_t *skipped_p, 44 const char *src_path, 45 const char *dst_path, 46 const char *file, 47 apr_pool_t *scratch_pool) 48{ 49 const svn_io_dirent2_t *src_dirent; 50 const svn_io_dirent2_t *dst_dirent; 51 const char *src_target; 52 const char *dst_target; 53 54 /* Does the destination already exist? If not, we must copy it. */ 55 dst_target = svn_dirent_join(dst_path, file, scratch_pool); 56 SVN_ERR(svn_io_stat_dirent2(&dst_dirent, dst_target, FALSE, TRUE, 57 scratch_pool, scratch_pool)); 58 if (dst_dirent->kind != svn_node_none) 59 { 60 /* If the destination's stat information indicates that the file 61 * is equal to the source, don't bother copying the file again. */ 62 src_target = svn_dirent_join(src_path, file, scratch_pool); 63 SVN_ERR(svn_io_stat_dirent2(&src_dirent, src_target, FALSE, FALSE, 64 scratch_pool, scratch_pool)); 65 if (src_dirent->kind == dst_dirent->kind && 66 src_dirent->special == dst_dirent->special && 67 src_dirent->filesize == dst_dirent->filesize && 68 src_dirent->mtime <= dst_dirent->mtime) 69 return SVN_NO_ERROR; 70 } 71 72 if (skipped_p) 73 *skipped_p = FALSE; 74 75 return svn_error_trace(svn_io_dir_file_copy(src_path, dst_path, file, 76 scratch_pool)); 77} 78 79/* Set *NAME_P to the UTF-8 representation of directory entry NAME. 80 * NAME is in the internal encoding used by APR; PARENT is in 81 * UTF-8 and in internal (not local) style. 82 * 83 * Use PARENT only for generating an error string if the conversion 84 * fails because NAME could not be represented in UTF-8. In that 85 * case, return a two-level error in which the outer error's message 86 * mentions PARENT, but the inner error's message does not mention 87 * NAME (except possibly in hex) since NAME may not be printable. 88 * Such a compound error at least allows the user to go looking in the 89 * right directory for the problem. 90 * 91 * If there is any other error, just return that error directly. 92 * 93 * If there is any error, the effect on *NAME_P is undefined. 94 * 95 * *NAME_P and NAME may refer to the same storage. 96 */ 97static svn_error_t * 98entry_name_to_utf8(const char **name_p, 99 const char *name, 100 const char *parent, 101 apr_pool_t *pool) 102{ 103 svn_error_t *err = svn_path_cstring_to_utf8(name_p, name, pool); 104 if (err && err->apr_err == APR_EINVAL) 105 { 106 return svn_error_createf(err->apr_err, err, 107 _("Error converting entry " 108 "in directory '%s' to UTF-8"), 109 svn_dirent_local_style(parent, pool)); 110 } 111 return err; 112} 113 114/* Like svn_io_copy_dir_recursively() but doesn't copy regular files that 115 * exist in the destination and do not differ from the source in terms of 116 * kind, size, and mtime. Set *SKIPPED_P to FALSE only if at least one 117 * file was copied, do not change the value in *SKIPPED_P otherwise. 118 * SKIPPED_P may be NULL if not required. */ 119static svn_error_t * 120hotcopy_io_copy_dir_recursively(svn_boolean_t *skipped_p, 121 const char *src, 122 const char *dst_parent, 123 const char *dst_basename, 124 svn_boolean_t copy_perms, 125 svn_cancel_func_t cancel_func, 126 void *cancel_baton, 127 apr_pool_t *pool) 128{ 129 svn_node_kind_t kind; 130 apr_status_t status; 131 const char *dst_path; 132 apr_dir_t *this_dir; 133 apr_finfo_t this_entry; 134 apr_int32_t flags = APR_FINFO_TYPE | APR_FINFO_NAME; 135 136 /* Make a subpool for recursion */ 137 apr_pool_t *subpool = svn_pool_create(pool); 138 139 /* The 'dst_path' is simply dst_parent/dst_basename */ 140 dst_path = svn_dirent_join(dst_parent, dst_basename, pool); 141 142 /* Sanity checks: SRC and DST_PARENT are directories, and 143 DST_BASENAME doesn't already exist in DST_PARENT. */ 144 SVN_ERR(svn_io_check_path(src, &kind, subpool)); 145 if (kind != svn_node_dir) 146 return svn_error_createf(SVN_ERR_NODE_UNEXPECTED_KIND, NULL, 147 _("Source '%s' is not a directory"), 148 svn_dirent_local_style(src, pool)); 149 150 SVN_ERR(svn_io_check_path(dst_parent, &kind, subpool)); 151 if (kind != svn_node_dir) 152 return svn_error_createf(SVN_ERR_NODE_UNEXPECTED_KIND, NULL, 153 _("Destination '%s' is not a directory"), 154 svn_dirent_local_style(dst_parent, pool)); 155 156 SVN_ERR(svn_io_check_path(dst_path, &kind, subpool)); 157 158 /* Create the new directory. */ 159 /* ### TODO: copy permissions (needs apr_file_attrs_get()) */ 160 SVN_ERR(svn_io_make_dir_recursively(dst_path, pool)); 161 162 /* Loop over the dirents in SRC. ('.' and '..' are auto-excluded) */ 163 SVN_ERR(svn_io_dir_open(&this_dir, src, subpool)); 164 165 for (status = apr_dir_read(&this_entry, flags, this_dir); 166 status == APR_SUCCESS; 167 status = apr_dir_read(&this_entry, flags, this_dir)) 168 { 169 if ((this_entry.name[0] == '.') 170 && ((this_entry.name[1] == '\0') 171 || ((this_entry.name[1] == '.') 172 && (this_entry.name[2] == '\0')))) 173 { 174 continue; 175 } 176 else 177 { 178 const char *entryname_utf8; 179 180 if (cancel_func) 181 SVN_ERR(cancel_func(cancel_baton)); 182 183 SVN_ERR(entry_name_to_utf8(&entryname_utf8, this_entry.name, 184 src, subpool)); 185 if (this_entry.filetype == APR_REG) /* regular file */ 186 { 187 SVN_ERR(hotcopy_io_dir_file_copy(skipped_p, src, dst_path, 188 entryname_utf8, subpool)); 189 } 190 else if (this_entry.filetype == APR_LNK) /* symlink */ 191 { 192 const char *src_target = svn_dirent_join(src, entryname_utf8, 193 subpool); 194 const char *dst_target = svn_dirent_join(dst_path, 195 entryname_utf8, 196 subpool); 197 SVN_ERR(svn_io_copy_link(src_target, dst_target, 198 subpool)); 199 } 200 else if (this_entry.filetype == APR_DIR) /* recurse */ 201 { 202 const char *src_target; 203 204 /* Prevent infinite recursion by filtering off our 205 newly created destination path. */ 206 if (strcmp(src, dst_parent) == 0 207 && strcmp(entryname_utf8, dst_basename) == 0) 208 continue; 209 210 src_target = svn_dirent_join(src, entryname_utf8, subpool); 211 SVN_ERR(hotcopy_io_copy_dir_recursively(skipped_p, 212 src_target, 213 dst_path, 214 entryname_utf8, 215 copy_perms, 216 cancel_func, 217 cancel_baton, 218 subpool)); 219 } 220 /* ### support other APR node types someday?? */ 221 222 } 223 } 224 225 if (! (APR_STATUS_IS_ENOENT(status))) 226 return svn_error_wrap_apr(status, _("Can't read directory '%s'"), 227 svn_dirent_local_style(src, pool)); 228 229 status = apr_dir_close(this_dir); 230 if (status) 231 return svn_error_wrap_apr(status, _("Error closing directory '%s'"), 232 svn_dirent_local_style(src, pool)); 233 234 /* Free any memory used by recursion */ 235 svn_pool_destroy(subpool); 236 237 return SVN_NO_ERROR; 238} 239 240/* Copy an un-packed revision or revprop file for revision REV from SRC_SUBDIR 241 * to DST_SUBDIR. Assume a sharding layout based on MAX_FILES_PER_DIR. 242 * Set *SKIPPED_P to FALSE only if the file was copied, do not change the 243 * value in *SKIPPED_P otherwise. SKIPPED_P may be NULL if not required. 244 * Use SCRATCH_POOL for temporary allocations. */ 245static svn_error_t * 246hotcopy_copy_shard_file(svn_boolean_t *skipped_p, 247 const char *src_subdir, 248 const char *dst_subdir, 249 svn_revnum_t rev, 250 int max_files_per_dir, 251 apr_pool_t *scratch_pool) 252{ 253 const char *src_subdir_shard = src_subdir, 254 *dst_subdir_shard = dst_subdir; 255 256 if (max_files_per_dir) 257 { 258 const char *shard = apr_psprintf(scratch_pool, "%ld", 259 rev / max_files_per_dir); 260 src_subdir_shard = svn_dirent_join(src_subdir, shard, scratch_pool); 261 dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); 262 263 if (rev % max_files_per_dir == 0) 264 { 265 SVN_ERR(svn_io_make_dir_recursively(dst_subdir_shard, scratch_pool)); 266 SVN_ERR(svn_io_copy_perms(dst_subdir, dst_subdir_shard, 267 scratch_pool)); 268 } 269 } 270 271 SVN_ERR(hotcopy_io_dir_file_copy(skipped_p, 272 src_subdir_shard, dst_subdir_shard, 273 apr_psprintf(scratch_pool, "%ld", rev), 274 scratch_pool)); 275 276 return SVN_NO_ERROR; 277} 278 279 280/* Copy a packed shard containing revision REV, and which contains 281 * MAX_FILES_PER_DIR revisions, from SRC_FS to DST_FS. 282 * Update *DST_MIN_UNPACKED_REV in case the shard is new in DST_FS. 283 * Do not re-copy data which already exists in DST_FS. 284 * Set *SKIPPED_P to FALSE only if at least one part of the shard 285 * was copied, do not change the value in *SKIPPED_P otherwise. 286 * SKIPPED_P may be NULL if not required. 287 * Use SCRATCH_POOL for temporary allocations. */ 288static svn_error_t * 289hotcopy_copy_packed_shard(svn_boolean_t *skipped_p, 290 svn_revnum_t *dst_min_unpacked_rev, 291 svn_fs_t *src_fs, 292 svn_fs_t *dst_fs, 293 svn_revnum_t rev, 294 int max_files_per_dir, 295 apr_pool_t *scratch_pool) 296{ 297 const char *src_subdir; 298 const char *dst_subdir; 299 const char *packed_shard; 300 const char *src_subdir_packed_shard; 301 svn_revnum_t revprop_rev; 302 apr_pool_t *iterpool; 303 fs_fs_data_t *src_ffd = src_fs->fsap_data; 304 305 /* Copy the packed shard. */ 306 src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, scratch_pool); 307 dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, scratch_pool); 308 packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD, 309 rev / max_files_per_dir); 310 src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard, 311 scratch_pool); 312 SVN_ERR(hotcopy_io_copy_dir_recursively(skipped_p, src_subdir_packed_shard, 313 dst_subdir, packed_shard, 314 TRUE /* copy_perms */, 315 NULL /* cancel_func */, NULL, 316 scratch_pool)); 317 318 /* Copy revprops belonging to revisions in this pack. */ 319 src_subdir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, scratch_pool); 320 dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, scratch_pool); 321 322 if ( src_ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT 323 || src_ffd->min_unpacked_rev < rev + max_files_per_dir) 324 { 325 /* copy unpacked revprops rev by rev */ 326 iterpool = svn_pool_create(scratch_pool); 327 for (revprop_rev = rev; 328 revprop_rev < rev + max_files_per_dir; 329 revprop_rev++) 330 { 331 svn_pool_clear(iterpool); 332 333 SVN_ERR(hotcopy_copy_shard_file(skipped_p, src_subdir, dst_subdir, 334 revprop_rev, max_files_per_dir, 335 iterpool)); 336 } 337 svn_pool_destroy(iterpool); 338 } 339 else 340 { 341 /* revprop for revision 0 will never be packed */ 342 if (rev == 0) 343 SVN_ERR(hotcopy_copy_shard_file(skipped_p, src_subdir, dst_subdir, 344 0, max_files_per_dir, 345 scratch_pool)); 346 347 /* packed revprops folder */ 348 packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD, 349 rev / max_files_per_dir); 350 src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard, 351 scratch_pool); 352 SVN_ERR(hotcopy_io_copy_dir_recursively(skipped_p, 353 src_subdir_packed_shard, 354 dst_subdir, packed_shard, 355 TRUE /* copy_perms */, 356 NULL /* cancel_func */, NULL, 357 scratch_pool)); 358 } 359 360 /* If necessary, update the min-unpacked rev file in the hotcopy. */ 361 if (*dst_min_unpacked_rev < rev + max_files_per_dir) 362 { 363 *dst_min_unpacked_rev = rev + max_files_per_dir; 364 SVN_ERR(svn_fs_fs__write_min_unpacked_rev(dst_fs, 365 *dst_min_unpacked_rev, 366 scratch_pool)); 367 } 368 369 return SVN_NO_ERROR; 370} 371 372/* Remove file PATH, if it exists - even if it is read-only. 373 * Use POOL for temporary allocations. */ 374static svn_error_t * 375hotcopy_remove_file(const char *path, 376 apr_pool_t *pool) 377{ 378 /* Make the rev file writable and remove it. */ 379 SVN_ERR(svn_io_set_file_read_write(path, TRUE, pool)); 380 SVN_ERR(svn_io_remove_file2(path, TRUE, pool)); 381 382 return SVN_NO_ERROR; 383} 384 385 386/* Remove revision or revprop files between START_REV (inclusive) and 387 * END_REV (non-inclusive) from folder DST_SUBDIR in DST_FS. Assume 388 * sharding as per MAX_FILES_PER_DIR. 389 * Use SCRATCH_POOL for temporary allocations. */ 390static svn_error_t * 391hotcopy_remove_files(svn_fs_t *dst_fs, 392 const char *dst_subdir, 393 svn_revnum_t start_rev, 394 svn_revnum_t end_rev, 395 int max_files_per_dir, 396 apr_pool_t *scratch_pool) 397{ 398 const char *shard; 399 const char *dst_subdir_shard; 400 svn_revnum_t rev; 401 apr_pool_t *iterpool; 402 403 /* Pre-compute paths for initial shard. */ 404 shard = apr_psprintf(scratch_pool, "%ld", start_rev / max_files_per_dir); 405 dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); 406 407 iterpool = svn_pool_create(scratch_pool); 408 for (rev = start_rev; rev < end_rev; rev++) 409 { 410 svn_pool_clear(iterpool); 411 412 /* If necessary, update paths for shard. */ 413 if (rev != start_rev && rev % max_files_per_dir == 0) 414 { 415 shard = apr_psprintf(iterpool, "%ld", rev / max_files_per_dir); 416 dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); 417 } 418 419 /* remove files for REV */ 420 SVN_ERR(hotcopy_remove_file(svn_dirent_join(dst_subdir_shard, 421 apr_psprintf(iterpool, 422 "%ld", rev), 423 iterpool), 424 iterpool)); 425 } 426 427 svn_pool_destroy(iterpool); 428 429 return SVN_NO_ERROR; 430} 431 432/* Remove revisions between START_REV (inclusive) and END_REV (non-inclusive) 433 * from DST_FS. Assume sharding as per MAX_FILES_PER_DIR. 434 * Use SCRATCH_POOL for temporary allocations. */ 435static svn_error_t * 436hotcopy_remove_rev_files(svn_fs_t *dst_fs, 437 svn_revnum_t start_rev, 438 svn_revnum_t end_rev, 439 int max_files_per_dir, 440 apr_pool_t *scratch_pool) 441{ 442 SVN_ERR_ASSERT(start_rev <= end_rev); 443 SVN_ERR(hotcopy_remove_files(dst_fs, 444 svn_dirent_join(dst_fs->path, 445 PATH_REVS_DIR, 446 scratch_pool), 447 start_rev, end_rev, 448 max_files_per_dir, scratch_pool)); 449 450 return SVN_NO_ERROR; 451} 452 453/* Remove revision properties between START_REV (inclusive) and END_REV 454 * (non-inclusive) from DST_FS. Assume sharding as per MAX_FILES_PER_DIR. 455 * Use SCRATCH_POOL for temporary allocations. Revision 0 revprops will 456 * not be deleted. */ 457static svn_error_t * 458hotcopy_remove_revprop_files(svn_fs_t *dst_fs, 459 svn_revnum_t start_rev, 460 svn_revnum_t end_rev, 461 int max_files_per_dir, 462 apr_pool_t *scratch_pool) 463{ 464 SVN_ERR_ASSERT(start_rev <= end_rev); 465 466 /* don't delete rev 0 props */ 467 SVN_ERR(hotcopy_remove_files(dst_fs, 468 svn_dirent_join(dst_fs->path, 469 PATH_REVPROPS_DIR, 470 scratch_pool), 471 start_rev ? start_rev : 1, end_rev, 472 max_files_per_dir, scratch_pool)); 473 474 return SVN_NO_ERROR; 475} 476 477/* Verify that DST_FS is a suitable destination for an incremental 478 * hotcopy from SRC_FS. */ 479static svn_error_t * 480hotcopy_incremental_check_preconditions(svn_fs_t *src_fs, 481 svn_fs_t *dst_fs, 482 apr_pool_t *pool) 483{ 484 fs_fs_data_t *src_ffd = src_fs->fsap_data; 485 fs_fs_data_t *dst_ffd = dst_fs->fsap_data; 486 487 /* We only support incremental hotcopy between the same format. */ 488 if (src_ffd->format != dst_ffd->format) 489 return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, 490 _("The FSFS format (%d) of the hotcopy source does not match the " 491 "FSFS format (%d) of the hotcopy destination; please upgrade " 492 "both repositories to the same format"), 493 src_ffd->format, dst_ffd->format); 494 495 /* Make sure the UUID of source and destination match up. 496 * We don't want to copy over a different repository. */ 497 if (strcmp(src_fs->uuid, dst_fs->uuid) != 0) 498 return svn_error_create(SVN_ERR_RA_UUID_MISMATCH, NULL, 499 _("The UUID of the hotcopy source does " 500 "not match the UUID of the hotcopy " 501 "destination")); 502 503 /* Also require same shard size. */ 504 if (src_ffd->max_files_per_dir != dst_ffd->max_files_per_dir) 505 return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL, 506 _("The sharding layout configuration " 507 "of the hotcopy source does not match " 508 "the sharding layout configuration of " 509 "the hotcopy destination")); 510 return SVN_NO_ERROR; 511} 512 513/* Remove folder PATH. Ignore errors due to the sub-tree not being empty. 514 * CANCEL_FUNC and CANCEL_BATON do the usual thing. 515 * Use POOL for temporary allocations. 516 */ 517static svn_error_t * 518remove_folder(const char *path, 519 svn_cancel_func_t cancel_func, 520 void *cancel_baton, 521 apr_pool_t *pool) 522{ 523 svn_error_t *err = svn_io_remove_dir2(path, TRUE, 524 cancel_func, cancel_baton, pool); 525 526 if (err && APR_STATUS_IS_ENOTEMPTY(err->apr_err)) 527 { 528 svn_error_clear(err); 529 err = SVN_NO_ERROR; 530 } 531 532 return svn_error_trace(err); 533} 534 535/* Copy the revision and revprop files (possibly sharded / packed) from 536 * SRC_FS to DST_FS. Do not re-copy data which already exists in DST_FS. 537 * When copying packed or unpacked shards, checkpoint the result in DST_FS 538 * for every shard by updating the 'current' file if necessary. Assume 539 * the >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT filesystem format without 540 * global next-ID counters. Indicate progress via the optional NOTIFY_FUNC 541 * callback using NOTIFY_BATON. Use POOL for temporary allocations. 542 */ 543static svn_error_t * 544hotcopy_revisions(svn_fs_t *src_fs, 545 svn_fs_t *dst_fs, 546 svn_revnum_t src_youngest, 547 svn_revnum_t dst_youngest, 548 svn_boolean_t incremental, 549 const char *src_revs_dir, 550 const char *dst_revs_dir, 551 const char *src_revprops_dir, 552 const char *dst_revprops_dir, 553 svn_fs_hotcopy_notify_t notify_func, 554 void* notify_baton, 555 svn_cancel_func_t cancel_func, 556 void* cancel_baton, 557 apr_pool_t *pool) 558{ 559 fs_fs_data_t *src_ffd = src_fs->fsap_data; 560 fs_fs_data_t *dst_ffd = dst_fs->fsap_data; 561 int max_files_per_dir = src_ffd->max_files_per_dir; 562 svn_revnum_t src_min_unpacked_rev; 563 svn_revnum_t dst_min_unpacked_rev; 564 svn_revnum_t rev; 565 apr_pool_t *iterpool; 566 567 /* Copy the min unpacked rev, and read its value. */ 568 if (src_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) 569 { 570 SVN_ERR(svn_fs_fs__read_min_unpacked_rev(&src_min_unpacked_rev, 571 src_fs, pool)); 572 SVN_ERR(svn_fs_fs__read_min_unpacked_rev(&dst_min_unpacked_rev, 573 dst_fs, pool)); 574 575 /* We only support packs coming from the hotcopy source. 576 * The destination should not be packed independently from 577 * the source. This also catches the case where users accidentally 578 * swap the source and destination arguments. */ 579 if (src_min_unpacked_rev < dst_min_unpacked_rev) 580 return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, 581 _("The hotcopy destination already contains " 582 "more packed revisions (%lu) than the " 583 "hotcopy source contains (%lu)"), 584 dst_min_unpacked_rev - 1, 585 src_min_unpacked_rev - 1); 586 587 SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path, 588 PATH_MIN_UNPACKED_REV, pool)); 589 } 590 else 591 { 592 src_min_unpacked_rev = 0; 593 dst_min_unpacked_rev = 0; 594 } 595 596 if (cancel_func) 597 SVN_ERR(cancel_func(cancel_baton)); 598 599 /* 600 * Copy the necessary rev files. 601 */ 602 603 iterpool = svn_pool_create(pool); 604 /* First, copy packed shards. */ 605 for (rev = 0; rev < src_min_unpacked_rev; rev += max_files_per_dir) 606 { 607 svn_boolean_t skipped = TRUE; 608 svn_revnum_t pack_end_rev; 609 610 svn_pool_clear(iterpool); 611 612 if (cancel_func) 613 SVN_ERR(cancel_func(cancel_baton)); 614 615 /* Copy the packed shard. */ 616 SVN_ERR(hotcopy_copy_packed_shard(&skipped, &dst_min_unpacked_rev, 617 src_fs, dst_fs, 618 rev, max_files_per_dir, 619 iterpool)); 620 621 pack_end_rev = rev + max_files_per_dir - 1; 622 623 /* Whenever this pack did not previously exist in the destination, 624 * update 'current' to the most recent packed rev (so readers can see 625 * new revisions which arrived in this pack). */ 626 if (pack_end_rev > dst_youngest) 627 { 628 SVN_ERR(svn_fs_fs__write_current(dst_fs, pack_end_rev, 0, 0, 629 iterpool)); 630 } 631 632 /* When notifying about packed shards, make things simpler by either 633 * reporting a full revision range, i.e [pack start, pack end] or 634 * reporting nothing. There is one case when this approach might not 635 * be exact (incremental hotcopy with a pack replacing last unpacked 636 * revisions), but generally this is good enough. */ 637 if (notify_func && !skipped) 638 notify_func(notify_baton, rev, pack_end_rev, iterpool); 639 640 /* Remove revision files which are now packed. */ 641 if (incremental) 642 { 643 SVN_ERR(hotcopy_remove_rev_files(dst_fs, rev, 644 rev + max_files_per_dir, 645 max_files_per_dir, iterpool)); 646 if (dst_ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) 647 SVN_ERR(hotcopy_remove_revprop_files(dst_fs, rev, 648 rev + max_files_per_dir, 649 max_files_per_dir, 650 iterpool)); 651 } 652 653 /* Now that all revisions have moved into the pack, the original 654 * rev dir can be removed. */ 655 SVN_ERR(remove_folder(svn_fs_fs__path_rev_shard(dst_fs, rev, iterpool), 656 cancel_func, cancel_baton, iterpool)); 657 if (rev > 0 && dst_ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) 658 SVN_ERR(remove_folder(svn_fs_fs__path_revprops_shard(dst_fs, rev, 659 iterpool), 660 cancel_func, cancel_baton, iterpool)); 661 } 662 663 if (cancel_func) 664 SVN_ERR(cancel_func(cancel_baton)); 665 666 SVN_ERR_ASSERT(rev == src_min_unpacked_rev); 667 SVN_ERR_ASSERT(src_min_unpacked_rev == dst_min_unpacked_rev); 668 669 /* Now, copy pairs of non-packed revisions and revprop files. 670 * If necessary, update 'current' after copying all files from a shard. */ 671 for (; rev <= src_youngest; rev++) 672 { 673 svn_boolean_t skipped = TRUE; 674 675 svn_pool_clear(iterpool); 676 677 if (cancel_func) 678 SVN_ERR(cancel_func(cancel_baton)); 679 680 /* Copying non-packed revisions is racy in case the source repository is 681 * being packed concurrently with this hotcopy operation. The race can 682 * happen with FS formats prior to SVN_FS_FS__MIN_PACK_LOCK_FORMAT that 683 * support packed revisions. With the pack lock, however, the race is 684 * impossible, because hotcopy and pack operations block each other. 685 * 686 * We assume that all revisions coming after 'min-unpacked-rev' really 687 * are unpacked and that's not necessarily true with concurrent packing. 688 * Don't try to be smart in this edge case, because handling it properly 689 * might require copying *everything* from the start. Just abort the 690 * hotcopy with an ENOENT (revision file moved to a pack, so it is no 691 * longer where we expect it to be). */ 692 693 /* Copy the rev file. */ 694 SVN_ERR(hotcopy_copy_shard_file(&skipped, 695 src_revs_dir, dst_revs_dir, rev, 696 max_files_per_dir, 697 iterpool)); 698 /* Copy the revprop file. */ 699 SVN_ERR(hotcopy_copy_shard_file(&skipped, 700 src_revprops_dir, dst_revprops_dir, 701 rev, max_files_per_dir, 702 iterpool)); 703 704 /* Whenever this revision did not previously exist in the destination, 705 * checkpoint the progress via 'current' (do that once per full shard 706 * in order not to slow things down). */ 707 if (rev > dst_youngest) 708 { 709 if (max_files_per_dir && (rev % max_files_per_dir == 0)) 710 { 711 SVN_ERR(svn_fs_fs__write_current(dst_fs, rev, 0, 0, 712 iterpool)); 713 } 714 } 715 716 if (notify_func && !skipped) 717 notify_func(notify_baton, rev, rev, iterpool); 718 } 719 svn_pool_destroy(iterpool); 720 721 /* We assume that all revisions were copied now, i.e. we didn't exit the 722 * above loop early. 'rev' was last incremented during exit of the loop. */ 723 SVN_ERR_ASSERT(rev == src_youngest + 1); 724 725 return SVN_NO_ERROR; 726} 727 728/* Shortcut for the revision and revprop copying for old (1 or 2) format 729 * filesystems without sharding and packing. Copy the non-sharded revision 730 * and revprop files from SRC_FS to DST_FS. Do not re-copy data which 731 * already exists in DST_FS. Do not somehow checkpoint the results in 732 * the 'current' file in DST_FS. Indicate progress via the optional 733 * NOTIFY_FUNC callback using NOTIFY_BATON. Use POOL for temporary 734 * allocations. Also see hotcopy_revisions(). 735 */ 736static svn_error_t * 737hotcopy_revisions_old(svn_fs_t *src_fs, 738 svn_fs_t *dst_fs, 739 svn_revnum_t src_youngest, 740 const char *src_revs_dir, 741 const char *dst_revs_dir, 742 const char *src_revprops_dir, 743 const char *dst_revprops_dir, 744 svn_fs_hotcopy_notify_t notify_func, 745 void* notify_baton, 746 svn_cancel_func_t cancel_func, 747 void* cancel_baton, 748 apr_pool_t *pool) 749{ 750 apr_pool_t *iterpool = svn_pool_create(pool); 751 svn_revnum_t rev; 752 753 for (rev = 0; rev <= src_youngest; rev++) 754 { 755 svn_boolean_t skipped = TRUE; 756 757 svn_pool_clear(iterpool); 758 759 if (cancel_func) 760 SVN_ERR(cancel_func(cancel_baton)); 761 762 SVN_ERR(hotcopy_io_dir_file_copy(&skipped, src_revs_dir, dst_revs_dir, 763 apr_psprintf(iterpool, "%ld", rev), 764 iterpool)); 765 SVN_ERR(hotcopy_io_dir_file_copy(&skipped, src_revprops_dir, 766 dst_revprops_dir, 767 apr_psprintf(iterpool, "%ld", rev), 768 iterpool)); 769 770 if (notify_func && !skipped) 771 notify_func(notify_baton, rev, rev, iterpool); 772 } 773 svn_pool_destroy(iterpool); 774 775 return SVN_NO_ERROR; 776} 777 778/* Baton for hotcopy_body(). */ 779struct hotcopy_body_baton { 780 svn_fs_t *src_fs; 781 svn_fs_t *dst_fs; 782 svn_boolean_t incremental; 783 svn_fs_hotcopy_notify_t notify_func; 784 void *notify_baton; 785 svn_cancel_func_t cancel_func; 786 void *cancel_baton; 787}; 788 789/* Perform a hotcopy, either normal or incremental. 790 * 791 * Normal hotcopy assumes that the destination exists as an empty 792 * directory. It behaves like an incremental hotcopy except that 793 * none of the copied files already exist in the destination. 794 * 795 * An incremental hotcopy copies only changed or new files to the destination, 796 * and removes files from the destination no longer present in the source. 797 * While the incremental hotcopy is running, readers should still be able 798 * to access the destination repository without error and should not see 799 * revisions currently in progress of being copied. Readers are able to see 800 * new fully copied revisions even if the entire incremental hotcopy procedure 801 * has not yet completed. 802 * 803 * Writers are blocked out completely during the entire incremental hotcopy 804 * process to ensure consistency. This function assumes that the repository 805 * write-lock is held. 806 */ 807static svn_error_t * 808hotcopy_body(void *baton, apr_pool_t *pool) 809{ 810 struct hotcopy_body_baton *hbb = baton; 811 svn_fs_t *src_fs = hbb->src_fs; 812 fs_fs_data_t *src_ffd = src_fs->fsap_data; 813 svn_fs_t *dst_fs = hbb->dst_fs; 814 fs_fs_data_t *dst_ffd = dst_fs->fsap_data; 815 svn_boolean_t incremental = hbb->incremental; 816 svn_fs_hotcopy_notify_t notify_func = hbb->notify_func; 817 void* notify_baton = hbb->notify_baton; 818 svn_cancel_func_t cancel_func = hbb->cancel_func; 819 void* cancel_baton = hbb->cancel_baton; 820 svn_revnum_t src_youngest; 821 apr_uint64_t src_next_node_id; 822 apr_uint64_t src_next_copy_id; 823 svn_revnum_t dst_youngest; 824 const char *src_revprops_dir; 825 const char *dst_revprops_dir; 826 const char *src_revs_dir; 827 const char *dst_revs_dir; 828 const char *src_subdir; 829 const char *dst_subdir; 830 svn_node_kind_t kind; 831 832 /* Try to copy the config. 833 * 834 * ### We try copying the config file before doing anything else, 835 * ### because higher layers will abort the hotcopy if we throw 836 * ### an error from this function, and that renders the hotcopy 837 * ### unusable anyway. */ 838 if (src_ffd->format >= SVN_FS_FS__MIN_CONFIG_FILE) 839 { 840 svn_error_t *err; 841 842 err = svn_io_dir_file_copy(src_fs->path, dst_fs->path, PATH_CONFIG, 843 pool); 844 if (err) 845 { 846 if (APR_STATUS_IS_ENOENT(err->apr_err)) 847 { 848 /* 1.6.0 to 1.6.11 did not copy the configuration file during 849 * hotcopy. So if we're hotcopying a repository which has been 850 * created as a hotcopy itself, it's possible that fsfs.conf 851 * does not exist. Ask the user to re-create it. 852 * 853 * ### It would be nice to make this a non-fatal error, 854 * ### but this function does not get an svn_fs_t object 855 * ### so we have no way of just printing a warning via 856 * ### the fs->warning() callback. */ 857 858 const char *src_abspath; 859 const char *dst_abspath; 860 const char *config_relpath; 861 svn_error_t *err2; 862 863 config_relpath = svn_dirent_join(src_fs->path, PATH_CONFIG, pool); 864 err2 = svn_dirent_get_absolute(&src_abspath, src_fs->path, pool); 865 if (err2) 866 return svn_error_trace(svn_error_compose_create(err, err2)); 867 err2 = svn_dirent_get_absolute(&dst_abspath, dst_fs->path, pool); 868 if (err2) 869 return svn_error_trace(svn_error_compose_create(err, err2)); 870 871 /* ### hack: strip off the 'db/' directory from paths so 872 * ### they make sense to the user */ 873 src_abspath = svn_dirent_dirname(src_abspath, pool); 874 dst_abspath = svn_dirent_dirname(dst_abspath, pool); 875 876 return svn_error_quick_wrapf(err, 877 _("Failed to create hotcopy at '%s'. " 878 "The file '%s' is missing from the source " 879 "repository. Please create this file, for " 880 "instance by running 'svnadmin upgrade %s'"), 881 dst_abspath, config_relpath, src_abspath); 882 } 883 else 884 return svn_error_trace(err); 885 } 886 } 887 888 if (cancel_func) 889 SVN_ERR(cancel_func(cancel_baton)); 890 891 /* Find the youngest revision in the source and destination. 892 * We only support hotcopies from sources with an equal or greater amount 893 * of revisions than the destination. 894 * This also catches the case where users accidentally swap the 895 * source and destination arguments. */ 896 SVN_ERR(svn_fs_fs__read_current(&src_youngest, &src_next_node_id, 897 &src_next_copy_id, src_fs, pool)); 898 if (incremental) 899 { 900 SVN_ERR(svn_fs_fs__youngest_rev(&dst_youngest, dst_fs, pool)); 901 if (src_youngest < dst_youngest) 902 return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, 903 _("The hotcopy destination already contains more revisions " 904 "(%lu) than the hotcopy source contains (%lu); are source " 905 "and destination swapped?"), 906 dst_youngest, src_youngest); 907 } 908 else 909 dst_youngest = 0; 910 911 src_revs_dir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, pool); 912 dst_revs_dir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, pool); 913 src_revprops_dir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, pool); 914 dst_revprops_dir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, pool); 915 916 /* Ensure that the required folders exist in the destination 917 * before actually copying the revisions and revprops. */ 918 SVN_ERR(svn_io_make_dir_recursively(dst_revs_dir, pool)); 919 SVN_ERR(svn_io_make_dir_recursively(dst_revprops_dir, pool)); 920 921 if (cancel_func) 922 SVN_ERR(cancel_func(cancel_baton)); 923 924 /* Split the logic for new and old FS formats. The latter is much simpler 925 * due to the absense of sharding and packing. However, it requires special 926 * care when updating the 'current' file (which contains not just the 927 * revision number, but also the next-ID counters). */ 928 if (src_ffd->format >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT) 929 { 930 SVN_ERR(hotcopy_revisions(src_fs, dst_fs, src_youngest, dst_youngest, 931 incremental, src_revs_dir, dst_revs_dir, 932 src_revprops_dir, dst_revprops_dir, 933 notify_func, notify_baton, 934 cancel_func, cancel_baton, pool)); 935 SVN_ERR(svn_fs_fs__write_current(dst_fs, src_youngest, 0, 0, pool)); 936 } 937 else 938 { 939 SVN_ERR(hotcopy_revisions_old(src_fs, dst_fs, src_youngest, 940 src_revs_dir, dst_revs_dir, 941 src_revprops_dir, dst_revprops_dir, 942 notify_func, notify_baton, 943 cancel_func, cancel_baton, pool)); 944 SVN_ERR(svn_fs_fs__write_current(dst_fs, src_youngest, src_next_node_id, 945 src_next_copy_id, pool)); 946 } 947 948 /* Replace the locks tree. 949 * This is racy in case readers are currently trying to list locks in 950 * the destination. However, we need to get rid of stale locks. 951 * This is the simplest way of doing this, so we accept this small race. */ 952 dst_subdir = svn_dirent_join(dst_fs->path, PATH_LOCKS_DIR, pool); 953 SVN_ERR(svn_io_remove_dir2(dst_subdir, TRUE, cancel_func, cancel_baton, 954 pool)); 955 src_subdir = svn_dirent_join(src_fs->path, PATH_LOCKS_DIR, pool); 956 SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); 957 if (kind == svn_node_dir) 958 SVN_ERR(svn_io_copy_dir_recursively(src_subdir, dst_fs->path, 959 PATH_LOCKS_DIR, TRUE, 960 cancel_func, cancel_baton, pool)); 961 962 /* Now copy the node-origins cache tree. */ 963 src_subdir = svn_dirent_join(src_fs->path, PATH_NODE_ORIGINS_DIR, pool); 964 SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); 965 if (kind == svn_node_dir) 966 SVN_ERR(hotcopy_io_copy_dir_recursively(NULL, src_subdir, dst_fs->path, 967 PATH_NODE_ORIGINS_DIR, TRUE, 968 cancel_func, cancel_baton, pool)); 969 970 /* 971 * NB: Data copied below is only read by writers, not readers. 972 * Writers are still locked out at this point. 973 */ 974 975 if (dst_ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT) 976 { 977 /* Copy the rep cache and then remove entries for revisions 978 * that did not make it into the destination. */ 979 src_subdir = svn_dirent_join(src_fs->path, REP_CACHE_DB_NAME, pool); 980 dst_subdir = svn_dirent_join(dst_fs->path, REP_CACHE_DB_NAME, pool); 981 SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); 982 if (kind == svn_node_file) 983 { 984 SVN_ERR(svn_sqlite__hotcopy(src_subdir, dst_subdir, pool)); 985 986 /* The source might have r/o flags set on it - which would be 987 carried over to the copy. */ 988 SVN_ERR(svn_io_set_file_read_write(dst_subdir, FALSE, pool)); 989 SVN_ERR(svn_fs_fs__del_rep_reference(dst_fs, src_youngest, pool)); 990 } 991 } 992 993 /* Copy the txn-current file. */ 994 if (dst_ffd->format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT) 995 SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path, 996 PATH_TXN_CURRENT, pool)); 997 998 /* Hotcopied FS is complete. Stamp it with a format file. */ 999 SVN_ERR(svn_fs_fs__write_format(dst_fs, TRUE, pool)); 1000 1001 return SVN_NO_ERROR; 1002} 1003 1004svn_error_t * 1005svn_fs_fs__hotcopy(svn_fs_t *src_fs, 1006 svn_fs_t *dst_fs, 1007 const char *src_path, 1008 const char *dst_path, 1009 svn_boolean_t incremental, 1010 svn_fs_hotcopy_notify_t notify_func, 1011 void *notify_baton, 1012 svn_cancel_func_t cancel_func, 1013 void *cancel_baton, 1014 svn_mutex__t *common_pool_lock, 1015 apr_pool_t *pool, 1016 apr_pool_t *common_pool) 1017{ 1018 struct hotcopy_body_baton hbb; 1019 1020 if (cancel_func) 1021 SVN_ERR(cancel_func(cancel_baton)); 1022 1023 SVN_ERR(svn_fs_fs__open(src_fs, src_path, pool)); 1024 1025 if (incremental) 1026 { 1027 const char *dst_format_abspath; 1028 svn_node_kind_t dst_format_kind; 1029 1030 /* Check destination format to be sure we know how to incrementally 1031 * hotcopy to the destination FS. */ 1032 dst_format_abspath = svn_dirent_join(dst_path, PATH_FORMAT, pool); 1033 SVN_ERR(svn_io_check_path(dst_format_abspath, &dst_format_kind, pool)); 1034 if (dst_format_kind == svn_node_none) 1035 { 1036 /* No destination? Fallback to a non-incremental hotcopy. */ 1037 incremental = FALSE; 1038 } 1039 } 1040 1041 if (incremental) 1042 { 1043 /* Check the existing repository. */ 1044 SVN_ERR(svn_fs_fs__open(dst_fs, dst_path, pool)); 1045 SVN_ERR(hotcopy_incremental_check_preconditions(src_fs, dst_fs, pool)); 1046 1047 SVN_ERR(svn_fs_fs__initialize_shared_data(dst_fs, common_pool_lock, 1048 pool, common_pool)); 1049 SVN_ERR(svn_fs_fs__initialize_caches(dst_fs, pool)); 1050 } 1051 else 1052 { 1053 /* Start out with an empty destination using the same configuration 1054 * as the source. */ 1055 fs_fs_data_t *src_ffd = src_fs->fsap_data; 1056 1057 /* Create the DST_FS repository with the same layout as SRC_FS. */ 1058 SVN_ERR(svn_fs_fs__create_file_tree(dst_fs, dst_path, src_ffd->format, 1059 src_ffd->max_files_per_dir, 1060 src_ffd->use_log_addressing, 1061 pool)); 1062 1063 /* Copy the UUID. Hotcopy destination receives a new instance ID, but 1064 * has the same filesystem UUID as the source. */ 1065 SVN_ERR(svn_fs_fs__set_uuid(dst_fs, src_fs->uuid, NULL, pool)); 1066 1067 /* Remove revision 0 contents. Otherwise, it may not get overwritten 1068 * due to having a newer timestamp. */ 1069 SVN_ERR(hotcopy_remove_file(svn_fs_fs__path_rev(dst_fs, 0, pool), 1070 pool)); 1071 SVN_ERR(hotcopy_remove_file(svn_fs_fs__path_revprops(dst_fs, 0, pool), 1072 pool)); 1073 1074 SVN_ERR(svn_fs_fs__initialize_shared_data(dst_fs, common_pool_lock, 1075 pool, common_pool)); 1076 SVN_ERR(svn_fs_fs__initialize_caches(dst_fs, pool)); 1077 } 1078 1079 if (cancel_func) 1080 SVN_ERR(cancel_func(cancel_baton)); 1081 1082 hbb.src_fs = src_fs; 1083 hbb.dst_fs = dst_fs; 1084 hbb.incremental = incremental; 1085 hbb.notify_func = notify_func; 1086 hbb.notify_baton = notify_baton; 1087 hbb.cancel_func = cancel_func; 1088 hbb.cancel_baton = cancel_baton; 1089 1090 /* Lock the destination in the incremental mode. For a non-incremental 1091 * hotcopy, don't take any locks. In that case the destination cannot be 1092 * opened until the hotcopy finishes, and we don't have to worry about 1093 * concurrency. */ 1094 if (incremental) 1095 SVN_ERR(svn_fs_fs__with_all_locks(dst_fs, hotcopy_body, &hbb, pool)); 1096 else 1097 SVN_ERR(hotcopy_body(&hbb, pool)); 1098 1099 return SVN_NO_ERROR; 1100} 1101