revprops.c revision 362181
1/* revprops.c --- everything needed to handle revprops in FSFS 2 * 3 * ==================================================================== 4 * Licensed to the Apache Software Foundation (ASF) under one 5 * or more contributor license agreements. See the NOTICE file 6 * distributed with this work for additional information 7 * regarding copyright ownership. The ASF licenses this file 8 * to you under the Apache License, Version 2.0 (the 9 * "License"); you may not use this file except in compliance 10 * with the License. You may obtain a copy of the License at 11 * 12 * http://www.apache.org/licenses/LICENSE-2.0 13 * 14 * Unless required by applicable law or agreed to in writing, 15 * software distributed under the License is distributed on an 16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 * KIND, either express or implied. See the License for the 18 * specific language governing permissions and limitations 19 * under the License. 20 * ==================================================================== 21 */ 22 23#include <assert.h> 24 25#include "svn_pools.h" 26#include "svn_hash.h" 27#include "svn_dirent_uri.h" 28#include "svn_sorts.h" 29 30#include "fs_fs.h" 31#include "revprops.h" 32#include "temp_serializer.h" 33#include "util.h" 34 35#include "private/svn_subr_private.h" 36#include "private/svn_string_private.h" 37#include "../libsvn_fs/fs-loader.h" 38 39#include "svn_private_config.h" 40 41svn_error_t * 42svn_fs_fs__upgrade_pack_revprops(svn_fs_t *fs, 43 svn_fs_upgrade_notify_t notify_func, 44 void *notify_baton, 45 svn_cancel_func_t cancel_func, 46 void *cancel_baton, 47 apr_pool_t *scratch_pool) 48{ 49 fs_fs_data_t *ffd = fs->fsap_data; 50 const char *revprops_shard_path; 51 const char *revprops_pack_file_dir; 52 apr_int64_t shard; 53 apr_int64_t first_unpacked_shard 54 = ffd->min_unpacked_rev / ffd->max_files_per_dir; 55 56 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 57 const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR, 58 scratch_pool); 59 int compression_level = ffd->compress_packed_revprops 60 ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT 61 : SVN_DELTA_COMPRESSION_LEVEL_NONE; 62 63 /* first, pack all revprops shards to match the packed revision shards */ 64 for (shard = 0; shard < first_unpacked_shard; ++shard) 65 { 66 svn_pool_clear(iterpool); 67 68 revprops_pack_file_dir = svn_dirent_join(revsprops_dir, 69 apr_psprintf(iterpool, 70 "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, 71 shard), 72 iterpool); 73 revprops_shard_path = svn_dirent_join(revsprops_dir, 74 apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard), 75 iterpool); 76 77 SVN_ERR(svn_fs_fs__pack_revprops_shard(revprops_pack_file_dir, 78 revprops_shard_path, 79 shard, ffd->max_files_per_dir, 80 (int)(0.9 * ffd->revprop_pack_size), 81 compression_level, 82 ffd->flush_to_disk, 83 cancel_func, cancel_baton, 84 iterpool)); 85 if (notify_func) 86 SVN_ERR(notify_func(notify_baton, shard, 87 svn_fs_upgrade_pack_revprops, iterpool)); 88 } 89 90 svn_pool_destroy(iterpool); 91 92 return SVN_NO_ERROR; 93} 94 95svn_error_t * 96svn_fs_fs__upgrade_cleanup_pack_revprops(svn_fs_t *fs, 97 svn_fs_upgrade_notify_t notify_func, 98 void *notify_baton, 99 svn_cancel_func_t cancel_func, 100 void *cancel_baton, 101 apr_pool_t *scratch_pool) 102{ 103 fs_fs_data_t *ffd = fs->fsap_data; 104 const char *revprops_shard_path; 105 apr_int64_t shard; 106 apr_int64_t first_unpacked_shard 107 = ffd->min_unpacked_rev / ffd->max_files_per_dir; 108 109 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 110 const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR, 111 scratch_pool); 112 113 /* delete the non-packed revprops shards afterwards */ 114 for (shard = 0; shard < first_unpacked_shard; ++shard) 115 { 116 svn_pool_clear(iterpool); 117 118 revprops_shard_path = svn_dirent_join(revsprops_dir, 119 apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard), 120 iterpool); 121 SVN_ERR(svn_fs_fs__delete_revprops_shard(revprops_shard_path, 122 shard, 123 ffd->max_files_per_dir, 124 cancel_func, cancel_baton, 125 iterpool)); 126 if (notify_func) 127 SVN_ERR(notify_func(notify_baton, shard, 128 svn_fs_upgrade_cleanup_revprops, iterpool)); 129 } 130 131 svn_pool_destroy(iterpool); 132 133 return SVN_NO_ERROR; 134} 135 136/* Container for all data required to access the packed revprop file 137 * for a given REVISION. This structure will be filled incrementally 138 * by read_pack_revprops() its sub-routines. 139 */ 140typedef struct packed_revprops_t 141{ 142 /* revision number to read (not necessarily the first in the pack) */ 143 svn_revnum_t revision; 144 145 /* the actual revision properties */ 146 apr_hash_t *properties; 147 148 /* their size when serialized to a single string 149 * (as found in PACKED_REVPROPS) */ 150 apr_size_t serialized_size; 151 152 153 /* name of the pack file (without folder path) */ 154 const char *filename; 155 156 /* packed shard folder path */ 157 const char *folder; 158 159 /* sum of values in SIZES */ 160 apr_size_t total_size; 161 162 /* first revision in the pack (>= MANIFEST_START) */ 163 svn_revnum_t start_revision; 164 165 /* size of the revprops in PACKED_REVPROPS */ 166 apr_array_header_t *sizes; 167 168 /* offset of the revprops in PACKED_REVPROPS */ 169 apr_array_header_t *offsets; 170 171 172 /* concatenation of the serialized representation of all revprops 173 * in the pack, i.e. the pack content without header and compression */ 174 svn_stringbuf_t *packed_revprops; 175 176 /* First revision covered by MANIFEST. 177 * Will equal the shard start revision or 1, for the 1st shard. */ 178 svn_revnum_t manifest_start; 179 180 /* content of the manifest. 181 * Maps long(rev - MANIFEST_START) to const char* pack file name */ 182 apr_array_header_t *manifest; 183} packed_revprops_t; 184 185/* Parse the serialized revprops in CONTENT and return them in *PROPERTIES. 186 * Also, put them into the revprop cache, if activated, for future use. 187 * 188 * The returned hash will be allocated in RESULT_POOL, SCRATCH_POOL is being 189 * used for temporary allocations. 190 */ 191static svn_error_t * 192parse_revprop(apr_hash_t **properties, 193 svn_fs_t *fs, 194 svn_revnum_t revision, 195 svn_string_t *content, 196 apr_pool_t *result_pool, 197 apr_pool_t *scratch_pool) 198{ 199 svn_stream_t *stream = svn_stream_from_string(content, scratch_pool); 200 *properties = apr_hash_make(result_pool); 201 202 SVN_ERR_W(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, 203 result_pool), 204 apr_psprintf(scratch_pool, "Failed to parse revprops for r%ld.", 205 revision)); 206 207 return SVN_NO_ERROR; 208} 209 210void 211svn_fs_fs__reset_revprop_cache(svn_fs_t *fs) 212{ 213 fs_fs_data_t *ffd = fs->fsap_data; 214 ffd->revprop_prefix = 0; 215} 216 217/* If FS has not a revprop cache prefix set, generate one. 218 * Always call this before accessing the revprop cache. 219 */ 220static svn_error_t * 221prepare_revprop_cache(svn_fs_t *fs, 222 apr_pool_t *scratch_pool) 223{ 224 fs_fs_data_t *ffd = fs->fsap_data; 225 if (!ffd->revprop_prefix) 226 SVN_ERR(svn_atomic__unique_counter(&ffd->revprop_prefix)); 227 228 return SVN_NO_ERROR; 229} 230 231/* Store the unparsed revprop hash CONTENT for REVISION in FS's revprop 232 * cache. If CACHED is not NULL, set *CACHED if there already is such 233 * an entry and skip the cache write in that case. Use SCRATCH_POOL for 234 * temporary allocations. */ 235static svn_error_t * 236cache_revprops(svn_boolean_t *is_cached, 237 svn_fs_t *fs, 238 svn_revnum_t revision, 239 svn_string_t *content, 240 apr_pool_t *scratch_pool) 241{ 242 fs_fs_data_t *ffd = fs->fsap_data; 243 pair_cache_key_t key; 244 245 /* Make sure prepare_revprop_cache() has been called. */ 246 SVN_ERR_ASSERT(ffd->revprop_prefix); 247 key.revision = revision; 248 key.second = ffd->revprop_prefix; 249 250 if (is_cached) 251 { 252 SVN_ERR(svn_cache__has_key(is_cached, ffd->revprop_cache, &key, 253 scratch_pool)); 254 if (*is_cached) 255 return SVN_NO_ERROR; 256 } 257 258 SVN_ERR(svn_cache__set(ffd->revprop_cache, &key, content, scratch_pool)); 259 260 return SVN_NO_ERROR; 261} 262 263/* Read the non-packed revprops for revision REV in FS, put them into the 264 * revprop cache if PROPULATE_CACHE is set and return them in *PROPERTIES. 265 * 266 * If the data could not be read due to an otherwise recoverable error, 267 * leave *PROPERTIES unchanged. No error will be returned in that case. 268 * 269 * Allocations will be done in POOL. 270 */ 271static svn_error_t * 272read_non_packed_revprop(apr_hash_t **properties, 273 svn_fs_t *fs, 274 svn_revnum_t rev, 275 svn_boolean_t populate_cache, 276 apr_pool_t *pool) 277{ 278 svn_stringbuf_t *content = NULL; 279 apr_pool_t *iterpool = svn_pool_create(pool); 280 svn_boolean_t missing = FALSE; 281 int i; 282 283 for (i = 0; 284 i < SVN_FS_FS__RECOVERABLE_RETRY_COUNT && !missing && !content; 285 ++i) 286 { 287 svn_pool_clear(iterpool); 288 SVN_ERR(svn_fs_fs__try_stringbuf_from_file(&content, 289 &missing, 290 svn_fs_fs__path_revprops(fs, rev, iterpool), 291 i + 1 < SVN_FS_FS__RECOVERABLE_RETRY_COUNT , 292 iterpool)); 293 } 294 295 if (content) 296 { 297 svn_string_t *as_string = svn_stringbuf__morph_into_string(content); 298 SVN_ERR(parse_revprop(properties, fs, rev, as_string, pool, iterpool)); 299 300 if (populate_cache) 301 SVN_ERR(cache_revprops(NULL, fs, rev, as_string, iterpool)); 302 } 303 304 svn_pool_clear(iterpool); 305 306 return SVN_NO_ERROR; 307} 308 309/* Return the minimum length of any packed revprop file name in REVPROPS. */ 310static apr_size_t 311get_min_filename_len(packed_revprops_t *revprops) 312{ 313 char number_buffer[SVN_INT64_BUFFER_SIZE]; 314 315 /* The revprop filenames have the format <REV>.<COUNT> - with <REV> being 316 * at least the first rev in the shard and <COUNT> having at least one 317 * digit. Thus, the minimum is 2 + #decimal places in the start rev. 318 */ 319 return svn__i64toa(number_buffer, revprops->manifest_start) + 2; 320} 321 322/* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST 323 * members. Use RESULT_POOL for allocating results and SCRATCH_POOL for 324 * temporaries. 325 */ 326static svn_error_t * 327get_revprop_packname(svn_fs_t *fs, 328 packed_revprops_t *revprops, 329 apr_pool_t *result_pool, 330 apr_pool_t *scratch_pool) 331{ 332 fs_fs_data_t *ffd = fs->fsap_data; 333 svn_stringbuf_t *content = NULL; 334 const char *manifest_file_path; 335 int idx, rev_count; 336 char *buffer, *buffer_end; 337 const char **filenames, **filenames_end; 338 apr_size_t min_filename_len; 339 340 /* Determine the dimensions. Rev 0 is excluded from the first shard. */ 341 rev_count = ffd->max_files_per_dir; 342 revprops->manifest_start 343 = revprops->revision - (revprops->revision % rev_count); 344 if (revprops->manifest_start == 0) 345 { 346 ++revprops->manifest_start; 347 --rev_count; 348 } 349 350 revprops->manifest = apr_array_make(result_pool, rev_count, 351 sizeof(const char*)); 352 353 /* No line in the file can be less than this number of chars long. */ 354 min_filename_len = get_min_filename_len(revprops); 355 356 /* Read the content of the manifest file */ 357 revprops->folder 358 = svn_fs_fs__path_revprops_pack_shard(fs, revprops->revision, 359 result_pool); 360 manifest_file_path 361 = svn_dirent_join(revprops->folder, PATH_MANIFEST, result_pool); 362 363 SVN_ERR(svn_fs_fs__read_content(&content, manifest_file_path, result_pool)); 364 365 /* There CONTENT must have a certain minimal size and there no 366 * unterminated lines at the end of the file. Both guarantees also 367 * simplify the parser loop below. 368 */ 369 if ( content->len < rev_count * (min_filename_len + 1) 370 || content->data[content->len - 1] != '\n') 371 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 372 _("Packed revprop manifest for r%ld not " 373 "properly terminated"), revprops->revision); 374 375 /* Chop (parse) the manifest CONTENT into filenames, one per line. 376 * We only have to replace all newlines with NUL and add all line 377 * starts to REVPROPS->MANIFEST. 378 * 379 * There must be exactly REV_COUNT lines and that is the number of 380 * lines we parse from BUFFER to FILENAMES. Set the end pointer for 381 * the source BUFFER such that BUFFER+MIN_FILENAME_LEN is still valid 382 * BUFFER_END is always valid due to CONTENT->LEN > MIN_FILENAME_LEN. 383 * 384 * Please note that this loop is performance critical for e.g. 'svn log'. 385 * It is run 1000x per revprop access, i.e. per revision and about 386 * 50 million times per sec (and CPU core). 387 */ 388 for (filenames = (const char **)revprops->manifest->elts, 389 filenames_end = filenames + rev_count, 390 buffer = content->data, 391 buffer_end = buffer + content->len - min_filename_len; 392 (filenames < filenames_end) && (buffer < buffer_end); 393 ++filenames) 394 { 395 /* BUFFER always points to the start of the next line / filename. */ 396 *filenames = buffer; 397 398 /* Find the next EOL. This is guaranteed to stay within the CONTENT 399 * buffer because we left enough room after BUFFER_END and we know 400 * we will always see a newline as the last non-NUL char. */ 401 buffer += min_filename_len; 402 while (*buffer != '\n') 403 ++buffer; 404 405 /* Found EOL. Turn it into the filename terminator and move BUFFER 406 * to the start of the next line or CONTENT buffer end. */ 407 *buffer = '\0'; 408 ++buffer; 409 } 410 411 /* We must have reached the end of both buffers. */ 412 if (buffer < content->data + content->len) 413 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 414 _("Packed revprop manifest for r%ld " 415 "has too many entries"), revprops->revision); 416 417 if (filenames < filenames_end) 418 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 419 _("Packed revprop manifest for r%ld " 420 "has too few entries"), revprops->revision); 421 422 /* The target array has now exactly one entry per revision. */ 423 revprops->manifest->nelts = rev_count; 424 425 /* Now get the file name */ 426 idx = (int)(revprops->revision - revprops->manifest_start); 427 revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*); 428 429 return SVN_NO_ERROR; 430} 431 432/* Return TRUE, if revision R1 and R2 refer to the same shard in FS. 433 */ 434static svn_boolean_t 435same_shard(svn_fs_t *fs, 436 svn_revnum_t r1, 437 svn_revnum_t r2) 438{ 439 fs_fs_data_t *ffd = fs->fsap_data; 440 return (r1 / ffd->max_files_per_dir) == (r2 / ffd->max_files_per_dir); 441} 442 443/* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS, 444 * fill the START_REVISION member, and make PACKED_REVPROPS point to the 445 * first serialized revprop. If READ_ALL is set, initialize the SIZES 446 * and OFFSETS members as well. If POPULATE_CACHE is set, cache all 447 * revprops found in this pack. 448 * 449 * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as 450 * well as the SERIALIZED_SIZE member. If revprop caching has been 451 * enabled, parse all revprops in the pack and cache them. 452 */ 453static svn_error_t * 454parse_packed_revprops(svn_fs_t *fs, 455 packed_revprops_t *revprops, 456 svn_boolean_t read_all, 457 svn_boolean_t populate_cache, 458 apr_pool_t *result_pool, 459 apr_pool_t *scratch_pool) 460{ 461 svn_stream_t *stream; 462 apr_int64_t first_rev, count, i; 463 apr_size_t offset; 464 const char *header_end; 465 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 466 467 /* Initial value for the "Leaking bucket" pattern. */ 468 int bucket = 4; 469 470 /* decompress (even if the data is only "stored", there is still a 471 * length header to remove) */ 472 svn_stringbuf_t *compressed = revprops->packed_revprops; 473 svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(result_pool); 474 SVN_ERR(svn__decompress_zlib(compressed->data, compressed->len, 475 uncompressed, APR_SIZE_MAX)); 476 477 /* read first revision number and number of revisions in the pack */ 478 stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); 479 SVN_ERR(svn_fs_fs__read_number_from_stream(&first_rev, NULL, stream, 480 iterpool)); 481 SVN_ERR(svn_fs_fs__read_number_from_stream(&count, NULL, stream, 482 iterpool)); 483 484 /* Check revision range for validity. */ 485 if ( !same_shard(fs, revprops->revision, first_rev) 486 || !same_shard(fs, revprops->revision, first_rev + count - 1) 487 || count < 1) 488 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 489 _("Revprop pack for revision r%ld" 490 " contains revprops for r%ld .. r%ld"), 491 revprops->revision, 492 (svn_revnum_t)first_rev, 493 (svn_revnum_t)(first_rev + count -1)); 494 495 /* Since start & end are in the same shard, it is enough to just test 496 * the FIRST_REV for being actually packed. That will also cover the 497 * special case of rev 0 never being packed. */ 498 if (!svn_fs_fs__is_packed_revprop(fs, first_rev)) 499 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 500 _("Revprop pack for revision r%ld" 501 " starts at non-packed revisions r%ld"), 502 revprops->revision, (svn_revnum_t)first_rev); 503 504 /* make PACKED_REVPROPS point to the first char after the header. 505 * This is where the serialized revprops are. */ 506 header_end = strstr(uncompressed->data, "\n\n"); 507 if (header_end == NULL) 508 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 509 _("Header end not found")); 510 511 offset = header_end - uncompressed->data + 2; 512 513 revprops->packed_revprops = svn_stringbuf_create_empty(result_pool); 514 revprops->packed_revprops->data = uncompressed->data + offset; 515 revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset); 516 revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize 517 - offset); 518 519 /* STREAM still points to the first entry in the sizes list. */ 520 revprops->start_revision = (svn_revnum_t)first_rev; 521 if (read_all) 522 { 523 /* Init / construct REVPROPS members. */ 524 revprops->sizes = apr_array_make(result_pool, (int)count, 525 sizeof(offset)); 526 revprops->offsets = apr_array_make(result_pool, (int)count, 527 sizeof(offset)); 528 } 529 530 /* Now parse, revision by revision, the size and content of each 531 * revisions' revprops. */ 532 for (i = 0, offset = 0, revprops->total_size = 0; i < count; ++i) 533 { 534 apr_int64_t size; 535 svn_string_t serialized; 536 svn_revnum_t revision = (svn_revnum_t)(first_rev + i); 537 svn_pool_clear(iterpool); 538 539 /* read & check the serialized size */ 540 SVN_ERR(svn_fs_fs__read_number_from_stream(&size, NULL, stream, 541 iterpool)); 542 if (size > (apr_int64_t)revprops->packed_revprops->len - offset) 543 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 544 _("Packed revprop size exceeds pack file size")); 545 546 /* Parse this revprops list, if necessary */ 547 serialized.data = revprops->packed_revprops->data + offset; 548 serialized.len = (apr_size_t)size; 549 550 if (revision == revprops->revision) 551 { 552 /* Parse (and possibly cache) the one revprop list we care about. */ 553 SVN_ERR(parse_revprop(&revprops->properties, fs, revision, 554 &serialized, result_pool, iterpool)); 555 revprops->serialized_size = serialized.len; 556 557 /* If we only wanted the revprops for REVISION then we are done. */ 558 if (!read_all && !populate_cache) 559 break; 560 } 561 562 if (populate_cache) 563 { 564 /* Adding all those revprops is expensive, in particular in a 565 * multi-threaded environment. There are situations where hit 566 * rates are low and revprops get evicted before re-using them. 567 * 568 * We try to detect thosse cases here. 569 * Only keep going while most (at least 2/3) aren't cached, yet. */ 570 svn_boolean_t already_cached; 571 SVN_ERR(cache_revprops(&already_cached, fs, revision, &serialized, 572 iterpool)); 573 574 /* Stop populating the cache once we encountered too many entries 575 * already present relative to the numbers being added. */ 576 if (!already_cached) 577 { 578 ++bucket; 579 } 580 else 581 { 582 bucket -= 2; 583 if (bucket < 0) 584 populate_cache = FALSE; 585 } 586 } 587 588 if (read_all) 589 { 590 /* fill REVPROPS data structures */ 591 APR_ARRAY_PUSH(revprops->sizes, apr_size_t) = serialized.len; 592 APR_ARRAY_PUSH(revprops->offsets, apr_size_t) = offset; 593 } 594 revprops->total_size += serialized.len; 595 596 offset += serialized.len; 597 } 598 599 return SVN_NO_ERROR; 600} 601 602/* In filesystem FS, read the packed revprops for revision REV into 603 * *REVPROPS. Populate the revprop cache, if POPULATE_CACHE is set. 604 * If you want to modify revprop contents / update REVPROPS, READ_ALL 605 * must be set. Otherwise, only the properties of REV are being provided. 606 * Allocate data in POOL. 607 */ 608static svn_error_t * 609read_pack_revprop(packed_revprops_t **revprops, 610 svn_fs_t *fs, 611 svn_revnum_t rev, 612 svn_boolean_t read_all, 613 svn_boolean_t populate_cache, 614 apr_pool_t *pool) 615{ 616 apr_pool_t *iterpool = svn_pool_create(pool); 617 svn_boolean_t missing = FALSE; 618 svn_error_t *err; 619 packed_revprops_t *result; 620 int i; 621 622 /* someone insisted that REV is packed. Double-check if necessary */ 623 if (!svn_fs_fs__is_packed_revprop(fs, rev)) 624 SVN_ERR(svn_fs_fs__update_min_unpacked_rev(fs, iterpool)); 625 626 if (!svn_fs_fs__is_packed_revprop(fs, rev)) 627 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, 628 _("No such packed revision %ld"), rev); 629 630 /* initialize the result data structure */ 631 result = apr_pcalloc(pool, sizeof(*result)); 632 result->revision = rev; 633 634 /* try to read the packed revprops. This may require retries if we have 635 * concurrent writers. */ 636 for (i = 0; 637 i < SVN_FS_FS__RECOVERABLE_RETRY_COUNT && !result->packed_revprops; 638 ++i) 639 { 640 const char *file_path; 641 svn_pool_clear(iterpool); 642 643 /* there might have been concurrent writes. 644 * Re-read the manifest and the pack file. 645 */ 646 SVN_ERR(get_revprop_packname(fs, result, pool, iterpool)); 647 file_path = svn_dirent_join(result->folder, 648 result->filename, 649 iterpool); 650 SVN_ERR(svn_fs_fs__try_stringbuf_from_file(&result->packed_revprops, 651 &missing, 652 file_path, 653 i + 1 < SVN_FS_FS__RECOVERABLE_RETRY_COUNT, 654 pool)); 655 } 656 657 /* the file content should be available now */ 658 if (!result->packed_revprops) 659 return svn_error_createf(SVN_ERR_FS_PACKED_REVPROP_READ_FAILURE, NULL, 660 _("Failed to read revprop pack file for r%ld"), rev); 661 662 /* parse it. RESULT will be complete afterwards. */ 663 err = parse_packed_revprops(fs, result, read_all, populate_cache, pool, 664 iterpool); 665 svn_pool_destroy(iterpool); 666 if (err) 667 return svn_error_createf(SVN_ERR_FS_CORRUPT, err, 668 _("Revprop pack file for r%ld is corrupt"), rev); 669 670 *revprops = result; 671 672 return SVN_NO_ERROR; 673} 674 675svn_error_t * 676svn_fs_fs__get_revision_props_size(apr_off_t *props_size_p, 677 svn_fs_t *fs, 678 svn_revnum_t rev, 679 apr_pool_t *scratch_pool) 680{ 681 fs_fs_data_t *ffd = fs->fsap_data; 682 683 /* should they be available at all? */ 684 SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, scratch_pool)); 685 686 /* if REV had not been packed when we began, try reading it from the 687 * non-packed shard. If that fails, we will fall through to packed 688 * shard reads. */ 689 if (!svn_fs_fs__is_packed_revprop(fs, rev)) 690 { 691 const char *path = svn_fs_fs__path_revprops(fs, rev, scratch_pool); 692 svn_error_t *err; 693 apr_file_t *file; 694 svn_filesize_t file_size; 695 696 err = svn_io_file_open(&file, path, APR_FOPEN_READ, APR_OS_DEFAULT, 697 scratch_pool); 698 if (!err) 699 err = svn_io_file_size_get(&file_size, file, scratch_pool); 700 if (!err) 701 { 702 *props_size_p = (apr_off_t)file_size; 703 return SVN_NO_ERROR; 704 } 705 else if (!APR_STATUS_IS_ENOENT(err->apr_err) 706 || ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) 707 { 708 return svn_error_trace(err); 709 } 710 711 /* fall through: maybe the revision got packed while we were looking */ 712 svn_error_clear(err); 713 } 714 715 /* Try reading packed revprops. If that fails, REV is most 716 * likely invalid (or its revprops highly contested). */ 717 { 718 packed_revprops_t *revprops; 719 720 /* ### This is inefficient -- reading all the revprops in a pack. We 721 should just read the index. */ 722 SVN_ERR(read_pack_revprop(&revprops, fs, rev, 723 TRUE /*read_all*/, FALSE /*populate_cache*/, 724 scratch_pool)); 725 *props_size_p = (apr_off_t)APR_ARRAY_IDX(revprops->sizes, 726 rev - revprops->start_revision, 727 apr_size_t); 728 } 729 730 return SVN_NO_ERROR; 731} 732 733/* Read the revprops for revision REV in FS and return them in *PROPERTIES_P. 734 * 735 * Allocations will be done in POOL. 736 */ 737svn_error_t * 738svn_fs_fs__get_revision_proplist(apr_hash_t **proplist_p, 739 svn_fs_t *fs, 740 svn_revnum_t rev, 741 svn_boolean_t refresh, 742 apr_pool_t *result_pool, 743 apr_pool_t *scratch_pool) 744{ 745 fs_fs_data_t *ffd = fs->fsap_data; 746 747 /* Only populate the cache if we did not just cross a sync barrier. 748 * This is to eliminate overhead from code that always sets REFRESH. 749 * For callers that want caching, the caching kicks in on read "later". */ 750 svn_boolean_t populate_cache = !refresh; 751 752 /* not found, yet */ 753 *proplist_p = NULL; 754 755 /* should they be available at all? */ 756 SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, scratch_pool)); 757 758 if (refresh) 759 { 760 /* Previous cache contents is invalid now. */ 761 svn_fs_fs__reset_revprop_cache(fs); 762 } 763 else 764 { 765 /* Try cache lookup first. */ 766 svn_boolean_t is_cached; 767 pair_cache_key_t key; 768 769 /* Auto-alloc prefix and construct the key. */ 770 SVN_ERR(prepare_revprop_cache(fs, scratch_pool)); 771 key.revision = rev; 772 key.second = ffd->revprop_prefix; 773 774 /* The only way that this might error out is due to parser error. */ 775 SVN_ERR_W(svn_cache__get((void **) proplist_p, &is_cached, 776 ffd->revprop_cache, &key, result_pool), 777 apr_psprintf(scratch_pool, 778 "Failed to parse revprops for r%ld.", 779 rev)); 780 if (is_cached) 781 return SVN_NO_ERROR; 782 } 783 784 /* if REV had not been packed when we began, try reading it from the 785 * non-packed shard. If that fails, we will fall through to packed 786 * shard reads. */ 787 if (!svn_fs_fs__is_packed_revprop(fs, rev)) 788 { 789 svn_error_t *err = read_non_packed_revprop(proplist_p, fs, rev, 790 populate_cache, result_pool); 791 if (err) 792 { 793 if (!APR_STATUS_IS_ENOENT(err->apr_err) 794 || ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) 795 return svn_error_trace(err); 796 797 svn_error_clear(err); 798 *proplist_p = NULL; /* in case read_non_packed_revprop changed it */ 799 } 800 } 801 802 /* if revprop packing is available and we have not read the revprops, yet, 803 * try reading them from a packed shard. If that fails, REV is most 804 * likely invalid (or its revprops highly contested). */ 805 if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT && !*proplist_p) 806 { 807 packed_revprops_t *revprops; 808 SVN_ERR(read_pack_revprop(&revprops, fs, rev, FALSE, populate_cache, 809 result_pool)); 810 *proplist_p = revprops->properties; 811 } 812 813 /* The revprops should have been there. Did we get them? */ 814 if (!*proplist_p) 815 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, 816 _("Could not read revprops for revision %ld"), 817 rev); 818 819 return SVN_NO_ERROR; 820} 821 822/* Serialize the revision property list PROPLIST of revision REV in 823 * filesystem FS to a non-packed file. Return the name of that temporary 824 * file in *TMP_PATH and the file path that it must be moved to in 825 * *FINAL_PATH. 826 * 827 * Use POOL for allocations. 828 */ 829static svn_error_t * 830write_non_packed_revprop(const char **final_path, 831 const char **tmp_path, 832 svn_fs_t *fs, 833 svn_revnum_t rev, 834 apr_hash_t *proplist, 835 apr_pool_t *pool) 836{ 837 fs_fs_data_t *ffd = fs->fsap_data; 838 apr_file_t *file; 839 svn_stream_t *stream; 840 *final_path = svn_fs_fs__path_revprops(fs, rev, pool); 841 842 /* ### do we have a directory sitting around already? we really shouldn't 843 ### have to get the dirname here. */ 844 SVN_ERR(svn_io_open_unique_file3(&file, tmp_path, 845 svn_dirent_dirname(*final_path, pool), 846 svn_io_file_del_none, pool, pool)); 847 stream = svn_stream_from_aprfile2(file, TRUE, pool); 848 SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); 849 SVN_ERR(svn_stream_close(stream)); 850 851 /* Flush temporary file to disk and close it. */ 852 if (ffd->flush_to_disk) 853 SVN_ERR(svn_io_file_flush_to_disk(file, pool)); 854 SVN_ERR(svn_io_file_close(file, pool)); 855 856 return SVN_NO_ERROR; 857} 858 859/* After writing the new revprop file(s), call this function to move the 860 * file at TMP_PATH to FINAL_PATH and give it the permissions from 861 * PERMS_REFERENCE. 862 * 863 * Finally, delete all the temporary files given in FILES_TO_DELETE. 864 * The latter may be NULL. 865 * 866 * Use POOL for temporary allocations. 867 */ 868static svn_error_t * 869switch_to_new_revprop(svn_fs_t *fs, 870 const char *final_path, 871 const char *tmp_path, 872 const char *perms_reference, 873 apr_array_header_t *files_to_delete, 874 apr_pool_t *pool) 875{ 876 fs_fs_data_t *ffd = fs->fsap_data; 877 878 SVN_ERR(svn_fs_fs__move_into_place(tmp_path, final_path, perms_reference, 879 ffd->flush_to_disk, pool)); 880 881 /* Clean up temporary files, if necessary. */ 882 if (files_to_delete) 883 { 884 apr_pool_t *iterpool = svn_pool_create(pool); 885 int i; 886 887 for (i = 0; i < files_to_delete->nelts; ++i) 888 { 889 const char *path = APR_ARRAY_IDX(files_to_delete, i, const char*); 890 891 svn_pool_clear(iterpool); 892 SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); 893 } 894 895 svn_pool_destroy(iterpool); 896 } 897 return SVN_NO_ERROR; 898} 899 900/* Write a pack file header to STREAM that starts at revision START_REVISION 901 * and contains the indexes [START,END) of SIZES. 902 */ 903static svn_error_t * 904serialize_revprops_header(svn_stream_t *stream, 905 svn_revnum_t start_revision, 906 apr_array_header_t *sizes, 907 int start, 908 int end, 909 apr_pool_t *pool) 910{ 911 apr_pool_t *iterpool = svn_pool_create(pool); 912 int i; 913 914 SVN_ERR_ASSERT(start < end); 915 916 /* start revision and entry count */ 917 SVN_ERR(svn_stream_printf(stream, pool, "%ld\n", start_revision)); 918 SVN_ERR(svn_stream_printf(stream, pool, "%d\n", end - start)); 919 920 /* the sizes array */ 921 for (i = start; i < end; ++i) 922 { 923 /* Non-standard pool usage. 924 * 925 * We only allocate a few bytes each iteration -- even with a 926 * million iterations we would still be in good shape memory-wise. 927 */ 928 apr_size_t size = APR_ARRAY_IDX(sizes, i, apr_size_t); 929 SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_SIZE_T_FMT "\n", 930 size)); 931 } 932 933 /* the double newline char indicates the end of the header */ 934 SVN_ERR(svn_stream_puts(stream, "\n")); 935 936 svn_pool_destroy(iterpool); 937 return SVN_NO_ERROR; 938} 939 940/* Writes the a pack file to FILE. It copies the serialized data 941 * from REVPROPS for the indexes [START,END) except for index CHANGED_INDEX. 942 * 943 * The data for the latter is taken from NEW_SERIALIZED. Note, that 944 * CHANGED_INDEX may be outside the [START,END) range, i.e. no new data is 945 * taken in that case but only a subset of the old data will be copied. 946 * 947 * NEW_TOTAL_SIZE is a hint for pre-allocating buffers of appropriate size. 948 * POOL is used for temporary allocations. 949 */ 950static svn_error_t * 951repack_revprops(svn_fs_t *fs, 952 packed_revprops_t *revprops, 953 int start, 954 int end, 955 int changed_index, 956 svn_stringbuf_t *new_serialized, 957 apr_size_t new_total_size, 958 apr_file_t *file, 959 apr_pool_t *pool) 960{ 961 fs_fs_data_t *ffd = fs->fsap_data; 962 svn_stream_t *stream; 963 int i; 964 965 /* create data empty buffers and the stream object */ 966 svn_stringbuf_t *uncompressed 967 = svn_stringbuf_create_ensure((apr_size_t)new_total_size, pool); 968 svn_stringbuf_t *compressed 969 = svn_stringbuf_create_empty(pool); 970 stream = svn_stream_from_stringbuf(uncompressed, pool); 971 972 /* write the header*/ 973 SVN_ERR(serialize_revprops_header(stream, revprops->start_revision + start, 974 revprops->sizes, start, end, pool)); 975 976 /* append the serialized revprops */ 977 for (i = start; i < end; ++i) 978 if (i == changed_index) 979 { 980 SVN_ERR(svn_stream_write(stream, 981 new_serialized->data, 982 &new_serialized->len)); 983 } 984 else 985 { 986 apr_size_t size = APR_ARRAY_IDX(revprops->sizes, i, apr_size_t); 987 apr_size_t offset = APR_ARRAY_IDX(revprops->offsets, i, apr_size_t); 988 989 SVN_ERR(svn_stream_write(stream, 990 revprops->packed_revprops->data + offset, 991 &size)); 992 } 993 994 /* flush the stream buffer (if any) to our underlying data buffer */ 995 SVN_ERR(svn_stream_close(stream)); 996 997 /* compress / store the data */ 998 SVN_ERR(svn__compress_zlib(uncompressed->data, uncompressed->len, 999 compressed, 1000 ffd->compress_packed_revprops 1001 ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT 1002 : SVN_DELTA_COMPRESSION_LEVEL_NONE)); 1003 1004 /* finally, write the content to the target file, flush and close it */ 1005 SVN_ERR(svn_io_file_write_full(file, compressed->data, compressed->len, 1006 NULL, pool)); 1007 if (ffd->flush_to_disk) 1008 SVN_ERR(svn_io_file_flush_to_disk(file, pool)); 1009 SVN_ERR(svn_io_file_close(file, pool)); 1010 1011 return SVN_NO_ERROR; 1012} 1013 1014/* Allocate a new pack file name for revisions 1015 * [REVPROPS->START_REVISION + START, REVPROPS->START_REVISION + END - 1] 1016 * of REVPROPS->MANIFEST. Add the name of old file to FILES_TO_DELETE, 1017 * auto-create that array if necessary. Return an open file *FILE that is 1018 * allocated in POOL. 1019 */ 1020static svn_error_t * 1021repack_file_open(apr_file_t **file, 1022 svn_fs_t *fs, 1023 packed_revprops_t *revprops, 1024 int start, 1025 int end, 1026 apr_array_header_t **files_to_delete, 1027 apr_pool_t *pool) 1028{ 1029 apr_int64_t tag; 1030 const char *tag_string; 1031 const char *new_filename; 1032 int i; 1033 int manifest_offset 1034 = (int)(revprops->start_revision - revprops->manifest_start); 1035 1036 /* get the old (= current) file name and enlist it for later deletion */ 1037 const char *old_filename = APR_ARRAY_IDX(revprops->manifest, 1038 start + manifest_offset, 1039 const char*); 1040 1041 if (*files_to_delete == NULL) 1042 *files_to_delete = apr_array_make(pool, 3, sizeof(const char*)); 1043 1044 APR_ARRAY_PUSH(*files_to_delete, const char*) 1045 = svn_dirent_join(revprops->folder, old_filename, pool); 1046 1047 /* increase the tag part, i.e. the counter after the dot */ 1048 tag_string = strchr(old_filename, '.'); 1049 if (tag_string == NULL) 1050 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 1051 _("Packed file '%s' misses a tag"), 1052 old_filename); 1053 1054 SVN_ERR(svn_cstring_atoi64(&tag, tag_string + 1)); 1055 new_filename = apr_psprintf(pool, "%ld.%" APR_INT64_T_FMT, 1056 revprops->start_revision + start, 1057 ++tag); 1058 1059 /* update the manifest to point to the new file */ 1060 for (i = start; i < end; ++i) 1061 APR_ARRAY_IDX(revprops->manifest, i + manifest_offset, const char*) 1062 = new_filename; 1063 1064 /* open the file */ 1065 SVN_ERR(svn_io_file_open(file, svn_dirent_join(revprops->folder, 1066 new_filename, 1067 pool), 1068 APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool)); 1069 1070 return SVN_NO_ERROR; 1071} 1072 1073/* For revision REV in filesystem FS, set the revision properties to 1074 * PROPLIST. Return a new file in *TMP_PATH that the caller shall move 1075 * to *FINAL_PATH to make the change visible. Files to be deleted will 1076 * be listed in *FILES_TO_DELETE which may remain unchanged / unallocated. 1077 * Use POOL for allocations. 1078 */ 1079static svn_error_t * 1080write_packed_revprop(const char **final_path, 1081 const char **tmp_path, 1082 apr_array_header_t **files_to_delete, 1083 svn_fs_t *fs, 1084 svn_revnum_t rev, 1085 apr_hash_t *proplist, 1086 apr_pool_t *pool) 1087{ 1088 fs_fs_data_t *ffd = fs->fsap_data; 1089 packed_revprops_t *revprops; 1090 svn_stream_t *stream; 1091 apr_file_t *file; 1092 svn_stringbuf_t *serialized; 1093 apr_size_t new_total_size; 1094 int changed_index; 1095 1096 /* read contents of the current pack file */ 1097 SVN_ERR(read_pack_revprop(&revprops, fs, rev, TRUE, FALSE, pool)); 1098 1099 /* serialize the new revprops */ 1100 serialized = svn_stringbuf_create_empty(pool); 1101 stream = svn_stream_from_stringbuf(serialized, pool); 1102 SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); 1103 SVN_ERR(svn_stream_close(stream)); 1104 1105 /* calculate the size of the new data */ 1106 changed_index = (int)(rev - revprops->start_revision); 1107 new_total_size = revprops->total_size - revprops->serialized_size 1108 + serialized->len 1109 + (revprops->offsets->nelts + 2) * SVN_INT64_BUFFER_SIZE; 1110 1111 APR_ARRAY_IDX(revprops->sizes, changed_index, apr_size_t) = serialized->len; 1112 1113 /* can we put the new data into the same pack as the before? */ 1114 if ( new_total_size < ffd->revprop_pack_size 1115 || revprops->sizes->nelts == 1) 1116 { 1117 /* simply replace the old pack file with new content as we do it 1118 * in the non-packed case */ 1119 1120 *final_path = svn_dirent_join(revprops->folder, revprops->filename, 1121 pool); 1122 SVN_ERR(svn_io_open_unique_file3(&file, tmp_path, revprops->folder, 1123 svn_io_file_del_none, pool, pool)); 1124 SVN_ERR(repack_revprops(fs, revprops, 0, revprops->sizes->nelts, 1125 changed_index, serialized, new_total_size, 1126 file, pool)); 1127 } 1128 else 1129 { 1130 /* split the pack file into two of roughly equal size */ 1131 int right_count, left_count, i; 1132 1133 int left = 0; 1134 int right = revprops->sizes->nelts - 1; 1135 apr_size_t left_size = 2 * SVN_INT64_BUFFER_SIZE; 1136 apr_size_t right_size = 2 * SVN_INT64_BUFFER_SIZE; 1137 1138 /* let left and right side grow such that their size difference 1139 * is minimal after each step. */ 1140 while (left <= right) 1141 if ( left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_size_t) 1142 < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_size_t)) 1143 { 1144 left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_size_t) 1145 + SVN_INT64_BUFFER_SIZE; 1146 ++left; 1147 } 1148 else 1149 { 1150 right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_size_t) 1151 + SVN_INT64_BUFFER_SIZE; 1152 --right; 1153 } 1154 1155 /* since the items need much less than SVN_INT64_BUFFER_SIZE 1156 * bytes to represent their length, the split may not be optimal */ 1157 left_count = left; 1158 right_count = revprops->sizes->nelts - left; 1159 1160 /* if new_size is large, one side may exceed the pack size limit. 1161 * In that case, split before and after the modified revprop.*/ 1162 if ( left_size > ffd->revprop_pack_size 1163 || right_size > ffd->revprop_pack_size) 1164 { 1165 left_count = changed_index; 1166 right_count = revprops->sizes->nelts - left_count - 1; 1167 } 1168 1169 /* write the new, split files */ 1170 if (left_count) 1171 { 1172 SVN_ERR(repack_file_open(&file, fs, revprops, 0, 1173 left_count, files_to_delete, pool)); 1174 SVN_ERR(repack_revprops(fs, revprops, 0, left_count, 1175 changed_index, serialized, new_total_size, 1176 file, pool)); 1177 } 1178 1179 if (left_count + right_count < revprops->sizes->nelts) 1180 { 1181 SVN_ERR(repack_file_open(&file, fs, revprops, changed_index, 1182 changed_index + 1, files_to_delete, 1183 pool)); 1184 SVN_ERR(repack_revprops(fs, revprops, changed_index, 1185 changed_index + 1, 1186 changed_index, serialized, new_total_size, 1187 file, pool)); 1188 } 1189 1190 if (right_count) 1191 { 1192 SVN_ERR(repack_file_open(&file, fs, revprops, 1193 revprops->sizes->nelts - right_count, 1194 revprops->sizes->nelts, 1195 files_to_delete, pool)); 1196 SVN_ERR(repack_revprops(fs, revprops, 1197 revprops->sizes->nelts - right_count, 1198 revprops->sizes->nelts, changed_index, 1199 serialized, new_total_size, file, 1200 pool)); 1201 } 1202 1203 /* write the new manifest */ 1204 *final_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool); 1205 SVN_ERR(svn_io_open_unique_file3(&file, tmp_path, revprops->folder, 1206 svn_io_file_del_none, pool, pool)); 1207 stream = svn_stream_from_aprfile2(file, TRUE, pool); 1208 for (i = 0; i < revprops->manifest->nelts; ++i) 1209 { 1210 const char *filename = APR_ARRAY_IDX(revprops->manifest, i, 1211 const char*); 1212 SVN_ERR(svn_stream_printf(stream, pool, "%s\n", filename)); 1213 } 1214 SVN_ERR(svn_stream_close(stream)); 1215 if (ffd->flush_to_disk) 1216 SVN_ERR(svn_io_file_flush_to_disk(file, pool)); 1217 SVN_ERR(svn_io_file_close(file, pool)); 1218 } 1219 1220 return SVN_NO_ERROR; 1221} 1222 1223/* Set the revision property list of revision REV in filesystem FS to 1224 PROPLIST. Use POOL for temporary allocations. */ 1225svn_error_t * 1226svn_fs_fs__set_revision_proplist(svn_fs_t *fs, 1227 svn_revnum_t rev, 1228 apr_hash_t *proplist, 1229 apr_pool_t *pool) 1230{ 1231 svn_boolean_t is_packed; 1232 const char *final_path; 1233 const char *tmp_path; 1234 const char *perms_reference; 1235 apr_array_header_t *files_to_delete = NULL; 1236 1237 SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, pool)); 1238 1239 /* this info will not change while we hold the global FS write lock */ 1240 is_packed = svn_fs_fs__is_packed_revprop(fs, rev); 1241 1242 /* Serialize the new revprop data */ 1243 if (is_packed) 1244 SVN_ERR(write_packed_revprop(&final_path, &tmp_path, &files_to_delete, 1245 fs, rev, proplist, pool)); 1246 else 1247 SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path, 1248 fs, rev, proplist, pool)); 1249 1250 /* Previous cache contents is invalid now. */ 1251 svn_fs_fs__reset_revprop_cache(fs); 1252 1253 /* We use the rev file of this revision as the perms reference, 1254 * because when setting revprops for the first time, the revprop 1255 * file won't exist and therefore can't serve as its own reference. 1256 * (Whereas the rev file should already exist at this point.) 1257 */ 1258 perms_reference = svn_fs_fs__path_rev_absolute(fs, rev, pool); 1259 1260 /* Now, switch to the new revprop data. */ 1261 SVN_ERR(switch_to_new_revprop(fs, final_path, tmp_path, perms_reference, 1262 files_to_delete, pool)); 1263 1264 return SVN_NO_ERROR; 1265} 1266 1267/* Return TRUE, if for REVISION in FS, we can find the revprop pack file. 1268 * Use POOL for temporary allocations. 1269 * Set *MISSING, if the reason is a missing manifest or pack file. 1270 */ 1271svn_boolean_t 1272svn_fs_fs__packed_revprop_available(svn_boolean_t *missing, 1273 svn_fs_t *fs, 1274 svn_revnum_t revision, 1275 apr_pool_t *pool) 1276{ 1277 fs_fs_data_t *ffd = fs->fsap_data; 1278 svn_stringbuf_t *content = NULL; 1279 1280 /* try to read the manifest file */ 1281 const char *folder 1282 = svn_fs_fs__path_revprops_pack_shard(fs, revision, pool); 1283 const char *manifest_path = svn_dirent_join(folder, PATH_MANIFEST, pool); 1284 1285 svn_error_t *err = svn_fs_fs__try_stringbuf_from_file(&content, 1286 missing, 1287 manifest_path, 1288 FALSE, 1289 pool); 1290 1291 /* if the manifest cannot be read, consider the pack files inaccessible 1292 * even if the file itself exists. */ 1293 if (err) 1294 { 1295 svn_error_clear(err); 1296 return FALSE; 1297 } 1298 1299 if (*missing) 1300 return FALSE; 1301 1302 /* parse manifest content until we find the entry for REVISION. 1303 * Revision 0 is never packed. */ 1304 revision = revision < ffd->max_files_per_dir 1305 ? revision - 1 1306 : revision % ffd->max_files_per_dir; 1307 while (content->data) 1308 { 1309 char *next = strchr(content->data, '\n'); 1310 if (next) 1311 { 1312 *next = 0; 1313 ++next; 1314 } 1315 1316 if (revision-- == 0) 1317 { 1318 /* the respective pack file must exist (and be a file) */ 1319 svn_node_kind_t kind; 1320 err = svn_io_check_path(svn_dirent_join(folder, content->data, 1321 pool), 1322 &kind, pool); 1323 if (err) 1324 { 1325 svn_error_clear(err); 1326 return FALSE; 1327 } 1328 1329 *missing = kind == svn_node_none; 1330 return kind == svn_node_file; 1331 } 1332 1333 content->data = next; 1334 } 1335 1336 return FALSE; 1337} 1338 1339 1340/****** Packing FSFS shards *********/ 1341 1342svn_error_t * 1343svn_fs_fs__copy_revprops(const char *pack_file_dir, 1344 const char *pack_filename, 1345 const char *shard_path, 1346 svn_revnum_t start_rev, 1347 svn_revnum_t end_rev, 1348 apr_array_header_t *sizes, 1349 apr_size_t total_size, 1350 int compression_level, 1351 svn_boolean_t flush_to_disk, 1352 svn_cancel_func_t cancel_func, 1353 void *cancel_baton, 1354 apr_pool_t *scratch_pool) 1355{ 1356 svn_stream_t *pack_stream; 1357 apr_file_t *pack_file; 1358 svn_revnum_t rev; 1359 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 1360 1361 /* create empty data buffer and a write stream on top of it */ 1362 svn_stringbuf_t *uncompressed 1363 = svn_stringbuf_create_ensure(total_size, scratch_pool); 1364 svn_stringbuf_t *compressed 1365 = svn_stringbuf_create_empty(scratch_pool); 1366 pack_stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); 1367 1368 /* write the pack file header */ 1369 SVN_ERR(serialize_revprops_header(pack_stream, start_rev, sizes, 0, 1370 sizes->nelts, iterpool)); 1371 1372 /* Some useful paths. */ 1373 SVN_ERR(svn_io_file_open(&pack_file, svn_dirent_join(pack_file_dir, 1374 pack_filename, 1375 scratch_pool), 1376 APR_WRITE | APR_CREATE, APR_OS_DEFAULT, 1377 scratch_pool)); 1378 1379 /* Iterate over the revisions in this shard, squashing them together. */ 1380 for (rev = start_rev; rev <= end_rev; rev++) 1381 { 1382 const char *path; 1383 svn_stream_t *stream; 1384 apr_file_t *file; 1385 1386 svn_pool_clear(iterpool); 1387 1388 /* Construct the file name. */ 1389 path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), 1390 iterpool); 1391 1392 /* Copy all the bits from the non-packed revprop file to the end of 1393 * the pack file. Use unbuffered apr_file_t since we're going to 1394 * write using 16kb chunks. */ 1395 SVN_ERR(svn_io_file_open(&file, path, APR_READ, APR_OS_DEFAULT, 1396 iterpool)); 1397 stream = svn_stream_from_aprfile2(file, FALSE, iterpool); 1398 SVN_ERR(svn_stream_copy3(stream, pack_stream, 1399 cancel_func, cancel_baton, iterpool)); 1400 } 1401 1402 /* flush stream buffers to content buffer */ 1403 SVN_ERR(svn_stream_close(pack_stream)); 1404 1405 /* compress the content (or just store it for COMPRESSION_LEVEL 0) */ 1406 SVN_ERR(svn__compress_zlib(uncompressed->data, uncompressed->len, 1407 compressed, compression_level)); 1408 1409 /* write the pack file content to disk */ 1410 SVN_ERR(svn_io_file_write_full(pack_file, compressed->data, compressed->len, 1411 NULL, scratch_pool)); 1412 if (flush_to_disk) 1413 SVN_ERR(svn_io_file_flush_to_disk(pack_file, scratch_pool)); 1414 SVN_ERR(svn_io_file_close(pack_file, scratch_pool)); 1415 1416 svn_pool_destroy(iterpool); 1417 1418 return SVN_NO_ERROR; 1419} 1420 1421svn_error_t * 1422svn_fs_fs__pack_revprops_shard(const char *pack_file_dir, 1423 const char *shard_path, 1424 apr_int64_t shard, 1425 int max_files_per_dir, 1426 apr_int64_t max_pack_size, 1427 int compression_level, 1428 svn_boolean_t flush_to_disk, 1429 svn_cancel_func_t cancel_func, 1430 void *cancel_baton, 1431 apr_pool_t *scratch_pool) 1432{ 1433 const char *manifest_file_path, *pack_filename = NULL; 1434 apr_file_t *manifest_file; 1435 svn_stream_t *manifest_stream; 1436 svn_revnum_t start_rev, end_rev, rev; 1437 apr_size_t total_size; 1438 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 1439 apr_array_header_t *sizes; 1440 1441 /* Sanitize config file values. */ 1442 apr_size_t max_size = (apr_size_t)MIN(MAX(max_pack_size, 1), 1443 SVN_MAX_OBJECT_SIZE); 1444 1445 /* Some useful paths. */ 1446 manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, 1447 scratch_pool); 1448 1449 /* Remove any existing pack file for this shard, since it is incomplete. */ 1450 SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton, 1451 scratch_pool)); 1452 1453 /* Create the new directory and manifest file stream. */ 1454 SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool)); 1455 1456 SVN_ERR(svn_io_file_open(&manifest_file, manifest_file_path, 1457 APR_WRITE | APR_BUFFERED | APR_CREATE | APR_EXCL, 1458 APR_OS_DEFAULT, scratch_pool)); 1459 manifest_stream = svn_stream_from_aprfile2(manifest_file, TRUE, 1460 scratch_pool); 1461 1462 /* revisions to handle. Special case: revision 0 */ 1463 start_rev = (svn_revnum_t) (shard * max_files_per_dir); 1464 end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1); 1465 if (start_rev == 0) 1466 ++start_rev; 1467 /* Special special case: if max_files_per_dir is 1, then at this point 1468 start_rev == 1 and end_rev == 0 (!). Fortunately, everything just 1469 works. */ 1470 1471 /* initialize the revprop size info */ 1472 sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_size_t)); 1473 total_size = 2 * SVN_INT64_BUFFER_SIZE; 1474 1475 /* Iterate over the revisions in this shard, determine their size and 1476 * squashing them together into pack files. */ 1477 for (rev = start_rev; rev <= end_rev; rev++) 1478 { 1479 apr_finfo_t finfo; 1480 const char *path; 1481 1482 svn_pool_clear(iterpool); 1483 1484 /* Get the size of the file. */ 1485 path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), 1486 iterpool); 1487 SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool)); 1488 1489 /* If we already have started a pack file and this revprop cannot be 1490 * appended to it, write the previous pack file. Note this overflow 1491 * check works because we enforced MAX_SIZE <= SVN_MAX_OBJECT_SIZE. */ 1492 if (sizes->nelts != 0 1493 && ( finfo.size > max_size 1494 || total_size > max_size 1495 || SVN_INT64_BUFFER_SIZE + finfo.size > max_size - total_size)) 1496 { 1497 SVN_ERR(svn_fs_fs__copy_revprops(pack_file_dir, pack_filename, 1498 shard_path, start_rev, rev-1, 1499 sizes, total_size, 1500 compression_level, flush_to_disk, 1501 cancel_func, cancel_baton, 1502 iterpool)); 1503 1504 /* next pack file starts empty again */ 1505 apr_array_clear(sizes); 1506 total_size = 2 * SVN_INT64_BUFFER_SIZE; 1507 start_rev = rev; 1508 } 1509 1510 /* Update the manifest. Allocate a file name for the current pack 1511 * file if it is a new one */ 1512 if (sizes->nelts == 0) 1513 pack_filename = apr_psprintf(scratch_pool, "%ld.0", rev); 1514 1515 SVN_ERR(svn_stream_printf(manifest_stream, iterpool, "%s\n", 1516 pack_filename)); 1517 1518 /* add to list of files to put into the current pack file */ 1519 APR_ARRAY_PUSH(sizes, apr_size_t) = finfo.size; 1520 total_size += SVN_INT64_BUFFER_SIZE + finfo.size; 1521 } 1522 1523 /* write the last pack file */ 1524 if (sizes->nelts != 0) 1525 SVN_ERR(svn_fs_fs__copy_revprops(pack_file_dir, pack_filename, 1526 shard_path, start_rev, rev-1, 1527 sizes, (apr_size_t)total_size, 1528 compression_level, flush_to_disk, 1529 cancel_func, cancel_baton, iterpool)); 1530 1531 /* flush the manifest file to disk and update permissions */ 1532 SVN_ERR(svn_stream_close(manifest_stream)); 1533 if (flush_to_disk) 1534 SVN_ERR(svn_io_file_flush_to_disk(manifest_file, iterpool)); 1535 SVN_ERR(svn_io_file_close(manifest_file, iterpool)); 1536 SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool)); 1537 1538 svn_pool_destroy(iterpool); 1539 1540 return SVN_NO_ERROR; 1541} 1542 1543svn_error_t * 1544svn_fs_fs__delete_revprops_shard(const char *shard_path, 1545 apr_int64_t shard, 1546 int max_files_per_dir, 1547 svn_cancel_func_t cancel_func, 1548 void *cancel_baton, 1549 apr_pool_t *scratch_pool) 1550{ 1551 if (shard == 0) 1552 { 1553 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 1554 int i; 1555 1556 /* delete all files except the one for revision 0 */ 1557 for (i = 1; i < max_files_per_dir; ++i) 1558 { 1559 const char *path; 1560 svn_pool_clear(iterpool); 1561 1562 path = svn_dirent_join(shard_path, 1563 apr_psprintf(iterpool, "%d", i), 1564 iterpool); 1565 if (cancel_func) 1566 SVN_ERR(cancel_func(cancel_baton)); 1567 1568 SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); 1569 } 1570 1571 svn_pool_destroy(iterpool); 1572 } 1573 else 1574 SVN_ERR(svn_io_remove_dir2(shard_path, TRUE, 1575 cancel_func, cancel_baton, scratch_pool)); 1576 1577 return SVN_NO_ERROR; 1578} 1579 1580